✨ Update LSP types to more recent version (#324)

See #324

---------

Signed-off-by: JonahSussman <sussmanjonah@gmail.com>
Signed-off-by: Jonah Sussman <42743659+JonahSussman@users.noreply.github.com>

README.md

@@ -61,9 +61,9 @@ Flags:
 
 ## Code Base Starting Point
 
- Using the LSP/Protocal from ACME https://github.com/fhs/acme-lsp and stripping out anything related to serving, proxy or anything. Just keeping the types for communication
+Using the LSP/Protocal from Golang https://github.com/golang/tools/tree/master/gopls/internal/lsp/protocol and stripping out anything related to serving, proxy or anything. Just keeping the types for communication
 
- Using JSONRPC2 from google.org/x/tools/internal. Copied and removed anything to do with serving.
+Using JSONRPC2 from google.org/x/tools/internal. Copied and removed anything to do with serving.
 
 
 ## Code of Conduct

external-providers/generic-external-provider/pkg/generic/service_client.go

@@ -48,7 +48,7 @@ func (p *genericServiceClient) Evaluate(ctx context.Context, cap string, conditi
 
 	incidents := []provider.IncidentContext{}
 	for _, s := range symbols {
-		references := p.GetAllReferences(ctx, s.Location)
+		references := p.GetAllReferences(ctx, s.Location.Value.(protocol.Location))
 		for _, ref := range references {
 			// Look for things that are in the location loaded, //Note may need to filter out vendor at some point
 			if strings.Contains(ref.URI, p.config.Location) {
@@ -100,8 +100,8 @@ func processFile(path string, regex *regexp.Regexp, positionsChan chan<- protoco
 						URI: fmt.Sprintf("file://%s", absPath),
 					},
 					Position: protocol.Position{
-						Line:      float64(lineNumber),
-						Character: float64(loc[1]),
+						Line:      uint32(lineNumber),
+						Character: uint32(loc[1]),
 					},
 				}
 			}
@@ -190,7 +190,12 @@ func (p *genericServiceClient) GetAllSymbols(ctx context.Context, query string)
 				fmt.Printf("Error rpc: %v", err)
 			}
 			for _, r := range res {
-				symbols = append(symbols, protocol.WorkspaceSymbol{Location: r})
+				symbols = append(symbols, protocol.WorkspaceSymbol{
+					Location: protocol.OrPLocation_workspace_symbol{
+						Value: r,
+					},
+					// Location: r
+				})
 			}
 		}
 	}
@@ -223,13 +228,12 @@ func (p *genericServiceClient) initialization(ctx context.Context, log logr.Logg
 		panic(1)
 	}
 
-	params := &protocol.InitializeParams{
-		//TODO(shawn-hurley): add ability to parse path to URI in a real supported way
-		RootURI:      fmt.Sprintf("file://%v", abs),
-		Capabilities: protocol.ClientCapabilities{},
-		ExtendedClientCapilities: map[string]interface{}{
-			"classFileContentsSupport": true,
-		},
+	//TODO(shawn-hurley): add ability to parse path to URI in a real supported way
+	params := &protocol.InitializeParams{}
+	params.RootURI = fmt.Sprintf("file://%v", abs)
+	params.Capabilities = protocol.ClientCapabilities{}
+	params.ExtendedClientCapilities = map[string]interface{}{
+		"classFileContentsSupport": true,
 	}
 
 	var result protocol.InitializeResult

lsp/protocol/README.md

@@ -0,0 +1,15 @@
+# Protocol generator tool
+
+Originally from: https://github.com/golang/tools/tree/master/gopls/internal/lsp/protocol
+Commit id: If4c85191760baef916911130ca315773d2adda1f
+
+## How to use
+
+If there is an update to the LSP specification, navigate to the `protocol` directory and run `go generate`. This will generate all the types needed for the analyzer.  You need to add `ExtendedClientCapilities map[string]interface{} `json:"extendedClientCapabilities"`` to XInitializeParams struct. Other than that, you should be good to go.
+
+## Changes
+
+- Commented out `writeclient()` and `writeserver()` in generate/main.go
+- Commented out `"Or_WorkspaceFoldersServerCapabilities_changeNotifications": "string",` in generate/tables.go
+- Changed `type DocumentURI string` to `type DocumentURI = string` in generate/output.go
+- Need to add `ExtendedClientCapilities map[string]interface{} `json:"extendedClientCapabilities"`` to XInitializeParams struct. Need to figure out why needed. Can remove if not.

lsp/protocol/codeactionkind.go

@@ -0,0 +1,11 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package protocol
+
+// Custom code actions that aren't explicitly stated in LSP
+const (
+	GoTest CodeActionKind = "goTest"
+	// TODO: Add GoGenerate, RegenerateCgo etc.
+)

lsp/protocol/doc.go

@@ -2,8 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package protocol contains the structs that map directly to the wire format
-// of the "Language Server Protocol".
+//go:generate go run ./generate
+
+// Package protocol contains the structs that map directly to the
+// request and response messages of the Language Server Protocol.
 //
 // It is a literal transcription, with unmodified comments, and only the changes
 // required to make it go code.

lsp/protocol/enums.go

@@ -10,7 +10,6 @@ import (
 
 var (
 	namesTextDocumentSyncKind   [int(Incremental) + 1]string
-	namesInitializeError        [int(UnknownProtocolVersion) + 1]string
 	namesMessageType            [int(Log) + 1]string
 	namesFileChangeType         [int(Deleted) + 1]string
 	namesWatchKind              [int(WatchDelete) + 1]string
@@ -29,8 +28,6 @@ func init() {
 	namesTextDocumentSyncKind[int(Full)] = "Full"
 	namesTextDocumentSyncKind[int(Incremental)] = "Incremental"
 
-	namesInitializeError[int(UnknownProtocolVersion)] = "UnknownProtocolVersion"
-
 	namesMessageType[int(Error)] = "Error"
 	namesMessageType[int(Warning)] = "Warning"
 	namesMessageType[int(Info)] = "Info"
@@ -149,14 +146,6 @@ func ParseTextDocumentSyncKind(s string) TextDocumentSyncKind {
 	return TextDocumentSyncKind(parseEnum(s, namesTextDocumentSyncKind[:]))
 }
 
-func (e InitializeError) Format(f fmt.State, c rune) {
-	formatEnum(f, c, int(e), namesInitializeError[:], "InitializeError")
-}
-
-func ParseInitializeError(s string) InitializeError {
-	return InitializeError(parseEnum(s, namesInitializeError[:]))
-}
-
 func (e MessageType) Format(f fmt.State, c rune) {
 	formatEnum(f, c, int(e), namesMessageType[:], "MessageType")
 }
@@ -173,10 +162,6 @@ func ParseFileChangeType(s string) FileChangeType {
 	return FileChangeType(parseEnum(s, namesFileChangeType[:]))
 }
 
-func (e WatchKind) Format(f fmt.State, c rune) {
-	formatEnum(f, c, int(e), namesWatchKind[:], "WatchKind")
-}
-
 func ParseWatchKind(s string) WatchKind {
 	return WatchKind(parseEnum(s, namesWatchKind[:]))
 }

lsp/protocol/generate/README.md

@@ -0,0 +1,144 @@
+# LSP Support for gopls
+
+## The protocol
+
+The LSP protocol exchanges json-encoded messages between the client and the server.
+(gopls is the server.) The messages are either Requests, which require Responses, or
+Notifications, which generate no response. Each Request or Notification has a method name
+such as "textDocument/hover" that indicates its meaning and determines which function in the server will handle it.
+The protocol is described in a
+[web page](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.18/specification/),
+in words, and in a json file (metaModel.json) available either linked towards the bottom of the
+web page, or in the vscode-languageserver-node repository. This code uses the latter so the
+exact version can be tied to a githash. By default, the command will download the `github.com/microsoft/vscode-languageserver-node` repository to a temporary directory.
+
+The specification has five sections
+
+1. Requests, which describe the Request and Response types for request methods (e.g., *textDocument/didChange*),
+2. Notifications, which describe the Request types for notification methods,
+3. Structures, which describe named struct-like types,
+4. TypeAliases, which describe type aliases,
+5. Enumerations, which describe named constants.
+
+Requests and Notifications are tagged with a Method (e.g., `"textDocument/hover"`).
+The specification does not specify the names of the functions that handle the messages. These
+names are specified by the `methodNames` map. Enumerations generate Go `const`s, but
+in Typescript they are scoped to namespaces, while in Go they are scoped to a package, so the Go names
+may need to be modified to avoid name collisions. (See the `disambiguate` map, and its use.)
+
+Finally, the specified types are Typescript types, which are quite different from Go types.
+
+### Optionality
+
+The specification can mark fields in structs as Optional. The client distinguishes between missing
+fields and `null` fields in some cases. The Go translation for an optional type
+should be making sure the field's value
+can be `nil`, and adding the json tag `,omitempty`. The former condition would be satisfied by
+adding `*` to the field's type if the type is not a reference type.
+
+### Types
+
+The specification uses a number of different types, only a few of which correspond directly to Go types.
+The specification's types are "base", "reference", "map", "literal", "stringLiteral", "tuple", "and", "or".
+The "base" types correspond directly to Go types, although some Go types needs to be chosen for `URI` and `DocumentUri`. (The "base" types`RegExp`, `BooleanLiteral`, `NumericLiteral` never occur.)
+
+"reference" types are the struct-like types in the Structures section of the specification. The given
+names are suitable for Go to use, except the code needs to change names like `_Initialze` to `XInitialize` so
+they are exported for json marshaling and unmarshaling.
+
+"map" types are just like Go. (The key type in all of them is `DocumentUri`.)
+
+"stringLiteral" types are types whose type name and value are a single string. The chosen Go equivalent
+is to make the type `string` and the value a constant. (The alternative would be to generate a new
+named type, which seemed redundant.)
+
+"literal" types are like Go anonymous structs, so they have to be given a name. (All instances
+of the remaining types have to be given names. One approach is to construct the name from the components
+of the type, but this leads to misleading punning, and is unstable if components are added. The other approach
+is to construct the name from the context of the definition, that is, from the types it is defined within.
+For instance `Lit__InitializeParams_clientInfo` is the "literal" type at the
+`clientInfo` field in the `_InitializeParams`
+struct. Although this choice is sensitive to the ordering of the components, the code uses this approach,
+presuming that reordering components is an unlikely protocol change.)
+
+"tuple" types are generated as Go structs. (There is only one, with two `uint32` fields.)
+
+"and" types are Go structs with embedded type names. (There is only one, `And_Param_workspace_configuration`.)
+
+"or" types are the most complicated. There are a lot of them and there is no simple Go equivalent.
+They are defined as structs with a single `Value interface{}` field and custom json marshaling
+and unmarshaling code. Users can assign anything to `Value` but the type will be checked, and
+correctly marshaled, by the custom marshaling code. The unmarshaling code checks types, so `Value`
+will have one of the permitted types. (`nil` is always allowed.) There are about 40 "or" types that
+have a single non-null component, and these are converted to the component type.
+
+## Processing
+
+The code parses the json specification file, and scans all the types. It assigns names, as described
+above, to the types that are unnamed in the specification, and constructs Go equivalents as required.
+(Most of this code is in typenames.go.)
+
+There are four output files. tsclient.go and tsserver.go contain the definition and implementation
+of the `protocol.Client` and `protocol.Server` types and the code that dispatches on the Method
+of the Request or Notification. tsjson.go contains the custom marshaling and unmarshaling code.
+And tsprotocol.go contains the type and const definitions.
+
+### Accommodating gopls
+
+As the code generates output, mostly in generateoutput.go and main.go,
+it makes adjustments so that no changes are required to the existing Go code.
+(Organizing the computation this way makes the code's structure simpler, but results in
+a lot of unused types.)
+There are three major classes of these adjustments, and leftover special cases.
+
+The first major
+adjustment is to change generated type names to the ones gopls expects. Some of these don't change the
+semantics of the type, just the name.
+But for historical reasons a lot of them replace "or" types by a single
+component of the type. (Until fairly recently Go only saw or used only one of components.)
+The `goplsType` map in tables.go controls this process.
+
+The second major adjustment is to the types of fields of structs, which is done using the
+`renameProp` map in tables.go.
+
+The third major adjustment handles optionality, controlling `*` and `,omitempty` placement when
+the default rules don't match what gopls is expecting. (The map is `goplsStar`, also in tables.go)
+(If the intermediate components in expressions of the form `A.B.C.S` were optional, the code would need
+a lot of useless checking for nils. Typescript has a language construct to avoid most checks.)
+
+Then there are some additional special cases. There are a few places with adjustments to avoid
+recursive types. For instance `LSPArray` is `[]LSPAny`, but `LSPAny` is an "or" type including `LSPArray`.
+The solution is to make `LSPAny` an `interface{}`. Another instance is `_InitializeParams.trace`
+whose type is an "or" of 3 stringLiterals, which just becomes a `string`.
+
+### Checking
+
+`TestAll(t *testing.T)` checks that there are no unexpected fields in the json specification.
+
+While the code is executing, it checks that all the entries in the maps in tables.go are used.
+It also checks that the entries in `renameProp` and `goplsStar` are not redundant.
+
+As a one-time check on the first release of this code, diff-ing the existing and generated tsclient.go
+and tsserver.go code results in only whitespace and comment diffs. The existing and generated
+tsprotocol.go differ in whitespace and comments, and in a substantial number of new type definitions
+that the older, more heuristic, code did not generate. (And the unused type `_InitializeParams` differs
+slightly between the new and the old, and is not worth fixing.)
+
+### Some history
+
+The original stub code was written by hand, but with the protocol under active development, that
+couldn't last. The web page existed before the json specification, but it lagged the implementation
+and was hard to process by machine. So the earlier version of the generating code was written in Typescript, and
+used the Typescript compiler's API to parse the protocol code in the repository.
+It then used a set of heuristics
+to pick out the elements of the protocol, and another set of overlapping heuristics to create the Go code.
+The output was functional, but idiosyncratic, and the code was fragile and barely maintainable.
+
+### The future
+
+Most of the adjustments using the maps in tables.go could be removed by making changes, mostly to names,
+in the gopls code. Using more "or" types in gopls requires more elaborate, but stereotyped, changes.
+But even without all the adjustments, making this its own module would face problems; a number of
+dependencies would have to be factored out. And, it is fragile. The custom unmarshaling code knows what
+types it expects. A design that return an 'any' on unexpected types would match the json
+'ignore unexpected values' philosophy better, but the Go code would need extra checking.