feat: local completion index (#419)

* feat: refactor symbols db

* feat: provide project id

* feat: build import context

* feat: provide monitoring range

* feat: use cache

* feat: implement GOROOT indexer

* fix: fix comment validator

* fix: fix package filter

* feat: update response model

* feat: debug parser

* fix: fix doc generation

* fix: fix doc gen for types

* feat: add filter for builtins

* feat: keep only func signature

* feat: finish docutil

* fix: fix IsGoSourceFile

* feat: use predefined buff sizes

* feat: restructure pkgindex

* chore: require protobuf for future developments

* feat: use structure of arrays to save space

* feat: add tests

* chore: goimports

* chore: add pprof

* chore: fix docs

* chore: update js types

* fix: respect build constraints during scan

* feat: flatten symbol source struct

* fix: fix indexing and fallbacks

* chore: make monaco-editor work in vitest

* fix: fix imports traversal

* fix: fix single import parse

* fix: fix start line number

* fix: add additional index

* chore: use built-in cache in setup-go

* feat: show completion after dot

* fix: fix PR job

* fix: linter

* fix: linter

* fix: linter

* chore: bump golangci-lint

* chore: bump golangci-lint

* chore: use node cache

* fix: tests

Author: x1unix

.github/workflows/pull_request.yml

@@ -12,21 +12,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - id: go-cache-paths
-        run: |
-          echo "::set-output name=go-build::$(go env GOCACHE)"
-          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
       - uses: actions/checkout@v4
-      - name: Go Build Cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ steps.go-cache-paths.outputs.go-build }}
-          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
-      - name: Go Mod Cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ steps.go-cache-paths.outputs.go-mod }}
-          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
       - uses: actions/cache@v4
         env:
           cache-name: npm-cache
@@ -35,6 +21,12 @@ jobs:
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/yarn.lock') }}
           restore-keys: |
             ${{ runner.os }}-build-${{ env.cache-name }}-
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          cache: 'yarn'
+          node-version-file: '.nvmrc'
+          cache-dependency-path: 'web/yarn.lock'
       - name: Install modules
         run: yarn
         working-directory: ./web
@@ -47,10 +39,11 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: "^1.21.0"
+          go-version: "^1.23.0"
+          cache-dependency-path: go.sum
       - name: golangci-lint
         uses: golangci/[email protected]
         with:
-          version: v1.54.2
+          version: v1.61.0
       - run: go version
       - run: go test ./...

HACKING.md

@@ -74,6 +74,9 @@ Please ensure that you have installed:
 * [Node Version Manager](https://github.com/nvm-sh/nvm) or Node.js 20 (`lts/iron`)
 * [Yarn](https://yarnpkg.com/) package manager.
 * Go 1.22+
+* Protobuf:
+    * [protoc](https://developers.google.com/protocol-buffers)
+    * [Protobuf Go plugins](https://grpc.io/docs/languages/go/quickstart/)
 
 ### First-time setup
 
@@ -83,7 +86,7 @@ Run following commands to configure a just cloned project:
 |-------------------|----------------------------------------------------------|
 | `make preinstall` | Installs NPM packages for a web app.                     |
 | `make wasm`       | Builds WebAssembly binaries used by the web app.         |
-| `make pkg-index`  | Generates Go packages index for autocomplete in web app. |
+| `make go-index`   | Generates Go packages index for autocomplete in web app. |
 
 ### Running Project
 

build.mk

@@ -32,10 +32,15 @@ check-go:
 		exit 1; \
 	fi
 
-.PHONY: pkg-index
-pkg-index:
-	@echo ":: Generating Go packages index..." && \
-	$(GO) run ./tools/pkgindexer -o $(UI)/public/data/imports.json
+.PHONY: imports-index
+imports-index:
+	@echo ":: Generating Go imports index..." && \
+	$(GO) run ./tools/pkgindexer imports -o $(UI)/public/data/imports.json $(OPTS)
+
+.PHONY: go-index
+go-index:
+	@echo ":: Generating Go symbols index..." && \
+	$(GO) run ./tools/pkgindexer index -o $(UI)/public/data/go-index.json $(OPTS)
 
 .PHONY:check-yarn
 check-yarn:
@@ -68,12 +73,12 @@ analyzer.wasm:
 wasm: wasm_exec.js analyzer.wasm 
 
 .PHONY: build
-build: check-go check-yarn clean preinstall gen build-server wasm pkg-index build-ui
+build: check-go check-yarn clean preinstall gen build-server wasm go-index imports-index build-ui
 	@echo ":: Copying assets..." && \
 	cp -rfv ./data $(TARGET)/data && \
 	mv -v $(UI)/build $(TARGET)/public && \
 	echo ":: Build done - $(TARGET)"
 
 .PHONY:gen
 gen:
-	@find . -name '*_gen.go' -exec go generate -v {} \;
+	@find . -name 'generate.go' -exec go generate -v {} \;

go.mod

@@ -1,6 +1,8 @@
 module github.com/x1unix/go-playground
 
-go 1.21
+go 1.23
+
+toolchain go1.23.1
 
 require (
 	github.com/TheZeroSlave/zapsentry v1.10.0
@@ -25,12 +27,15 @@ require (
 	github.com/benbjohnson/clock v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/getsentry/sentry-go v0.13.0 // indirect
+	github.com/hashicorp/go-set/v3 v3.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -14,6 +14,8 @@ github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/hashicorp/go-set/v3 v3.0.0 h1:CaJBQvQCOWoftrBcDt7Nwgo0kdpmrKxar/x2o6pV9JA=
+github.com/hashicorp/go-set/v3 v3.0.0/go.mod h1:IEghM2MpE5IaNvL+D7X480dfNtxjRXZ6VMpK3C8s2ok=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
@@ -88,6 +90,8 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -100,6 +104,12 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 h1:F29+wU6Ee6qgu9TddPgooOdaqsxTMunOoj8KA5yuS5A=
+google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1/go.mod h1:5KF+wpkbTSbGcR9zteSqZV6fqFOWBl4Yde8En8MryZA=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/analyzer/docfmt.go

@@ -1,6 +1,11 @@
 package analyzer
 
-import "strings"
+import (
+	"bytes"
+	"go/ast"
+	"go/doc/comment"
+	"strings"
+)
 
 const (
 	newLineChar = '\n'
@@ -27,7 +32,27 @@ func isDocLine(line string) bool {
 	return false
 }
 
+// ParseCommentGroup parses comments from AST and returns them in Markdown format.
+func ParseCommentGroup(group *ast.CommentGroup) []byte {
+	if group == nil || len(group.List) == 0 {
+		return nil
+	}
+
+	var (
+		parser  comment.Parser
+		printer comment.Printer
+	)
+
+	str := group.Text()
+	parsedDoc := parser.Parse(str)
+	mdDoc := printer.Markdown(parsedDoc)
+	mdDoc = bytes.TrimSuffix(mdDoc, []byte("\n"))
+	return mdDoc
+}
+
 // FormatDocString parses Go comment and returns a markdown-formatted string.
+//
+// Deprecated: use ParseCommentGroup instead.
 func FormatDocString(str string) MarkdownString {
 	if str == "" {
 		return MarkdownString{Value: str}

internal/pkgindex/cmd/common.go

@@ -0,0 +1,46 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func writeOutput(flags importsFlags, data any) error {
+	if flags.outFile == "" {
+		return getEncoder(os.Stdout, true).Encode(data)
+	}
+
+	if err := os.MkdirAll(filepath.Dir(flags.outFile), 0755); err != nil {
+		return fmt.Errorf("failed to pre-create parent directories: %w", err)
+	}
+
+	f, err := os.OpenFile(flags.outFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	defer silentClose(f)
+
+	if err != nil {
+		return fmt.Errorf("can't create output file: %w", err)
+	}
+
+	if err := getEncoder(f, flags.prettyPrint).Encode(data); err != nil {
+		return fmt.Errorf("can't write JSON to file %q: %w", flags.outFile, err)
+	}
+
+	return nil
+}
+
+func getEncoder(dst io.Writer, pretty bool) *json.Encoder {
+	enc := json.NewEncoder(dst)
+	if pretty {
+		enc.SetIndent("", "  ")
+	}
+
+	return enc
+}
+
+func silentClose(c io.Closer) {
+	// I don't care
+	_ = c.Close()
+}

internal/pkgindex/cmd/flags.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/x1unix/go-playground/internal/pkgindex/imports"
+)
+
+type globalFlags struct {
+	goRoot       string
+	heapProfFile string
+}
+
+func (f globalFlags) withDefaults() (globalFlags, error) {
+	if f.goRoot != "" {
+		return f, nil
+	}
+
+	goRoot, err := imports.ResolveGoRoot()
+	if err != nil {
+		return f, fmt.Errorf(
+			"cannot find GOROOT, please set GOROOT path or check if Go is installed.\nError: %w",
+			err,
+		)
+	}
+	f.goRoot = goRoot
+	return f, err
+}
+
+type importsFlags struct {
+	*globalFlags
+	verbose     bool
+	prettyPrint bool
+	stdout      bool
+	outFile     string
+}
+
+func (f importsFlags) validate() error {
+	if f.outFile == "" && !f.stdout {
+		return fmt.Errorf("missing output file flag. Use --stdout flag to print into stdout")
+	}
+
+	if f.stdout && f.outFile != "" {
+		return fmt.Errorf("ambiguous output flag: --stdout and output file flag can't be together")
+	}
+
+	return nil
+}

internal/pkgindex/cmd/imports.go

@@ -0,0 +1,45 @@
+package cmd
+
+import (
+	"log"
+
+	"github.com/spf13/cobra"
+	"github.com/x1unix/go-playground/internal/pkgindex/imports"
+)
+
+func newCmdImports(g *globalFlags) *cobra.Command {
+	flags := importsFlags{
+		globalFlags: g,
+	}
+	cmd := &cobra.Command{
+		Use:   "imports [-r goroot] [-o output]",
+		Short: "Generate imports.json file for old Playground version",
+		Long:  "Generate imports file which contains list of all importable packages. Used in legacy app versions",
+		PreRunE: func(_ *cobra.Command, _ []string) error {
+			return flags.validate()
+		},
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return runGenImports(flags)
+		},
+	}
+
+	cmd.Flags().StringVarP(&flags.outFile, "output", "o", "", "Path to output file. When enpty, prints to stdout")
+	cmd.Flags().BoolVarP(&flags.prettyPrint, "pretty", "P", false, "Add indents to JSON output")
+	cmd.Flags().BoolVar(&flags.stdout, "stdout", false, "Dump result into stdout")
+	return cmd
+}
+
+func runGenImports(flags importsFlags) error {
+	scanner := imports.NewGoRootScanner(flags.goRoot)
+	results, err := scanner.Scan()
+	if err != nil {
+		return err
+	}
+
+	if err := writeOutput(flags, results); err != nil {
+		return err
+	}
+
+	log.Printf("Scanned %d packages", len(results.Packages))
+	return nil
+}

internal/pkgindex/cmd/index.go

@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	"log"
+
+	"github.com/spf13/cobra"
+	"github.com/x1unix/go-playground/internal/pkgindex/index"
+)
+
+func newCmdIndex(g *globalFlags) *cobra.Command {
+	flags := importsFlags{
+		globalFlags: g,
+	}
+
+	cmd := &cobra.Command{
+		Use:   "index [-r goroot] [-o output]",
+		Short: "Generate index file with standard Go packages and symbols",
+		Long:  "Generate a JSON file that contains list of all standard Go packages and its symbols. Used in new version of app",
+		PreRunE: func(_ *cobra.Command, _ []string) error {
+			index.Debug = flags.verbose
+			return flags.validate()
+		},
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return runGenIndex(flags)
+		},
+	}
+
+	cmd.Flags().StringVarP(&flags.outFile, "output", "o", "", "Path to output file. When enpty, prints to stdout")
+	cmd.Flags().BoolVarP(&flags.prettyPrint, "pretty", "P", false, "Add indents to JSON output")
+	cmd.Flags().BoolVar(&flags.stdout, "stdout", false, "Dump result into stdout")
+	cmd.Flags().BoolVarP(&flags.verbose, "verbose", "v", false, "Enable verbose logging")
+	return cmd
+}
+
+func runGenIndex(flags importsFlags) error {
+	entries, err := index.ScanRoot(flags.goRoot)
+	if err != nil {
+		return err
+	}
+
+	if err := writeOutput(flags, entries); err != nil {
+		return err
+	}
+
+	log.Printf("Scanned %d packages and %d symbols", len(entries.Packages.Names), len(entries.Symbols.Names))
+	return nil
+}