First class dagger shell (#736)

In order to natively support Dagger's shell Runme needs to take the
entire notebook as opposed to an atomic cell into account. This PR
introduces the APIs necessary to transform a notebook into a Dagger
Shell script using some basic AST transformation.

Integration with extension in
stateful/vscode-runme#1947.

PS: The protos are not published yet and need local-gen to work.

The new service will turn a deserialized version of the following
markdown into a runnable Dagger shell script.

## Simple example notebook

Unamed cell:

```sh {"closeTerminalOnSuccess":"false","interactive":"true","openTerminalOnError":"true","terminalRows":"23"}
git github.com/stateful/runme |
    head |
    tree |
    file examples/README.md
```

And the same as a named cell:

```sh {"name":"README"}
# Named in README in cell attributes
git github.com/stateful/runme |
    head |
    tree |
    file examples/README.md
```

## Resulting Dagger Shell Script

Running the unamed cell:

```sh
DAGGER_01JKE40B6A0F5NBESE49X122RD()
{
  git github.com/stateful/runme \
    | head \
    | tree \
    | file examples/README.md
}
DAGGER_01JKE40B6A0F5NBESE49X122RD
```

Running the named cell:

```sh
DAGGER_01JKE4K781F87XRK9SQZ9T1GQD()
{
  git github.com/stateful/runme \
    | head \
    | tree \
    | file examples/README.md
}
README()
{
  git github.com/stateful/runme | head | tree | file examples/README.md
}
README
```

internal/cmd/server.go

@@ -14,11 +14,13 @@ import (
 	"google.golang.org/grpc/reflection"
 
 	"github.com/stateful/runme/v3/internal/command"
+	notebookservice "github.com/stateful/runme/v3/internal/notebook"
 	"github.com/stateful/runme/v3/internal/project/projectservice"
 	"github.com/stateful/runme/v3/internal/runner"
 	runnerv2service "github.com/stateful/runme/v3/internal/runnerv2service"
 	"github.com/stateful/runme/v3/internal/telemetry"
 	runmetls "github.com/stateful/runme/v3/internal/tls"
+	notebookv1alpha1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/notebook/v1alpha1"
 	parserv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/parser/v1"
 	projectv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/project/v1"
 	runnerv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/runner/v1"
@@ -107,6 +109,7 @@ The kernel is used to run long running processes like shells and interacting wit
 			)
 			parserv1.RegisterParserServiceServer(server, editorservice.NewParserServiceServer(logger))
 			projectv1.RegisterProjectServiceServer(server, projectservice.NewProjectServiceServer(logger))
+			notebookv1alpha1.RegisterNotebookServiceServer(server, notebookservice.NewNotebookService(logger))
 			// todo(sebastian): decided to forgo the reporter service for now
 			// reporterv1alpha1.RegisterReporterServiceServer(server, reporterservice.NewReporterServiceServer(logger))
 			if enableRunner {

internal/notebook/daggershell/script.go

@@ -0,0 +1,115 @@
+package daggershell
+
+import (
+	"errors"
+	"io"
+	"strings"
+
+	"mvdan.cc/sh/v3/syntax"
+)
+
+type Script struct {
+	stmts   []*syntax.Stmt
+	printer *syntax.Printer
+	parser  *syntax.Parser
+}
+
+func NewScript() *Script {
+	return &Script{
+		stmts: []*syntax.Stmt{},
+		printer: syntax.NewPrinter(
+			syntax.Indent(2),
+			syntax.BinaryNextLine(true),
+			syntax.FunctionNextLine(true),
+		),
+		parser: syntax.NewParser(),
+	}
+}
+
+func (s *Script) DeclareFunc(name, body string) error {
+	stmt := &syntax.Stmt{
+		Cmd: &syntax.FuncDecl{
+			Parens: true,
+			Name: &syntax.Lit{
+				Value: name,
+			},
+			Body: &syntax.Stmt{
+				Cmd: &syntax.Block{},
+			},
+		},
+	}
+
+	snippet, err := s.parser.Parse(strings.NewReader(body), name)
+	if err != nil {
+		return err
+	}
+
+	// assign stmts to insert function body
+	syntax.Walk(stmt, func(node syntax.Node) bool {
+		if block, ok := node.(*syntax.Block); ok {
+			block.Stmts = snippet.Stmts
+			return false
+		}
+
+		// todo(sebastian): check for validity, e.g. func def inside itself
+		return true
+	})
+
+	s.stmts = append(s.stmts, stmt)
+
+	return nil
+}
+
+func (s *Script) Render(w io.Writer) error {
+	return s.RenderWithCall(w, "")
+}
+
+func (s *Script) RenderWithCall(w io.Writer, name string) error {
+	if name == "" {
+		return s.printer.Print(w, &syntax.File{
+			Name:  "DaggerShellScript",
+			Stmts: s.stmts,
+		})
+	}
+
+	stmts := make([]*syntax.Stmt, len(s.stmts))
+	copy(stmts, s.stmts)
+	f := &syntax.File{
+		Name:  "DaggerShellScript",
+		Stmts: stmts,
+	}
+
+	validFuncName := false
+	// check if func name was previously declared
+	syntax.Walk(f, func(node syntax.Node) bool {
+		decl, ok := node.(*syntax.FuncDecl)
+		if !ok {
+			return true
+		}
+
+		if decl.Name.Value == name {
+			validFuncName = true
+			return false
+		}
+
+		return true
+	})
+
+	if !validFuncName {
+		return errors.New("undeclared function name")
+	}
+
+	f.Stmts = append(f.Stmts, &syntax.Stmt{
+		Cmd: &syntax.CallExpr{
+			Args: []*syntax.Word{
+				{
+					Parts: []syntax.WordPart{
+						&syntax.Lit{Value: name},
+					},
+				},
+			},
+		},
+	})
+
+	return s.printer.Print(w, f)
+}

internal/notebook/daggershell/script_test.go

@@ -0,0 +1,106 @@
+package daggershell
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/go-playground/assert/v2"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDaggerShell_FuncDecl(t *testing.T) {
+	script := NewScript()
+
+	err := script.DeclareFunc("DAGGER_FUNCTION", `echo "Dagger Function Placeholder"`)
+	require.NoError(t, err)
+
+	var rendered bytes.Buffer
+	err = script.Render(&rendered)
+	require.NoError(t, err)
+
+	const expected = `DAGGER_FUNCTION()
+{
+  echo "Dagger Function Placeholder"
+}
+`
+	assert.Equal(t,
+		expected,
+		rendered.String(),
+	)
+}
+
+func TestDaggerShell_Script(t *testing.T) {
+	// can't use map because order is not guaranteed
+	fakeCells := []struct {
+		Name string
+		Body string
+	}{
+		{"DAGGER_01JJDCG2SQSGV0DP55X86EJFSZ", `echo "Use known ID"; date;`},
+		{"PRESETUP", `echo "This is PRESETUP" | xxd`},
+		{"EXTENSION", `echo "This is EXTENSION" | less`},
+		{"KERNEL_BINARY", `echo "This is KERNEL_BINARY"`},
+	}
+
+	expected := `DAGGER_01JJDCG2SQSGV0DP55X86EJFSZ()
+{
+  echo "Use known ID"
+  date
+}
+PRESETUP()
+{
+  echo "This is PRESETUP" | xxd
+}
+EXTENSION()
+{
+  echo "This is EXTENSION" | less
+}
+KERNEL_BINARY()
+{
+  echo "This is KERNEL_BINARY"
+}
+`
+
+	t.Run("Render", func(t *testing.T) {
+		script := NewScript()
+		for _, entry := range fakeCells {
+			script.DeclareFunc(entry.Name, entry.Body)
+		}
+
+		var rendered bytes.Buffer
+		err := script.Render(&rendered)
+		require.NoError(t, err)
+
+		assert.Equal(t, expected, rendered.String())
+	})
+
+	t.Run("RenderWithCall", func(t *testing.T) {
+		script := NewScript()
+		for _, entry := range fakeCells {
+			err := script.DeclareFunc(entry.Name, entry.Body)
+			require.NoError(t, err)
+		}
+
+		for _, entry := range fakeCells {
+			var renderedWithCall bytes.Buffer
+			err := script.RenderWithCall(&renderedWithCall, entry.Name)
+			require.NoError(t, err)
+
+			// add function call padded by new lines
+			expectedBytesWithCall := strings.Join([]string{expected[:len(expected)-1], entry.Name, ""}, "\n")
+			assert.Equal(t, expectedBytesWithCall, renderedWithCall.String())
+		}
+	})
+
+	t.Run("RenderWithCall_Invalid", func(t *testing.T) {
+		script := NewScript()
+		for _, entry := range fakeCells {
+			err := script.DeclareFunc(entry.Name, entry.Body)
+			require.NoError(t, err)
+		}
+
+		var renderedWithCall bytes.Buffer
+		err := script.RenderWithCall(&renderedWithCall, "INVALID")
+		require.Error(t, err)
+	})
+}

internal/notebook/resolver.go

@@ -0,0 +1,127 @@
+package notebook
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/stateful/runme/v3/internal/notebook/daggershell"
+	parserv1 "github.com/stateful/runme/v3/pkg/api/gen/proto/go/runme/parser/v1"
+	"github.com/stateful/runme/v3/pkg/document/editor/editorservice"
+	"go.uber.org/zap"
+)
+
+type NotebookResolver struct {
+	notebook *parserv1.Notebook
+	editor   parserv1.ParserServiceServer
+}
+
+func NewNotebookResolver(notebook *parserv1.Notebook) *NotebookResolver {
+	return &NotebookResolver{
+		notebook: notebook,
+		editor:   editorservice.NewParserServiceServer(zap.NewNop()),
+	}
+}
+
+func (r *NotebookResolver) parseNotebook(context context.Context) (*parserv1.Notebook, error) {
+	// make id sticky only for resolving purposes
+	for _, cell := range r.notebook.Cells {
+		if cell.GetKind() != parserv1.CellKind_CELL_KIND_CODE {
+			continue
+		}
+
+		_, ok := cell.Metadata["id"]
+		if ok {
+			continue
+		}
+
+		if cell.Metadata == nil {
+			return nil, fmt.Errorf("cell metadata is missing")
+		}
+
+		cell.Metadata["id"] = cell.Metadata["runme.dev/id"]
+	}
+
+	// properly parse frontmatter and notebook/cell metadata
+	ser, err := r.editor.Serialize(context, &parserv1.SerializeRequest{Notebook: r.notebook})
+	if err != nil {
+		return nil, err
+	}
+	des, err := r.editor.Deserialize(context, &parserv1.DeserializeRequest{Source: ser.Result})
+	if err != nil {
+		return nil, err
+	}
+
+	return des.Notebook, nil
+}
+
+func (r *NotebookResolver) ResolveDaggerShell(context context.Context, cellIndex uint32) (string, error) {
+	notebook, err := r.parseNotebook(context)
+	if err != nil {
+		return "", err
+	}
+
+	var targetCell *parserv1.Cell
+	targetName := ""
+	if int(cellIndex) < 0 || int(cellIndex) >= len(notebook.Cells) {
+		return "", fmt.Errorf("cell index out of range")
+	}
+
+	cell := notebook.Cells[cellIndex]
+	id, okID := cell.Metadata["runme.dev/id"]
+	known, okKnown := cell.Metadata["name"]
+	generated := cell.Metadata["runme.dev/nameGenerated"]
+	if !okID && !okKnown {
+		return "", fmt.Errorf("cell metadata is missing required fields")
+	}
+
+	isGenerated, err := strconv.ParseBool(generated)
+	if !okKnown || isGenerated || err != nil {
+		known = fmt.Sprintf("DAGGER_%s", id)
+	}
+
+	targetCell = cell
+	targetName = known
+
+	if notebook.Frontmatter == nil || !strings.Contains(strings.Trim(notebook.Frontmatter.Shell, " \t\r\n"), "dagger shell") {
+		return targetCell.GetValue(), nil
+	}
+
+	script := daggershell.NewScript()
+	for _, cell := range notebook.Cells {
+		if cell.GetKind() != parserv1.CellKind_CELL_KIND_CODE {
+			continue
+		}
+
+		languageID := cell.GetLanguageId()
+		if languageID != "sh" && languageID != "dagger" {
+			continue
+		}
+
+		id, okID := cell.Metadata["runme.dev/id"]
+		known, okName := cell.Metadata["runme.dev/name"]
+		generated := cell.Metadata["runme.dev/nameGenerated"]
+		if !okID && !okName {
+			continue
+		}
+
+		isGenerated, err := strconv.ParseBool(generated)
+		if !okName || isGenerated || err != nil {
+			known = fmt.Sprintf("DAGGER_%s", id)
+		}
+
+		snippet := cell.GetValue()
+		if err := script.DeclareFunc(known, snippet); err != nil {
+			return "", err
+		}
+	}
+
+	var rendered bytes.Buffer
+	if err := script.RenderWithCall(&rendered, targetName); err != nil {
+		return "", err
+	}
+
+	return rendered.String(), nil
+}