Skip to content

Commit

Permalink
Add session metadata to serialization + frontmatter (#458)
Browse files Browse the repository at this point in the history 
* Baseline

* Respect runme session frontmatter for deserialization

* Fix tests

* Add test cases

* Typo
  • Loading branch information
@sourishkrout
sourishkrout authored Dec 28, 2023
1 parent 36193ed commit f5e7a93
Show file tree
Hide file tree
Showing 25 changed files with 593 additions and 831 deletions.
12 changes: 12 additions & 0 deletions internal/api/runme/parser/v1/parser.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,16 +73,27 @@ enum RunmeIdentity {
RUNME_IDENTITY_CELL = 3;
}

message RunmeSessionDocument {
string relative_path = 1;
}

message RunmeSession {
string id = 1;
RunmeSessionDocument document = 2;
}

message FrontmatterRunme {
string id = 1;
string version = 2;
RunmeSession session = 3;
}

message Frontmatter {
string shell = 1;
string cwd = 2;
bool skip_prompts = 3;
FrontmatterRunme runme = 4;
string category = 5;
}

message DeserializeRequestOptions {
Expand All @@ -105,6 +116,7 @@ message SerializeRequestOutputOptions {

message SerializeRequestOptions {
SerializeRequestOutputOptions outputs = 1;
RunmeSession session = 2;
}

message SerializeRequest {
Expand Down
26 changes: 22 additions & 4 deletions internal/document/editor/editor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package editor
import (
"bytes"
"strconv"
"time"

"github.com/pkg/errors"
"github.com/stateful/runme/internal/document"
Expand Down Expand Up @@ -43,15 +44,32 @@ func Deserialize(data []byte, identityResolver *identity.IdentityResolver) (*Not
return notebook, nil
}

func Serialize(notebook *Notebook) ([]byte, error) {
func Serialize(notebook *Notebook, outputMetadata *document.RunmeMetadata) ([]byte, error) {
var result []byte

// Serialize frontmatter.
if intro, ok := notebook.Metadata[PrefixAttributeName(InternalAttributePrefix, FrontmatterKey)]; ok {
intro := []byte(intro)
lb := document.DetectLineBreak(intro)
raw := []byte(intro)

if outputMetadata != nil {
frontmatter, err := document.ParseFrontmatter(raw)
if err != nil {
return nil, err
}
frontmatter.Runme.Session = outputMetadata.Session
frontmatter.Runme.Session.Updated = prettyTime(time.Now())
frontmatter.Runme.Document = outputMetadata.Document

// true because no outputs serialization without lifecycle identity
raw, err = frontmatter.Marshal(true)
if err != nil {
return nil, err
}
}

lb := document.DetectLineBreak(raw)
result = append(
intro,
raw,
bytes.Repeat(lb, 2)...,
)
}
Expand Down
66 changes: 57 additions & 9 deletions internal/document/editor/editor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/stateful/runme/internal/document"
"github.com/stateful/runme/internal/document/constants"
"github.com/stateful/runme/internal/document/identity"
ulid "github.com/stateful/runme/internal/ulid"
Expand All @@ -32,7 +33,7 @@ func TestMain(m *testing.M) {
func TestEditor(t *testing.T) {
notebook, err := Deserialize(testDataNested, identityResolverNone)
require.NoError(t, err)
result, err := Serialize(notebook)
result, err := Serialize(notebook, nil)
require.NoError(t, err)
assert.Equal(
t,
Expand All @@ -51,7 +52,7 @@ func TestEditor_List(t *testing.T) {

notebook.Cells[0].Value = "1. Item 1\n2. Item 2\n"

newData, err := Serialize(notebook)
newData, err := Serialize(notebook, nil)
require.NoError(t, err)
assert.Equal(
t,
Expand All @@ -61,7 +62,7 @@ func TestEditor_List(t *testing.T) {
string(newData),
)

newData, err = Serialize(notebook)
newData, err = Serialize(notebook, nil)
require.NoError(t, err)
assert.Equal(
t,
Expand All @@ -88,7 +89,7 @@ func TestEditor_CodeBlock(t *testing.T) {
t,
cell.Metadata["name"],
)
result, err := Serialize(notebook)
result, err := Serialize(notebook, nil)
require.NoError(t, err)
assert.Equal(t, string(data), string(result))
})
Expand All @@ -109,7 +110,7 @@ func TestEditor_CodeBlock(t *testing.T) {
cell.Metadata["name"],
"name1",
)
result, err := Serialize(notebook)
result, err := Serialize(notebook, nil)
require.NoError(t, err)
assert.Equal(t, string(data), string(result))
})
Expand Down Expand Up @@ -177,7 +178,7 @@ A paragraph
`, testMockID, version.BaseVersion()))
notebook, err := Deserialize(data, identityResolverNone)
require.NoError(t, err)
result, err := Serialize(notebook)
result, err := Serialize(notebook, nil)
require.NoError(t, err)
assert.Equal(
t,
Expand All @@ -186,6 +187,53 @@ A paragraph
)
}

func TestEditor_SessionOutput(t *testing.T) {
data := []byte(fmt.Sprintf(`+++
prop1 = 'val1'
prop2 = 'val2'
[runme]
id = '%s'
version = '%s'
+++
# Example
A paragraph
`, testMockID, version.BaseVersion()))
notebook, err := Deserialize(data, identityResolverNone)
require.NoError(t, err)

sid := "01HJP23P1R57BPGEA17QDJXJE"
rpath := "README.md"
invalidTs := "invalid-timestamp-should-be-overwritten"
outputMetadata := &document.RunmeMetadata{
Session: document.RunmeMetadataSession{
ID: sid,
Updated: invalidTs,
},
Document: document.RunmeMetadataDocument{RelativePath: rpath},
}
result, err := Serialize(notebook, outputMetadata)
require.NoError(t, err)
assert.Contains(
t,
string(result),
string(sid),
)

sessionNb, err := Deserialize(result, identityResolverAll)
require.NoError(t, err)

sess := sessionNb.Frontmatter.Runme.Session
assert.Equal(t, sid, sess.ID)
assert.NotEqual(t, sess.Updated, invalidTs)
assert.Greater(t, len(sess.Updated), 0)

doc := sessionNb.Frontmatter.Runme.Document
assert.Equal(t, doc.RelativePath, rpath)
}

func TestEditor_Newlines(t *testing.T) {
data := []byte(`## Newline debugging
Expand All @@ -201,7 +249,7 @@ This will test final line breaks`)
"0",
)

actual, err := Serialize(notebook)
actual, err := Serialize(notebook, nil)
require.NoError(t, err)
assert.Equal(
t,
Expand All @@ -222,7 +270,7 @@ This will test final line breaks`)
"1",
)

actual, err := Serialize(notebook)
actual, err := Serialize(notebook, nil)
require.NoError(t, err)
assert.Equal(
t,
Expand All @@ -243,7 +291,7 @@ This will test final line breaks`)
"7",
)

actual, err := Serialize(notebook)
actual, err := Serialize(notebook, nil)
require.NoError(t, err)
assert.Equal(
t,
Expand Down
34 changes: 32 additions & 2 deletions internal/document/editor/editorservice/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/base64"
"strings"

"github.com/stateful/runme/internal/document"
"github.com/stateful/runme/internal/document/editor"
"github.com/stateful/runme/internal/document/identity"
parserv1 "github.com/stateful/runme/internal/gen/proto/go/runme/parser/v1"
Expand Down Expand Up @@ -59,6 +60,8 @@ func (s *parserServiceServer) Deserialize(_ context.Context, req *parserv1.Deser
Shell: notebook.Frontmatter.Shell,
Cwd: notebook.Frontmatter.Cwd,
SkipPrompts: notebook.Frontmatter.SkipPrompts,
// todo(sebastian): impl logic for doc category (runme#369)
Category: notebook.Frontmatter.Category,
}

runme := parserv1.FrontmatterRunme{}
Expand All @@ -71,7 +74,16 @@ func (s *parserServiceServer) Deserialize(_ context.Context, req *parserv1.Deser
runme.Version = notebook.Frontmatter.Runme.Version
}

if runme.Id != "" || runme.Version != "" {
if notebook.Frontmatter.Runme.Session.ID != "" {
runme.Session = &parserv1.RunmeSession{
Id: notebook.Frontmatter.Runme.Session.ID,
Document: &parserv1.RunmeSessionDocument{
RelativePath: notebook.Frontmatter.Runme.Document.RelativePath,
},
}
}

if runme.Id != "" || runme.Version != "" || runme.Session != nil {
frontmatter.Runme = &runme
}
}
Expand Down Expand Up @@ -103,10 +115,28 @@ func (s *parserServiceServer) Serialize(_ context.Context, req *parserv1.Seriali
})
}

var outputMetadata *document.RunmeMetadata
if req.Options != nil && req.Options.Session != nil {
relativePath := ""
if req.Options.Session.Document != nil {
relativePath = req.Options.Session.Document.GetRelativePath()
}

outputMetadata = &document.RunmeMetadata{
Session: document.RunmeMetadataSession{
ID: req.Options.Session.GetId(),
},
Document: document.RunmeMetadataDocument{
RelativePath: relativePath,
},
}

}

data, err := editor.Serialize(&editor.Notebook{
Cells: cells,
Metadata: req.Notebook.Metadata,
})
}, outputMetadata)
if err != nil {
s.logger.Info("failed to call Serialize", zap.Error(err))
return nil, err
Expand Down
23 changes: 20 additions & 3 deletions internal/document/editor/editorservice/service_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,13 +323,30 @@ func Test_parserServiceServer_Outputs(t *testing.T) {
},
},
}
notebook := &parserv1.Notebook{Cells: []*parserv1.Cell{cell}}

serializeOptions := &parserv1.SerializeRequestOptions{Outputs: &parserv1.SerializeRequestOutputOptions{Enabled: true, Summary: true}}
parsedFm := "---\nrunme:\n id: 01HF7B0KK32HBQ9X4AC2GPMZG5\n version: v2.0\nsidebar_position: 1\ntitle: Examples\n---"
notebook := &parserv1.Notebook{Cells: []*parserv1.Cell{cell}, Metadata: map[string]string{"runme.dev/frontmatter": parsedFm}}

serializeOptions := &parserv1.SerializeRequestOptions{
Outputs: &parserv1.SerializeRequestOutputOptions{Enabled: true, Summary: true},
Session: &parserv1.RunmeSession{
Id: "01HJP23P1R57BPGEA17QDJXJE",
Document: &parserv1.RunmeSessionDocument{RelativePath: "README.md"},
},
}
resp, err := serializeWithOutputs(client, notebook, serializeOptions)
assert.NoError(t, err)

assert.Equal(t, "```sh {\"background\":\"false\",\"id\":\"01HF7B0KJPF469EG9ZVX256S75\",\"interactive\":\"true\"}\n$ printf \"\\u001b[34mDoes it work?\\n\"\n$ sleep 2\n$ printf \"\\u001b[32mYes, success!\\x1b[0m\\n\"\n$ exit 16\n\n# Ran on 2023-11-29 17:58:19Z for 2.296s exited with 16\nDoes it work?\r\nYes, success!\n```\n", string(resp.Result))
var content []string
lines := strings.Split(string(resp.Result), "\n")
for _, line := range lines {
if strings.HasPrefix(line, " updated:") {
content = append(content, " updated: 0000-00-00 00:00:00Z")
continue
}
content = append(content, line)
}
assert.Equal(t, "---\nrunme:\n id: 01HF7B0KK32HBQ9X4AC2GPMZG5\n version: v2.0\n document:\n relativePath: README.md\n session:\n id: 01HJP23P1R57BPGEA17QDJXJE\n updated: 0000-00-00 00:00:00Z\nsidebar_position: 1\ntitle: Examples\n---\n\n```sh {\"background\":\"false\",\"id\":\"01HF7B0KJPF469EG9ZVX256S75\",\"interactive\":\"true\"}\n$ printf \"\\u001b[34mDoes it work?\\n\"\n$ sleep 2\n$ printf \"\\u001b[32mYes, success!\\x1b[0m\\n\"\n$ exit 16\n\n# Ran on 2023-11-29 17:58:19Z for 2.296s exited with 16\nDoes it work?\r\nYes, success!\n```\n", strings.Join(content, "\n"))
})
}

Expand Down
16 changes: 14 additions & 2 deletions internal/document/frontmatter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,27 @@ const (
frontmatterFormatTOML = "toml"
)

type RunmeMetadata struct {
type RunmeMetadataDocument struct {
RelativePath string `yaml:"relativePath,omitempty" json:"relativePath,omitempty" toml:"relativePath,omitempty"`
}

type RunmeMetadataSession struct {
ID string `yaml:"id,omitempty" json:"id,omitempty" toml:"id,omitempty"`
Version string `yaml:"version,omitempty" json:"version,omitempty" toml:"version,omitempty"`
Updated string `yaml:"updated,omitempty" json:"updated,omitempty" toml:"updated,omitempty"`
}

type RunmeMetadata struct {
ID string `yaml:"id,omitempty" json:"id,omitempty" toml:"id,omitempty"`
Version string `yaml:"version,omitempty" json:"version,omitempty" toml:"version,omitempty"`
Document RunmeMetadataDocument `yaml:"document,omitempty" json:"document,omitempty" toml:"document,omitempty"`
Session RunmeMetadataSession `yaml:"session,omitempty" json:"session,omitempty" toml:"session,omitempty"`
}

type Frontmatter struct {
Runme RunmeMetadata `yaml:"runme,omitempty"`
Shell string `yaml:"shell"`
Cwd string `yaml:"cwd"`
Category string `yaml:"category"`
SkipPrompts bool `yaml:"skipPrompts,omitempty"`

format string
Expand Down
Loading

0 comments on commit f5e7a93

Please sign in to comment.