Skip to content

Commit 856bd58

Browse files
angusleessbarzowski
anguslees
authored and
sbarzowski
committed
Add 'importbin' statement
Add `importbin` statement. Similar to `importstr` but the result is an array of numbers (all integers 0-255).
1 parent 880ac99 commit 856bd58

28 files changed

+177
-23
lines changed

‎ast/ast.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,14 @@ type ImportStr struct {
423423

424424
// ---------------------------------------------------------------------------
425425

426+
// ImportBin represents importbin "file".
427+
type ImportBin struct {
428+
NodeBase
429+
File *LiteralString
430+
}
431+
432+
// ---------------------------------------------------------------------------
433+
426434
// Index represents both e[e] and the syntax sugar e.f.
427435
//
428436
// One of index and id will be nil before desugaring. After desugaring id

‎ast/clone.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,13 @@ func clone(astPtr *Node) {
169169
r.File = new(LiteralString)
170170
*r.File = *node.File
171171

172+
case *ImportBin:
173+
r := new(ImportBin)
174+
*astPtr = r
175+
*r = *node
176+
r.File = new(LiteralString)
177+
*r.File = *node.File
178+
172179
case *Index:
173180
r := new(Index)
174181
*astPtr = r

‎imports.go

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"io/ioutil"
2222
"os"
2323
"path"
24+
"unsafe"
2425

2526
"github.com/google/go-jsonnet/ast"
2627
"github.com/google/go-jsonnet/internal/program"
@@ -58,19 +59,33 @@ type Importer interface {
5859
}
5960

6061
// Contents is a representation of imported data. It is a simple
61-
// string wrapper, which makes it easier to enforce the caching policy.
62+
// byte wrapper, which makes it easier to enforce the caching policy.
6263
type Contents struct {
63-
data *string
64+
data *[]byte
6465
}
6566

6667
func (c Contents) String() string {
68+
// Construct string without copying underlying bytes.
69+
// NB: This only works because c.data is not modified.
70+
return *(*string)(unsafe.Pointer(c.data))
71+
}
72+
73+
func (c Contents) Data() []byte {
6774
return *c.data
6875
}
6976

7077
// MakeContents creates Contents from a string.
7178
func MakeContents(s string) Contents {
79+
data := []byte(s)
7280
return Contents{
73-
data: &s,
81+
data: &data,
82+
}
83+
}
84+
85+
// MakeContentsRaw creates Contents from (possibly non-utf8) []byte data.
86+
func MakeContentsRaw(bytes []byte) Contents {
87+
return Contents{
88+
data: &bytes,
7489
}
7590
}
7691

@@ -139,6 +154,20 @@ func (cache *importCache) importString(importedFrom, importedPath string, i *int
139154
return makeValueString(data.String()), nil
140155
}
141156

157+
// ImportString imports an array of bytes, caches it and then returns it.
158+
func (cache *importCache) importBinary(importedFrom, importedPath string, i *interpreter) (*valueArray, error) {
159+
data, _, err := cache.importData(importedFrom, importedPath)
160+
if err != nil {
161+
return nil, i.Error(err.Error())
162+
}
163+
bytes := data.Data()
164+
elements := make([]*cachedThunk, len(bytes))
165+
for i := range bytes {
166+
elements[i] = readyThunk(intToValue(int(bytes[i])))
167+
}
168+
return makeValueArray(elements), nil
169+
}
170+
142171
func nodeToPV(i *interpreter, filename string, node ast.Node) *cachedThunk {
143172
env := makeInitialEnv(filename, i.baseStd)
144173
return &cachedThunk{
@@ -223,7 +252,7 @@ func (importer *FileImporter) tryPath(dir, importedPath string) (found bool, con
223252
} else {
224253
entry = &fsCacheEntry{
225254
exists: true,
226-
contents: MakeContents(string(contentBytes)),
255+
contents: MakeContentsRaw(contentBytes),
227256
}
228257
}
229258
importer.fsCache[absPath] = entry

‎internal/formatter/fix_indentation.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -574,6 +574,11 @@ func (c *FixIndentation) Visit(expr ast.Node, currIndent indent, crowded bool) {
574574
newIndent := c.newIndent(*openFodder(node.File), currIndent, c.column+1)
575575
c.Visit(node.File, newIndent, true)
576576

577+
case *ast.ImportBin:
578+
c.column += 9 // importbin
579+
newIndent := c.newIndent(*openFodder(node.File), currIndent, c.column+1)
580+
c.Visit(node.File, newIndent, true)
581+
577582
case *ast.InSuper:
578583
c.Visit(node.Index, currIndent, crowded)
579584
c.fill(node.InFodder, true, true, currIndent.lineUp)

‎internal/formatter/unparser.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,10 @@ func (u *unparser) unparse(expr ast.Node, crowded bool) {
370370
u.write("importstr")
371371
u.unparse(node.File, true)
372372

373+
case *ast.ImportBin:
374+
u.write("importbin")
375+
u.unparse(node.File, true)
376+
373377
case *ast.Index:
374378
u.unparse(node.Target, crowded)
375379
u.fill(node.LeftBracketFodder, false, false) // Can also be DotFodder

‎internal/parser/context.go

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,7 @@ func DirectChildren(node ast.Node) []ast.Node {
6464
return []ast.Node{node.Expr}
6565
case *ast.Function:
6666
return nil
67-
case *ast.Import:
68-
return nil
69-
case *ast.ImportStr:
67+
case *ast.Import, *ast.ImportStr, *ast.ImportBin:
7068
return nil
7169
case *ast.Index:
7270
if node.Id != nil {
@@ -181,9 +179,7 @@ func thunkChildren(node ast.Node) []ast.Node {
181179
return nil
182180
case *ast.Function:
183181
return nil
184-
case *ast.Import:
185-
return nil
186-
case *ast.ImportStr:
182+
case *ast.Import, *ast.ImportStr, *ast.ImportBin:
187183
return nil
188184
case *ast.Index:
189185
return nil
@@ -304,9 +300,7 @@ func specialChildren(node ast.Node) []ast.Node {
304300
}
305301
}
306302
return children
307-
case *ast.Import:
308-
return nil
309-
case *ast.ImportStr:
303+
case *ast.Import, *ast.ImportStr, *ast.ImportBin:
310304
return nil
311305
case *ast.Index:
312306
return nil

‎internal/parser/lexer.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ const (
6565
tokenIf
6666
tokenImport
6767
tokenImportStr
68+
tokenImportBin
6869
tokenIn
6970
tokenLocal
7071
tokenNullLit
@@ -112,6 +113,7 @@ var tokenKindStrings = []string{
112113
tokenIf: "if",
113114
tokenImport: "import",
114115
tokenImportStr: "importstr",
116+
tokenImportBin: "importbin",
115117
tokenIn: "in",
116118
tokenLocal: "local",
117119
tokenNullLit: "null",
@@ -580,6 +582,8 @@ func getTokenKindFromID(str string) tokenKind {
580582
return tokenImport
581583
case "importstr":
582584
return tokenImportStr
585+
case "importbin":
586+
return tokenImportBin
583587
case "in":
584588
return tokenIn
585589
case "local":

‎internal/parser/lexer_test.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,12 @@ func TestImportstr(t *testing.T) {
424424
})
425425
}
426426

427+
func TestImportbin(t *testing.T) {
428+
SingleTest(t, "importbin", "", Tokens{
429+
{kind: tokenImportBin, data: "importbin"},
430+
})
431+
}
432+
427433
func TestIn(t *testing.T) {
428434
SingleTest(t, "in", "", Tokens{
429435
{kind: tokenIn, data: "in"},

‎internal/parser/parser.go

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -870,7 +870,7 @@ func (p *parser) parseTerminal() (ast.Node, errors.StaticError) {
870870
tok := p.pop()
871871
switch tok.kind {
872872
case tokenAssert, tokenBraceR, tokenBracketR, tokenComma, tokenDot, tokenElse,
873-
tokenError, tokenFor, tokenFunction, tokenIf, tokenIn, tokenImport, tokenImportStr,
873+
tokenError, tokenFor, tokenFunction, tokenIf, tokenIn, tokenImport, tokenImportStr, tokenImportBin,
874874
tokenLocal, tokenOperator, tokenParenR, tokenSemicolon, tokenTailStrict, tokenThen:
875875
return nil, makeUnexpectedError(tok, "parsing terminal")
876876

@@ -1122,6 +1122,23 @@ func (p *parser) parse(prec precedence) (ast.Node, errors.StaticError) {
11221122
}
11231123
return nil, errors.MakeStaticError("Computed imports are not allowed", *body.Loc())
11241124

1125+
case tokenImportBin:
1126+
p.pop()
1127+
body, err := p.parse(maxPrecedence)
1128+
if err != nil {
1129+
return nil, err
1130+
}
1131+
if lit, ok := body.(*ast.LiteralString); ok {
1132+
if lit.Kind == ast.StringBlock {
1133+
return nil, errors.MakeStaticError("Block string literals not allowed in imports", *body.Loc())
1134+
}
1135+
return &ast.ImportBin{
1136+
NodeBase: ast.NewNodeBaseLoc(locFromTokenAST(begin, body), begin.fodder),
1137+
File: lit,
1138+
}, nil
1139+
}
1140+
return nil, errors.MakeStaticError("Computed imports are not allowed", *body.Loc())
1141+
11251142
case tokenLocal:
11261143
p.pop()
11271144
var binds ast.LocalBinds

‎internal/parser/parser_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ var tests = []string{
9898

9999
`import 'foo.jsonnet'`,
100100
`importstr 'foo.text'`,
101+
`importbin 'foo.bin'`,
101102

102103
`{a: b} + {c: d}`,
103104
`{a: b}{c: d}`,
@@ -230,6 +231,8 @@ var errorTests = []testError{
230231
{`import (a+b)`, `test:1:8-13 Computed imports are not allowed`},
231232
{`importstr (a b)`, `test:1:14-15 Expected token ")" but got (IDENTIFIER, "b")`},
232233
{`importstr (a+b)`, `test:1:11-16 Computed imports are not allowed`},
234+
{`importbin (a b)`, `test:1:14-15 Expected token ")" but got (IDENTIFIER, "b")`},
235+
{`importbin (a+b)`, `test:1:11-16 Computed imports are not allowed`},
233236

234237
{`local a = b ()`, `test:1:15 Expected , or ; but got end of file`},
235238
{`local a = b; (a b)`, `test:1:17-18 Expected token ")" but got (IDENTIFIER, "b")`},

0 commit comments

Comments
 (0)