Upstreaming various features implemented for ZetaSQL

compiler/compiler.go

@@ -578,6 +578,7 @@ func generateTables(source *syntax.Model, out *grammar.Grammar, opts genOptions,
 					Report: report,
 					Code:   command,
 					Origin: cmdOrigin,
+		            NtName: nt.Name,
 				}
 				if args != nil {
 					act.Vars = &grammar.ActionVars{CmdArgs: *args, Remap: actualPos}
@@ -723,6 +724,7 @@ func (e *commandExtractor) extract(n *syntax.Nonterm, command string, vars *gram
 		Code:   command,
 		Vars:   vars,
 		Origin: cmdOrigin,
+		NtName: nt.Name,
 	}
 	rule.Action = len(e.actions)
 	e.actions = append(e.actions, act)

compiler/compiler_test.go

@@ -39,6 +39,7 @@ var testFiles = []string{
 	"flexmode.tmerr",
 	"max_la.tmerr",
 	"disabled_syntax.tmerr",
+ 	"expansion_limit.tmerr",
 }
 
 func TestErrors(t *testing.T) {

compiler/options.go

@@ -20,11 +20,15 @@ type optionsParser struct {
 func newOptionsParser(s *status.Status) *optionsParser {
 	return &optionsParser{
 		out: &grammar.Options{
-			TokenLine:              true,
-			GenParser:              true,
-			AbslIncludePrefix:      "absl",
-			SkipByteOrderMark:      true,
-			OptInstantiationSuffix: "opt",
+			TokenLine:                true,
+			GenParser:                true,
+			AbslIncludePrefix:        "absl",
+			SkipByteOrderMark:        true,
+			OptInstantiationSuffix:   "opt",
+			ExpansionLimit:           65_536,
+			ExpansionWarn:            256,
+			MaxRuleSizeForOrdinalRef: 16,
+			VariantStackEntry:        true,
 		},
 		Status: s,
 	}
@@ -105,6 +109,10 @@ func (p *optionsParser) parseFrom(file ast.File) {
 			opts.MaxLookahead = p.parseExpr(opt.Value(), opts.MaxLookahead).(int)
 		case "disableSyntax":
 			opts.DisableSyntax = p.parseExpr(opt.Value(), opts.DisableSyntax).([]string)
+ 		case "expansionLimit":
+ 			opts.ExpansionLimit = p.parseExpr(opt.Value(), opts.ExpansionLimit).(int)
+ 		case "expansionWarn":
+ 			opts.ExpansionWarn = p.parseExpr(opt.Value(), opts.ExpansionWarn).(int)
 		case "eventFields":
 			p.validLangs(opt.Key(), "go")
 			opts.EventFields = p.parseExpr(opt.Value(), opts.EventFields).(bool)
@@ -143,6 +151,11 @@ func (p *optionsParser) parseFrom(file ast.File) {
 		case "variantStackEntry":
 			p.validLangs(opt.Key(), "cc")
 			opts.VariantStackEntry = p.parseExpr(opt.Value(), opts.VariantStackEntry).(bool)
+		case "trackReduces":
+			p.validLangs(opt.Key(), "cc")
+			opts.TrackReduces = p.parseExpr(opt.Value(), opts.TrackReduces).(bool)
+		case "maxRuleSizeForOrdinalRef":
+			opts.MaxRuleSizeForOrdinalRef = p.parseExpr(opt.Value(), opts.MaxRuleSizeForOrdinalRef).(int)
 		case "skipByteOrderMark":
 			opts.SkipByteOrderMark = p.parseExpr(opt.Value(), opts.SkipByteOrderMark).(bool)
 		default:

compiler/syntax.go

@@ -47,8 +47,10 @@ func newSyntaxLoader(resolver *resolver, targetLang string, opts *grammar.Option
 	case "cc":
 		expandOpts = syntax.CcExpandOptions()
 	default:
-		expandOpts = &syntax.ExpandOptions{}
+		expandOpts = syntax.DefaultExpandOptions()
 	}
+	expandOpts.ExpansionLimit = opts.ExpansionLimit
+	expandOpts.ExpansionWarn = opts.ExpansionWarn
 
 	return &syntaxLoader{
 		resolver:     resolver,

compiler/testdata/expansion_limit.tmerr

@@ -0,0 +1,36 @@
+language parser(go);
+
+expansionWarn = 2
+expansionLimit = 8
+
+:: lexer
+
+a: /a/
+b: /b/
+c: /c/
+d: /d/
+e: /e/
+f: /f/
+g: /g/
+
+:: parser
+
+input: a b c d;
+
+A1: a? b c d;
+
+A2: a? b? c d;
+
+A3: a? b? c? d;
+
+A4: «a? b? c? d?»;
+# err: expanding rule produced 16 rules which exeeds the limit of 8. Refactor the rule to reduce expansion or increase the `expansionLimit` value
+
+A5: «a? b? c? d? e?»;
+# err: expanding rule produced 32 rules which exeeds the limit of 8. Refactor the rule to reduce expansion or increase the `expansionLimit` value
+
+A6: b | «a b? c? d? e?»;
+# err: expanding rule produced 16 rules which exeeds the limit of 8. Refactor the rule to reduce expansion or increase the `expansionLimit` value
+
+A7: «a ( b c? (d | e f? | f | g) | c )»;
+# err: expanding rule produced 11 rules which exeeds the limit of 8. Refactor the rule to reduce expansion or increase the `expansionLimit` value

gen/funcs.go

@@ -39,6 +39,7 @@ var funcMap = template.FuncMap{
 	"sub":                 sub,
 	"go_parser_action":    goParserAction,
 	"cc_parser_action":    ccParserAction,
+ 	"cc_action_func":      ccActionFunc,
 	"bison_parser_action": bisonParserAction,
 	"short_pkg":           shortPkg,
 	"list":                func(vals ...string) []string { return vals },
@@ -285,9 +286,9 @@ func goParserAction(s string, args *grammar.ActionVars, origin status.SourceNode
 				}
 			}
 
-			ref, ok := args.Resolve(id)
-			if !ok {
-				return "", status.Errorf(origin, "invalid reference %q", id)
+			ref, err := args.Resolve(id, origin)
+			if err != nil {
+				return "", err
 			}
 			index = ref.Index
 			pos = ref.Pos
@@ -355,14 +356,15 @@ func ccTypeFromUnion(unionField string) string {
 	return strings.TrimSpace(unionField[:len(unionField)-len(lastID(unionField))])
 }
 
-func ccParserAction(s string, args *grammar.ActionVars, origin status.SourceNode, variantStackEntry bool) (ret string, err error) {
+func ccParserAction(s string, args *grammar.ActionVars, origin status.SourceNode, opts *grammar.Options) (ret string, err error) {
 	defer func(s string) {
 		if r := recover(); r != nil {
 			err = fmt.Errorf("crashed with %v in %q with %v", err, s, args)
 		}
 	}(s)
 
 	var sb strings.Builder
+	sb.WriteString("#line " + strconv.Itoa(origin.SourceRange().Line) + " \"" + origin.SourceRange().Filename + "\"\n")
 	for len(s) > 0 {
 		next := strings.IndexAny(s, "@$")
 		if next == -1 {
@@ -389,7 +391,7 @@ func ccParserAction(s string, args *grammar.ActionVars, origin status.SourceNode
 				if t == "" {
 					return "", status.Errorf(origin, "$$ cannot be used inside a nonterminal semantic action without a type")
 				}
-				if variantStackEntry {
+				if opts.VariantStackEntry {
 					replacement = "std::get<" + t + ">(lhs.value)"
 				} else {
 					replacement = "lhs.value." + lastID(t)
@@ -414,9 +416,9 @@ func ccParserAction(s string, args *grammar.ActionVars, origin status.SourceNode
 		s = s[d:]
 
 		// cc uses 1-based indexing.
-		ref, ok := args.ResolveOneBased(val)
-		if !ok {
-			return "", status.Errorf(origin, "invalid reference %c%q", ch, val)
+ 		ref, err := args.ResolveOneBased(val, opts.MaxRuleSizeForOrdinalRef, string(ch)+val, origin)
+ 		if err != nil {
+ 			return "", err
 		}
 
 		index := ref.Index
@@ -436,7 +438,7 @@ func ccParserAction(s string, args *grammar.ActionVars, origin status.SourceNode
 			if argType == "" {
 				return "", status.Errorf(origin, "symbol %c%q does not have an associated type", ch, val)
 			}
-			if !variantStackEntry {
+			if !opts.VariantStackEntry {
 				argType = ccTypeFromUnion(argType)
 			}
 			sb.WriteString(ccWrapInOptional(argType, ""))
@@ -453,7 +455,7 @@ func ccParserAction(s string, args *grammar.ActionVars, origin status.SourceNode
 			if argType == "" {
 				return "", status.Errorf(origin, "%c%q does not have an associated type", ch, val)
 			}
-			if variantStackEntry {
+			if opts.VariantStackEntry {
 				replacement = "std::get<" + argType + ">(" + target + ".value)"
 			} else {
 				replacement = target + ".value." + lastID(argType)
@@ -471,6 +473,16 @@ func ccParserAction(s string, args *grammar.ActionVars, origin status.SourceNode
 	return sb.String(), nil
 }
 
+// Generate a C++ function name for a reduce action that encodes information useful for
+// understanding stack traces including the nonterminal name and line number. We also include the
+// rule index because there can be multiple actions on the same line in the grammar file.
+func ccActionFunc(idx int, act *grammar.SemanticAction) string {
+	// Include the index in the name to differentiate between actions of extracted non-terminals and
+	// also actions of named non-terminals that happen to be on the same line of the grammar file.
+	return fmt.Sprintf("Action%v__ReduceOf_%v__AtLine_%v_Column_%v", idx, act.NtName, act.Origin.SourceRange().Line, act.Origin.SourceRange().Column)
+}
+
+
 func bisonParserAction(s string, args *grammar.ActionVars, origin status.SourceNode) (string, error) {
 	var sb strings.Builder
 	for len(s) > 0 {
@@ -514,9 +526,9 @@ func bisonParserAction(s string, args *grammar.ActionVars, origin status.SourceN
 				}
 			}
 
-			ref, ok := args.Resolve(id)
-			if !ok {
-				return "", status.Errorf(origin, "invalid reference %q", id)
+			ref, err := args.Resolve(id, origin)
+			if err != nil {
+				return "", err
 			}
 			index = ref.Index
 			pos = ref.Pos

gen/funcs_test.go

@@ -195,15 +195,19 @@ func TestCcParserAction(t *testing.T) {
 		want       string
 		useVariant bool
 	}{
-		{"abc", vars(), "abc", false},
+		{"abc", vars(), "#line 0 \"abc\"\nabc", false},
 		// The 1-based index for "a" is 2.
-		{"$$ = $2", vars("%node", "a:0:expr"), "lhs.value.node = rhs[0].value.expr", false},
-		{"$$ = @$ @2", vars("%node", "a:0:expr"), "lhs.value.node = lhs.sym.location rhs[0].sym.location", false},
-		{"$$ = $2", vars("%node", "a:0:expr"), "std::get<node>(lhs.value) = std::get<expr>(rhs[0].value)", true},
+		{"$$ = $2", vars("%node", "a:0:expr"), "#line 0 \"$$ = $2\"\nlhs.value.node = rhs[0].value.expr", false},
+		{"$$ = @$ @2", vars("%node", "a:0:expr"), "#line 0 \"$$ = @$ @2\"\nlhs.value.node = lhs.sym.location rhs[0].sym.location", false},
+		{"$$ = $2", vars("%node", "a:0:expr"), "#line 0 \"$$ = $2\"\nstd::get<node>(lhs.value) = std::get<expr>(rhs[0].value)", true},
 	}
 
 	for _, tc := range tests {
-		got, err := ccParserAction(tc.input, tc.args, node(tc.input), tc.useVariant)
+		opts := &grammar.Options{
+			VariantStackEntry:        tc.useVariant,
+			MaxRuleSizeForOrdinalRef: 2,
+		}
+		got, err := ccParserAction(tc.input, tc.args, node(tc.input), opts)
 		if err != nil {
 			t.Errorf("parserAction(%v, %v) failed with %v", tc.input, tc.args, err)
 			continue
@@ -214,6 +218,33 @@ func TestCcParserAction(t *testing.T) {
 	}
 }
 
+func TestCcParserActionSymbolLookupErrors(t *testing.T) {
+	tests := []struct {
+		input string
+		args  *grammar.ActionVars
+		want  string
+	}{
+		{"$abc", vars(), "$abc:0:0: invalid reference \"$abc\". Cannot find symbol \"abc\" in rule"},
+		{"$$ = $12", vars("%node", "a:0:expr"), "$$ = $12:0:0: index 12 is out of range [1, 3]"},
+		{"$$ = $1", vars("%node", "a:0:expr", "b:1:expr", "c:2:expr"), "$$ = $1:0:0: invalid reference \"$1\". Ordinal references disabled for rules with more than 2 symbols, use the symbol alias instead"},
+	}
+	for _, tc := range tests {
+		opts := &grammar.Options{
+			VariantStackEntry:        true,
+			MaxRuleSizeForOrdinalRef: 2,
+		}
+		got, err := ccParserAction(tc.input, tc.args, node(tc.input), opts)
+		if err == nil {
+			t.Errorf("parserAction(%v, %v) expected to fail with %v", tc.input, tc.args, got)
+			continue
+		}
+		got = err.Error()
+		if diff := diff.LineDiff(tc.want, got); diff != "" {
+			t.Errorf("parserAction(%v, %v) failed, but with diff:\n--- want\n+++ got\n%v", tc.input, tc.args, diff)
+		}
+	}
+}
+
 func vars(list ...string) *grammar.ActionVars {
 	ret := &grammar.ActionVars{
 		CmdArgs: syntax.CmdArgs{

gen/templates/cc_parser_cc.go.tmpl

@@ -707,15 +707,15 @@ void Parser::fetchNext(Lexer& lexer, std::vector<stackEntry>& stack) {
 {{ range $index, $rule := .Parser.Rules -}}
 {{ $act := index $.Parser.Actions $rule.Action -}}
 {{ if (ne $act.Code "") -}}
-absl::Status Parser::action{{$index}}([[maybe_unused]] stackEntry& lhs,
+absl::Status Parser::{{cc_action_func $index $act}}([[maybe_unused]] stackEntry& lhs,
                         [[maybe_unused]] const stackEntry* rhs) {
 {{ if and $.Options.VariantStackEntry (ne $act.Vars.LHSType "") -}}
 {{/* Initialize the variant tag so that semantic action code can use $$ as if it
      was strongly typed. This allows convenient patterns like `$$ = nullptr;`.
 */}}
   lhs.value.emplace<{{$act.Vars.LHSType}}>();
 {{ end -}}
-{{ cc_parser_action $act.Code $act.Vars $act.Origin $.Options.VariantStackEntry -}}
+{{ cc_parser_action $act.Code $act.Vars $act.Origin $.Options -}}
   return absl::OkStatus();
 }
 {{ end -}}
@@ -745,7 +745,7 @@ absl::Status Parser::applyRule(int32_t rule, int32_t ruleLen, stackEntry& lhs,
 {{ end -}}
 {{ if $act.Code -}}
 {
-  absl::Status action_result = action{{$index}}(lhs, rhs);
+  absl::Status action_result = {{cc_action_func $index $act}}(lhs, rhs);
   if (!action_result.ok()) {
     return action_result;
   }
@@ -869,6 +869,9 @@ absl::Status Parser::Parse(int{{$stateType}}_t start, int{{$stateType}}_t end,
 
     if (action >= 0) {
       // Reduce.
+{{ if .Options.TrackReduces -}}
+      reduced_states.push_back(stack.back().state);
+{{ end -}}
       int32_t rule = action;
       int32_t ln = tmRuleLen[rule];
       stackEntry entry;
@@ -899,6 +902,9 @@ absl::Status Parser::Parse(int{{$stateType}}_t start, int{{$stateType}}_t end,
 
     } else if (action {{if .Parser.Tables.Optimized}}<{{else}}=={{end}} -1) {
       // Shift.
+{{ if .Options.TrackReduces -}}
+      reduced_states.clear();
+{{ end -}}
 {{ if .Parser.Tables.Optimized -}}
       state = -2-action;
 {{ else -}}

gen/templates/cc_parser_h.go.tmpl

@@ -168,7 +168,7 @@ class Parser final {
 {{ range $index, $rule := .Parser.Rules -}}
 {{ $act := index $.Parser.Actions $rule.Action -}}
 {{ if (ne $act.Code "") -}}
-  absl::Status action{{$index}}([[maybe_unused]] stackEntry& lhs,
+  absl::Status {{cc_action_func $index $act}}([[maybe_unused]] stackEntry& lhs,
                         [[maybe_unused]] const stackEntry* rhs);
 {{ end -}}
 {{ end -}}