From e247bbdd242fee771d384ff43bae464344002da2 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Tue, 14 Jul 2020 23:24:54 +1000 Subject: [PATCH 01/15] start on new syntax --- lib/snippet-body-old.pegjs | 82 ++++++++++++++++ lib/snippet-body.pegjs | 187 +++++++++++++++++++++---------------- package.json | 11 +++ 3 files changed, 198 insertions(+), 82 deletions(-) create mode 100644 lib/snippet-body-old.pegjs diff --git a/lib/snippet-body-old.pegjs b/lib/snippet-body-old.pegjs new file mode 100644 index 00000000..476c65af --- /dev/null +++ b/lib/snippet-body-old.pegjs @@ -0,0 +1,82 @@ +{ + // Joins all consecutive strings in a collection without clobbering any + // non-string members. + function coalesce (parts) { + const result = []; + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + const ri = result.length - 1; + if (typeof part === 'string' && typeof result[ri] === 'string') { + result[ri] = result[ri] + part; + } else { + result.push(part); + } + } + return result; + } + + function flatten (parts) { + return parts.reduce(function (flat, rest) { + return flat.concat(Array.isArray(rest) ? flatten(rest) : rest); + }, []); + } +} +bodyContent = content:(tabStop / bodyContentText)* { return content; } +bodyContentText = text:bodyContentChar+ { return text.join(''); } +bodyContentChar = escaped / !tabStop char:. { return char; } + +escaped = '\\' char:. { return char; } +tabStop = tabStopWithTransformation / tabStopWithPlaceholder / tabStopWithoutPlaceholder / simpleTabStop + +simpleTabStop = '$' index:[0-9]+ { + return { index: parseInt(index.join("")), content: [] }; +} +tabStopWithoutPlaceholder = '${' index:[0-9]+ '}' { + return { index: parseInt(index.join("")), content: [] }; +} +tabStopWithPlaceholder = '${' index:[0-9]+ ':' content:placeholderContent '}' { + return { index: parseInt(index.join("")), content: content }; +} +tabStopWithTransformation = '${' index:[0-9]+ substitution:transformationSubstitution '}' { + return { + index: parseInt(index.join(""), 10), + content: [], + substitution: substitution + }; +} + +placeholderContent = content:(tabStop / placeholderContentText / variable )* { return flatten(content); } +placeholderContentText = text:placeholderContentChar+ { return coalesce(text); } +placeholderContentChar = escaped / placeholderVariableReference / !tabStop !variable char:[^}] { return char; } + +placeholderVariableReference = '$' digit:[0-9]+ { + return { index: parseInt(digit.join(""), 10), content: [] }; +} + +variable = '${' variableContent '}' { + return ''; // we eat variables and do nothing with them for now +} +variableContent = content:(variable / variableContentText)* { return content; } +variableContentText = text:variableContentChar+ { return text.join(''); } +variableContentChar = !variable char:('\\}' / [^}]) { return char; } + +escapedForwardSlash = pair:'\\/' { return pair; } + +// A pattern and replacement for a transformed tab stop. +transformationSubstitution = '/' find:(escapedForwardSlash / [^/])* '/' replace:formatString* '/' flags:[imy]* { + let reFind = new RegExp(find.join(''), flags.join('') + 'g'); + return { find: reFind, replace: replace[0] }; +} + +formatString = content:(formatStringEscape / formatStringReference / escapedForwardSlash / [^/])+ { + return content; +} +// Backreferencing a substitution. Different from a tab stop. +formatStringReference = '$' digits:[0-9]+ { + return { backreference: parseInt(digits.join(''), 10) }; +}; +// One of the special control flags in a format string for case folding and +// other tasks. +formatStringEscape = '\\' flag:[ULulErn$] { + return { escape: flag }; +} diff --git a/lib/snippet-body.pegjs b/lib/snippet-body.pegjs index 476c65af..59a912c1 100644 --- a/lib/snippet-body.pegjs +++ b/lib/snippet-body.pegjs @@ -1,82 +1,105 @@ -{ - // Joins all consecutive strings in a collection without clobbering any - // non-string members. - function coalesce (parts) { - const result = []; - for (let i = 0; i < parts.length; i++) { - const part = parts[i]; - const ri = result.length - 1; - if (typeof part === 'string' && typeof result[ri] === 'string') { - result[ri] = result[ri] + part; - } else { - result.push(part); - } - } - return result; - } - - function flatten (parts) { - return parts.reduce(function (flat, rest) { - return flat.concat(Array.isArray(rest) ? flatten(rest) : rest); - }, []); - } -} -bodyContent = content:(tabStop / bodyContentText)* { return content; } -bodyContentText = text:bodyContentChar+ { return text.join(''); } -bodyContentChar = escaped / !tabStop char:. { return char; } - -escaped = '\\' char:. { return char; } -tabStop = tabStopWithTransformation / tabStopWithPlaceholder / tabStopWithoutPlaceholder / simpleTabStop - -simpleTabStop = '$' index:[0-9]+ { - return { index: parseInt(index.join("")), content: [] }; -} -tabStopWithoutPlaceholder = '${' index:[0-9]+ '}' { - return { index: parseInt(index.join("")), content: [] }; -} -tabStopWithPlaceholder = '${' index:[0-9]+ ':' content:placeholderContent '}' { - return { index: parseInt(index.join("")), content: content }; -} -tabStopWithTransformation = '${' index:[0-9]+ substitution:transformationSubstitution '}' { - return { - index: parseInt(index.join(""), 10), - content: [], - substitution: substitution - }; -} - -placeholderContent = content:(tabStop / placeholderContentText / variable )* { return flatten(content); } -placeholderContentText = text:placeholderContentChar+ { return coalesce(text); } -placeholderContentChar = escaped / placeholderVariableReference / !tabStop !variable char:[^}] { return char; } - -placeholderVariableReference = '$' digit:[0-9]+ { - return { index: parseInt(digit.join(""), 10), content: [] }; -} - -variable = '${' variableContent '}' { - return ''; // we eat variables and do nothing with them for now -} -variableContent = content:(variable / variableContentText)* { return content; } -variableContentText = text:variableContentChar+ { return text.join(''); } -variableContentChar = !variable char:('\\}' / [^}]) { return char; } - -escapedForwardSlash = pair:'\\/' { return pair; } - -// A pattern and replacement for a transformed tab stop. -transformationSubstitution = '/' find:(escapedForwardSlash / [^/])* '/' replace:formatString* '/' flags:[imy]* { - let reFind = new RegExp(find.join(''), flags.join('') + 'g'); - return { find: reFind, replace: replace[0] }; -} - -formatString = content:(formatStringEscape / formatStringReference / escapedForwardSlash / [^/])+ { - return content; -} -// Backreferencing a substitution. Different from a tab stop. -formatStringReference = '$' digits:[0-9]+ { - return { backreference: parseInt(digits.join(''), 10) }; -}; -// One of the special control flags in a format string for case folding and -// other tasks. -formatStringEscape = '\\' flag:[ULulErn$] { - return { escape: flag }; -} +/* + +Target grammar: + +(Based on VS Code and TextMate, with particular emphasis on supporting LSP snippets) +See https://microsoft.github.io/language-server-protocol/specification#snippet_syntax + +any ::= (text | tabstop | choice | variable)* + +text ::= anything that's not something else + +tabstop ::= '$' int | '${' int '}' | '${' int transform '}' | '${' int ':' any '}' + +choice ::= '${' int '|' text (',' text)* '|}' + +variable ::= '$' var | '${' var '}' | '${' var ':' any '}' | '${' var transform '}' + +transform ::= '/' regex '/' replace '/' options + +replace ::= (format | text)* + +format ::= '$' int | '${' int '}' | '${' int ':' modifier '}' | '${' int ':+' if:replace '}' | '${' int ':?' if:replace ':' else:replace '}' | '${' int ':-' else:replace '}' | '${' int ':' else:replace '}' + +regex ::= JS regex value + +options ::= JS regex options // NOTE: Unrecognised options should be ignored for the best fault tolerance (can log a warning though) + +var ::= [a-zA-Z_][a-zA-Z_0-9]* + +int ::= [0-9]+ + +*/ + +// Grab anything that isn't \ or $, then try to build a special node out of it, and (at the top level) if that fails then just accept it as text +topLevelContent = content:(text / escapedTopLevel / tabStop / choice / variable / any)* { return content; } + +tabStopContent = content:(text / escapedTabStop / tabStop / choice / variable)* { return content; } + +tabStop = tabStopSimple / tabStopWithoutPlaceholder / tabStopWithPlaceholder / tabStopWithTransform + +tabStopSimple = '$' n:integer { return { index: n }; } + +tabStopWithoutPlaceholder = '${' n:integer '}' { return { index: n }; } + +tabStopWithPlaceholder = '${' n:integer ':' content:tabStopContent '}' { return { index: n, content }; } + +tabStopWithTransform = '${' n:integer t:transformation '}' { return { index: n, transformation: t }; } + +transformation = '/' capture:regexString '/' replace:replace '/' flags:flags { return { capture, flags, replace }; } + +regexString = r:[^/]* { return r.join(""); } + +replace = (format / replaceText)* + +format = formatSimple / formatPlain / formatWithModifier / formatWithIf / formatWithIfElse / formatWithElse + +formatSimple = '$' n:integer { return { backreference: n }; } + +formatPlain = '${' n:integer '}' { return { backreference: n }; } + +formatWithModifier = '${' n:integer ':' modifier:modifier '}' { return { backreference: n, modifier }; } + +formatWithIf = '${' n:integer ':+' ifContent:replace '}' { return { backreference: n, ifContent }; } + +formatWithIfElse = '${' n:integer ':?' ifContent:replace ':' elseContent:replace '}' { return { backreference: n, ifContent, elseContent }; } + +formatWithElse = '${' n:integer ':' '-'? elseContent:replace { return { backreference: n, elseContent }; } + +modifier = '/' modifier:var { return modifier; } + +flags = f:[a-z]* { return f; } + +choice = '${' n:integer '|' choiceText (',' choiceText)* '|}' + +variable = variableSimple / variablePlain / variableWithPlaceholder / variableWithTransform + +variableSimple = '$' v:var { return { variable: v }; } + +variablePlain = '${' v:var '}' { return { variable: v }; } + +variableWithPlaceholder = '${' v:var ':' content:tabStopContent '}' { return { variable: v, content }; } + +variableWithTransform = '${' v:var t:transformation '}' { return { variable: v, transformation: t }; } + +text = t:[^$\\}]+ { return t.join("") } + +choiceText = t:[^,|]+ { return t.join(""); } + +replaceText = t:[^$\\}/]+ { return t.join(""); } + +// Match an escaped character. The set of characters that can be escaped is based on context, generally restricted to the minimum set that enables expressing any text content +escapedTopLevel = '\\' c:[$\\}] { return c; } + +escapedTabStop = escapedTopLevel + +escapedChoice = '\\' c:[$\\,|] { return c; } + +// Match nonnegative integers like those used for tab stop ordering +integer = digits:[0-9]+ { return parseInt(digits.join(""), 10); } + +// Match variable names like TM_SELECTED_TEXT +var = a:[a-zA-Z_] b:[a-zA-Z_0-9]* { return a + b.join(""); } + +// Match any single character. Useful to resolve any parse errors where something that looked like it would be special had malformed syntax. +any = a:. { return a; } diff --git a/package.json b/package.json index 7ced0272..ac4584e2 100644 --- a/package.json +++ b/package.json @@ -29,5 +29,16 @@ }, "devDependencies": { "coffeelint": "^1.9.7" + }, + "configSchema": { + "snippetSyntax": { + "type": "enum", + "description": "Configures the syntax used for snippets. LSP is mostly a superset of original, with support for more features.", + "default": "LSP", + "enum": [ + "LSP", + "original" + ] + } } } From 26c5d90d68ebbe76a9cb2ac79e08b2b1078881d3 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Wed, 15 Jul 2020 00:01:27 +1000 Subject: [PATCH 02/15] tweaks --- lib/snippet-body.pegjs | 39 ++++++++++++++++++++++++++++++++------- lib/snippets.js | 8 ++++++++ package.json | 4 ++-- spec/body-parser-spec.js | 4 ++++ spec/parsing-todo.txt | 6 ++++++ 5 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 spec/parsing-todo.txt diff --git a/lib/snippet-body.pegjs b/lib/snippet-body.pegjs index 59a912c1..4a8a142f 100644 --- a/lib/snippet-body.pegjs +++ b/lib/snippet-body.pegjs @@ -25,22 +25,40 @@ regex ::= JS regex value options ::= JS regex options // NOTE: Unrecognised options should be ignored for the best fault tolerance (can log a warning though) +modifier = '/' var + var ::= [a-zA-Z_][a-zA-Z_0-9]* int ::= [0-9]+ */ +{ + function coalesce (parts) { + const result = []; + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + const ri = result.length - 1; + if (typeof part === 'string' && typeof result[ri] === 'string') { + result[ri] = result[ri] + part; + } else { + result.push(part); + } + } + return result; + } +} + // Grab anything that isn't \ or $, then try to build a special node out of it, and (at the top level) if that fails then just accept it as text -topLevelContent = content:(text / escapedTopLevel / tabStop / choice / variable / any)* { return content; } +topLevelContent = content:(text / escapedTopLevel / tabStop / choice / variable / any)* { return coalesce(content); } -tabStopContent = content:(text / escapedTabStop / tabStop / choice / variable)* { return content; } +tabStopContent = content:(tabStopText / escapedTabStop / tabStop / choice / variable)* { return coalesce(content); } tabStop = tabStopSimple / tabStopWithoutPlaceholder / tabStopWithPlaceholder / tabStopWithTransform -tabStopSimple = '$' n:integer { return { index: n }; } +tabStopSimple = '$' n:integer { return { index: n, content: [] }; } -tabStopWithoutPlaceholder = '${' n:integer '}' { return { index: n }; } +tabStopWithoutPlaceholder = '${' n:integer '}' { return { index: n, content: [] }; } tabStopWithPlaceholder = '${' n:integer ':' content:tabStopContent '}' { return { index: n, content }; } @@ -48,9 +66,10 @@ tabStopWithTransform = '${' n:integer t:transformation '}' { return { index: n, transformation = '/' capture:regexString '/' replace:replace '/' flags:flags { return { capture, flags, replace }; } -regexString = r:[^/]* { return r.join(""); } +// TODO: enforce this is a valid regex, or fail (can do at transform level where we make regex though) +regexString = r:([^/\\] / '\\' c:. { return '\\' + c } )* { return r.join(""); } -replace = (format / replaceText)* +replace = (format / replaceText / replaceModifier / escapedReplace)* format = formatSimple / formatPlain / formatWithModifier / formatWithIf / formatWithIfElse / formatWithElse @@ -82,7 +101,9 @@ variableWithPlaceholder = '${' v:var ':' content:tabStopContent '}' { return { v variableWithTransform = '${' v:var t:transformation '}' { return { variable: v, transformation: t }; } -text = t:[^$\\}]+ { return t.join("") } +text = t:([^$\\}])+ { return t.join("") } + +tabStopText = text choiceText = t:[^,|]+ { return t.join(""); } @@ -95,6 +116,10 @@ escapedTabStop = escapedTopLevel escapedChoice = '\\' c:[$\\,|] { return c; } +replaceModifier = '\\' m:[uUlL] { return { modifier: m }; } + +escapedReplace = '\\' c:[$\\] { return c; } + // Match nonnegative integers like those used for tab stop ordering integer = digits:[0-9]+ { return parseInt(digits.join(""), 10); } diff --git a/lib/snippets.js b/lib/snippets.js index 8e67ec0b..2b433099 100644 --- a/lib/snippets.js +++ b/lib/snippets.js @@ -45,6 +45,10 @@ module.exports = { this.handleDisabledPackagesDidChange(newValue, oldValue) })) + this.subscriptions.add(atom.config.observe('snippets.snippetSyntax', ({newValue, oldValue}) => { + this.handleSnippetSyntaxDidChange(newValue, oldValue) + })) + const snippets = this this.subscriptions.add(atom.commands.add('atom-text-editor', { @@ -439,6 +443,10 @@ module.exports = { return this.bodyParser }, + handleSnippetSyntaxDidChange(oldValue, newValue) { + // TODO: Clear out all snippet parse trees && set the parser to the correct syntax + }, + // Get an {Object} with these keys: // * `snippetPrefix`: the possible snippet prefix text preceding the cursor // * `wordPrefix`: the word preceding the cursor diff --git a/package.json b/package.json index ac4584e2..ff09f7bc 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,8 @@ }, "configSchema": { "snippetSyntax": { - "type": "enum", - "description": "Configures the syntax used for snippets. LSP is mostly a superset of original, with support for more features.", + "type": "string", + "description": "Configures the syntax used for snippets. LSP is mostly a superset of original, with support for more features (but not all features may be implemented yet).", "default": "LSP", "enum": [ "LSP", diff --git a/spec/body-parser-spec.js b/spec/body-parser-spec.js index 35492ded..af82af73 100644 --- a/spec/body-parser-spec.js +++ b/spec/body-parser-spec.js @@ -1,6 +1,10 @@ const BodyParser = require('../lib/snippet-body-parser'); describe("Snippet Body Parser", () => { + function t(snippetBody) { + return BodyParser.parse(snippetBody); + } + it("breaks a snippet body into lines, with each line containing tab stops at the appropriate position", () => { const bodyTree = BodyParser.parse(`\ the quick brown $1fox \${2:jumped \${3:over} diff --git a/spec/parsing-todo.txt b/spec/parsing-todo.txt new file mode 100644 index 00000000..09d2a614 --- /dev/null +++ b/spec/parsing-todo.txt @@ -0,0 +1,6 @@ +foo bar +$1 ${1} ${1:/foo} ${1/foo/bar/baz} +foo ${1 bar +\\ \\\\ \\$ \\{ \\} \\, \\| \\: \\/ \\a \\n \\r +${1:\\ \\\\ \\$ \\{ \\} \\, \\| \\: \\/ \\a \\n \\r} +${1|\\ \\\\ \\$ \\{ \\} \\, \\| \\: \\/ \\a \\n \\r|} From f3bb0e9d336f4eaf3a56c8b052d8a3402615ea2a Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Thu, 16 Jul 2020 00:18:54 +1000 Subject: [PATCH 03/15] comments --- lib/snippet-body.pegjs | 101 +++++++++++++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 24 deletions(-) diff --git a/lib/snippet-body.pegjs b/lib/snippet-body.pegjs index 4a8a142f..b7fb43c2 100644 --- a/lib/snippet-body.pegjs +++ b/lib/snippet-body.pegjs @@ -23,7 +23,7 @@ format ::= '$' int | '${' int '}' | '${' int ':' modifier '}' | '${' int ':+' if regex ::= JS regex value -options ::= JS regex options // NOTE: Unrecognised options should be ignored for the best fault tolerance (can log a warning though) +options ::= JS regex options modifier = '/' var @@ -52,73 +52,123 @@ int ::= [0-9]+ // Grab anything that isn't \ or $, then try to build a special node out of it, and (at the top level) if that fails then just accept it as text topLevelContent = content:(text / escapedTopLevel / tabStop / choice / variable / any)* { return coalesce(content); } -tabStopContent = content:(tabStopText / escapedTabStop / tabStop / choice / variable)* { return coalesce(content); } +// Placeholder content. The same as top level, except we need to fail on '}' so that it can end the tab stop (the `any` rule would eat it if we used it here) +tabStopContent = content:(tabStopText / escapedTabStop / tabStop / choice / variable / notCloseBrace)* { return coalesce(content); } -tabStop = tabStopSimple / tabStopWithoutPlaceholder / tabStopWithPlaceholder / tabStopWithTransform +// The forms of a tab stop. They all start with '$', so we pull that out here. +tabStop = '$' t:(tabStopSimple / tabStopWithoutPlaceholder / tabStopWithPlaceholder / tabStopWithTransform) { return t; } -tabStopSimple = '$' n:integer { return { index: n, content: [] }; } +// The simplest form is just $n for some integer `n` +tabStopSimple = n:integer { return { index: n, content: [] }; } -tabStopWithoutPlaceholder = '${' n:integer '}' { return { index: n, content: [] }; } +// The next simplest form is equivalent to the above, but wrapped in `{}` +tabStopWithoutPlaceholder = '{' n:integer '}' { return { index: n, content: [] }; } -tabStopWithPlaceholder = '${' n:integer ':' content:tabStopContent '}' { return { index: n, content }; } +// When a ':' follows `n`, the content after the ':' is the placeholder and it can be anything +tabStopWithPlaceholder = '{' n:integer ':' content:tabStopContent '}' { return { index: n, content }; } -tabStopWithTransform = '${' n:integer t:transformation '}' { return { index: n, transformation: t }; } +// When a transform follows `n` (indicated by '${n:/...') +tabStopWithTransform = '{' n:integer t:transformation '}' { return { index: n, transformation: t }; } -transformation = '/' capture:regexString '/' replace:replace '/' flags:flags { return { capture, flags, replace }; } +// Builds a capture regex and substitution tree. If the capture is not a valid regex, then the match fails +transformation = '/' find:regexString '/' replace:replace '/' flags:flags & { + // Predicate: only succeed if the `find` + `flags` values make a valid regex + // If so, then store the regex into `find` to be used in the following + // match transformation action + try { + find = new RegExp(find, flags); + return true; + } catch(e) { + return false; + } +} { + return { find, replace }; +} -// TODO: enforce this is a valid regex, or fail (can do at transform level where we make regex though) +// Pulls out the portion that would be for the find regex. Validation is done +// higher up, where we also have access to the flags. regexString = r:([^/\\] / '\\' c:. { return '\\' + c } )* { return r.join(""); } -replace = (format / replaceText / replaceModifier / escapedReplace)* +// The form of a substitution for a transformation. It is a mix of plain text + modifiers + backreferences to the find capture groups +// It cannot access tab stop values. +replace = (replaceText / format / replaceModifier / escapedReplace)* -format = formatSimple / formatPlain / formatWithModifier / formatWithIf / formatWithIfElse / formatWithElse +// A reference to a capture group of the find regex of a transformation. Can conditionally +// resolve based on if the match occurred, and have arbitrary modifiers applied to it. +// The common '$' prefix has been pulled out. +format = '$' f:(formatSimple / formatPlain / formatWithModifier / formatWithIf / formatWithIfElse / formatWithElse) { return f; } -formatSimple = '$' n:integer { return { backreference: n }; } +// The simplest format form, resembles a simpel tab stop except `n` refers to the capture group index, not a tab stop +formatSimple = n:integer { return { backreference: n }; } -formatPlain = '${' n:integer '}' { return { backreference: n }; } +// The same as the simple variant, but `n` is enclosed in {} +formatPlain = '{' n:integer '}' { return { backreference: n }; } -formatWithModifier = '${' n:integer ':' modifier:modifier '}' { return { backreference: n, modifier }; } +// A modifier is something like "/upcase", "/pascalcase". If recognised, it resolves to the +// application of a JS function to the `n`th captured group. +formatWithModifier = '{' n:integer ':' modifier:modifier '}' { return { backreference: n, modifier }; } -formatWithIf = '${' n:integer ':+' ifContent:replace '}' { return { backreference: n, ifContent }; } +// If the `n`th capture group is non-empty, then resolve to the `ifContent` value, else an empty string +// Note that ifContent is a replace itself; it's formats still refer to the original transformation find though, +// as transformations cannot be nested. +formatWithIf = '{' n:integer ':+' ifContent:replace '}' { return { backreference: n, ifContent }; } -formatWithIfElse = '${' n:integer ':?' ifContent:replace ':' elseContent:replace '}' { return { backreference: n, ifContent, elseContent }; } +// Same as the if case, but resolve to `elseContent` if empty instead of the empty string +formatWithIfElse = '{' n:integer ':?' ifContent:replace ':' elseContent:replace '}' { return { backreference: n, ifContent, elseContent }; } -formatWithElse = '${' n:integer ':' '-'? elseContent:replace { return { backreference: n, elseContent }; } +// Same as the if case, but reversed behaviour with empty vs non-empty `n`th match +formatWithElse = '{' n:integer ':' '-'? elseContent:replace { return { backreference: n, elseContent }; } +// Used in `format`s to transform a string using a JS function modifier = '/' modifier:var { return modifier; } +// Regex flags. Validation is performed when the regex itself is also known. flags = f:[a-z]* { return f; } +// A tab stop that offers a choice between several fixed values. These values are plain text only. +// This feature is not implemented, but the syntax is parsed to reserve it for future use. +// It will currently just default to a regular tab stop with the first value as it's placeholder. choice = '${' n:integer '|' choiceText (',' choiceText)* '|}' -variable = variableSimple / variablePlain / variableWithPlaceholder / variableWithTransform +// Syntactically looks like a named tab stop. Variables are resolved in JS and may be +// further processed with a transformation. Unrecognised variables are transformed into +// tab stops with the variable name as a placeholder. +variable = '$' v:(variableSimple / variablePlain / variableWithPlaceholder / variableWithTransform) { return v; } -variableSimple = '$' v:var { return { variable: v }; } +variableSimple = v:var { return { variable: v }; } -variablePlain = '${' v:var '}' { return { variable: v }; } +variablePlain = '{' v:var '}' { return { variable: v }; } -variableWithPlaceholder = '${' v:var ':' content:tabStopContent '}' { return { variable: v, content }; } +variableWithPlaceholder = '{' v:var ':' content:tabStopContent '}' { return { variable: v, content }; } -variableWithTransform = '${' v:var t:transformation '}' { return { variable: v, transformation: t }; } +variableWithTransform = '{' v:var t:transformation '}' { return { variable: v, transformation: t }; } +// Top level text. Anything that cannot be the start of something special. False negatives are handled later by the `any` rule text = t:([^$\\}])+ { return t.join("") } +// None-special text inside a tab stop placeholder. Should be no different to regular top level text. tabStopText = text +// None-special text inside a choice. $, {, }, etc. are all regular text in this context. choiceText = t:[^,|]+ { return t.join(""); } +// None-special text inside a replace (substitution part of transformation). Same as normal text, but `/` is special (the end of the regex-like pattern) replaceText = t:[^$\\}/]+ { return t.join(""); } // Match an escaped character. The set of characters that can be escaped is based on context, generally restricted to the minimum set that enables expressing any text content escapedTopLevel = '\\' c:[$\\}] { return c; } +// Characters that can be escaped in a tab stop placeholder are the same as top level escapedTabStop = escapedTopLevel +// Only `,` and `|` can be escaped in a choice, as everything else is plain text escapedChoice = '\\' c:[$\\,|] { return c; } -replaceModifier = '\\' m:[uUlL] { return { modifier: m }; } +// Same as top level, but `/` can also be escaped +escapedReplace = '\\' c:[$\\/] { return c; } -escapedReplace = '\\' c:[$\\] { return c; } +// We handle 'modifiers' separately to escapes. These indicate a change in state when building the replacement (e.g., capitalisation) +replaceModifier = '\\' m:[uUlL] { return { modifier: m }; } // Match nonnegative integers like those used for tab stop ordering integer = digits:[0-9]+ { return parseInt(digits.join(""), 10); } @@ -128,3 +178,6 @@ var = a:[a-zA-Z_] b:[a-zA-Z_0-9]* { return a + b.join(""); } // Match any single character. Useful to resolve any parse errors where something that looked like it would be special had malformed syntax. any = a:. { return a; } + +// Match anything that isn't a '}'. Useful for parse errors inside placeholder text, as `}` should be used to end the tab stop region +notCloseBrace = a:[^}] { return a; } From d6abd4e2c45f688d82760e3aaaaf3293bc167cb5 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Thu, 16 Jul 2020 10:38:50 +1000 Subject: [PATCH 04/15] progress on tests --- lib/snippet-body.pegjs | 18 ++++--- spec/body-parser-spec.js | 106 +++++++++++++++++++++++++++++---------- 2 files changed, 91 insertions(+), 33 deletions(-) diff --git a/lib/snippet-body.pegjs b/lib/snippet-body.pegjs index b7fb43c2..537a0ff5 100644 --- a/lib/snippet-body.pegjs +++ b/lib/snippet-body.pegjs @@ -2,9 +2,6 @@ Target grammar: -(Based on VS Code and TextMate, with particular emphasis on supporting LSP snippets) -See https://microsoft.github.io/language-server-protocol/specification#snippet_syntax - any ::= (text | tabstop | choice | variable)* text ::= anything that's not something else @@ -31,6 +28,15 @@ var ::= [a-zA-Z_][a-zA-Z_0-9]* int ::= [0-9]+ +(Based on VS Code and TextMate, with particular emphasis on supporting LSP snippets) +See https://microsoft.github.io/language-server-protocol/specification#snippet_syntax + +Parse issues (such as unclosed tab stops or invalid regexes) are simply treated +as plain text. + +NOTE: PEG.js is not designed for efficiency. With appropriate benchmarks, it should +be a significant gain to hand write a parser. + */ { @@ -128,7 +134,7 @@ flags = f:[a-z]* { return f; } // A tab stop that offers a choice between several fixed values. These values are plain text only. // This feature is not implemented, but the syntax is parsed to reserve it for future use. // It will currently just default to a regular tab stop with the first value as it's placeholder. -choice = '${' n:integer '|' choiceText (',' choiceText)* '|}' +choice = '${' n:integer '|' a:choiceText b:(',' c:choiceText { return c; } )* '|}' { return { index: n, choices: [a, ...b] }; } // Syntactically looks like a named tab stop. Variables are resolved in JS and may be // further processed with a transformation. Unrecognised variables are transformed into @@ -150,7 +156,7 @@ text = t:([^$\\}])+ { return t.join("") } tabStopText = text // None-special text inside a choice. $, {, }, etc. are all regular text in this context. -choiceText = t:[^,|]+ { return t.join(""); } +choiceText = b:(t:[^,|\\]+ { return t.join(""); } / '\\' c:[,|\\] { return c; } / '\\' c:. { return '\\' + c; } )* { return b.join(""); } // None-special text inside a replace (substitution part of transformation). Same as normal text, but `/` is special (the end of the regex-like pattern) replaceText = t:[^$\\}/]+ { return t.join(""); } @@ -168,7 +174,7 @@ escapedChoice = '\\' c:[$\\,|] { return c; } escapedReplace = '\\' c:[$\\/] { return c; } // We handle 'modifiers' separately to escapes. These indicate a change in state when building the replacement (e.g., capitalisation) -replaceModifier = '\\' m:[uUlL] { return { modifier: m }; } +replaceModifier = '\\' m:[uUlLeEnr] { return { modifier: m }; } // Match nonnegative integers like those used for tab stop ordering integer = digits:[0-9]+ { return parseInt(digits.join(""), 10); } diff --git a/spec/body-parser-spec.js b/spec/body-parser-spec.js index af82af73..b6279e32 100644 --- a/spec/body-parser-spec.js +++ b/spec/body-parser-spec.js @@ -1,18 +1,76 @@ const BodyParser = require('../lib/snippet-body-parser'); describe("Snippet Body Parser", () => { - function t(snippetBody) { - return BodyParser.parse(snippetBody); + function expectMatch(input, tree) { + expect(BodyParser.parse(input)).toEqual(tree); } - it("breaks a snippet body into lines, with each line containing tab stops at the appropriate position", () => { - const bodyTree = BodyParser.parse(`\ -the quick brown $1fox \${2:jumped \${3:over} -}the \${4:lazy} dog\ -` - ); + it("parses snippets with no special behaviour as plain text", () => { + const plainSnippets = [ + "foo $ bar", + "$% $ 1 ${/upcase} \n ${|world|} ${3foo}", + ]; - expect(bodyTree).toEqual([ + for (const plain of plainSnippets) { + expectMatch(plain, [plain]); + } + }); + + it("parses simple tab stops", () => { + expectMatch("hello$1world${2}", [ + "hello", {index: 1, content: []}, "world", {index: 2, content: []}, + ]); + }); + + it("doesn't find escaped tab stops", () => { + expectMatch("\\$1", ["$1"]); + }) + + it("parses simple variables", () => { + expectMatch("hello$foo2__bar&baz${abc}d", [ + "hello", {variable: "foo2__bar"}, "&baz", {variable: "abc"}, "d" + ]); + }); + + describe("only escapes a select few characters", () => { + const escapeTest = "\\$ \\\\ \\} \\% \\* \\, \\| \\{ \\n \\r \\:"; + + const escapeResolveTop = "$ \\ } \\% \\* \\, \\| \\{ \\n \\r \\:"; + + const escapeResolveChoice = "\\$ \\ \\} \\% \\* , | \\{ \\n \\r \\:"; + + it("only escapes '$', '\\', and '}' in top level text", () => { + expectMatch(escapeTest, [ + escapeResolveTop + ]); + }); + + it("escapes the same characters inside tab stop placeholders as in top level text", () => { + expectMatch(`\${1:${escapeTest}}`, [ + {index: 1, content: [escapeResolveTop]}, + ]); + }); + + it("escapes the same characters inside variable placeholders as in top level text", () => { + expectMatch(`\${foo:${escapeTest}}`, [ + {variable: "foo", content: [escapeResolveTop]}, + ]); + }); + + it("escapes ',', '|', and '\\' in choice text", () => { + expectMatch(`\${1|${escapeTest}|}`, [ + {index: 1, choices: [escapeResolveChoice]}, + ]); + }); + }); + + it("does not recognise a tab stop with transformation if the transformation is invalid regex", () => { + expectMatch("${1/foo/bar/a}", ["${1/foo/bar/a}"]); // invalid flag + expectMatch("${1/fo)o/bar/}", ["${1/fo)o/bar/}"]); // invalid body + }); + + it("breaks a snippet body into lines, with each line containing tab stops at the appropriate position", () => { + expectMatch("the quick brown $1fox ${2:jumped ${3:over}\n}the ${4:lazy} dog", [ "the quick brown ", {index: 1, content: []}, "fox ", @@ -31,35 +89,29 @@ the quick brown $1fox \${2:jumped \${3:over} }); it("removes interpolated variables in placeholder text (we don't currently support it)", () => { - const bodyTree = BodyParser.parse("module ${1:ActiveRecord::${TM_FILENAME/(?:\\A|_)([A-Za-z0-9]+)(?:\\.rb)?/(?2::\\u$1)/g}}"); - expect(bodyTree).toEqual([ + expect(t("module ${1:ActiveRecord::${TM_FILENAME/(?:\\A|_)([A-Za-z0-9]+)(?:\\.rb)?/(?2::\\u$1)/g}}")).toEqual([ "module ", { - "index": 1, - "content": ["ActiveRecord::", ""] + index: 1, + content: ["ActiveRecord::", ""] } ]); }); it("skips escaped tabstops", () => { - const bodyTree = BodyParser.parse("snippet $1 escaped \\$2 \\\\$3"); - expect(bodyTree).toEqual([ - "snippet ", - { - index: 1, - content: [] - }, - " escaped $2 \\", - { - index: 3, - content: [] - } + expectMatch("$1 \\$2 $3 \\\\$4 \\\\\\$5 $6", [ + {index: 1, content: []}, + ' $2 ', + {index: 3, content: []}, + ' \\', + {index: 4, content: []}, + ' \\$5 ', + {index: 6, content: []} ]); }); it("includes escaped right-braces", () => { - const bodyTree = BodyParser.parse("snippet ${1:{\\}}"); - expect(bodyTree).toEqual([ + expectMatch("snippet ${1:{\\}}", [ "snippet ", { index: 1, From 25cb10da47e20219b36332322563bb282fe1aab2 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Thu, 16 Jul 2020 11:52:44 +1000 Subject: [PATCH 05/15] more test progress --- lib/snippet-body.pegjs | 6 +- spec/body-parser-spec.js | 303 ++++++++++++++++++++++----------------- 2 files changed, 176 insertions(+), 133 deletions(-) diff --git a/lib/snippet-body.pegjs b/lib/snippet-body.pegjs index 537a0ff5..dad72f53 100644 --- a/lib/snippet-body.pegjs +++ b/lib/snippet-body.pegjs @@ -79,8 +79,8 @@ tabStopWithTransform = '{' n:integer t:transformation '}' { return { index: n, t // Builds a capture regex and substitution tree. If the capture is not a valid regex, then the match fails transformation = '/' find:regexString '/' replace:replace '/' flags:flags & { // Predicate: only succeed if the `find` + `flags` values make a valid regex - // If so, then store the regex into `find` to be used in the following - // match transformation action + // TODO: find a way to not build the same RegExp twice. May need to wait until + // hand written parser. try { find = new RegExp(find, flags); return true; @@ -88,7 +88,7 @@ transformation = '/' find:regexString '/' replace:replace '/' flags:flags & { return false; } } { - return { find, replace }; + return { find: new RegExp(find, flags), replace }; } // Pulls out the portion that would be for the find regex. Validation is done diff --git a/spec/body-parser-spec.js b/spec/body-parser-spec.js index b6279e32..6e92254d 100644 --- a/spec/body-parser-spec.js +++ b/spec/body-parser-spec.js @@ -5,6 +5,88 @@ describe("Snippet Body Parser", () => { expect(BodyParser.parse(input)).toEqual(tree); } + describe("tab stops", () => { + it("parses simple tab stops", () => { + expectMatch("hello$1world${2}", [ + "hello", { index: 1, content: [] }, "world", { index: 2, content: [] }, + ]); + }); + + it("skips escaped tab stops", () => { + expectMatch("$1 \\$2 $3", [ + { index: 1, content: [] }, + " $2 ", + { index: 3, content: [] }, + ]); + }); + + describe("with placeholders", () => { + it("allows placeholders to be arbitrary", () => { + expectMatch("${1:${2}$foo${3|a,b|}}", [ + { + index: 1, + content: [ + { index: 2, content: [] }, + { variable: "foo" }, + { index: 3, choices: ["a", "b"] }, + ] + } + ]); + }); + + it("allows escaping '}' in placeholders", () => { + expectMatch("${1:\\}}", [{ index: 1, content: ["}"] }]); + }); + }); + + describe("with transformations", () => { + it("parses simple transformations", () => { + expectMatch("${1/foo/bar/}", [ + { + index: 1, + transformation: { + find: /foo/, + replace: [ + "bar" + ] + } + } + ]); + }); + + it("does not parse invalid regex as transformations", () => { + expectMatch("${1/foo/bar/a}", ["${1/foo/bar/a}"]); // invalid flag + expectMatch("${1/fo)o$1/$bar/}", [ + "${1/fo)o", + { index: 1, content: [] }, + "/", + { variable: "bar" }, + "/}" + ]); + }); + }); + }); + + describe("variables", () => { + it("parses simple variables", () => { + expectMatch("hello$foo2__bar&baz${abc}d", [ + "hello", + { variable: "foo2__bar" }, + "&baz", + { variable: "abc" }, + "d" + ]); + }); + + it("skips escaped variables", () => { + expectMatch("\\$foo $b\\ar $\\{baz}", [ + "$foo ", + { variable: "b" }, + "\\ar $\\{baz}", + ]); + }); + }); + it("parses snippets with no special behaviour as plain text", () => { const plainSnippets = [ "foo $ bar", @@ -16,22 +98,6 @@ describe("Snippet Body Parser", () => { } }); - it("parses simple tab stops", () => { - expectMatch("hello$1world${2}", [ - "hello", {index: 1, content: []}, "world", {index: 2, content: []}, - ]); - }); - - it("doesn't find escaped tab stops", () => { - expectMatch("\\$1", ["$1"]); - }) - - it("parses simple variables", () => { - expectMatch("hello$foo2__bar&baz${abc}d", [ - "hello", {variable: "foo2__bar"}, "&baz", {variable: "abc"}, "d" - ]); - }); - describe("only escapes a select few characters", () => { const escapeTest = "\\$ \\\\ \\} \\% \\* \\, \\| \\{ \\n \\r \\:"; @@ -47,75 +113,66 @@ describe("Snippet Body Parser", () => { it("escapes the same characters inside tab stop placeholders as in top level text", () => { expectMatch(`\${1:${escapeTest}}`, [ - {index: 1, content: [escapeResolveTop]}, + { index: 1, content: [escapeResolveTop] }, ]); }); it("escapes the same characters inside variable placeholders as in top level text", () => { expectMatch(`\${foo:${escapeTest}}`, [ - {variable: "foo", content: [escapeResolveTop]}, + { variable: "foo", content: [escapeResolveTop] }, ]); }); it("escapes ',', '|', and '\\' in choice text", () => { expectMatch(`\${1|${escapeTest}|}`, [ - {index: 1, choices: [escapeResolveChoice]}, + { index: 1, choices: [escapeResolveChoice] }, ]); }); }); - it("does not recognise a tab stop with transformation if the transformation is invalid regex", () => { - expectMatch("${1/foo/bar/a}", ["${1/foo/bar/a}"]); // invalid flag - expectMatch("${1/fo)o/bar/}", ["${1/fo)o/bar/}"]); // invalid body - }); - it("breaks a snippet body into lines, with each line containing tab stops at the appropriate position", () => { expectMatch("the quick brown $1fox ${2:jumped ${3:over}\n}the ${4:lazy} dog", [ "the quick brown ", - {index: 1, content: []}, + { index: 1, content: [] }, "fox ", { index: 2, content: [ "jumped ", - {index: 3, content: ["over"]}, + { index: 3, content: ["over"] }, "\n" ], }, "the ", - {index: 4, content: ["lazy"]}, + { index: 4, content: ["lazy"] }, " dog" ]); }); - it("removes interpolated variables in placeholder text (we don't currently support it)", () => { - expect(t("module ${1:ActiveRecord::${TM_FILENAME/(?:\\A|_)([A-Za-z0-9]+)(?:\\.rb)?/(?2::\\u$1)/g}}")).toEqual([ + it("supports interpolated variables in placeholder text", () => { + expectMatch("module ${1:ActiveRecord::${TM_FILENAME/(?:\\A|_)([A-Za-z0-9]+)(?:\\.rb)?/(?2::\\u$1)/g}}", [ "module ", { index: 1, - content: ["ActiveRecord::", ""] - } - ]); - }); - - it("skips escaped tabstops", () => { - expectMatch("$1 \\$2 $3 \\\\$4 \\\\\\$5 $6", [ - {index: 1, content: []}, - ' $2 ', - {index: 3, content: []}, - ' \\', - {index: 4, content: []}, - ' \\$5 ', - {index: 6, content: []} - ]); - }); - - it("includes escaped right-braces", () => { - expectMatch("snippet ${1:{\\}}", [ - "snippet ", - { - index: 1, - content: ["{}"] + content: [ + "ActiveRecord::", + { + variable: "TM_FILENAME", + transformation: { + find: /(?:\A|_)([A-Za-z0-9]+)(?:\.rb)?/g, + replace: [ + "(?2::", + { + modifier: "u", + }, + { + backreference: 1, + }, + ")", + ] + } + } + ], } ]); }); @@ -124,160 +181,146 @@ describe("Snippet Body Parser", () => { const bodyTree = BodyParser.parse("<${1:p}>$0"); expect(bodyTree).toEqual([ '<', - {index: 1, content: ['p']}, + { index: 1, content: ['p'] }, '>', - {index: 0, content: []}, + { index: 0, content: [] }, '' + { index: 1, transformation: { find: /f/, replace: ['F'] } }, + '>', ]); }); it("parses a snippet with multiple tab stops with transformations", () => { - const bodyTree = BodyParser.parse("${1:placeholder} ${1/(.)/\\u$1/} $1 ${2:ANOTHER} ${2/^(.*)$/\\L$1/} $2"); - expect(bodyTree).toEqual([ - {index: 1, content: ['placeholder']}, + expectMatch("${1:placeholder} ${1/(.)/\\u$1/} $1 ${2:ANOTHER} ${2/^(.*)$/\\L$1/} $2", [ + { index: 1, content: ['placeholder'] }, ' ', { index: 1, - content: [], - substitution: { - find: /(.)/g, + transformation: { + find: /(.)/, replace: [ - {escape: 'u'}, - {backreference: 1} - ] - } + { modifier: 'u' }, + { backreference: 1 }, + ], + }, }, ' ', - {index: 1, content: []}, + { index: 1, content: [] }, ' ', - {index: 2, content: ['ANOTHER']}, + { index: 2, content: ['ANOTHER'] }, ' ', { index: 2, - content: [], - substitution: { - find: /^(.*)$/g, + transformation: { + find: /^(.*)$/, replace: [ - {escape: 'L'}, - {backreference: 1} - ] - } + { modifier: 'L' }, + { backreference: 1 }, + ], + }, }, ' ', - {index: 2, content: []}, + { index: 2, content: [] }, ]); }); - it("parses a snippet with transformations and mirrors", () => { - const bodyTree = BodyParser.parse("${1:placeholder}\n${1/(.)/\\u$1/}\n$1"); - expect(bodyTree).toEqual([ - {index: 1, content: ['placeholder']}, + expectMatch("${1:placeholder}\n${1/(.)/\\u$1/}\n$1", [ + { index: 1, content: ['placeholder'] }, '\n', { index: 1, - content: [], - substitution: { - find: /(.)/g, + transformation: { + find: /(.)/, replace: [ - {escape: 'u'}, - {backreference: 1} - ] - } + { modifier: 'u' }, + { backreference: 1 }, + ], + }, }, '\n', - {index: 1, content: []} + { index: 1, content: [] }, ]); }); it("parses a snippet with a format string and case-control flags", () => { - const bodyTree = BodyParser.parse("<${1:p}>$0"); - expect(bodyTree).toEqual([ + expectMatch("<${1:p}>$0", [ '<', - {index: 1, content: ['p']}, + { index: 1, content: ['p'] }, '>', - {index: 0, content: []}, + { index: 0, content: [] }, '' + '>', ]); }); it("parses a snippet with an escaped forward slash in a transform", () => { - // Annoyingly, a forward slash needs to be double-backslashed just like the - // other escapes. - const bodyTree = BodyParser.parse("<${1:p}>$0"); - expect(bodyTree).toEqual([ + expectMatch("<${1:p}>$0", [ '<', - {index: 1, content: ['p']}, + { index: 1, content: ['p'] }, '>', - {index: 0, content: []}, + { index: 0, content: [] }, '' + '>', ]); }); it("parses a snippet with a placeholder that mirrors another tab stop's content", () => { - const bodyTree = BodyParser.parse("$4console.${3:log}('${2:$1}', $1);$0"); - expect(bodyTree).toEqual([ - {index: 4, content: []}, + expectMatch("$4console.${3:log}('${2:$1}', $1);$0", [ + { index: 4, content: [] }, 'console.', - {index: 3, content: ['log']}, + { index: 3, content: ['log'] }, '(\'', { index: 2, content: [ - {index: 1, content: []} + { index: 1, content: [] } ] }, '\', ', - {index: 1, content: []}, + { index: 1, content: [] }, ');', - {index: 0, content: []} + { index: 0, content: [] } ]); }); it("parses a snippet with a placeholder that mixes text and tab stop references", () => { - const bodyTree = BodyParser.parse("$4console.${3:log}('${2:uh $1}', $1);$0"); - expect(bodyTree).toEqual([ - {index: 4, content: []}, + expectMatch("$4console.${3:log}('${2:uh $1}', $1);$0", [ + { index: 4, content: [] }, 'console.', - {index: 3, content: ['log']}, + { index: 3, content: ['log'] }, '(\'', { index: 2, content: [ 'uh ', - {index: 1, content: []} + { index: 1, content: [] } ] }, '\', ', - {index: 1, content: []}, + { index: 1, content: [] }, ');', - {index: 0, content: []} + { index: 0, content: [] } ]); }); }); From 95a050aaf0ad0f49bbdd11d1f59f16a1a738b2f5 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Thu, 16 Jul 2020 12:25:13 +1000 Subject: [PATCH 06/15] moreee --- lib/snippet-body.pegjs | 4 +- spec/body-parser-spec.js | 117 ++++++++++++++++++++++++++++++++++----- 2 files changed, 106 insertions(+), 15 deletions(-) diff --git a/lib/snippet-body.pegjs b/lib/snippet-body.pegjs index dad72f53..5be14799 100644 --- a/lib/snippet-body.pegjs +++ b/lib/snippet-body.pegjs @@ -129,7 +129,7 @@ formatWithElse = '{' n:integer ':' '-'? elseContent:replace { return { backrefer modifier = '/' modifier:var { return modifier; } // Regex flags. Validation is performed when the regex itself is also known. -flags = f:[a-z]* { return f; } +flags = f:[a-z]* { return f.join(""); } // A tab stop that offers a choice between several fixed values. These values are plain text only. // This feature is not implemented, but the syntax is parsed to reserve it for future use. @@ -156,7 +156,7 @@ text = t:([^$\\}])+ { return t.join("") } tabStopText = text // None-special text inside a choice. $, {, }, etc. are all regular text in this context. -choiceText = b:(t:[^,|\\]+ { return t.join(""); } / '\\' c:[,|\\] { return c; } / '\\' c:. { return '\\' + c; } )* { return b.join(""); } +choiceText = b:(t:[^,|\\]+ { return t.join(""); } / '\\' c:[,|\\] { return c; } / '\\' c:. { return '\\' + c; } )+ { return b.join(""); } // None-special text inside a replace (substitution part of transformation). Same as normal text, but `/` is special (the end of the regex-like pattern) replaceText = t:[^$\\}/]+ { return t.join(""); } diff --git a/spec/body-parser-spec.js b/spec/body-parser-spec.js index 6e92254d..4524a10d 100644 --- a/spec/body-parser-spec.js +++ b/spec/body-parser-spec.js @@ -1,8 +1,8 @@ -const BodyParser = require('../lib/snippet-body-parser'); +const SnippetParser = require('../lib/snippet-body-parser'); describe("Snippet Body Parser", () => { function expectMatch(input, tree) { - expect(BodyParser.parse(input)).toEqual(tree); + expect(SnippetParser.parse(input)).toEqual(tree); } describe("tab stops", () => { @@ -20,6 +20,14 @@ describe("Snippet Body Parser", () => { ]); }); + it("only allows non-negative integer stop numbers", () => { + expectMatch("$99999", [{ index: 99999, content: [] }]); + expectMatch("$-1", ["$-1"]); + expectMatch("${-1}", ["${-1}"]); + expectMatch("$1.5", [{ index: 1, content: [] }, ".5"]); + expectMatch("${1.5}", ["${1.5}"]); + }); + describe("with placeholders", () => { it("allows placeholders to be arbitrary", () => { expectMatch("${1:${2}$foo${3|a,b|}}", [ @@ -54,6 +62,20 @@ describe("Snippet Body Parser", () => { ]); }); + it("applies flags to the find regex", () => { + expectMatch("${1/foo/bar/gimsuy}", [ + { + index: 1, + transformation: { + find: /foo/gimsuy, + replace: [ + "bar" + ] + } + } + ]); + }); + it("does not parse invalid regex as transformations", () => { expectMatch("${1/foo/bar/a}", ["${1/foo/bar/a}"]); // invalid flag expectMatch("${1/fo)o$1/$bar/}", [ @@ -85,20 +107,90 @@ describe("Snippet Body Parser", () => { "\\ar $\\{baz}", ]); }); + + describe("naming", () => { + it("only allows ASCII letters, numbers, and underscores in names", () => { + expectMatch("$abc_123-not", [{ variable: "abc_123" }, "-not"]); + }); + + it("allows names to start with underscores", () => { + expectMatch("$__properties", [{ variable: "__properties" }]); + }); + + it("doesn't allow names to start with a number", () => { + expectMatch("$1foo", [{ index: 1, content: [] }, "foo"]); + }); + }); + + describe("with placeholders", () => { + it("allows placeholders to be arbitrary", () => { + expectMatch("${foo:${2}$bar${3|a,b|}}", [ + { + variable: "foo", + content: [ + { index: 2, content: [] }, + { variable: "bar" }, + { index: 3, choices: ["a", "b"] }, + ] + } + ]); + }); + + it("allows escaping '}' in placeholders", () => { + expectMatch("${foo:\\}}", [{ variable: "foo", content: ["}"] }]); + }); + }); + + describe("with transformations", () => { + it("parses simple transformations", () => { + expectMatch("${var/foo/bar/}", [ + { + variable: "var", + transformation: { + find: /foo/, + replace: [ + "bar" + ] + } + } + ]); + }); + }); }); - it("parses snippets with no special behaviour as plain text", () => { - const plainSnippets = [ - "foo $ bar", - "$% $ 1 ${/upcase} \n ${|world|} ${3foo}", - ]; + describe("choices", () => { + it("parses choices", () => { + expectMatch("${1|a,b,c|}", [{ index: 1, choices: ["a", "b", "c"] }]); + }); + + it("skips empty choices", () => { + expectMatch("${1||}", ["${1||}"]); + }); + + it("skips escaped choices", () => { + expectMatch("\\${1|a|}", ["${1|a|}"]); + }); - for (const plain of plainSnippets) { - expectMatch(plain, [plain]); - } + it("treats choice items as plain text", () => { + expectMatch("${1|$2,$foo|}", [{ index: 1, choices: ["$2", "$foo"] }]); + }); + + it("only allows ',' and '|' to be escaped in choice text", () => { + expectMatch("${1|a,b\\,c,d\\|},e\\$f|}", [ + { + index: 1, + choices: [ + "a", + "b,c", + "d|}", + "e\\$f" + ] + } + ]); + }); }); - describe("only escapes a select few characters", () => { + describe("escaped characters", () => { const escapeTest = "\\$ \\\\ \\} \\% \\* \\, \\| \\{ \\n \\r \\:"; const escapeResolveTop = "$ \\ } \\% \\* \\, \\| \\{ \\n \\r \\:"; @@ -178,8 +270,7 @@ describe("Snippet Body Parser", () => { }); it("parses a snippet with transformations", () => { - const bodyTree = BodyParser.parse("<${1:p}>$0"); - expect(bodyTree).toEqual([ + expectMatch("<${1:p}>$0", [ '<', { index: 1, content: ['p'] }, '>', From e4822b6fb6737b6b4979f95d1cb939ca5f0542f1 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Thu, 16 Jul 2020 20:46:21 +1000 Subject: [PATCH 07/15] more tests --- lib/snippet-body.pegjs | 35 ++- spec/body-parser-spec.js | 446 +++++++++++++++++++++------------------ 2 files changed, 254 insertions(+), 227 deletions(-) diff --git a/lib/snippet-body.pegjs b/lib/snippet-body.pegjs index 5be14799..d28c72cb 100644 --- a/lib/snippet-body.pegjs +++ b/lib/snippet-body.pegjs @@ -35,7 +35,7 @@ Parse issues (such as unclosed tab stops or invalid regexes) are simply treated as plain text. NOTE: PEG.js is not designed for efficiency. With appropriate benchmarks, it should -be a significant gain to hand write a parser. +be a significant gain to hand write a parser (and remove the PEG.js dependency). */ @@ -55,11 +55,11 @@ be a significant gain to hand write a parser. } } -// Grab anything that isn't \ or $, then try to build a special node out of it, and (at the top level) if that fails then just accept it as text -topLevelContent = content:(text / escapedTopLevel / tabStop / choice / variable / any)* { return coalesce(content); } +// Grab anything that isn't \ or $, then try to build a special node out of it, and (at the top level) if that fails then just accept the first character as text and continue +topLevelContent = c:(text / escapedTopLevel / tabStop / choice / variable / .)* { return coalesce(c); } -// Placeholder content. The same as top level, except we need to fail on '}' so that it can end the tab stop (the `any` rule would eat it if we used it here) -tabStopContent = content:(tabStopText / escapedTabStop / tabStop / choice / variable / notCloseBrace)* { return coalesce(content); } +// Placeholder content. The same as top level, except we need to fail on '}' so that it can end the tab stop (the any matcher would eat it if we used it here) +tabStopContent = c:(tabStopText / escapedTabStop / tabStop / choice / variable / [^}])* { return coalesce(c); } // The forms of a tab stop. They all start with '$', so we pull that out here. tabStop = '$' t:(tabStopSimple / tabStopWithoutPlaceholder / tabStopWithPlaceholder / tabStopWithTransform) { return t; } @@ -97,7 +97,7 @@ regexString = r:([^/\\] / '\\' c:. { return '\\' + c } )* { return r.join(""); } // The form of a substitution for a transformation. It is a mix of plain text + modifiers + backreferences to the find capture groups // It cannot access tab stop values. -replace = (replaceText / format / replaceModifier / escapedReplace)* +replace = r:(replaceText / format / replaceModifier / escapedReplace / [^}/])* { return coalesce(r); } // A reference to a capture group of the find regex of a transformation. Can conditionally // resolve based on if the match occurred, and have arbitrary modifiers applied to it. @@ -126,7 +126,7 @@ formatWithIfElse = '{' n:integer ':?' ifContent:replace ':' elseContent:replace formatWithElse = '{' n:integer ':' '-'? elseContent:replace { return { backreference: n, elseContent }; } // Used in `format`s to transform a string using a JS function -modifier = '/' modifier:var { return modifier; } +modifier = '/' modifier:name { return modifier; } // Regex flags. Validation is performed when the regex itself is also known. flags = f:[a-z]* { return f.join(""); } @@ -134,20 +134,21 @@ flags = f:[a-z]* { return f.join(""); } // A tab stop that offers a choice between several fixed values. These values are plain text only. // This feature is not implemented, but the syntax is parsed to reserve it for future use. // It will currently just default to a regular tab stop with the first value as it's placeholder. -choice = '${' n:integer '|' a:choiceText b:(',' c:choiceText { return c; } )* '|}' { return { index: n, choices: [a, ...b] }; } +// Empty choices are still parsed, as we may wish to assign meaning to it in future. +choice = '${' n:integer '|' c:(a:choiceText b:(',' c:choiceText { return c; } )* { return [a, ...b] })? '|}' { return { index: n, choices: c || [] }; } // Syntactically looks like a named tab stop. Variables are resolved in JS and may be // further processed with a transformation. Unrecognised variables are transformed into // tab stops with the variable name as a placeholder. variable = '$' v:(variableSimple / variablePlain / variableWithPlaceholder / variableWithTransform) { return v; } -variableSimple = v:var { return { variable: v }; } +variableSimple = v:name { return { variable: v }; } -variablePlain = '{' v:var '}' { return { variable: v }; } +variablePlain = '{' v:name '}' { return { variable: v }; } -variableWithPlaceholder = '{' v:var ':' content:tabStopContent '}' { return { variable: v, content }; } +variableWithPlaceholder = '{' v:name ':' content:tabStopContent '}' { return { variable: v, content }; } -variableWithTransform = '{' v:var t:transformation '}' { return { variable: v, transformation: t }; } +variableWithTransform = '{' v:name t:transformation '}' { return { variable: v, transformation: t }; } // Top level text. Anything that cannot be the start of something special. False negatives are handled later by the `any` rule text = t:([^$\\}])+ { return t.join("") } @@ -174,16 +175,10 @@ escapedChoice = '\\' c:[$\\,|] { return c; } escapedReplace = '\\' c:[$\\/] { return c; } // We handle 'modifiers' separately to escapes. These indicate a change in state when building the replacement (e.g., capitalisation) -replaceModifier = '\\' m:[uUlLeEnr] { return { modifier: m }; } +replaceModifier = '\\' m:[ElLuUnr] { return { modifier: m }; } // Match nonnegative integers like those used for tab stop ordering integer = digits:[0-9]+ { return parseInt(digits.join(""), 10); } // Match variable names like TM_SELECTED_TEXT -var = a:[a-zA-Z_] b:[a-zA-Z_0-9]* { return a + b.join(""); } - -// Match any single character. Useful to resolve any parse errors where something that looked like it would be special had malformed syntax. -any = a:. { return a; } - -// Match anything that isn't a '}'. Useful for parse errors inside placeholder text, as `}` should be used to end the tab stop region -notCloseBrace = a:[^}] { return a; } +name = a:[a-zA-Z_] b:[a-zA-Z_0-9]* { return a + b.join(""); } diff --git a/spec/body-parser-spec.js b/spec/body-parser-spec.js index 4524a10d..91c8684e 100644 --- a/spec/body-parser-spec.js +++ b/spec/body-parser-spec.js @@ -5,7 +5,7 @@ describe("Snippet Body Parser", () => { expect(SnippetParser.parse(input)).toEqual(tree); } - describe("tab stops", () => { + describe("When parsing tab stops", () => { it("parses simple tab stops", () => { expectMatch("hello$1world${2}", [ "hello", { index: 1, content: [] }, "world", { index: 2, content: [] }, @@ -21,6 +21,7 @@ describe("Snippet Body Parser", () => { }); it("only allows non-negative integer stop numbers", () => { + expectMatch("$0", [{ index: 0, content: [] }]); expectMatch("$99999", [{ index: 99999, content: [] }]); expectMatch("$-1", ["$-1"]); expectMatch("${-1}", ["${-1}"]); @@ -45,29 +46,27 @@ describe("Snippet Body Parser", () => { it("allows escaping '}' in placeholders", () => { expectMatch("${1:\\}}", [{ index: 1, content: ["}"] }]); }); - }); - describe("with transformations", () => { - it("parses simple transformations", () => { - expectMatch("${1/foo/bar/}", [ + it("allows '$' in placeholders", () => { + expectMatch("${1:$}", [ { index: 1, - transformation: { - find: /foo/, - replace: [ - "bar" - ] - } + content: [ + "$" + ] } ]); }); + }); - it("applies flags to the find regex", () => { - expectMatch("${1/foo/bar/gimsuy}", [ + // See the transformations section for more thorough testing + describe("with transformations", () => { + it("parses simple transformations", () => { + expectMatch("${1/foo/bar/}", [ { index: 1, transformation: { - find: /foo/gimsuy, + find: /foo/, replace: [ "bar" ] @@ -75,21 +74,10 @@ describe("Snippet Body Parser", () => { } ]); }); - - it("does not parse invalid regex as transformations", () => { - expectMatch("${1/foo/bar/a}", ["${1/foo/bar/a}"]); // invalid flag - expectMatch("${1/fo)o$1/$bar/}", [ - "${1/fo)o", - { index: 1, content: [] }, - "/", - { variable: "bar" }, - "/}" - ]); - }); }); }); - describe("variables", () => { + describe("When parsing variables", () => { it("parses simple variables", () => { expectMatch("hello$foo2__bar&baz${abc}d", [ "hello", @@ -141,6 +129,7 @@ describe("Snippet Body Parser", () => { }); }); + // See the transformations section for more thorough testing describe("with transformations", () => { it("parses simple transformations", () => { expectMatch("${var/foo/bar/}", [ @@ -158,13 +147,13 @@ describe("Snippet Body Parser", () => { }); }); - describe("choices", () => { - it("parses choices", () => { + describe("When parsing choices", () => { + it("parses simple choices", () => { expectMatch("${1|a,b,c|}", [{ index: 1, choices: ["a", "b", "c"] }]); }); - it("skips empty choices", () => { - expectMatch("${1||}", ["${1||}"]); + it("parses empty choices", () => { + expectMatch("${1||}", [{ index: 1, choices: [] }]); }); it("skips escaped choices", () => { @@ -190,7 +179,38 @@ describe("Snippet Body Parser", () => { }); }); - describe("escaped characters", () => { + describe("When parsing transformations", () => { + it("allows an empty transformation", () => { + expectMatch("${1///}", [{ index: 1, transformation: { find: new RegExp(""), replace: [] } }]); + }); + + it("applies flags to the find regex", () => { + expectMatch("${1/foo/bar/gimsuy}", [ + { + index: 1, + transformation: { + find: /foo/gimsuy, + replace: [ + "bar" + ] + } + } + ]); + }); + + it("does not parse invalid regex as transformations", () => { + expectMatch("${1/foo/bar/a}", ["${1/foo/bar/a}"]); // invalid flag + expectMatch("${1/fo)o$1/$bar/}", [ + "${1/fo)o", + { index: 1, content: [] }, + "/", + { variable: "bar" }, + "/}" + ]); + }); + }); + + describe("When parsing escaped characters", () => { const escapeTest = "\\$ \\\\ \\} \\% \\* \\, \\| \\{ \\n \\r \\:"; const escapeResolveTop = "$ \\ } \\% \\* \\, \\| \\{ \\n \\r \\:"; @@ -222,196 +242,208 @@ describe("Snippet Body Parser", () => { }); }); - it("breaks a snippet body into lines, with each line containing tab stops at the appropriate position", () => { - expectMatch("the quick brown $1fox ${2:jumped ${3:over}\n}the ${4:lazy} dog", [ - "the quick brown ", - { index: 1, content: [] }, - "fox ", - { - index: 2, - content: [ - "jumped ", - { index: 3, content: ["over"] }, - "\n" - ], - }, - "the ", - { index: 4, content: ["lazy"] }, - " dog" - ]); + describe("When a potential non-text parse fails", () => { + it("accepts the first character as text and resumes", () => { + expectMatch("${1:${2}${3}", [ + "${1:", + { index: 2, content: [] }, + { index: 3, content: [] } + ]); + }); }); - it("supports interpolated variables in placeholder text", () => { - expectMatch("module ${1:ActiveRecord::${TM_FILENAME/(?:\\A|_)([A-Za-z0-9]+)(?:\\.rb)?/(?2::\\u$1)/g}}", [ - "module ", - { - index: 1, - content: [ - "ActiveRecord::", - { - variable: "TM_FILENAME", - transformation: { - find: /(?:\A|_)([A-Za-z0-9]+)(?:\.rb)?/g, - replace: [ - "(?2::", - { - modifier: "u", - }, - { - backreference: 1, - }, - ")", - ] + describe("examples", () => { + it("breaks a snippet body into lines, with each line containing tab stops at the appropriate position", () => { + expectMatch("the quick brown $1fox ${2:jumped ${3:over}\n}the ${4:lazy} dog", [ + "the quick brown ", + { index: 1, content: [] }, + "fox ", + { + index: 2, + content: [ + "jumped ", + { index: 3, content: ["over"] }, + "\n" + ], + }, + "the ", + { index: 4, content: ["lazy"] }, + " dog" + ]); + }); + + it("supports interpolated variables in placeholder text", () => { + expectMatch("module ${1:ActiveRecord::${TM_FILENAME/(?:\\A|_)([A-Za-z0-9]+)(?:\\.rb)?/(?2::\\u$1)/g}}", [ + "module ", + { + index: 1, + content: [ + "ActiveRecord::", + { + variable: "TM_FILENAME", + transformation: { + find: /(?:\A|_)([A-Za-z0-9]+)(?:\.rb)?/g, + replace: [ + "(?2::", + { + modifier: "u", + }, + { + backreference: 1, + }, + ")", + ] + } } - } - ], - } - ]); - }); + ], + } + ]); + }); - it("parses a snippet with transformations", () => { - expectMatch("<${1:p}>$0", [ - '<', - { index: 1, content: ['p'] }, - '>', - { index: 0, content: [] }, - '', - ]); - }); + it("parses a snippet with transformations", () => { + expectMatch("<${1:p}>$0", [ + '<', + { index: 1, content: ['p'] }, + '>', + { index: 0, content: [] }, + '', + ]); + }); - it("parses a snippet with multiple tab stops with transformations", () => { - expectMatch("${1:placeholder} ${1/(.)/\\u$1/} $1 ${2:ANOTHER} ${2/^(.*)$/\\L$1/} $2", [ - { index: 1, content: ['placeholder'] }, - ' ', - { - index: 1, - transformation: { - find: /(.)/, - replace: [ - { modifier: 'u' }, - { backreference: 1 }, - ], + it("parses a snippet with multiple tab stops with transformations", () => { + expectMatch("${1:placeholder} ${1/(.)/\\u$1/} $1 ${2:ANOTHER} ${2/^(.*)$/\\L$1/} $2", [ + { index: 1, content: ['placeholder'] }, + ' ', + { + index: 1, + transformation: { + find: /(.)/, + replace: [ + { modifier: 'u' }, + { backreference: 1 }, + ], + }, }, - }, - ' ', - { index: 1, content: [] }, - ' ', - { index: 2, content: ['ANOTHER'] }, - ' ', - { - index: 2, - transformation: { - find: /^(.*)$/, - replace: [ - { modifier: 'L' }, - { backreference: 1 }, - ], + ' ', + { index: 1, content: [] }, + ' ', + { index: 2, content: ['ANOTHER'] }, + ' ', + { + index: 2, + transformation: { + find: /^(.*)$/, + replace: [ + { modifier: 'L' }, + { backreference: 1 }, + ], + }, }, - }, - ' ', - { index: 2, content: [] }, - ]); - }); + ' ', + { index: 2, content: [] }, + ]); + }); - it("parses a snippet with transformations and mirrors", () => { - expectMatch("${1:placeholder}\n${1/(.)/\\u$1/}\n$1", [ - { index: 1, content: ['placeholder'] }, - '\n', - { - index: 1, - transformation: { - find: /(.)/, - replace: [ - { modifier: 'u' }, - { backreference: 1 }, - ], + it("parses a snippet with transformations and mirrors", () => { + expectMatch("${1:placeholder}\n${1/(.)/\\u$1/}\n$1", [ + { index: 1, content: ['placeholder'] }, + '\n', + { + index: 1, + transformation: { + find: /(.)/, + replace: [ + { modifier: 'u' }, + { backreference: 1 }, + ], + }, }, - }, - '\n', - { index: 1, content: [] }, - ]); - }); + '\n', + { index: 1, content: [] }, + ]); + }); - it("parses a snippet with a format string and case-control flags", () => { - expectMatch("<${1:p}>$0", [ - '<', - { index: 1, content: ['p'] }, - '>', - { index: 0, content: [] }, - ' { + expectMatch("<${1:p}>$0", [ + '<', + { index: 1, content: ['p'] }, + '>', + { index: 0, content: [] }, + '', - ]); - }); + '>', + ]); + }); - it("parses a snippet with an escaped forward slash in a transform", () => { - expectMatch("<${1:p}>$0", [ - '<', - { index: 1, content: ['p'] }, - '>', - { index: 0, content: [] }, - ' { + expectMatch("<${1:p}>$0", [ + '<', + { index: 1, content: ['p'] }, + '>', + { index: 0, content: [] }, + '', - ]); - }); + '>', + ]); + }); - it("parses a snippet with a placeholder that mirrors another tab stop's content", () => { - expectMatch("$4console.${3:log}('${2:$1}', $1);$0", [ - { index: 4, content: [] }, - 'console.', - { index: 3, content: ['log'] }, - '(\'', - { - index: 2, content: [ - { index: 1, content: [] } - ] - }, - '\', ', - { index: 1, content: [] }, - ');', - { index: 0, content: [] } - ]); - }); + it("parses a snippet with a placeholder that mirrors another tab stop's content", () => { + expectMatch("$4console.${3:log}('${2:$1}', $1);$0", [ + { index: 4, content: [] }, + 'console.', + { index: 3, content: ['log'] }, + '(\'', + { + index: 2, content: [ + { index: 1, content: [] } + ] + }, + '\', ', + { index: 1, content: [] }, + ');', + { index: 0, content: [] } + ]); + }); - it("parses a snippet with a placeholder that mixes text and tab stop references", () => { - expectMatch("$4console.${3:log}('${2:uh $1}', $1);$0", [ - { index: 4, content: [] }, - 'console.', - { index: 3, content: ['log'] }, - '(\'', - { - index: 2, content: [ - 'uh ', - { index: 1, content: [] } - ] - }, - '\', ', - { index: 1, content: [] }, - ');', - { index: 0, content: [] } - ]); + it("parses a snippet with a placeholder that mixes text and tab stop references", () => { + expectMatch("$4console.${3:log}('${2:uh $1}', $1);$0", [ + { index: 4, content: [] }, + 'console.', + { index: 3, content: ['log'] }, + '(\'', + { + index: 2, content: [ + 'uh ', + { index: 1, content: [] } + ] + }, + '\', ', + { index: 1, content: [] }, + ');', + { index: 0, content: [] } + ]); + }); }); }); From 09aa82a7e8e6daa49d61d28a750ee83381fdc601 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Thu, 16 Jul 2020 22:39:11 +1000 Subject: [PATCH 08/15] even more tests --- lib/snippet-body.pegjs | 36 ++++++-- spec/body-parser-spec.js | 187 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 9 deletions(-) diff --git a/lib/snippet-body.pegjs b/lib/snippet-body.pegjs index d28c72cb..1cc9f0a9 100644 --- a/lib/snippet-body.pegjs +++ b/lib/snippet-body.pegjs @@ -97,7 +97,16 @@ regexString = r:([^/\\] / '\\' c:. { return '\\' + c } )* { return r.join(""); } // The form of a substitution for a transformation. It is a mix of plain text + modifiers + backreferences to the find capture groups // It cannot access tab stop values. -replace = r:(replaceText / format / replaceModifier / escapedReplace / [^}/])* { return coalesce(r); } +replace = r:(replaceText / format / replaceModifier / escapedReplace / [^/])* { return coalesce(r); } + +// Same as replace, but we disallow plain '}' instead of plain '/' because we are inside a format (ended by '}') +// NOTE: Diallowing escape of '/' is consistent with VS Code. The general rule is "if it's not got a special meaning, it can't be escaped" +// Inside a format there is no special meaning to '/', so we can't escape it. +formatReplace = r:(replaceText / format / replaceModifier / escapedFormatReplace / [^}])* { return coalesce(r); } + +// Another special case; the if half of an if-else format is terminated by ':' +ifElseReplace = r:(replaceText / format / replaceModifier / escapedIfElseReplace / [^:])* { return coalesce(r); } + // A reference to a capture group of the find regex of a transformation. Can conditionally // resolve based on if the match occurred, and have arbitrary modifiers applied to it. @@ -117,13 +126,16 @@ formatWithModifier = '{' n:integer ':' modifier:modifier '}' { return { backrefe // If the `n`th capture group is non-empty, then resolve to the `ifContent` value, else an empty string // Note that ifContent is a replace itself; it's formats still refer to the original transformation find though, // as transformations cannot be nested. -formatWithIf = '{' n:integer ':+' ifContent:replace '}' { return { backreference: n, ifContent }; } +formatWithIf = '{' n:integer ':+' ifContent:formatReplace '}' { return { backreference: n, ifContent }; } // Same as the if case, but resolve to `elseContent` if empty instead of the empty string -formatWithIfElse = '{' n:integer ':?' ifContent:replace ':' elseContent:replace '}' { return { backreference: n, ifContent, elseContent }; } +formatWithIfElse = '{' n:integer ':?' ifContent:ifElseReplace ':' elseContent:formatReplace '}' { return { backreference: n, ifContent, elseContent }; } // Same as the if case, but reversed behaviour with empty vs non-empty `n`th match -formatWithElse = '{' n:integer ':' '-'? elseContent:replace { return { backreference: n, elseContent }; } +// NOTE: The ':' form can cause ambiguities when the contents starts with '/', '+', etc. +// However, instead of allowing them to be escaped, just tell issue raisers to use the +// less ambiguous ':-' form (which also has nice symmetry with the ':+' form). +formatWithElse = '{' n:integer ':' '-'? elseContent:formatReplace '}' { return { backreference: n, elseContent }; } // Used in `format`s to transform a string using a JS function modifier = '/' modifier:name { return modifier; } @@ -153,14 +165,14 @@ variableWithTransform = '{' v:name t:transformation '}' { return { variable: v, // Top level text. Anything that cannot be the start of something special. False negatives are handled later by the `any` rule text = t:([^$\\}])+ { return t.join("") } -// None-special text inside a tab stop placeholder. Should be no different to regular top level text. +// Non-special text inside a tab stop placeholder. Should be no different to regular top level text. tabStopText = text -// None-special text inside a choice. $, {, }, etc. are all regular text in this context. +// Non-special text inside a choice. $, {, }, etc. are all regular text in this context. choiceText = b:(t:[^,|\\]+ { return t.join(""); } / '\\' c:[,|\\] { return c; } / '\\' c:. { return '\\' + c; } )+ { return b.join(""); } -// None-special text inside a replace (substitution part of transformation). Same as normal text, but `/` is special (the end of the regex-like pattern) -replaceText = t:[^$\\}/]+ { return t.join(""); } +// Non-special text inside a replace (substitution part of transformation). Same as normal text, but `/` and ':' is special (the end of the regex-like pattern and if half terminator for if-else format) +replaceText = t:[^$\\}/:]+ { return t.join(""); } // Match an escaped character. The set of characters that can be escaped is based on context, generally restricted to the minimum set that enables expressing any text content escapedTopLevel = '\\' c:[$\\}] { return c; } @@ -174,8 +186,14 @@ escapedChoice = '\\' c:[$\\,|] { return c; } // Same as top level, but `/` can also be escaped escapedReplace = '\\' c:[$\\/] { return c; } +// Format terminated by '}' instead of '/' +escapedFormatReplace = '\\' c:[$\\}] { return c; } + +// If half of if-else format terminated by ':' instead of '}' +escapedIfElseReplace = '\\' c:[$\\:] { return c; } + // We handle 'modifiers' separately to escapes. These indicate a change in state when building the replacement (e.g., capitalisation) -replaceModifier = '\\' m:[ElLuUnr] { return { modifier: m }; } +replaceModifier = '\\' m:[ElLuU] { return { modifier: m }; } // Match nonnegative integers like those used for tab stop ordering integer = digits:[0-9]+ { return parseInt(digits.join(""), 10); } diff --git a/spec/body-parser-spec.js b/spec/body-parser-spec.js index 91c8684e..da495ace 100644 --- a/spec/body-parser-spec.js +++ b/spec/body-parser-spec.js @@ -1,6 +1,9 @@ const SnippetParser = require('../lib/snippet-body-parser'); describe("Snippet Body Parser", () => { + // Helper for testing a snippet parse tree. The `input` + // is the snippet string, the `tree` is the expected + // parse tree. function expectMatch(input, tree) { expect(SnippetParser.parse(input)).toEqual(tree); } @@ -30,6 +33,10 @@ describe("Snippet Body Parser", () => { }); describe("with placeholders", () => { + it("allows placeholders to be empty", () => { + expectMatch("${1:}", [{ index: 1, content: [] }]); + }); + it("allows placeholders to be arbitrary", () => { expectMatch("${1:${2}$foo${3|a,b|}}", [ { @@ -43,6 +50,37 @@ describe("Snippet Body Parser", () => { ]); }); + it("even lets placeholders contain placeholders", () => { + expectMatch("${1:${2:${3:levels}}}", [ + { + index: 1, + content: [ + { + index: 2, + content: [ + { + index: 3, + content: [ + "levels" + ] + } + ] + } + ] + } + ]); + }); + + it("ends the placeholder at an unmatched '}'", () => { + expectMatch("${1:}}", [ + { + index: 1, + content: [] + }, + "}" + ]); + }); + it("allows escaping '}' in placeholders", () => { expectMatch("${1:\\}}", [{ index: 1, content: ["}"] }]); }); @@ -110,6 +148,8 @@ describe("Snippet Body Parser", () => { }); }); + // The placeholder implementation is expected to be the same as for tab stops, so + // see the tab stop placeholder section for more thorough tests describe("with placeholders", () => { it("allows placeholders to be arbitrary", () => { expectMatch("${foo:${2}$bar${3|a,b|}}", [ @@ -208,6 +248,153 @@ describe("Snippet Body Parser", () => { "/}" ]); }); + + it("allows and preserves all escapes in regex strings", () => { + expectMatch("${1/foo\\/\\$\\:\\n\\r/baz/}", [ + { + index: 1, + transformation: { + find: /foo\/\$\:\n\r/, + replace: [ + "baz" + ] + } + } + ]); + }); + + describe("When parsing the replace section", () => { + // Helper for testing the relacement part of + // transformations, which are relatively deep in + // the tree and have a lot of behaviour to cover + // NOTE: Only use when the replace section is expected to + // be valid, or else you will be testing against the + // boilerplate (which is not a good idea) + function expectReplaceMatch(replace, tree) { + expectMatch(`\${1/foo/${replace}/}`, [ + { + index: 1, + transformation: { + find: /foo/, + replace: tree, + } + } + ]); + } + + it("allows '$' and '}' as plain text if not part of a format", () => { + expectReplaceMatch("$}", ["$}"]); + }); + + it("allows inline 'escaped modifiers'", () => { + expectReplaceMatch("foo\\E\\l\\L\\u\\Ubar", [ + "foo", + { modifier: "E" }, + { modifier: "l" }, + { modifier: "L" }, + { modifier: "u" }, + { modifier: "U" }, + "bar" + ]); + }); + + it("allows '$', '\\', and '/' to be escaped", () => { + expectReplaceMatch("\\$1 \\\\ \\/", [ + "$1 \\ /" + ]); + }); + + describe("When parsing formats", () => { + it("parses simple formats", () => { + expectReplaceMatch("$1${2}", [ + { backreference: 1 }, + { backreference: 2 } + ]); + }); + + it("parses formats with modifiers", () => { + expectReplaceMatch("${1:/upcase}", [ + { + backreference: 1, + modifier: "upcase", + } + ]); + }); + + it("parses formats with an if branch", () => { + expectReplaceMatch("${1:+foo$2$bar}", [ + { + backreference: 1, + ifContent: [ + "foo", + { backreference: 2, }, + "$bar" // no variables inside a replace / format + ] + } + ]); + }); + + it("parses formats with if and else branches", () => { + expectReplaceMatch("${1:?foo\\:stillIf:bar\\}stillElse}", [ + { + backreference: 1, + ifContent: [ + "foo:stillIf" + ], + elseContent: [ + "bar}stillElse" + ] + } + ]); + }); + + it("parses formats with an else branch", () => { + expectReplaceMatch("${1:-foo}", [ + { + backreference: 1, + elseContent: [ + "foo" + ] + } + ]); + }); + + it("parses formats with the old else branch syntax", () => { + expectReplaceMatch("${1:foo}", [ + { + backreference: 1, + elseContent: [ + "foo" + ] + } + ]); + }); + + it("allows nested replacements inside of formats", () => { + expectReplaceMatch("${1:+${2:-${3:?a lot of:layers}}}", [ + { + backreference: 1, + ifContent: [ + { + backreference: 2, + elseContent: [ + { + backreference: 3, + ifContent: [ + "a lot of" + ], + elseContent: [ + "layers" + ] + } + ] + } + ] + } + ]); + }); + }); + }); }); describe("When parsing escaped characters", () => { From c657c924016f8e83654eeb6e8bb3e52a22bc9629 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Thu, 16 Jul 2020 22:42:37 +1000 Subject: [PATCH 09/15] caps --- spec/body-parser-spec.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/spec/body-parser-spec.js b/spec/body-parser-spec.js index da495ace..e7c2aeb6 100644 --- a/spec/body-parser-spec.js +++ b/spec/body-parser-spec.js @@ -8,7 +8,7 @@ describe("Snippet Body Parser", () => { expect(SnippetParser.parse(input)).toEqual(tree); } - describe("When parsing tab stops", () => { + describe("when parsing tab stops", () => { it("parses simple tab stops", () => { expectMatch("hello$1world${2}", [ "hello", { index: 1, content: [] }, "world", { index: 2, content: [] }, @@ -115,7 +115,7 @@ describe("Snippet Body Parser", () => { }); }); - describe("When parsing variables", () => { + describe("when parsing variables", () => { it("parses simple variables", () => { expectMatch("hello$foo2__bar&baz${abc}d", [ "hello", @@ -187,7 +187,7 @@ describe("Snippet Body Parser", () => { }); }); - describe("When parsing choices", () => { + describe("when parsing choices", () => { it("parses simple choices", () => { expectMatch("${1|a,b,c|}", [{ index: 1, choices: ["a", "b", "c"] }]); }); @@ -219,7 +219,7 @@ describe("Snippet Body Parser", () => { }); }); - describe("When parsing transformations", () => { + describe("when parsing transformations", () => { it("allows an empty transformation", () => { expectMatch("${1///}", [{ index: 1, transformation: { find: new RegExp(""), replace: [] } }]); }); @@ -263,7 +263,7 @@ describe("Snippet Body Parser", () => { ]); }); - describe("When parsing the replace section", () => { + describe("when parsing the replace section", () => { // Helper for testing the relacement part of // transformations, which are relatively deep in // the tree and have a lot of behaviour to cover @@ -304,7 +304,7 @@ describe("Snippet Body Parser", () => { ]); }); - describe("When parsing formats", () => { + describe("when parsing formats", () => { it("parses simple formats", () => { expectReplaceMatch("$1${2}", [ { backreference: 1 }, @@ -397,7 +397,7 @@ describe("Snippet Body Parser", () => { }); }); - describe("When parsing escaped characters", () => { + describe("when parsing escaped characters", () => { const escapeTest = "\\$ \\\\ \\} \\% \\* \\, \\| \\{ \\n \\r \\:"; const escapeResolveTop = "$ \\ } \\% \\* \\, \\| \\{ \\n \\r \\:"; @@ -429,7 +429,7 @@ describe("Snippet Body Parser", () => { }); }); - describe("When a potential non-text parse fails", () => { + describe("when a potential non-text parse fails", () => { it("accepts the first character as text and resumes", () => { expectMatch("${1:${2}${3}", [ "${1:", From 9be72d381662560d84d7d6bc25f28d93066b89a5 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Thu, 16 Jul 2020 23:43:32 +1000 Subject: [PATCH 10/15] pass specs --- lib/insertion.js | 23 ++++++++--------- lib/snippet-body.pegjs | 4 +-- lib/snippet.js | 6 ++--- lib/tab-stop.js | 10 ++++---- spec/body-parser-spec.js | 54 ++++++++++++++++++++-------------------- spec/insertion-spec.js | 34 ++++++++++++------------- spec/parsing-todo.txt | 6 ----- spec/snippets-spec.js | 10 ++++---- 8 files changed, 69 insertions(+), 78 deletions(-) delete mode 100644 spec/parsing-todo.txt diff --git a/lib/insertion.js b/lib/insertion.js index 96065d1e..f9427805 100644 --- a/lib/insertion.js +++ b/lib/insertion.js @@ -45,19 +45,16 @@ function transformText (str, flags) { } class Insertion { - constructor ({ range, substitution }) { + constructor ({ range, transformation }) { this.range = range - this.substitution = substitution - if (substitution) { - if (substitution.replace === undefined) { - substitution.replace = '' - } - this.replacer = this.makeReplacer(substitution.replace) + this.transformation = transformation + if (transformation) { + this.replacer = this.makeReplacer(transformation.replace) } } isTransformation () { - return !!this.substitution + return !!this.transformation } makeReplacer (replace) { @@ -73,8 +70,8 @@ class Insertion { replace.forEach(token => { if (typeof token === 'string') { result.push(transformText(token, flags)) - } else if (token.escape) { - ESCAPES[token.escape](flags, result) + } else if (token.modifier) { + ESCAPES[token.modifier](flags, result) } else if (token.backreference) { let transformed = transformText(match[token.backreference], flags) result.push(transformed) @@ -85,9 +82,9 @@ class Insertion { } transform (input) { - let { substitution } = this - if (!substitution) { return input } - return input.replace(substitution.find, this.replacer) + let { transformation } = this + if (!transformation) { return input } + return input.replace(transformation.find, this.replacer) } } diff --git a/lib/snippet-body.pegjs b/lib/snippet-body.pegjs index 1cc9f0a9..d9b6c484 100644 --- a/lib/snippet-body.pegjs +++ b/lib/snippet-body.pegjs @@ -65,10 +65,10 @@ tabStopContent = c:(tabStopText / escapedTabStop / tabStop / choice / variable / tabStop = '$' t:(tabStopSimple / tabStopWithoutPlaceholder / tabStopWithPlaceholder / tabStopWithTransform) { return t; } // The simplest form is just $n for some integer `n` -tabStopSimple = n:integer { return { index: n, content: [] }; } +tabStopSimple = n:integer { return { index: n }; } // The next simplest form is equivalent to the above, but wrapped in `{}` -tabStopWithoutPlaceholder = '{' n:integer '}' { return { index: n, content: [] }; } +tabStopWithoutPlaceholder = '{' n:integer '}' { return { index: n }; } // When a ':' follows `n`, the content after the ':' is the placeholder and it can be anything tabStopWithPlaceholder = '{' n:integer ':' content:tabStopContent '}' { return { index: n, content }; } diff --git a/lib/snippet.js b/lib/snippet.js index fcdfed90..b5428576 100644 --- a/lib/snippet.js +++ b/lib/snippet.js @@ -24,16 +24,16 @@ module.exports = class Snippet { let extractTabStops = bodyTree => { for (const segment of bodyTree) { if (segment.index != null) { - let {index, content, substitution} = segment + let {index, content, transformation} = segment if (index === 0) { index = Infinity; } const start = [row, column] - extractTabStops(content) + if (content) { extractTabStops(content); } const range = new Range(start, [row, column]) const tabStop = this.tabStopList.findOrCreate({ index, snippet: this }) - tabStop.addInsertion({ range, substitution }) + tabStop.addInsertion({ range, transformation }) } else if (typeof segment === 'string') { bodyText.push(segment) var segmentLines = segment.split('\n') diff --git a/lib/tab-stop.js b/lib/tab-stop.js index 61a423e4..02f568a2 100644 --- a/lib/tab-stop.js +++ b/lib/tab-stop.js @@ -1,5 +1,5 @@ const {Range} = require('atom') -const Insertion = require('./insertion') +const Insertion = require('./insertion').default // A tab stop: // * belongs to a snippet @@ -20,8 +20,8 @@ class TabStop { return !all } - addInsertion ({ range, substitution }) { - let insertion = new Insertion({ range, substitution }) + addInsertion ({ range, transformation }) { + let insertion = new Insertion({ range, transformation }) let insertions = this.insertions insertions.push(insertion) insertions = insertions.sort((i1, i2) => { @@ -38,7 +38,7 @@ class TabStop { copyWithIndent (indent) { let { snippet, index, insertions } = this let newInsertions = insertions.map(insertion => { - let { range, substitution } = insertion + let { range, transformation } = insertion let newRange = Range.fromObject(range, true) if (newRange.start.row) { newRange.start.column += indent.length @@ -46,7 +46,7 @@ class TabStop { } return new Insertion({ range: newRange, - substitution + transformation }) }) diff --git a/spec/body-parser-spec.js b/spec/body-parser-spec.js index e7c2aeb6..089c1920 100644 --- a/spec/body-parser-spec.js +++ b/spec/body-parser-spec.js @@ -11,24 +11,24 @@ describe("Snippet Body Parser", () => { describe("when parsing tab stops", () => { it("parses simple tab stops", () => { expectMatch("hello$1world${2}", [ - "hello", { index: 1, content: [] }, "world", { index: 2, content: [] }, + "hello", { index: 1 }, "world", { index: 2 }, ]); }); it("skips escaped tab stops", () => { expectMatch("$1 \\$2 $3", [ - { index: 1, content: [] }, + { index: 1 }, " $2 ", - { index: 3, content: [] }, + { index: 3 }, ]); }); it("only allows non-negative integer stop numbers", () => { - expectMatch("$0", [{ index: 0, content: [] }]); - expectMatch("$99999", [{ index: 99999, content: [] }]); + expectMatch("$0", [{ index: 0 }]); + expectMatch("$99999", [{ index: 99999 }]); expectMatch("$-1", ["$-1"]); expectMatch("${-1}", ["${-1}"]); - expectMatch("$1.5", [{ index: 1, content: [] }, ".5"]); + expectMatch("$1.5", [{ index: 1 }, ".5"]); expectMatch("${1.5}", ["${1.5}"]); }); @@ -42,7 +42,7 @@ describe("Snippet Body Parser", () => { { index: 1, content: [ - { index: 2, content: [] }, + { index: 2 }, { variable: "foo" }, { index: 3, choices: ["a", "b"] }, ] @@ -144,7 +144,7 @@ describe("Snippet Body Parser", () => { }); it("doesn't allow names to start with a number", () => { - expectMatch("$1foo", [{ index: 1, content: [] }, "foo"]); + expectMatch("$1foo", [{ index: 1 }, "foo"]); }); }); @@ -156,7 +156,7 @@ describe("Snippet Body Parser", () => { { variable: "foo", content: [ - { index: 2, content: [] }, + { index: 2 }, { variable: "bar" }, { index: 3, choices: ["a", "b"] }, ] @@ -242,7 +242,7 @@ describe("Snippet Body Parser", () => { expectMatch("${1/foo/bar/a}", ["${1/foo/bar/a}"]); // invalid flag expectMatch("${1/fo)o$1/$bar/}", [ "${1/fo)o", - { index: 1, content: [] }, + { index: 1 }, "/", { variable: "bar" }, "/}" @@ -433,8 +433,8 @@ describe("Snippet Body Parser", () => { it("accepts the first character as text and resumes", () => { expectMatch("${1:${2}${3}", [ "${1:", - { index: 2, content: [] }, - { index: 3, content: [] } + { index: 2 }, + { index: 3 } ]); }); }); @@ -443,7 +443,7 @@ describe("Snippet Body Parser", () => { it("breaks a snippet body into lines, with each line containing tab stops at the appropriate position", () => { expectMatch("the quick brown $1fox ${2:jumped ${3:over}\n}the ${4:lazy} dog", [ "the quick brown ", - { index: 1, content: [] }, + { index: 1 }, "fox ", { index: 2, @@ -492,7 +492,7 @@ describe("Snippet Body Parser", () => { '<', { index: 1, content: ['p'] }, '>', - { index: 0, content: [] }, + { index: 0 }, '', @@ -514,7 +514,7 @@ describe("Snippet Body Parser", () => { }, }, ' ', - { index: 1, content: [] }, + { index: 1 }, ' ', { index: 2, content: ['ANOTHER'] }, ' ', @@ -529,7 +529,7 @@ describe("Snippet Body Parser", () => { }, }, ' ', - { index: 2, content: [] }, + { index: 2 }, ]); }); @@ -548,7 +548,7 @@ describe("Snippet Body Parser", () => { }, }, '\n', - { index: 1, content: [] }, + { index: 1 }, ]); }); @@ -557,7 +557,7 @@ describe("Snippet Body Parser", () => { '<', { index: 1, content: ['p'] }, '>', - { index: 0, content: [] }, + { index: 0 }, ' { '<', { index: 1, content: ['p'] }, '>', - { index: 0, content: [] }, + { index: 0 }, ' { it("parses a snippet with a placeholder that mirrors another tab stop's content", () => { expectMatch("$4console.${3:log}('${2:$1}', $1);$0", [ - { index: 4, content: [] }, + { index: 4 }, 'console.', { index: 3, content: ['log'] }, '(\'', { index: 2, content: [ - { index: 1, content: [] } + { index: 1 } ] }, '\', ', - { index: 1, content: [] }, + { index: 1 }, ');', - { index: 0, content: [] } + { index: 0 } ]); }); it("parses a snippet with a placeholder that mixes text and tab stop references", () => { expectMatch("$4console.${3:log}('${2:uh $1}', $1);$0", [ - { index: 4, content: [] }, + { index: 4 }, 'console.', { index: 3, content: ['log'] }, '(\'', { index: 2, content: [ 'uh ', - { index: 1, content: [] } + { index: 1 } ] }, '\', ', - { index: 1, content: [] }, + { index: 1 }, ');', - { index: 0, content: [] } + { index: 0 } ]); }); }); diff --git a/spec/insertion-spec.js b/spec/insertion-spec.js index 83fac925..dd997d99 100644 --- a/spec/insertion-spec.js +++ b/spec/insertion-spec.js @@ -4,10 +4,10 @@ const { Range } = require('atom') const range = new Range(0, 0) describe('Insertion', () => { - it('returns what it was given when it has no substitution', () => { + it('returns what it was given when it has no transformation', () => { let insertion = new Insertion({ range, - substitution: undefined + transformation: undefined }) let transformed = insertion.transform('foo!') @@ -17,7 +17,7 @@ describe('Insertion', () => { it('transforms what it was given when it has a regex transformation', () => { let insertion = new Insertion({ range, - substitution: { + transformation: { find: /foo/g, replace: ['bar'] } @@ -30,11 +30,11 @@ describe('Insertion', () => { it('transforms the case of the next character when encountering a \\u or \\l flag', () => { let uInsertion = new Insertion({ range, - substitution: { + transformation: { find: /(.)(.)(.*)/g, replace: [ { backreference: 1 }, - { escape: 'u' }, + { modifier: 'u' }, { backreference: 2 }, { backreference: 3 } ] @@ -47,11 +47,11 @@ describe('Insertion', () => { let lInsertion = new Insertion({ range, - substitution: { + transformation: { find: /(.{2})(.)(.*)/g, replace: [ { backreference: 1 }, - { escape: 'l' }, + { modifier: 'l' }, { backreference: 2 }, { backreference: 3 } ] @@ -67,11 +67,11 @@ describe('Insertion', () => { it('transforms the case of all remaining characters when encountering a \\U or \\L flag, up until it sees a \\E flag', () => { let uInsertion = new Insertion({ range, - substitution: { + transformation: { find: /(.)(.*)/, replace: [ { backreference: 1 }, - { escape: 'U' }, + { modifier: 'U' }, { backreference: 2 } ] } @@ -83,13 +83,13 @@ describe('Insertion', () => { let ueInsertion = new Insertion({ range, - substitution: { + transformation: { find: /(.)(.{3})(.*)/, replace: [ { backreference: 1 }, - { escape: 'U' }, + { modifier: 'U' }, { backreference: 2 }, - { escape: 'E' }, + { modifier: 'E' }, { backreference: 3 } ] } @@ -101,11 +101,11 @@ describe('Insertion', () => { let lInsertion = new Insertion({ range, - substitution: { + transformation: { find: /(.{4})(.)(.*)/, replace: [ { backreference: 1 }, - { escape: 'L' }, + { modifier: 'L' }, { backreference: 2 }, 'WHAT' ] @@ -116,13 +116,13 @@ describe('Insertion', () => { let leInsertion = new Insertion({ range, - substitution: { + transformation: { find: /^([A-Fa-f])(.*)(.)$/, replace: [ { backreference: 1 }, - { escape: 'L' }, + { modifier: 'L' }, { backreference: 2 }, - { escape: 'E' }, + { modifier: 'E' }, { backreference: 3 } ] } diff --git a/spec/parsing-todo.txt b/spec/parsing-todo.txt deleted file mode 100644 index 09d2a614..00000000 --- a/spec/parsing-todo.txt +++ /dev/null @@ -1,6 +0,0 @@ -foo bar -$1 ${1} ${1:/foo} ${1/foo/bar/baz} -foo ${1 bar -\\ \\\\ \\$ \\{ \\} \\, \\| \\: \\/ \\a \\n \\r -${1:\\ \\\\ \\$ \\{ \\} \\, \\| \\: \\/ \\a \\n \\r} -${1|\\ \\\\ \\$ \\{ \\} \\, \\| \\: \\/ \\a \\n \\r|} diff --git a/spec/snippets-spec.js b/spec/snippets-spec.js index 68994478..c41ea9b6 100644 --- a/spec/snippets-spec.js +++ b/spec/snippets-spec.js @@ -263,19 +263,19 @@ third tabstop $3\ }, "transform with non-transforming mirrors": { prefix: "t13", - body: "${1:placeholder}\n${1/(.)/\\u$1/}\n$1" + body: "${1:placeholder}\n${1/(.)/\\u$1/g}\n$1" }, "multiple tab stops, some with transforms and some without": { prefix: "t14", - body: "${1:placeholder} ${1/(.)/\\u$1/} $1 ${2:ANOTHER} ${2/^(.*)$/\\L$1/} $2" + body: "${1:placeholder} ${1/(.)/\\u$1/g} $1 ${2:ANOTHER} ${2/^(.*)$/\\L$1/} $2" }, "has a transformed tab stop without a corresponding ordinary tab stop": { prefix: 't15', - body: "${1/(.)/\\u$1/} & $2" + body: "${1/(.)/\\u$1/g} & $2" }, "has a transformed tab stop that occurs before the corresponding ordinary tab stop": { prefix: 't16', - body: "& ${1/(.)/\\u$1/} & ${1:q}" + body: "& ${1/(.)/\\u$1/g} & ${1:q}" }, "has a placeholder that mirrors another tab stop's content": { prefix: 't17', @@ -283,7 +283,7 @@ third tabstop $3\ }, "has a transformed tab stop such that it is possible to move the cursor between the ordinary tab stop and its transformed version without an intermediate step": { prefix: 't18', - body: '// $1\n// ${1/./=/}' + body: '// $1\n// ${1/./=/g}' } } }); From e8ab10c5cd0676527d63437404df40b35e90af31 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Thu, 16 Jul 2020 23:47:12 +1000 Subject: [PATCH 11/15] undo code action --- lib/tab-stop.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tab-stop.js b/lib/tab-stop.js index 02f568a2..24755e77 100644 --- a/lib/tab-stop.js +++ b/lib/tab-stop.js @@ -1,5 +1,5 @@ const {Range} = require('atom') -const Insertion = require('./insertion').default +const Insertion = require('./insertion') // A tab stop: // * belongs to a snippet From fb5e9127a80bcc071c2c0dac477a2d980a59d10d Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Thu, 16 Jul 2020 23:52:53 +1000 Subject: [PATCH 12/15] match newer grammar naming --- lib/snippet-body-old.pegjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/snippet-body-old.pegjs b/lib/snippet-body-old.pegjs index 476c65af..75e0be88 100644 --- a/lib/snippet-body-old.pegjs +++ b/lib/snippet-body-old.pegjs @@ -37,11 +37,11 @@ tabStopWithoutPlaceholder = '${' index:[0-9]+ '}' { tabStopWithPlaceholder = '${' index:[0-9]+ ':' content:placeholderContent '}' { return { index: parseInt(index.join("")), content: content }; } -tabStopWithTransformation = '${' index:[0-9]+ substitution:transformationSubstitution '}' { +tabStopWithTransformation = '${' index:[0-9]+ transformation:transformationSubstitution '}' { return { index: parseInt(index.join(""), 10), content: [], - substitution: substitution + transformation, }; } @@ -71,12 +71,12 @@ transformationSubstitution = '/' find:(escapedForwardSlash / [^/])* '/' replace: formatString = content:(formatStringEscape / formatStringReference / escapedForwardSlash / [^/])+ { return content; } -// Backreferencing a substitution. Different from a tab stop. +// Backreferencing a transformation capture group. Different from a tab stop. formatStringReference = '$' digits:[0-9]+ { return { backreference: parseInt(digits.join(''), 10) }; }; // One of the special control flags in a format string for case folding and // other tasks. formatStringEscape = '\\' flag:[ULulErn$] { - return { escape: flag }; + return { modifier: flag }; } From 8b67413ffe158e545e8df187f466daef9e91e5be Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Fri, 17 Jul 2020 00:04:15 +1000 Subject: [PATCH 13/15] add syntax switch --- lib/snippet-body-parser.js | 7 +++++-- package.json | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/snippet-body-parser.js b/lib/snippet-body-parser.js index d4293ecb..cf3a1600 100644 --- a/lib/snippet-body-parser.js +++ b/lib/snippet-body-parser.js @@ -1,12 +1,15 @@ +const syntax = atom.config.get("snippets.snippetSyntax"); +const parserName = syntax === "LSP" ? "snippet-body" : "snippet-body-old"; + let parser try { - parser = require('./snippet-body') + parser = require(`./${parserName}`) } catch (error) { const {allowUnsafeEval} = require('loophole') const fs = require('fs-plus') const PEG = require('pegjs') - const grammarSrc = fs.readFileSync(require.resolve('./snippet-body.pegjs'), 'utf8') + const grammarSrc = fs.readFileSync(require.resolve(`./${parserName}.pegjs`), 'utf8') parser = null allowUnsafeEval(() => parser = PEG.buildParser(grammarSrc)) } diff --git a/package.json b/package.json index ff09f7bc..54832f06 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "configSchema": { "snippetSyntax": { "type": "string", - "description": "Configures the syntax used for snippets. LSP is mostly a superset of original, with support for more features (but not all features may be implemented yet).", + "description": "(Requires restart) Configures the syntax used for snippets. LSP is mostly a superset of original, with support for more features (but not all features may be implemented yet).", "default": "LSP", "enum": [ "LSP", From 0880ec435ed08831a1334b4ba171dc2bd89947a8 Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Fri, 17 Jul 2020 00:56:54 +1000 Subject: [PATCH 14/15] tweak --- lib/snippet-body.pegjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/snippet-body.pegjs b/lib/snippet-body.pegjs index d9b6c484..a3e5ebed 100644 --- a/lib/snippet-body.pegjs +++ b/lib/snippet-body.pegjs @@ -40,6 +40,8 @@ be a significant gain to hand write a parser (and remove the PEG.js dependency). */ { + // Joins all consecutive strings in a collection without clobbering any + // non-string members. function coalesce (parts) { const result = []; for (let i = 0; i < parts.length; i++) { @@ -73,7 +75,7 @@ tabStopWithoutPlaceholder = '{' n:integer '}' { return { index: n }; } // When a ':' follows `n`, the content after the ':' is the placeholder and it can be anything tabStopWithPlaceholder = '{' n:integer ':' content:tabStopContent '}' { return { index: n, content }; } -// When a transform follows `n` (indicated by '${n:/...') +// When a transform follows `n` (indicated by '${n/...') tabStopWithTransform = '{' n:integer t:transformation '}' { return { index: n, transformation: t }; } // Builds a capture regex and substitution tree. If the capture is not a valid regex, then the match fails From 9630b66b73c8e42311cd0bf835fdcc80c74d18ad Mon Sep 17 00:00:00 2001 From: Benjamin Gray Date: Fri, 17 Jul 2020 11:13:57 +1000 Subject: [PATCH 15/15] swap condition --- lib/snippet-body-parser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/snippet-body-parser.js b/lib/snippet-body-parser.js index cf3a1600..d49a115e 100644 --- a/lib/snippet-body-parser.js +++ b/lib/snippet-body-parser.js @@ -1,5 +1,5 @@ const syntax = atom.config.get("snippets.snippetSyntax"); -const parserName = syntax === "LSP" ? "snippet-body" : "snippet-body-old"; +const parserName = syntax === "original" ? "snippet-body-old" : "snippet-body"; let parser try {