diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000000..faf15d33b9
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,3 @@
+# do not modify line endings of our hat test golden files
+packages/cursorless-engine/src/test/fixtures/hat-stats/*.golden -text
+packages/cursorless-engine/src/test/fixtures/hat-stats/*.stats -text
diff --git a/.vscode/launch.json b/.vscode/launch.json
index 0be1e310e7..59b9b6e82e 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -156,6 +156,23 @@
"!**/node_modules/**"
]
},
+ {
+ "name": "Update fixtures, unit tests only",
+ "type": "node",
+ "request": "launch",
+ "program": "${workspaceFolder}/packages/test-harness/out/scripts/runUnitTestsOnly",
+ "env": {
+ "CURSORLESS_TEST": "true",
+ "CURSORLESS_TEST_UPDATE_FIXTURES": "true",
+ "CURSORLESS_REPO_ROOT": "${workspaceFolder}"
+ },
+ "outFiles": ["${workspaceFolder}/**/out/**/*.js"],
+ "preLaunchTask": "${defaultBuildTask}",
+ "resolveSourceMapLocations": [
+ "${workspaceFolder}/**",
+ "!**/node_modules/**"
+ ]
+ },
{
"name": "Docusaurus start",
"type": "node",
diff --git a/packages/common/package.json b/packages/common/package.json
index ee5da97be2..ff9a04f6d3 100644
--- a/packages/common/package.json
+++ b/packages/common/package.json
@@ -20,6 +20,7 @@
"@types/lodash": "4.14.181",
"@types/mocha": "^8.0.4",
"@types/sinon": "^10.0.2",
+ "fast-check": "3.12.0",
"js-yaml": "^4.1.0",
"mocha": "^10.2.0",
"sinon": "^11.1.1"
diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts
index f42a47ee7e..95ca164ee5 100644
--- a/packages/common/src/index.ts
+++ b/packages/common/src/index.ts
@@ -87,3 +87,4 @@ export * from "./getFakeCommandServerApi";
export * from "./types/TestCaseFixture";
export * from "./util/getEnvironmentVariableStrict";
export * from "./util/CompositeKeyDefaultMap";
+export { MockTextDocument, MockTextEditor } from "./testUtil/mockEditor";
diff --git a/packages/common/src/testUtil/mockEditor.test.ts b/packages/common/src/testUtil/mockEditor.test.ts
new file mode 100644
index 0000000000..354e91a085
--- /dev/null
+++ b/packages/common/src/testUtil/mockEditor.test.ts
@@ -0,0 +1,78 @@
+import * as assert from "assert";
+import { MockTextDocument, Range } from "..";
+import * as fc from "fast-check";
+
+suite("mockEditor", () => {
+ test("basic", () => {
+ const s = "abc\n\n123\n";
+ const doc: MockTextDocument = new MockTextDocument(
+ "test.txt",
+ "plaintext",
+ s,
+ );
+
+ for (let i = 0; i < s.length; i++) {
+ const pos = doc.positionAt(i);
+ const offset = doc.offsetAt(pos);
+ assert.equal(offset, i);
+ }
+ const line0 = doc.lineAt(0);
+ assert.equal(line0.text, "abc");
+ assert.equal(line0.firstNonWhitespaceCharacterIndex, 0);
+ assert.equal(line0.isEmptyOrWhitespace, false);
+ assert.equal(line0.lineNumber, 0);
+ assert.ok(line0.range.isEqual(new Range(0, 0, 0, 3)));
+ assert.equal(line0.rangeIncludingLineBreak.start.character, 0);
+ assert.equal(line0.lastNonWhitespaceCharacterIndex, 2);
+
+ const line1 = doc.lineAt(1);
+ assert.equal(line1.text, "");
+ assert.equal(line1.firstNonWhitespaceCharacterIndex, 0);
+ assert.equal(line1.isEmptyOrWhitespace, true);
+ assert.equal(line1.lineNumber, 1);
+ assert.ok(line1.range.isEqual(new Range(1, 0, 1, 0)));
+ assert.equal(line1.rangeIncludingLineBreak.start.character, 0);
+ assert.equal(line1.lastNonWhitespaceCharacterIndex, 0);
+ });
+
+ test("fastcheck", () => {
+ fc.assert(
+ fc.property(fc.string(), (contents) => {
+ const doc: MockTextDocument = new MockTextDocument(
+ "test.txt",
+ "plaintext",
+ contents,
+ );
+ let tot: number = 0;
+ for (let lineno = 0; lineno < doc.lineCount; lineno++) {
+ const line = doc.lineAt(lineno);
+ tot += line.rangeIncludingLineBreak.end.character;
+ assert.equal(line.lineNumber, lineno);
+ assert.equal(line.range.start.line, lineno);
+ assert.equal(line.range.end.line, lineno);
+ assert.equal(line.rangeIncludingLineBreak.start.line, lineno);
+ assert.equal(line.rangeIncludingLineBreak.end.line, lineno);
+ assert.equal(
+ line.rangeIncludingLineBreak.end.character,
+ line.text.length,
+ );
+ assert.equal(
+ line.rangeIncludingLineBreak.end.character,
+ line.range.end.character,
+ );
+ }
+ assert.equal(tot, contents.length);
+
+ for (let i = 0; i < contents.length; i++) {
+ const pos = doc.positionAt(i);
+ // positions must be within the range of a line
+ assert.ok(pos.character <= doc.lineAt(pos.line).range.end.character);
+ const offset = doc.offsetAt(pos);
+ // positionAt and offsetAt are inverses
+ assert.equal(offset, i);
+ return true;
+ }
+ }),
+ );
+ });
+});
diff --git a/packages/common/src/testUtil/mockEditor.ts b/packages/common/src/testUtil/mockEditor.ts
new file mode 100644
index 0000000000..cf014d90ff
--- /dev/null
+++ b/packages/common/src/testUtil/mockEditor.ts
@@ -0,0 +1,169 @@
+import { URI } from "vscode-uri";
+import {
+ EndOfLine,
+ Position,
+ Range,
+ Selection,
+ TextDocument,
+ TextEditor,
+ TextEditorOptions,
+ TextLine,
+} from "..";
+
+// See the TextLine, TextEditor, and TextDocument interfaces
+// for documentation of these classes and their fields.
+
+export class MockTextLine implements TextLine {
+ readonly lineNumber: number;
+ readonly text: string;
+ readonly range: Range;
+ readonly rangeIncludingLineBreak: Range;
+ readonly firstNonWhitespaceCharacterIndex: number;
+ readonly lastNonWhitespaceCharacterIndex: number;
+ readonly isEmptyOrWhitespace: boolean;
+
+ constructor(lineNumber: number, text: string) {
+ if (lineNumber < 0) {
+ throw new Error("lineNumber must be non-negative");
+ }
+ this.lineNumber = lineNumber;
+ // capture any trailing \r\n or \n as eol (possibly neither is present)
+ const eol = text.match(/(\r?\n)$/)?.[1] ?? "";
+ if (eol.length > 0) {
+ this.text = text.slice(0, -eol.length);
+ } else {
+ this.text = text;
+ }
+ this.range = new Range(
+ this.lineNumber,
+ 0,
+ this.lineNumber,
+ this.text.length,
+ );
+ this.rangeIncludingLineBreak = new Range(
+ this.lineNumber,
+ 0,
+ this.lineNumber,
+ this.text.length + eol.length,
+ );
+ const first = this.text.search(/\S/);
+ this.firstNonWhitespaceCharacterIndex =
+ first === -1 ? this.text.length : first;
+ const all = this.text.match(/\S/g);
+ this.lastNonWhitespaceCharacterIndex = all
+ ? this.text.lastIndexOf(all[all.length - 1])
+ : 0;
+ this.isEmptyOrWhitespace =
+ this.firstNonWhitespaceCharacterIndex === this.text.length;
+ }
+}
+
+export class MockTextDocument implements TextDocument {
+ readonly uri: URI;
+ readonly languageId: string;
+ readonly version: number;
+ readonly range: Range;
+ readonly eol: EndOfLine;
+ private lines: MockTextLine[];
+ private contents: string;
+
+ constructor(filename: string, languageId: string, contents: string) {
+ this.uri = URI.file(filename);
+ this.languageId = languageId;
+ this.version = 1;
+ this.contents = contents;
+ const rawLines: string[] = contents.match(/[^\n]*\n|[^\n]+/g) ?? [];
+ this.lines = rawLines.map((line, i) => {
+ return new MockTextLine(i, line);
+ });
+ if (this.lines.length === 0) {
+ this.range = new Range(0, 0, 0, 0);
+ } else {
+ const lastLineRange = this.lines[this.lines.length - 1].range;
+ this.range = new Range(
+ 0,
+ 0,
+ lastLineRange.end.line,
+ lastLineRange.end.character,
+ );
+ }
+ this.eol = "LF";
+ }
+
+ get lineCount(): number {
+ return this.lines.length;
+ }
+
+ public lineAt(x: number | Position): TextLine {
+ if (typeof x === "number") {
+ return this.lines[x];
+ }
+ return this.lines[x.line];
+ }
+
+ public offsetAt(position: Position): number {
+ let offset = 0;
+ for (let i = 0; i < position.line; i++) {
+ offset += this.lineAt(i).rangeIncludingLineBreak.end.character;
+ }
+ offset += position.character;
+ return offset;
+ }
+
+ public positionAt(offset: number): Position {
+ let line = 0;
+ while (offset >= this.lineAt(line).rangeIncludingLineBreak.end.character) {
+ offset -= this.lineAt(line).rangeIncludingLineBreak.end.character;
+ line++;
+ }
+ return new Position(line, offset);
+ }
+
+ public getText(range?: Range): string {
+ if (range === undefined) {
+ return this.contents;
+ }
+ const startOffset = this.offsetAt(range.start);
+ const endOffset = this.offsetAt(range.end);
+ return this.contents.slice(startOffset, endOffset);
+ }
+}
+
+export class MockTextEditor implements TextEditor {
+ public primarySelection: Selection;
+ readonly id: string;
+ readonly document: TextDocument;
+ readonly options: TextEditorOptions;
+ readonly isActive: boolean;
+
+ constructor(document: TextDocument, active: boolean) {
+ this.id = document.uri.toString();
+ this.document = document;
+ this.primarySelection = new Selection(0, 0, 0, 0);
+ this.options = new DefaultTextEditorOptions();
+ this.isActive = active;
+ // TODO: support visible ranges, multiple selections, options
+ }
+
+ get visibleRanges(): Range[] {
+ return [this.document.range];
+ }
+
+ get selections(): Selection[] {
+ return [this.primarySelection];
+ }
+
+ isEqual(other: TextEditor): boolean {
+ return this.id === other.id;
+ }
+}
+
+class DefaultTextEditorOptions implements TextEditorOptions {
+ get tabSize(): number | string {
+ return 4;
+ }
+
+ get insertSpaces(): boolean | string {
+ return true;
+ }
+}
diff --git a/packages/common/src/types/Range.ts b/packages/common/src/types/Range.ts
index b0ffe057f3..899fd6a18c 100644
--- a/packages/common/src/types/Range.ts
+++ b/packages/common/src/types/Range.ts
@@ -64,6 +64,17 @@ export class Range {
return this.start.isEqual(this.end);
}
+ /**
+ * Check if this range is equal to `other`.
+ *
+ * @param other A range.
+ * @return `true` if the start and end of the given range are equal to
+ * the start and end of this range.
+ */
+ public isEqual(other: Range): boolean {
+ return this.start.isEqual(other.start) && this.end.isEqual(other.end);
+ }
+
/**
* `true` if `start.line` and `end.line` are equal.
*/
diff --git a/packages/cursorless-engine/src/core/hatStats.test.ts b/packages/cursorless-engine/src/core/hatStats.test.ts
new file mode 100644
index 0000000000..3df8aa69d5
--- /dev/null
+++ b/packages/cursorless-engine/src/core/hatStats.test.ts
@@ -0,0 +1,364 @@
+import {
+ HatStability,
+ HatStyleMap,
+ HatStyleName,
+ MockTextDocument,
+ MockTextEditor,
+ Selection,
+ TokenHat,
+ TokenHatSplittingMode,
+ getCursorlessRepoRoot,
+ shouldUpdateFixtures,
+} from "@cursorless/common";
+import * as fs from "fs";
+import { unitTestSetup } from "../test/unitTestSetup";
+import { TokenGraphemeSplitter } from "../tokenGraphemeSplitter";
+import { allocateHats } from "../util/allocateHats";
+import path = require("path");
+import {
+ RankedToken,
+ getRankedTokens,
+} from "../util/allocateHats/getRankedTokens";
+import assert = require("assert");
+
+// We use special hat "colors"/"shapes" for nice ASCII art output.
+const HAT_COLORS = ["default", ..."ABCDEF"];
+const HAT_NON_DEFAULT_SHAPES = [..."123456"];
+const allHatStyles: HatStyleMap = {
+ ...Object.fromEntries(
+ HAT_COLORS.map((color) => [
+ color,
+ {
+ color,
+ shape: "default",
+ penalty: penaltyForColorShape(color, "default"),
+ },
+ ]),
+ ),
+ ...Object.fromEntries(
+ HAT_COLORS.flatMap((color) =>
+ HAT_NON_DEFAULT_SHAPES.map((shape) => [
+ `${color}-${shape}`,
+ {
+ color,
+ shape,
+ penalty: penaltyForColorShape(color, shape),
+ },
+ ]),
+ ),
+ ),
+};
+
+const tokenHatSplittingDefaults: TokenHatSplittingMode = {
+ preserveCase: false,
+ lettersToPreserve: [],
+ symbolsToPreserve: [],
+};
+
+// map of file extension to language id
+// TODO: is there a canonical list of these somewhere else?
+const languageIdMap: { [key: string]: string } = {
+ txt: "plaintext",
+ js: "javascript",
+ ts: "typescript",
+ go: "go",
+ py: "python",
+ rs: "rust",
+ java: "java",
+ c: "c",
+};
+
+suite("hatStats", () => {
+ unitTestSetup(({ configuration }) => {
+ configuration.mockConfiguration("tokenHatSplittingMode", {
+ ...tokenHatSplittingDefaults,
+ });
+ });
+
+ const fixturePath = path.join(
+ getCursorlessRepoRoot(),
+ "packages",
+ "cursorless-engine",
+ "src",
+ "test",
+ "fixtures",
+ "hat-stats",
+ );
+
+ fs.readdirSync(fixturePath).forEach((file) => {
+ if (
+ file.endsWith(".stats") ||
+ file.endsWith(".golden") ||
+ file === "readme.md" ||
+ file.startsWith(".") // silly dot files
+ ) {
+ return;
+ }
+
+ test(file, () => {
+ const filepath = path.join(fixturePath, file);
+ const extension = file.slice(file.lastIndexOf("."));
+ const languageId = languageIdMap[extension.slice(1)];
+
+ const contents = fs.readFileSync(filepath, "utf-8");
+ const doc: MockTextDocument = new MockTextDocument(
+ filepath.toString(),
+ languageId,
+ contents,
+ );
+
+ // get a list of all tokens so that we can iterate over them,
+ // placing the primary selection before each one in turn
+ const editor = new MockTextEditor(doc, true);
+ const allTokens = getRankedTokens(editor, [editor]);
+
+ const nHats: number[] = [];
+ const nPenalty0: number[] = [];
+ const nPenalty1: number[] = [];
+ const nPenalty2: number[] = [];
+ // Generate hats at ~16 evenly spaced tokens from allTokens.
+ // It's too slow to do more, even though that would give us more interesting statistics.
+ const someTokens = allTokens.filter(
+ (_, index) => index % Math.floor(allTokens.length / 16) === 0,
+ );
+
+ let tokenSpark: string = "";
+
+ someTokens.forEach((token, index) => {
+ editor.primarySelection = new Selection(
+ token.token.range.start,
+ token.token.range.start,
+ );
+
+ const tokenHat = allocateHats(
+ new TokenGraphemeSplitter(),
+ allHatStyles,
+ [], // for now, no old token hats
+ HatStability.greedy, // doesn't matter for now, because there are no old hats
+ editor,
+ [editor],
+ );
+
+ if (index === 0) {
+ const golden = goldenHatFile(contents, allTokens, tokenHat);
+ tokenSpark = makeTokenSpark(allTokens, tokenHat);
+ const goldenPath = filepath + ".golden";
+ if (shouldUpdateFixtures()) {
+ fs.writeFileSync(goldenPath, golden);
+ } else {
+ const actual = fs.readFileSync(goldenPath, "utf-8");
+ assert.equal(actual, golden);
+ }
+ }
+
+ nHats.push((100 * tokenHat.length) / allTokens.length);
+ const nTokensWithPenalty: number[] = [0, 0, 0];
+ tokenHat.forEach((tokenHat) => {
+ const hatStyle = tokenHat.hatStyle;
+ const penalty = penaltyForHatStyle(hatStyle);
+ nTokensWithPenalty[penalty] += 1;
+ });
+ nPenalty0.push((100 * nTokensWithPenalty[0]) / allTokens.length);
+ nPenalty1.push((100 * nTokensWithPenalty[1]) / allTokens.length);
+ nPenalty2.push((100 * nTokensWithPenalty[2]) / allTokens.length);
+
+ // todo: do another allocation nearby with balanced/stable,
+ // and track % of hats that move
+ // TODO: test with fewer hats enabled also?
+ });
+
+ let s = "";
+ s += `nTokens: ${allTokens.length}\n\n`;
+ s += `tokenSpark:\n`;
+ for (let i = 0; i < tokenSpark.length; i += 60) {
+ s += "\t" + tokenSpark.slice(i, i + 60) + "\n";
+ }
+ s += "\n";
+ s += describeDistribution("nHats", nHats) + "\n";
+ s += describeDistribution("nPenalty0", nPenalty0) + "\n";
+ s += describeDistribution("nPenalty1", nPenalty1) + "\n";
+ s += describeDistribution("nPenalty2", nPenalty2) + "\n";
+ // replace multiple trailing newlines with just one to placate pre-commit
+ s = s.replace(/\n+$/, "\n");
+ const statsPath = filepath + ".stats";
+ if (shouldUpdateFixtures()) {
+ fs.writeFileSync(filepath + ".stats", s);
+ } else {
+ const actual = fs.readFileSync(statsPath, "utf-8");
+ assert.equal(actual, s);
+ }
+ });
+ });
+});
+
+function colorShapeForHatStyle(hatStyle: HatStyleName): [string, string] {
+ const [color, shape] = hatStyle.split("-");
+ return [color, shape ?? "default"];
+}
+
+function penaltyForHatStyle(hatStyle: HatStyleName): number {
+ const [color, shape] = colorShapeForHatStyle(hatStyle);
+ return penaltyForColorShape(color, shape ?? "default");
+}
+
+function penaltyForColorShape(color: string, shape: string): number {
+ return (shape === "default" ? 0 : 1) + (color === "default" ? 0 : 1);
+}
+
+function describeDistribution(name: string, x: number[]): string {
+ const n = x.length;
+ const mean = x.reduce((a, b) => a + b, 0) / n;
+ const variance =
+ x.map((x) => (x - mean) ** 2).reduce((a, b) => a + b, 0) / (n - 1);
+ const std = Math.sqrt(variance);
+ const min = Math.min(...x);
+ const max = Math.max(...x);
+ // create a sparkline histogram with 50 bins
+ const binCounts = new Array(51).fill(0);
+ x.forEach((x) => {
+ const bin = Math.floor((x / 100) * 50);
+ binCounts[bin] += 1;
+ });
+ const spark = sparkline(binCounts);
+ return `${name}:\n\tmean: ${asPercent(mean)}\n\tstd: ${asPercent(
+ std,
+ )}\n\tmin: ${asPercent(min)}\n\tmax: ${asPercent(max)}\n\tspark: ${spark}\n`;
+}
+
+function asPercent(n: number): string {
+ return (
+ n.toLocaleString("en-US", {
+ minimumFractionDigits: 0,
+ maximumFractionDigits: 0,
+ }) + "%"
+ );
+}
+
+function sparkline(pcts: number[]) {
+ const bars = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"];
+ const max = Math.max(...pcts);
+ const chars = pcts.map((pct) => {
+ if (pct === 0) {
+ return " ";
+ }
+ const idx = Math.ceil((pct / max) * bars.length) - 1;
+ return bars[idx];
+ });
+ const chunk = chars.length / 4;
+ return (
+ `0 ${chars.slice(0 * chunk, 1 * chunk).join("")} ` +
+ `25 ${chars.slice(1 * chunk, 2 * chunk).join("")} ` +
+ `50 ${chars.slice(2 * chunk, 3 * chunk).join("")} ` +
+ `75 ${chars.slice(3 * chunk, 4 * chunk).join("")} ` +
+ `100`
+ );
+}
+
+function makeTokenSpark(
+ allTokens: RankedToken[],
+ tokenHats: TokenHat[],
+): string {
+ const bars = ["▁", "▃", "▅", "█"];
+ const penalties = allTokens.map((token) => {
+ const hat = tokenHats.find((hat) =>
+ hat.token.range.isEqual(token.token.range),
+ ) as TokenHat;
+ if (hat === undefined) {
+ return 3;
+ }
+ const hatStyle = hat.hatStyle;
+ return penaltyForHatStyle(hatStyle);
+ });
+ const chars = penalties.map((penalty) => {
+ return bars[penalty];
+ });
+ return chars.join("");
+}
+
+function goldenHatFile(
+ contents: string,
+ allTokens: RankedToken[],
+ tokenHats: TokenHat[],
+): string {
+ // Iterate over all tokens, writing out hats, ranges, and content.
+ let out: string = "";
+ const lines = contents.split(/\r?\n/);
+ lines.forEach((line, lineno) => {
+ // Use only one empty line per empty input line, rather than three
+ if (line.length === 0) {
+ return;
+ }
+ // TODO: this is wasteful. oh well?
+ const lineTokens = allTokens.filter(
+ (token) => token.token.range.end.line === lineno,
+ );
+ let line1 = "";
+ let line2 = "";
+ let rangeLine = "";
+ lineTokens.forEach((token) => {
+ const tokenRange = token.token.range;
+ if (tokenRange.start.line !== tokenRange.end.line) {
+ throw new Error(
+ `multi-line tokens not supported, have ${tokenRange.concise()}`,
+ );
+ }
+
+ const hat = tokenHats.find((hat) =>
+ hat.token.range.isEqual(token.token.range),
+ ) as TokenHat;
+ if (hat === undefined) {
+ // TODO: visually call out tokens without hats?
+ return;
+ }
+ const hatRange = hat.hatRange;
+ const [color, shape] = colorShapeForHatStyle(hat.hatStyle);
+ const penalty = penaltyForHatStyle(hat.hatStyle);
+ if (penalty === 0) {
+ line1 += " ".repeat(hatRange.start.character - line1.length);
+ line1 += "_";
+ } else if (penalty === 1) {
+ const char = color === "default" ? shape : color;
+ line1 += " ".repeat(hatRange.start.character - line1.length);
+ line1 += char;
+ } else if (penalty === 2) {
+ line1 += " ".repeat(hatRange.start.character - line1.length);
+ line1 += shape;
+ line2 += " ".repeat(hatRange.start.character - line2.length);
+ line2 += color;
+ } else {
+ throw new Error(`unexpected penalty: ${penalty}`);
+ }
+ const width = tokenRange.end.character - tokenRange.start.character;
+ let rangeStr = "";
+ if (width === 1) {
+ rangeStr = "|";
+ } else if (width === 2) {
+ rangeStr = "[]";
+ } else if (width > 2) {
+ rangeStr = "[" + "-".repeat(width - 2) + "]";
+ } else {
+ throw new Error(`unexpected width: ${width}`);
+ }
+ rangeLine += " ".repeat(tokenRange.start.character - rangeLine.length);
+ rangeLine += rangeStr;
+ });
+ if (line2.length !== 0) {
+ out += line2 + "\n";
+ }
+ if (line1.length !== 0) {
+ out += line1 + "\n";
+ }
+ if (line.length !== 0) {
+ // TODO: tabs, emoji, sigh
+ line = line.replaceAll(/\t/g, "␉");
+ out += line + "\n";
+ }
+ if (rangeLine.length !== 0) {
+ out += rangeLine + "\n";
+ }
+ out += "\n";
+ });
+ // replace multiple trailing newlines with just one to placate pre-commit
+ out = out.replace(/\n+$/, "\n");
+ return out;
+}
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/intern.go b/packages/cursorless-engine/src/test/fixtures/hat-stats/intern.go
new file mode 100644
index 0000000000..5af665a4aa
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/intern.go
@@ -0,0 +1,42 @@
+// Package intern interns strings.
+// Interning is best effort only.
+// Interned strings may be removed automatically
+// at any time without notification.
+// All functions may be called concurrently
+// with themselves and each other.
+package intern
+
+import "sync"
+
+var pool sync.Pool = sync.Pool{
+ New: func() interface{} {
+ return make(map[string]string)
+ },
+}
+
+// String returns s, interned.
+func String(s string) string {
+ m := pool.Get().(map[string]string)
+ c, ok := m[s]
+ if ok {
+ pool.Put(m)
+ return c
+ }
+ m[s] = s
+ pool.Put(m)
+ return s
+}
+
+// Bytes returns b converted to a string, interned.
+func Bytes(b []byte) string {
+ m := pool.Get().(map[string]string)
+ c, ok := m[string(b)]
+ if ok {
+ pool.Put(m)
+ return c
+ }
+ s := string(b)
+ m[s] = s
+ pool.Put(m)
+ return s
+}
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/intern.go.golden b/packages/cursorless-engine/src/test/fixtures/hat-stats/intern.go.golden
new file mode 100644
index 0000000000..21a742a314
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/intern.go.golden
@@ -0,0 +1,160 @@
+_ _ _ _ _ _
+// Package intern interns strings.
+[] [-----] [----] [-----] [-----]|
+
+A _ A _ _ _ A
+// Interning is best effort only.
+[] [-------] [] [--] [----] [--]|
+
+B _ _ _ A _ _
+// Interned strings may be removed automatically
+[] [------] [-----] [-] [] [-----] [-----------]
+
+C A _ A _ _ B
+// at any time without notification.
+[] [] [-] [--] [-----] [----------]|
+
+D _ _ A B _ A
+// All functions may be called concurrently
+[] [-] [-------] [-] [] [----] [----------]
+
+E _ B _ A A C
+// with themselves and each other.
+[] [--] [--------] [-] [--] [---]|
+
+ _ B
+package intern
+[-----] [----]
+
+C _A A
+import "sync"
+[----] |[--]|
+
+A A B DB _ C EC _
+var pool sync.Pool = sync.Pool{
+[-] [--] [--]|[--] | [--]|[--]|
+
+ A _ A __ D A_ B
+␉New: func() interface{} {
+ [-]| [--]|| [-------]|| |
+
+ A B AC _D _E A
+␉␉return make(map[string]string)
+ [----] [--]|[-]|[----]|[----]|
+
+ A_
+␉},
+ ||
+
+B
+}
+|
+
+F F B 1A E F
+// String returns s, interned.
+[] [----] [-----] || [------]|
+
+B 2 B3 4 B 5 C
+func String(s string) string {
+[--] [----]|| [----]| [----] |
+
+ D AA D 1A CC2DE A6 A C D
+␉m := pool.Get().(map[string]string)
+ | || [--]|[-]||||[-]|[----]|[----]|
+
+ A
+ BB B BB FB1B
+␉c, ok := m[s]
+ || [] || ||||
+
+ F C D
+␉if ok {
+ [] [] |
+
+ E 3F E1E
+␉␉pool.Put(m)
+ [--]|[-]|||
+
+ C C
+␉␉return c
+ [----] |
+
+ C
+␉}
+ |
+
+ A A
+ 2C2C C 3
+␉m[s] = s
+ |||| | |
+
+ 1 42 F3F
+␉pool.Put(m)
+ [--]|[-]|||
+
+ A
+ D 4
+␉return s
+ [----] |
+
+D
+}
+|
+
+1 C E D D D B E C 1 5
+// Bytes returns b converted to a string, interned.
+[] [---] [-----] | [-------] [] | [----]| [------]|
+
+C E 1F DD1 1 F E
+func Bytes(b []byte) string {
+[--] [---]|| ||[--]| [----] |
+
+ A
+ 4 CD 3 6B 22135 E 1 E 2 3
+␉m := pool.Get().(map[string]string)
+ | || [--]|[-]||||[-]|[----]|[----]|
+
+ ED D DE 6F 3 424F
+␉c, ok := m[string(b)]
+ || [] || ||[----]||||
+
+ 2 E F
+␉if ok {
+ [] [] |
+
+ A A
+ 4 25 515
+␉␉pool.Put(m)
+ [--]|[-]|||
+
+ F F
+␉␉return c
+ [----] |
+
+ E
+␉}
+ |
+
+ A
+ 5 EF 4 636
+␉s := string(b)
+ | || [----]|||
+
+ A A B
+ 2161 1 1
+␉m[s] = s
+ |||| | |
+
+ A AAA
+ 6 3 A 131
+␉pool.Put(m)
+ [--]|[-]|||
+
+ B
+ 1 2
+␉return s
+ [----] |
+
+F
+}
+|
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/intern.go.stats b/packages/cursorless-engine/src/test/fixtures/hat-stats/intern.go.stats
new file mode 100644
index 0000000000..593f7576d8
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/intern.go.stats
@@ -0,0 +1,35 @@
+nTokens: 228
+
+tokenSpark:
+ ▁▁▁▁▁▁▃▁▃▁▁▁▃▃▁▁▁▃▁▁▃▃▁▃▁▁▃▃▁▁▃▃▁▃▃▁▃▁▃▃▃▁▃▃▁▃▃▃▃▃▃▃▁▃▃▃▁▃▁▃
+ ▁▁▃▃▁▃▃▃▃▃▁▃▁▃▃▃▁▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▅▃▃▃
+ ▃▃▃▃▃▃▃▃▃▃▃▃▅▃▃▅▃▃▃▃▃▃▃▅▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▅▃▃▃▃▃
+ ▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▅▃▃▅▃▃▃▃▅▃▃▃▃▃▃▅▃▅▃▃▅▃▅▃▅▅▅▃▅▃
+
+nHats:
+ mean: 100%
+ std: 0%
+ min: 100%
+ max: 100%
+ spark: 0 25 50 75 █ 100
+
+nPenalty0:
+ mean: 15%
+ std: 0%
+ min: 15%
+ max: 15%
+ spark: 0 █ 25 50 75 100
+
+nPenalty1:
+ mean: 80%
+ std: 2%
+ min: 78%
+ max: 82%
+ spark: 0 25 50 75 █▇█ 100
+
+nPenalty2:
+ mean: 5%
+ std: 2%
+ min: 3%
+ max: 7%
+ spark: 0 █▇▅ 25 50 75 100
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/mending_wall.txt b/packages/cursorless-engine/src/test/fixtures/hat-stats/mending_wall.txt
new file mode 100644
index 0000000000..6a1ad9c8bc
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/mending_wall.txt
@@ -0,0 +1,48 @@
+Mending Wall
+by Robert Frost, 1914 (public domain)
+
+Something there is that doesn’t love a wall,
+That sends the frozen-ground-swell under it,
+And spills the upper boulders in the sun;
+And makes gaps even two can pass abreast.
+The work of hunters is another thing:
+I have come after them and made repair
+Where they have left not one stone on a stone,
+But they would have the rabbit out of hiding,
+To please the yelping dogs. The gaps I mean,
+No one has seen them made or heard them made,
+But at spring mending-time we find them there.
+I let my neighbor know beyond the hill;
+And on a day we meet to walk the line
+And set the wall between us once again.
+We keep the wall between us as we go.
+To each the boulders that have fallen to each.
+And some are loaves and some so nearly balls
+We have to use a spell to make them balance:
+‘Stay where you are until our backs are turned!’
+We wear our fingers rough with handling them.
+Oh, just another kind of out-door game,
+One on a side. It comes to little more:
+There where it is we do not need the wall:
+He is all pine and I am apple orchard.
+My apple trees will never get across
+And eat the cones under his pines, I tell him.
+He only says, ‘Good fences make good neighbors.’
+Spring is the mischief in me, and I wonder
+If I could put a notion in his head:
+‘Why do they make good neighbors? Isn’t it
+Where there are cows? But here there are no cows.
+Before I built a wall I’d ask to know
+What I was walling in or walling out,
+And to whom I was like to give offense.
+Something there is that doesn’t love a wall,
+That wants it down.’ I could say ‘Elves’ to him,
+But it’s not elves exactly, and I’d rather
+He said it for himself. I see him there
+Bringing a stone grasped firmly by the top
+In each hand, like an old-stone savage armed.
+He moves in darkness as it seems to me,
+Not of woods only and the shade of trees.
+He will not go behind his father’s saying,
+And he likes having thought of it so well
+He says again, ‘Good fences make good neighbors.’
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/mending_wall.txt.golden b/packages/cursorless-engine/src/test/fixtures/hat-stats/mending_wall.txt.golden
new file mode 100644
index 0000000000..015bf0f1fd
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/mending_wall.txt.golden
@@ -0,0 +1,213 @@
+_ _
+Mending Wall
+[-----] [--]
+
+_ _ _ _ _ __ _ _
+by Robert Frost, 1914 (public domain)
+[] [----] [---]| [--] |[----] [----]|
+
+_ _ _ _ _ _A _ _ A A
+Something there is that doesn’t love a wall,
+[-------] [---] [] [--] [---]|| [--] | [--]|
+
+B _ C _ __ AA _ A B
+That sends the frozen-ground-swell under it,
+[--] [---] [-] [----]|[----]|[---] [---] []|
+
+ _ B D A A B E C _
+And spills the upper boulders in the sun;
+[-] [----] [-] [---] [------] [] [-] [-]|
+
+A _ A _ F _ A B _
+And makes gaps even two can pass abreast.
+[-] [---] [--] [--] [-] [-] [--] [-----]|
+
+1 A A A C C 2 _
+The work of hunters is another thing:
+[-] [--] [] [-----] [] [-----] [---]|
+
+D B A D 3 E A A
+I have come after them and made repair
+| [--] [--] [---] [--] [-] [--] [----]
+
+B _ C A A B D C F E C
+Where they have left not one stone on a stone,
+[---] [--] [--] [--] [-] [-] [---] [] | [---]|
+
+B 4 C D 5 B D E E D
+But they would have the rabbit out of hiding,
+[-] [--] [---] [--] [-] [----] [-] [] [----]|
+
+6 B F A A A 1 B E B E
+To please the yelping dogs. The gaps I mean,
+[] [----] [-] [-----] [--]| [-] [--] | [--]|
+
+B F 2 F 3 C 1 4 5 D F
+No one has seen them made or heard them made,
+[] [-] [-] [--] [--] [--] [] [---] [--] [--]|
+
+C 1 1 E B F D A 6 A B
+But at spring mending-time we find them there.
+[-] [] [----] [-----]|[--] [] [--] [--] [---]|
+
+1 B F C B D B 2 A
+I let my neighbor know beyond the hill;
+| [-] [] [------] [--] [----] [-] [--]|
+
+2 2 3 B E 1 3 F C C
+And on a day we meet to walk the line
+[-] [] | [-] [] [--] [] [--] [-] [--]
+
+4 2 D 1 E B 4 5 C
+And set the wall between us once again.
+[-] [-] [-] [--] [-----] [] [--] [---]|
+
+2 C E 3 F C 6 4 C D
+We keep the wall between us as we go.
+[] [--] [-] [--] [-----] [] [] [] []|
+
+ A
+ 5 F 1 1 1 A B 6 2 E
+To each the boulders that have fallen to each.
+[] [--] [-] [------] [--] [--] [----] [] [--]|
+
+ D 3 C D E 4 5 F 2
+And some are loaves and some so nearly balls
+[-] [--] [-] [----] [-] [--] [] [----] [---]
+
+ A A A
+5 B 2 D 1 6 3 2 3 3 A
+We have to use a spell to make them balance:
+[] [--] [] [-] | [---] [] [--] [--] [-----]|
+
+A B 6 C D E F 4 E 1 _B
+‘Stay where you are until our backs are turned!’
+|[--] [---] [-] [-] [---] [-] [---] [-] [----]||
+
+ 4 5 2 C F 3 1 6 F
+We wear our fingers rough with handling them.
+[] [--] [-] [-----] [---] [--] [------] [--]|
+
+A
+1 1 _ 2 D D 3 CC D 2
+Oh, just another kind of out-door game,
+[]| [--] [-----] [--] [] [-]|[--] [--]|
+
+ A A
+ 3 4 2 4 1 5 B 4 E 3 B
+One on a side. It comes to little more:
+[-] [] | [--]| [] [---] [] [----] [--]|
+
+ A A A
+ 1 2 6 1 1 D 5 6 5 F C
+There where it is we do not need the wall:
+[---] [---] [] [] [] [] [-] [--] [-] [--]|
+
+A A A
+1 2 1 C E 3 4 D 3 2
+He is all pine and I am apple orchard.
+[] [] [-] [--] [-] | [] [---] [-----]|
+
+5 E 4 2 C E C
+My apple trees will never get across
+[] [---] [---] [--] [---] [-] [----]
+
+ A A A A
+ F 1 6 D 4 2 F 3 4 3 63
+And eat the cones under his pines, I tell him.
+[-] [-] [-] [---] [---] [-] [---]| | [--] [-]|
+
+A
+3 4 D 4 CF E E 1 2 4D
+He only says, ‘Good fences make good neighbors.’
+[] [--] [--]| |[--] [----] [--] [--] [-------]||
+
+ A B A A B
+ 1 5 1 E 6 1 5 1 1 2
+Spring is the mischief in me, and I wonder
+[----] [] [-] [------] [] []| [-] | [----]
+
+ B A A B A
+ F 2 F 2 3 1 3 4 3D
+If I could put a notion in his head:
+[] | [---] [-] | [----] [] [-] [--]|
+
+ B B B
+E E 4 F F 3 4 _ 4 F2 5
+‘Why do they make good neighbors? Isn’t it
+|[-] [] [--] [--] [--] [-------]| [-]|| []
+
+ A A B A A
+ 5 6 4 1 A 5 5 3 5 2 2 5
+Where there are cows? But here there are no cows.
+[---] [---] [-] [--]| [-] [--] [---] [-] [] [--]|
+
+ B A C B
+6 6 5 6 5 115 1 4 2
+Before I built a wall I’d ask to know
+[----] | [---] | [--] ||| [-] [] [--]
+
+A C A C A
+2 2 3 6 3 2 5 6 6
+What I was walling in or walling out,
+[--] | [-] [-----] [] [] [-----] [-]|
+
+ B A C A B
+ 6 5 4 4 5 3 6 6 1 6
+And to whom I was like to give offense.
+[-] [] [--] | [-] [--] [] [--] [-----]|
+
+A C C C A C B A A
+1 1 5 2 1 23 D 1 6 1
+Something there is that doesn’t love a wall,
+[-------] [---] [] [--] [---]|| [--] | [--]|
+
+C B C A A D C A A
+4 1 6 2 13 1 3 1 4 E 5 5 6 2
+That wants it down.’ I could say ‘Elves’ to him,
+[--] [---] [] [--]|| | [---] [-] |[---]| [] [-]|
+
+A D A A A B DAA A
+1 2 62 3 F _ 3 2 313 1
+But it’s not elves exactly, and I’d rather
+[-] []|| [-] [---] [-----]| [-] ||| [----]
+
+B A D A D A B C
+1 3 4 2 32 5 4 2 6
+He said it for himself. I see him there
+[] [--] [] [-] [-----]| | [-] [-] [---]
+
+A B A D
+2 3 5 3 4 2 1 4
+Bringing a stone grasped firmly by the top
+[------] | [---] [-----] [----] [] [-] [-]
+
+D B A B A A B A
+6 4 3 4 4 4 3 D6 1 5 3
+In each hand, like an old-stone savage armed.
+[] [--] [--]| [--] [] [-]|[---] [----] [---]|
+
+B E B E B D A A
+4 2 1 5 6 2 1 2 2 5
+He moves in darkness as it seems to me,
+[] [---] [] [------] [] [] [---] [] []|
+
+A B C D B D A
+4 5 2 3 1 3 2 6 4 4
+Not of woods only and the shade of trees.
+[-] [] [---] [--] [-] [-] [---] [] [---]|
+
+B B A A A B A AB A
+5 3 5 1 3 6 1 23 4 6
+He will not go behind his father’s saying,
+[] [--] [-] [] [----] [-] [----]|| [----]|
+
+C C D A E B B
+2 1 6 3 5 4 3 4 4
+And he likes having thought of it so well
+[-] [] [---] [----] [-----] [] [] [] [--]
+
+C C B AA A A A AA
+2 5 3 1 32 5 3 3 6 54
+He says again, ‘Good fences make good neighbors.’
+[] [--] [---]| |[--] [----] [--] [--] [-------]||
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/mending_wall.txt.stats b/packages/cursorless-engine/src/test/fixtures/hat-stats/mending_wall.txt.stats
new file mode 100644
index 0000000000..0f11994513
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/mending_wall.txt.stats
@@ -0,0 +1,39 @@
+nTokens: 479
+
+tokenSpark:
+ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▃▁▁▃▃▃▁▃▁▁▁▃▃▁▃▃▁▃▃▃▃▃▃▃▁▃▁▃▁▃▁▃▃▁▃▃▃▃▃▃▃▁▃
+ ▃▃▃▃▃▃▃▃▁▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃
+ ▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▅▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃
+ ▅▃▅▃▅▃▃▃▃▃▃▃▃▃▃▃▃▃▃▁▃▃▃▃▃▃▃▃▃▃▅▃▁▃▃▃▃▃▃▃▃▃▃▅▃▃▃▃▅▃▃▃▃▃▃▅▅▃▃▃
+ ▅▃▃▅▅▃▃▃▅▃▃▃▃▃▃▃▃▃▃▃▃▅▅▃▃▅▃▃▅▃▃▃▅▃▃▃▃▃▃▃▃▃▃▃▃▅▅▃▅▅▃▃▅▃▃▅▃▃▅▅
+ ▅▅▃▃▃▃▃▃▃▃▃▁▅▃▅▅▃▃▅▃▃▃▅▅▅▅▃▃▃▅▃▅▃▅▃▃▃▅▃▅▅▅▃▅▅▃▃▃▃▅▅▅▅▃▅▃▃▃▅▅
+ ▅▅▅▃▅▃▅▅▅▅▅▅▅▅▃▅▃▃▃▃▃▅▅▅▅▅▃▅▅▃▁▅▅▅▅▅▅▅▅▅▃▃▅▅▅▅▅▅▅▅▃▃▃▅▃▅▃▅▅▃
+ ▅▅▃▅▃▅▅▅▃▅▃▅▅▅▅▅▅▅▃▅▃▅▅▅▃▅▅▅▅▅▅▅▅▅▅▅▃▅▅▅▃▃▅▅▅▅▅▅▃▅▅▅▅▃▅▅▅▅▅
+
+nHats:
+ mean: 100%
+ std: 0%
+ min: 100%
+ max: 100%
+ spark: 0 25 50 75 █ 100
+
+nPenalty0:
+ mean: 8%
+ std: 0%
+ min: 7%
+ max: 8%
+ spark: 0 █ 25 50 75 100
+
+nPenalty1:
+ mean: 63%
+ std: 1%
+ min: 62%
+ max: 63%
+ spark: 0 25 50 ▂█ 75 100
+
+nPenalty2:
+ mean: 30%
+ std: 1%
+ min: 29%
+ max: 31%
+ spark: 0 25 ██ 50 75 100
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/readme.md b/packages/cursorless-engine/src/test/fixtures/hat-stats/readme.md
new file mode 100644
index 0000000000..0494651ff5
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/readme.md
@@ -0,0 +1,9 @@
+This directory contains hat allocation fixtures.
+
+For any given fixture, it also contains:
+
+- A .golden output file indicating hat allocations and ranges. The cursor is at the beginning of the file. The line below the content has the token ranges. The line(s) above the content are the hats. An underscore indicates a default hat, any other single character is a "color" (letter) or a "shape" (number), and two stacked characters are a color and a shape. This layout lets you see at a glance the penalty associated with a token by looking at the hat's height. Tabs are replaced by a ␉ character so that fixed-width ASCII alignment works.
+
+- A .stats output file providing a summary of the hat allocation within the file. It contains a sparkline showing the hat penalty per token, when the cursor is at the beginning of the file. (You can think of it as a concise summary of the golden hat heights.) And for a variety of metrics, it contains distribution information about hat allocations, as sampled with the cursor in several different locations in the file. For example, nHats is the percentage of tokens that have a hat at all. The distribution information includes min, max, mean, and a sparkline.
+
+If these files have changed, you have changed something about the hat allocator. You should make sure that you are happy with the change.
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/river.ex b/packages/cursorless-engine/src/test/fixtures/hat-stats/river.ex
new file mode 100644
index 0000000000..32c630acbb
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/river.ex
@@ -0,0 +1,69 @@
+defmodule MyApp.Mod do
+ alias MyApp.Car
+
+ @model_regex ~r/^[A-Z][a-Z]*$/
+
+ @doc """
+ This is a function.
+
+ ## People use markdown in docstrings usually
+ * So lists
+ * And inline code for arguments like `color` is common
+ """
+ def database_call(color) do
+ query =
+ from(c in Car,
+ where:
+ c.make == :toyota or
+ fragment("lower(?) like ?", s.color, ^"#{color}%")
+ )
+
+ results =
+ query
+ |> Repo.all()
+ |> Enum.filter(fn
+ %{color: :unknown} -> false
+ c -> c.make == Application.fetch_env!(__MODULE__, :make_filter)
+ end)
+ |> Enum.map(&[&1.model, &1.color])
+
+ Enum.at(results, 0).model =~ @model_regex
+ end
+end
+
+defmodule MyApp.Car do
+ @moduledoc """
+ A database model
+ """
+ use Ecto.Schema
+ import Ecto.Changeset
+
+ schema "cars" do
+ field(:color, Ecto.Enum,
+ values: [:blue, :unknown, :green, :silver, :black, :yellow, :red, :white]
+ )
+
+ field(:model, :string)
+ field(:make, :string)
+
+ belongs_to(:owner, MyApp.Person)
+
+ timestamps()
+ end
+
+ def changeset(sign, attrs) do
+ required_fields = [
+ :make,
+ :model
+ ]
+
+ optional_fields = [
+ :owner
+ ]
+
+ sign
+ |> cast(attrs, required_fields ++ optional_fields)
+ |> validate_required(required_fields)
+ |> foreign_key_constraint(:people, name: :cars_owned_by_fkey)
+ end
+end
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/river.ex.golden b/packages/cursorless-engine/src/test/fixtures/hat-stats/river.ex.golden
new file mode 100644
index 0000000000..c604bc66c0
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/river.ex.golden
@@ -0,0 +1,266 @@
+_ _ _ _ A
+defmodule MyApp.Mod do
+[-------] [---]|[-] []
+
+ _ _ A_
+ alias MyApp.Car
+ [---] [---]|[-]
+
+
+
+ _ _ _____A___ABAAA__A
+ @model_regex ~r/^[A-Z][a-Z]*$/
+ |[---------] |||||||||||||||||
+
+
+
+ AB _
+ @doc """
+ |[-] [-]
+
+ _ _ C _ B
+ This is a function.
+ [--] [] | [------]|
+
+
+
+ _ _ _ _ A _ _
+ ## People use markdown in docstrings usually
+ [] [----] [-] [------] [] [--------] [-----]
+
+ A A A
+ * So lists
+ | [] [---]
+
+ B B C A _ _ A _B A A D
+ * And inline code for arguments like `color` is common
+ | [-] [----] [--] [-] [-------] [--] |[---]| [] [----]
+
+ A
+ """
+ [-]
+
+ C _ _C _ D
+ def database_call(color) do
+ [-] [-----------]|[---]| []
+
+ _ _
+ query =
+ [---] |
+
+ A AD B E _
+ from(c in Car,
+ [--]|| [] [-]|
+
+ _ _
+ where:
+ [---]|
+
+ FCA A AB A
+ c.make == :toyota or
+ ||[--] [] |[----] []
+
+ B BBA C_A B ACA BD1 B ADA_2 __EB
+ fragment("lower(?) like ?", s.color, ^"#{color}%")
+ [------]||[---]||| [--] ||| ||[---]| ||||[---]||||
+
+ C
+ )
+ |
+
+
+
+ A B
+ results =
+ [-----] |
+
+ A
+ query
+ [---]
+
+ __ B ED DD
+ |> Repo.all()
+ || [--]|[-]||
+
+ AA A FC ED
+ |> Enum.filter(fn
+ || [--]|[----]|[]
+
+ AA3 B CA A B E
+ %{color: :unknown} -> false
+ ||[---]| |[-----]| [] [---]
+
+ 4 C 51B C E 2 __F_ C D A E
+ c -> c.make == Application.fetch_env!(__MODULE__, :make_filter)
+ | [] ||[--] [] [---------]|[-------]||[--------]| |[---------]|
+
+ B F
+ end)
+ [-]|
+
+ BD C 3C 1_BA_4D D BA56 B1
+ |> Enum.map(&[&1.model, &1.color])
+ || [--]|[-]||||||[---]| |||[---]||
+
+
+
+ A
+ D 6F 2C E _21E DA B _
+ Enum.at(results, 0).model =~ @model_regex
+ [--]|[]|[-----]| |||[---] || |[---------]
+
+ E
+ end
+ [-]
+
+F
+end
+[-]
+
+
+
+ A
+E F 2 1 F
+defmodule MyApp.Car do
+[-------] [---]|[-] []
+
+ C1 F
+ @moduledoc """
+ |[-------] [-]
+
+ 2 A 2
+ A database model
+ | [------] [---]
+
+ 1
+ """
+ [-]
+
+ A
+ B 1 3 _
+ use Ecto.Schema
+ [-] [--]|[----]
+
+ A
+ C 2 4 A
+ import Ecto.Changeset
+ [----] [--]|[-------]
+
+
+
+ C 2 3 3 1
+ schema "cars" do
+ [----] |[--]| []
+
+ A
+ F 3E B F 3 54 1
+ field(:color, Ecto.Enum,
+ [---]||[---]| [--]|[--]|
+
+ A A A A
+ A F C1B 2 2C 3 3A 4 4 B 5 5C 6 6A 1 1D 2 2A C
+ values: [:blue, :unknown, :green, :silver, :black, :yellow, :red, :white]
+ [----]| ||[--]| |[-----]| |[---]| |[----]| |[---]| |[----]| |[-]| |[---]|
+
+ 3
+ )
+ |
+
+
+
+ A A A
+ 1 433 3 4D 4
+ field(:model, :string)
+ [---]||[---]| |[----]|
+
+ A A A
+ 2 554 4 6E 5
+ field(:make, :string)
+ [---]||[--]| |[----]|
+
+
+
+ B A A
+ D 61C 5 5 6A 6
+ belongs_to(:owner, MyApp.Person)
+ [--------]||[---]| [---]|[----]|
+
+
+
+ AA
+ C 11
+ timestamps()
+ [--------]||
+
+ 5
+ end
+ [-]
+
+
+
+ A A A
+ 2 B 2F 6 4 2 3
+ def changeset(sign, attrs) do
+ [-] [-------]|[--]| [---]| []
+
+ B E D
+ required_fields = [
+ [-------------] | |
+
+ B B
+ 26 1
+ :make,
+ |[--]|
+
+ B
+ 3 D
+ :model
+ |[---]
+
+ D
+ ]
+ |
+
+
+
+ B F E
+ optional_fields = [
+ [-------------] | |
+
+ B
+ 4E
+ :owner
+ |[---]
+
+ E
+ ]
+ |
+
+
+
+ 1
+ sign
+ [--]
+
+ A B A
+ CE 5 36 2 C _ C 3
+ |> cast(attrs, required_fields ++ optional_fields)
+ || [--]|[---]| [-------------] [] [-------------]|
+
+ A A
+ DF C 4 D 4
+ |> validate_required(required_fields)
+ || [---------------]|[-------------]|
+
+ AB B B C A
+ E1 D 55B 3 E 6 1 E 5
+ |> foreign_key_constraint(:people, name: :cars_owned_by_fkey)
+ || [--------------------]||[----]| [--]| |[----------------]|
+
+ 6
+ end
+ [-]
+
+ F
+end
+[-]
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/river.ex.stats b/packages/cursorless-engine/src/test/fixtures/hat-stats/river.ex.stats
new file mode 100644
index 0000000000..e71b091c63
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/river.ex.stats
@@ -0,0 +1,37 @@
+nTokens: 344
+
+tokenSpark:
+ ▁▁▁▁▃▁▁▃▁█▁▁▁▁▁▁▁▃▁▁▁▃▃▃▃▃▁▁▃█▃▃▁▁▁▃▁▃█▁▁▁▁▃▁▁▃▃▃▃▃▃▃▁▁▃▁▃▃▃
+ ▃▃▃▁▁▃▁▃▁▁▃▃▃▃▃▁▁▁▃▃▃▃▃▃▃▃▃▃▃▃▁▃▃▃▃▃▃▃▃▃▃▃▃▁▃▁▁▃▃▃█▃▃▃▁▁▃▃▃▃
+ ▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▁▁▃▁▃▃▃▃▃▃▃▃▃▃▃▃▁▃▃▁▃▃▃▃▃▃▃▃▃█▃▃▃▃▃
+ ▃▁▃▅▃▃▃▃▁▃▃█▃▃▅▃▃▃▃▃▃▃▃▃▃▃▅▁▃▃▅▃█▃▃▃▃▃▃▃▃▃▃▃▅▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃
+ ▃▃▃▃▃▃▃▅▅▃▅▅▃▃▃█▃▃▅▃▅▅▃▃▃▃▅▃▅▅▃▃█▃▃▅▃▅▃▅▃▃█▃▅▅▃█▃▃▅▃▅▃▅▃▃▃▃▅
+ ▃▅▅▃▃█▃▃▃▅▃▃█▃▃▃▃▅▃▅▃▁▃▅▃▃▃▅▃▅▃▃▃▅▅▃▅▃▅▅▃▅▃▃
+
+nHats:
+ mean: 96%
+ std: 0%
+ min: 96%
+ max: 96%
+ spark: 0 25 50 75 █ 100
+
+nPenalty0:
+ mean: 16%
+ std: 0%
+ min: 16%
+ max: 16%
+ spark: 0 █ 25 50 75 100
+
+nPenalty1:
+ mean: 69%
+ std: 0%
+ min: 68%
+ max: 69%
+ spark: 0 25 50 █ 75 100
+
+nPenalty2:
+ mean: 11%
+ std: 0%
+ min: 11%
+ max: 12%
+ spark: 0 █▁ 25 50 75 100
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/the_bells.txt b/packages/cursorless-engine/src/test/fixtures/hat-stats/the_bells.txt
new file mode 100644
index 0000000000..04cc3a21ab
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/the_bells.txt
@@ -0,0 +1,127 @@
+The Bells
+by Edgar Allan Poe 1809 – 1849 (public domain)
+
+I.
+
+ Hear the sledges with the bells—
+ Silver bells!
+What a world of merriment their melody foretells!
+ How they tinkle, tinkle, tinkle,
+ In the icy air of night!
+ While the stars that oversprinkle
+ All the heavens, seem to twinkle
+ With a crystalline delight;
+ Keeping time, time, time,
+ In a sort of Runic rhyme,
+To the tintinabulation that so musically wells
+ From the bells, bells, bells, bells,
+ Bells, bells, bells—
+ From the jingling and the tinkling of the bells.
+
+II.
+
+ Hear the mellow wedding bells,
+ Golden bells!
+What a world of happiness their harmony foretells!
+ Through the balmy air of night
+ How they ring out their delight!
+ From the molten-golden notes,
+ And all in tune,
+ What a liquid ditty floats
+ To the turtle-dove that listens, while she gloats
+ On the moon!
+ Oh, from out the sounding cells,
+What a gush of euphony voluminously wells!
+ How it swells!
+ How it dwells
+ On the Future! how it tells
+ Of the rapture that impels
+ To the swinging and the ringing
+ Of the bells, bells, bells,
+ Of the bells, bells, bells, bells,
+ Bells, bells, bells—
+ To the rhyming and the chiming of the bells!
+
+III.
+
+ Hear the loud alarum bells—
+ Brazen bells!
+What tale of terror, now, their turbulency tells!
+ In the startled ear of night
+ How they scream out their affright!
+ Too much horrified to speak,
+ They can only shriek, shriek,
+ Out of tune,
+In a clamorous appealing to the mercy of the fire,
+In a mad expostulation with the deaf and frantic fire,
+ Leaping higher, higher, higher,
+ With a desperate desire,
+ And a resolute endeavor
+ Now—now to sit or never,
+ By the side of the pale-faced moon.
+ Oh, the bells, bells, bells!
+ What a tale their terror tells
+ Of Despair!
+ How they clang, and clash, and roar!
+ What a horror they outpour
+On the bosom of the palpitating air!
+ Yet the ear it fully knows,
+ By the twanging,
+ And the clanging,
+ How the danger ebbs and flows;
+ Yet the ear distinctly tells,
+ In the jangling,
+ And the wrangling.
+ How the danger sinks and swells,
+By the sinking or the swelling in the anger of the bells—
+ Of the bells—
+ Of the bells, bells, bells, bells,
+ Bells, bells, bells—
+ In the clamor and the clangor of the bells!
+
+IV.
+
+ Hear the tolling of the bells—
+ Iron bells!
+What a world of solemn thought their monody compels!
+ In the silence of the night,
+ How we shiver with affright
+ At the melancholy menace of their tone!
+ For every sound that floats
+ From the rust within their throats
+ Is a groan.
+ And the people—ah, the people—
+ They that dwell up in the steeple,
+ All alone,
+ And who tolling, tolling, tolling,
+ In that muffled monotone,
+ Feel a glory in so rolling
+ On the human heart a stone—
+ They are neither man nor woman—
+ They are neither brute nor human—
+ They are Ghouls:
+ And their king it is who tolls;
+ And he rolls, rolls, rolls,
+ Rolls
+ A pæan from the bells!
+ And his merry bosom swells
+ With the pæan of the bells!
+ And he dances, and he yells;
+ Keeping time, time, time,
+ In a sort of Runic rhyme,
+ To the pæan of the bells—
+ Of the bells:
+ Keeping time, time, time,
+ In a sort of Runic rhyme,
+ To the throbbing of the bells—
+ Of the bells, bells, bells—
+ To the sobbing of the bells;
+ Keeping time, time, time,
+ As he knells, knells, knells,
+ In a happy Runic rhyme,
+ To the rolling of the bells—
+ Of the bells, bells, bells—
+ To the tolling of the bells,
+ Of the bells, bells, bells, bells—
+ Bells, bells, bells—
+ To the moaning and the groaning of the bells.
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/the_bells.txt.golden b/packages/cursorless-engine/src/test/fixtures/hat-stats/the_bells.txt.golden
new file mode 100644
index 0000000000..8fdbdf7236
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/the_bells.txt.golden
@@ -0,0 +1,566 @@
+_ _
+The Bells
+[-] [---]
+
+ _ _ _ _ _ _ _ _ _ _ _
+by Edgar Allan Poe 1809 – 1849 (public domain)
+[] [---] [---] [-] [--] | [--] |[----] [----]|
+
+__
+I.
+||
+
+ _ A _ _ B _ A
+ Hear the sledges with the bells—
+ [--] [-] [-----] [--] [-] [---]|
+
+ _ A _
+ Silver bells!
+ [----] [---]|
+
+A A _ _ _ _ A A A
+What a world of merriment their melody foretells!
+[--] | [---] [] [-------] [---] [----] [-------]|
+
+ B C _ _ _ A A B
+ How they tinkle, tinkle, tinkle,
+ [-] [--] [----]| [----]| [----]|
+
+ A D _ B B _ B
+ In the icy air of night!
+ [] [-] [-] [-] [] [---]|
+
+ C E A F A
+ While the stars that oversprinkle
+ [---] [-] [---] [--] [----------]
+
+ C 1 A C B 2 3
+ All the heavens, seem to twinkle
+ [-] [-] [-----]| [--] [] [-----]
+
+ D D A A _
+ With a crystalline delight;
+ [--] | [---------] [-----]|
+
+ B 4 D 5 E 6 F
+ Keeping time, time, time,
+ [-----] [--]| [--]| [--]|
+
+ B E C A A A 1
+ In a sort of Runic rhyme,
+ [] | [--] [] [---] [---]|
+
+ B B B C D C E
+To the tintinabulation that so musically wells
+[] [-] [-------------] [--] [] [-------] [---]
+
+ C D B 2 C 3 D 4 E 5
+ From the bells, bells, bells, bells,
+ [--] [-] [---]| [---]| [---]| [---]|
+
+ A
+ F 6 1 1 2 B
+ Bells, bells, bells—
+ [---]| [---]| [---]|
+
+ D E _ F F C C 1 3 A
+ From the jingling and the tinkling of the bells.
+ [--] [-] [------] [-] [-] [------] [] [-] [---]|
+
+D B
+II.
+[]|
+
+ A
+ 2 3 B F 4 2
+ Hear the mellow wedding bells,
+ [--] [-] [----] [-----] [---]|
+
+ A 5 C
+ Golden bells!
+ [----] [---]|
+
+1 1 2 D 4 5 6 E D
+What a world of happiness their harmony foretells!
+[--] | [---] [] [-------] [---] [-----] [-------]|
+
+ D A 6 2 E A
+ Through the balmy air of night
+ [-----] [-] [---] [-] [] [---]
+
+ F B B E C B E
+ How they ring out their delight!
+ [-] [--] [--] [-] [---] [-----]|
+
+ A
+ F D C _B B 3
+ From the molten-golden notes,
+ [--] [-] [----]|[----] [---]|
+
+ A
+ 3 4 E F 4
+ And all in tune,
+ [-] [-] [] [--]|
+
+ 3 5 _ C 1
+ What a liquid ditty floats
+ [--] | [----] [---] [----]
+
+ A
+ 1 E 1 AD 6 A 5 4 E C
+ To the turtle-dove that listens, while she gloats
+ [] [-] [----]|[--] [--] [-----]| [---] [-] [----]
+
+ 2 F D F
+ On the moon!
+ [] [-] [--]|
+
+ A B
+ 3 6 2 2 1 3 B 1
+ Oh, from out the sounding cells,
+ []| [--] [-] [-] [------] [---]|
+
+ A
+5 1 4 4 5 6 6 1
+What a gush of euphony voluminously wells!
+[--] | [--] [] [-----] [----------] [---]|
+
+ 5 F F 2
+ How it swells!
+ [-] [] [----]|
+
+ 6 1 E
+ How it dwells
+ [-] [] [----]
+
+ A
+ C 2 3 3 1 2 3
+ On the Future! how it tells
+ [] [-] [----]| [-] [] [---]
+
+ A
+ 4 4 C 1 3
+ Of the rapture that impels
+ [] [-] [-----] [--] [----]
+
+ A
+ 2 5 1 D 6 D
+ To the swinging and the ringing
+ [] [-] [------] [-] [-] [-----]
+
+ A B B B
+ 5 3 B 2 C 3 D 4
+ Of the bells, bells, bells,
+ [] [-] [---]| [---]| [---]|
+
+ A B B C C
+ 6 4 E 5 F 6 1 1 2 2
+ Of the bells, bells, bells, bells,
+ [] [-] [---]| [---]| [---]| [---]|
+
+ C C
+ 3 3 4 4 5 C
+ Bells, bells, bells—
+ [---]| [---]| [---]|
+
+ A A B A B
+ 5 6 E E 1 C 1 2 6 4
+ To the rhyming and the chiming of the bells!
+ [] [-] [-----] [-] [-] [-----] [] [-] [---]|
+
+4 C
+III.
+[-]|
+
+ B
+ F 3 F 1 2D
+ Hear the loud alarum bells—
+ [--] [-] [--] [----] [---]|
+
+ _ 35
+ Brazen bells!
+ [----] [---]|
+
+A B A C C
+1 4 2 2 5 F 6 5 3 46
+What tale of terror, now, their turbulency tells!
+[--] [--] [] [----]| [-]| [---] [--------] [---]|
+
+ B A
+ 6 5 5 4 3 1
+ In the startled ear of night
+ [] [-] [------] [-] [] [---]
+
+ A A A
+ 2 A 6 1 5 6 1
+ How they scream out their affright!
+ [-] [--] [----] [-] [---] [------]|
+
+ B C D
+ 6 E 1 1 A 1
+ Too much horrified to speak,
+ [-] [--] [-------] [] [---]|
+
+ D D
+ B D 2 C2 D3
+ They can only shriek, shriek,
+ [--] [-] [--] [----]| [----]|
+
+ A A D
+ 2 4 3 4
+ Out of tune,
+ [-] [] [--]|
+
+ A C C A C A D
+ 4 2 E B 2 3 F 5 4 1 5
+In a clamorous appealing to the mercy of the fire,
+[] | [-------] [-------] [] [-] [---] [] [-] [--]|
+
+ A A C A D
+ 5 3 1 _ 2 5 2 6 F 2 6
+In a mad expostulation with the deaf and frantic fire,
+[] | [-] [-----------] [--] [-] [--] [-] [-----] [--]|
+
+ E E E
+ C D 1 E 2 F 3
+ Leaping higher, higher, higher,
+ [-----] [----]| [----]| [----]|
+
+ A A E
+ 3 4 3 4 4
+ With a desperate desire,
+ [--] | [-------] [----]|
+
+ A A
+ 5 5 3 6
+ And a resolute endeavor
+ [-] | [------] [------]
+
+ A A C A A E
+ 1 E2 6 1 6 B 5
+ Now—now to sit or never,
+ [-]|[-] [] [-] [] [---]|
+
+ D A B D
+ C 1 2 1 2 D B 1 2 D
+ By the side of the pale-faced moon.
+ [] [-] [--] [] [-] [--]|[---] [--]|
+
+ B E D A F A F A A
+ 2 6 3 1 1 2 2 3 2
+ Oh, the bells, bells, bells!
+ []| [-] [---]| [---]| [---]|
+
+ A A D D D E
+ 4 6 4 5 6 1
+ What a tale their terror tells
+ [--] | [--] [---] [----] [---]
+
+ B A
+ 3 E 3
+ Of Despair!
+ [] [-----]|
+
+ A F B F B A A
+ 3 D 2 3 1 3 4 2 1 4
+ How they clang, and clash, and roar!
+ [-] [--] [---]| [-] [---]| [-] [--]|
+
+ A B A
+ 5 3 4 E F
+ What a horror they outpour
+ [--] | [----] [--] [-----]
+
+B E B E B A
+4 2 3 5 3 1 4 5
+On the bosom of the palpitating air!
+[] [-] [---] [] [-] [---------] [-]|
+
+ E A A F
+ F 4 1 1 1 E 5
+ Yet the ear it fully knows,
+ [-] [-] [-] [] [---] [---]|
+
+ E F
+ 2 5 1 6
+ By the twanging,
+ [] [-] [------]|
+
+ B E
+ 5 6 4
+ And the clanging,
+ [-] [-] [------]
+
+ A F A B A
+ 5 1 2 2 6 3 A
+ How the danger ebbs and flows;
+ [-] [-] [----] [--] [-] [---]|
+
+ F A F
+ 3 2 3 5 3
+ Yet the ear distinctly tells,
+ [-] [-] [-] [--------] [---]
+
+ A F
+ 2 4 A
+ In the jangling,
+ [] [-] [------]
+
+ C F
+ 1 5 3 E
+ And the wrangling.
+ [-] [-] [-------]|
+
+ A F C A
+ 6 6 4 F 2 3
+ How the danger sinks and swells,
+ [-] [-] [----] [---] [-] [----]
+
+ B B B A B C B A
+ 4 1 1 6 2 5 3 3 6 1 4 4 F
+By the sinking or the swelling in the anger of the bells—
+[] [-] [-----] [] [-] [------] [] [-] [---] [] [-] [---]|
+
+ C B A
+ 2 5 5 1
+ Of the bells—
+ [] [-] [---]|
+
+ C B A B B B
+ 3 6 6 1 2 3
+ Of the bells, bells, bells, bells,
+ [] [-] [---] [---] [---] [---]
+
+ B B B
+ 4 5 6 2
+ Bells, bells, bells—
+ [---] [---] [---]|
+
+ A C C C A C C C A
+ 4 1 6 3 2 1 4 3 1 6
+ In the clamor and the clangor of the bells!
+ [] [-] [----] [-] [-] [-----] [] [-] [---]|
+
+ CF
+IV.
+[]|
+
+ C C C C C C
+ 4 5 5 6 6 2 3
+ Hear the tolling of the bells—
+ [--] [-] [-----] [] [-] [---]|
+
+ A C B
+ 5 3 1
+ Iron bells!
+ [--] [---]|
+
+A C B D A D B
+6 4 1 1 4 4 1 5 6 2
+What a world of solemn thought their monody compels!
+[--] | [---] [] [----] [-----] [---] [----] [-----]|
+
+ A D A D D A
+ 6 2 4 2 3 3
+ In the silence of the night,
+ [] [-] [-----] [] [-] [---]
+
+ D B B C
+ 4 2 D 3 5
+ How we shiver with affright
+ [-] [] [----] [--] [------]
+
+ C D A D D D B
+ 6 5 5 1 3 6 4 3
+ At the melancholy menace of their tone!
+ [] [-] [--------] [----] [] [---] [--]|
+
+ A A E A
+ 4 E 5 1 5
+ For every sound that floats
+ [-] [---] [---] [--] [----]
+
+ A E A B E E
+ 6 2 6 4 3 4
+ From the rust within their throats
+ [--] [-] [--] [----] [---] [-----]
+
+ B D A
+ 1 1 1 1
+ Is a groan.
+ [] | [---]|
+
+ D E D E
+ 2 5 2 43 6 3 5
+ And the people—ah, the people—
+ [-] [-] [----]|[] [-] [----]|
+
+ F A B F
+ 6 1 1 4 2 2 5
+ They that dwell up in the steeple,
+ [--] [--] [---] [] [] [-] [-----]
+
+ D D
+ 4 5
+ All alone,
+ [-] [---]
+
+ D B D D E
+ 6 5 5 6 1
+ And who tolling, tolling, tolling,
+ [-] [-] [-----] [-----] [-----]
+
+ B F B A
+ 3 3 1 2
+ In that muffled monotone,
+ [] [--] [-----] [------]
+
+ B E A B A A
+ 1 1 2 4 5 2
+ Feel a glory in so rolling
+ [--] | [---] [] [] [-----]
+
+ E F B F E A
+ 2 4 2 5 2 6 6
+ On the human heart a stone—
+ [] [-] [---] [---] | [---]|
+
+ F E A A A B A
+ 6 3 4 3 5 6 1
+ They are neither man nor woman—
+ [--] [-] [-----] [-] [-] [---]|
+
+ A E A B B B A
+ 4 4 6 3 1 4 2
+ They are neither brute nor human—
+ [--] [-] [-----] [---] [-] [---]|
+
+ A E B
+ 5 5 5 _
+ They are Ghouls:
+ [--] [-] [----]|
+
+ E A B B C E
+ 6 6 2 5 6 1 3 B
+ And their king it is who tolls;
+ [-] [---] [--] [] [] [-] [---]|
+
+ F B A A A
+ 1 1 3 4 5
+ And he rolls, rolls, rolls,
+ [-] [] [---] [---] [---]
+
+ A
+ 6
+ Rolls
+ [---]
+
+ F B B C B
+ 2 6 2 2 4 4
+ A pæan from the bells!
+ | [--] [--] [-] [---]|
+
+ F C A C B
+ 3 1 4 5 1
+ And his merry bosom swells
+ [-] [-] [---] [---] [----]
+
+ C B A E B C B
+ 2 3 1 4 4 6 5
+ With the pæan of the bells!
+ [--] [-] [--] [] [-] [---]|
+
+ F B A F B A
+ 4 5 2 5 6 1 C
+ And he dances, and he yells;
+ [-] [] [----] [-] [] [---]|
+
+ C C C
+ 3 2 3 4
+ Keeping time, time, time,
+ [-----] [--] [--] [--]
+
+ C F B E B B
+ 5 6 2 5 6 1
+ In a sort of Runic rhyme,
+ [] | [--] [] [---] [---]
+
+ E C A F C D A
+ 6 1 2 1 2 1 3
+ To the pæan of the bells—
+ [] [-] [--] [] [-] [---]|
+
+ F C D
+ 2 3 2 A
+ Of the bells:
+ [] [-] [---]|
+
+ C D D
+ 4 6 1 2
+ Keeping time, time, time,
+ [-----] [--] [--] [--]
+
+ D B F C B
+ 3 3 3 1 2
+ In a sort of Runic rhyme,
+ [] [--] [] [---] [---]
+
+ F C B F C D A
+ 4 4 3 5 5 3 4
+ To the throbbing of the bells—
+ [] [-] [-------] [] [-] [---]|
+
+ F C D D D A
+ 6 6 4 5 6 5
+ Of the bells, bells, bells—
+ [] [-] [---] [---] [---]|
+
+ D B B D E
+ 1 4 3 2 1 D
+ To the sobbing of the bells;
+ [-] [-----] [] [-] [---]|
+
+ D D D
+ 5 4 5 6
+ Keeping time, time, time,
+ [-----] [--] [--] [--]
+
+ B D A A
+ 5 3 6 1 2
+ As he knells, knells, knells,
+ [] [] [----] [----] [----]
+
+ E A C B
+ 1 3 2 4
+ In a happy Runic rhyme,
+ [] [---] [---] [---]
+
+ D B B D E A
+ 4 5 4 5 2 6
+ To the rolling of the bells—
+ [-] [-----] [] [-] [---]|
+
+ B D E E E B
+ 5 6 3 4 5 1
+ Of the bells, bells, bells—
+ [] [-] [---] [---] [---]|
+
+ E A B E E
+ 1 1 6 2 6
+ To the tolling of the bells,
+ [-] [-----] [] [-] [---]
+
+ C E F F F F B
+ 1 3 1 2 3 4 2
+ Of the bells, bells, bells, bells—
+ [] [-] [---] [---] [---] [---]|
+
+ F F E B
+ 5 6 4 3
+ Bells, bells, bells—
+ [---] [---] [---]|
+
+ E A B E A C F F
+ 5 5 2 6 3 2 1 2 2
+ To the moaning and the groaning of the bells.
+ [-] [-----] [-] [-] [------] [] [-] [---]|
diff --git a/packages/cursorless-engine/src/test/fixtures/hat-stats/the_bells.txt.stats b/packages/cursorless-engine/src/test/fixtures/hat-stats/the_bells.txt.stats
new file mode 100644
index 0000000000..fe5ea9988b
--- /dev/null
+++ b/packages/cursorless-engine/src/test/fixtures/hat-stats/the_bells.txt.stats
@@ -0,0 +1,45 @@
+nTokens: 782
+
+tokenSpark:
+ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▃▁▁▃▁▃▁▃▁▃▃▁▁▁▁▃▃▃▃▃▁▁▁▃▃▃▃▃▁▃▃▁▃▃▃▃▃▃▃▃▃▃▃▃
+ ▃▃▃▃▃▁▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▅▃▃▃▃▁▃▃▃▃▃▃▃▃▃▃▃▃▃▃
+ ▅▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▁▃▃▅▃▃▃▃▅▃▃▁▃▃▃▃▃▃▃▃▃▅▃▃▃▃▃▃▃▃▅
+ ▃▃▃▃▃▅▃▅▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▃▅▃▃▃▃▃▅▃▅▃▃▃▃▃▃▅▃▅▃▅▃▅▃▅▃▅▃▅▃▅▃▅▃▅▃
+ ▅▃▃▅▅▃▃▅▃▅▅▃▃▃▃▃▅▃▃▃▃▁▃▃▅▅▅▃▅▃▅▃▃▃▃▃▅▃▃▅▃▅▃▃▅▃▃▅▅▃▃▅▃▅▃▃▃▃▅▃
+ ▅▅▅▃▅▃▅▃▃▅▅▃▅▅▅▅▃▅▃▁▅▅▃▃▃▅▅▃▃▅▃▅▃▅▅▅▃▃▅▃▅▅▃▅▃▅▅▅▅▃▅▃▅▅▅▅▃▃▃▃
+ ▃▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▃▅▅▃▃▅▅▃▅▅▅▅▅▅▅▃▃▅▅▃▅▅▃▅▅▃▅▅▅▃▃▅▃▅▃▅▅▅▃█▅▅▃
+ ▅▅▅▃▃▅▅▃▅█▅▅▃█▅▅▃▃▅▅▃▃▅▅█▃▅▃▅▅▃▅▅▃▅▅▅▃▅▅▅▃▅▅▅█▅█▅█▅█▅█▅█▅▃▅▅
+ ▃▅▅▅▅▅▅▅▃▃▅▅▅▅▅▅▃▅▅▅▅▅▅▅▃▅▅▃▃▅▅▅▅▅▅▅█▅▅▃▅▅▅▅▃▅▅▅▅▅▅▃▅▅▅▅▅▅▅▅
+ ▅▅▅▅▃▅▅▃▃▅█▅▃▃▃▅▅▃▅▅▃█▅▅█▅▅▅█▅█▅█▅▅▅▅█▅▅▅▅▅▅▅▅▅▅▅▅▃▅▅▅▅▅▅▅▅▅
+ ▅▅▅▅▅▅▅▅▁▅▅▃▅▅▅▅▃▅▅▅█▅█▅█▅▅▃▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅█▅▅▅▃▃▅█▅█▅█▅
+ ▅▅▅▅▅█▅▅▅▅▅▅▅▅▅▅▃▃▅█▅█▅█▅█▅▅▅▅█▅▅▅▅▅▅▅▅▅▅█▅█▅▅█▅▅▅▅▅▃▃▅█▅█▅█
+ ▅▅▃█▅█▅█▅█▅▅▅██▅▅▅▅▅▅▅▅▅█▅█▅▅█▅▅▅▅▅█▅▅▅█▅█▅█▅▅▅█▅█▅▅█▅▅▅▅▅▅▅
+ ▅▃
+
+nHats:
+ mean: 93%
+ std: 0%
+ min: 93%
+ max: 94%
+ spark: 0 25 50 75 █ 100
+
+nPenalty0:
+ mean: 5%
+ std: 0%
+ min: 5%
+ max: 5%
+ spark: 0 █ 25 50 75 100
+
+nPenalty1:
+ mean: 40%
+ std: 0%
+ min: 40%
+ max: 40%
+ spark: 0 25 █▃ 50 75 100
+
+nPenalty2:
+ mean: 49%
+ std: 0%
+ min: 48%
+ max: 49%
+ spark: 0 25 █ 50 75 100
diff --git a/packages/cursorless-engine/src/util/allocateHats/allocateHats.ts b/packages/cursorless-engine/src/util/allocateHats/allocateHats.ts
index 09bb49fc00..7b619313ae 100644
--- a/packages/cursorless-engine/src/util/allocateHats/allocateHats.ts
+++ b/packages/cursorless-engine/src/util/allocateHats/allocateHats.ts
@@ -86,7 +86,7 @@ export function allocateHats(
// Iterate through tokens in order of decreasing rank, assigning each one a
// hat
return rankedTokens
- .map<TokenHat | undefined>(({ token, rank: tokenRank }) => {
+ .map<TokenHat | undefined>(({ token, score: tokenScore }) => {
/**
* All hats for the graphemes in this token that weren't taken by a
* higher ranked token
@@ -101,7 +101,7 @@ export function allocateHats(
const chosenHat = chooseTokenHat(
context,
hatStability,
- tokenRank,
+ tokenScore,
tokenOldHatMap.get(token),
tokenRemainingHatCandidates,
);
diff --git a/packages/cursorless-engine/src/util/allocateHats/chooseTokenHat.ts b/packages/cursorless-engine/src/util/allocateHats/chooseTokenHat.ts
index c1e583677c..b3b60c8c1e 100644
--- a/packages/cursorless-engine/src/util/allocateHats/chooseTokenHat.ts
+++ b/packages/cursorless-engine/src/util/allocateHats/chooseTokenHat.ts
@@ -50,13 +50,17 @@ import { maxByFirstDiffering } from "./maxByFirstDiffering";
export function chooseTokenHat(
{ hatOldTokenRanks, graphemeTokenRanks }: RankingContext,
hatStability: HatStability,
- tokenRank: number,
+ tokenScore: number,
oldTokenHat: TokenHat | undefined,
candidates: HatCandidate[],
): HatCandidate | undefined {
// We narrow down the candidates by a series of criteria until there is only
// one left
return maxByFirstDiffering(candidates, [
+ // 0. TODO: https://github.com/cursorless-dev/cursorless/issues/1278
+ // Today, when choosing a hat for a token in chooseTokenHat, we first discard any tokens that are less than ideal according to our hat equivalence class mapping, as defined by user setting
+ // We should add a metric before that step that returns 0 if the hat would be stolen from another token with the same rank as the token getting its hat assigned, and return 1 otherwise
+
// 1. Discard any hats that are sufficiently worse than the best hat that we
// wouldn't use them even if they were our old hat
penaltyEquivalenceClass(hatStability),
@@ -73,6 +77,6 @@ export function chooseTokenHat(
// 5. Prefer hats that sit on a grapheme that doesn't appear in any highly
// ranked token
- minimumTokenRankContainingGrapheme(tokenRank, graphemeTokenRanks),
+ minimumTokenRankContainingGrapheme(tokenScore, graphemeTokenRanks),
])!;
}
diff --git a/packages/cursorless-engine/src/util/allocateHats/getHatRankingContext.ts b/packages/cursorless-engine/src/util/allocateHats/getHatRankingContext.ts
index 8a5aa42d4c..fa331e33b2 100644
--- a/packages/cursorless-engine/src/util/allocateHats/getHatRankingContext.ts
+++ b/packages/cursorless-engine/src/util/allocateHats/getHatRankingContext.ts
@@ -43,10 +43,10 @@ export function getHatRankingContext(
number
>(({ grapheme, hatStyle }) => [grapheme, hatStyle]);
- tokens.forEach(({ token, rank }) => {
+ tokens.forEach(({ token, score }) => {
const existingTokenHat = oldTokenHatMap.get(token);
if (existingTokenHat != null) {
- hatOldTokenRanks.set(existingTokenHat, rank);
+ hatOldTokenRanks.set(existingTokenHat, score);
}
tokenGraphemeSplitter
.getTokenGraphemes(token.text)
@@ -60,7 +60,7 @@ export function getHatRankingContext(
graphemeTokenRanks[graphemeText] = tokenRanksForGrapheme;
}
- tokenRanksForGrapheme.push(rank);
+ tokenRanksForGrapheme.push(score);
});
});
diff --git a/packages/cursorless-engine/src/util/allocateHats/getRankedTokens.ts b/packages/cursorless-engine/src/util/allocateHats/getRankedTokens.ts
index 3649502aa6..d49165e4ce 100644
--- a/packages/cursorless-engine/src/util/allocateHats/getRankedTokens.ts
+++ b/packages/cursorless-engine/src/util/allocateHats/getRankedTokens.ts
@@ -2,7 +2,7 @@ import { TextEditor } from "@cursorless/common";
import { flatten } from "lodash";
import { Token } from "@cursorless/common";
import { getDisplayLineMap } from "./getDisplayLineMap";
-import { getTokenComparator } from "./getTokenComparator";
+import { getTokenComparator, tokenScore } from "./getTokenComparator";
import { getTokensInRange } from "./getTokensInRange";
/**
@@ -37,14 +37,16 @@ export function getRankedTokens(
),
);
- tokens.sort(
- getTokenComparator(
- displayLineMap.get(referencePosition.line)!,
- referencePosition.character,
- ),
- );
+ const displayLine = displayLineMap.get(referencePosition.line)!;
+
+ // Sort tokens, with the closest token to the reference position first.
+ tokens.sort(getTokenComparator(displayLine, referencePosition.character));
- return tokens.map((token, index) => ({ token, rank: -index }));
+ // Apply a coarse score to each token.
+ return tokens.map((token) => ({
+ token,
+ score: tokenScore(token, editor === activeTextEditor, displayLine),
+ }));
});
}
@@ -69,9 +71,9 @@ export interface RankedToken {
token: Token;
/**
- * A number indicating how likely the token is to be used. Tokens closer to
- * the cursor will be considered more likely to be used, and will receive a
- * higher rank, causing them to be assigned better hats.
+ * A number indicating how important the token is.
+ * Broadly speaking, tokens closer to the cursor will have a higher score,
+ * causing them to be assigned better hats.
*/
- rank: number;
+ score: number;
}
diff --git a/packages/cursorless-engine/src/util/allocateHats/getTokenComparator.ts b/packages/cursorless-engine/src/util/allocateHats/getTokenComparator.ts
index c309f058fe..ac84041460 100644
--- a/packages/cursorless-engine/src/util/allocateHats/getTokenComparator.ts
+++ b/packages/cursorless-engine/src/util/allocateHats/getTokenComparator.ts
@@ -6,7 +6,7 @@ interface TokenWithDisplayLine extends Token {
/**
* Gets a comparison function that can be used to sort tokens based on their
- * distance from the current cursor in terms of display lines.
+ * distance from the current cursor in terms of display lines and characters.
* @param selectionDisplayLine The display line of the cursor location
* @param selectionCharacterIndex The character index of current cursor within line
*/
@@ -37,3 +37,32 @@ export function getTokenComparator(
return token1CharacterDiff - token2CharacterDiff;
};
}
+
+/**
+ * Scores tokens based on their
+ * distance from the current cursor in terms of display lines.
+ * @param selectionDisplayLine The display line of the cursor location
+ * @param selectionCharacterIndex The character index of current cursor within line
+ */
+export function tokenScore(
+ token: TokenWithDisplayLine,
+ isActiveEditor: boolean,
+ selectionDisplayLine: number,
+): number {
+ // See https://github.com/cursorless-dev/cursorless/issues/1278
+ // for a discussion of the scoring function
+ if (!isActiveEditor) {
+ // worst score
+ // todo: differentiate between near and far in inactive editors?
+ return -3;
+ }
+ const lineDiff = Math.abs(token.displayLine - selectionDisplayLine);
+ if (lineDiff <= 3) {
+ return 0;
+ }
+ if (lineDiff <= 10) {
+ return -1;
+ }
+ // Same editor, but far away
+ return -2;
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 0aabb70b61..a02e4b8007 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -205,6 +205,9 @@ importers:
'@types/sinon':
specifier: ^10.0.2
version: 10.0.13
+ fast-check:
+ specifier: 3.12.0
+ version: 3.12.0
js-yaml:
specifier: ^4.1.0
version: 4.1.0