memoize partial parsers

Author: neverRare

src/parser/lexer.ts

@@ -1,4 +1,5 @@
 import { sumOf } from "@std/collections/sum-of";
+import { throwError } from "../../misc/misc.ts";
 import { settings } from "../settings.ts";
 import {
   all,
@@ -39,7 +40,7 @@ import {
   UCSUR_CHARACTER_REGEX,
   UCSUR_TO_LATIN,
 } from "./ucsur.ts";
-import { throwError } from "../../misc/misc.ts";
+import { memoize } from "@std/cache/memoize";
 
 const spacesWithoutNewline = match(/[^\S\n]*?(?=\S|\r?\n|$)/, "spaces");
 const newline = match(/\r?\n\s*/, "newline");
@@ -80,10 +81,10 @@ const multipleA = specificWord("a")
   .with(count(allAtLeastOnce(specificWord("a"))))
   .map((count) => ({ type: "multiple a", count: count + 1 }) as const);
 const repeatingLetter = match(/[a-zA-Z]/, "latin letter")
-  .then((letter) =>
+  .then(memoize((letter) =>
     count(all(matchString(letter)))
       .map((count) => [letter, count + 1] as const)
-  );
+  ));
 const longWord = allAtLeastOnce(repeatingLetter)
   .skip(spaces)
   .map((letters) => {
@@ -95,9 +96,9 @@ const longWord = allAtLeastOnce(repeatingLetter)
   .filter(({ length }) => length > 1);
 const xAlaX = lazy(() =>
   settings.xAlaXPartialParsing ? empty : word
-    .then((word) =>
+    .then(memoize((word) =>
       sequence(specificWord("ala"), specificWord(word)).map(() => word)
-    )
+    ))
 )
   .map((word) => ({ type: "x ala x", word }) as const);
 const punctuation = choiceOnlyOne(

src/parser/parser.ts

@@ -1,4 +1,9 @@
 import { memoize } from "@std/cache/memoize";
+import {
+  lazy as lazyEval,
+  nullableAsArray,
+  throwError,
+} from "../../misc/misc.ts";
 import {
   contentWordSet,
   fillerSet,
@@ -7,7 +12,6 @@ import {
   preverbSet,
   tokiPonaWordSet,
 } from "../dictionary.ts";
-import { nullableAsArray, throwError } from "../../misc/misc.ts";
 import {
   Clause,
   ContextClause,
@@ -58,7 +62,6 @@ import {
   UnrecognizedError,
 } from "./parser_lib.ts";
 import { describe, Token } from "./token.ts";
-import { lazy as lazyEval } from "../../misc/misc.ts";
 
 const spaces = match(/\s*/, "spaces");
 
@@ -125,6 +128,15 @@ const emphasis = choice<Emphasis>(
   specificWord("a").map((word) => ({ type: "word", word })),
 );
 const optionalEmphasis = optional(emphasis);
+const alaXLongGlyph = memoize((word: string) =>
+  specificWord(word)
+    .skip(specificToken("headless long glyph end"))
+    .map(() => ({ type: "x ala x", word }) as const)
+);
+const alaX = memoize((word: string) =>
+  sequence(specificWord("ala"), specificWord(word))
+    .map(() => ({ type: "x ala x", word }) as const)
+);
 function xAlaX(
   useWord: Set<string>,
   description: string,
@@ -136,34 +148,30 @@ function xAlaX(
         specificToken("inside long glyph")
           .filter(({ words }) => filterCombinedGlyphs(words, "ala")),
       )
-      .then((word) =>
-        specificWord(word)
-          .skip(specificToken("headless long glyph end"))
-          .map(() => ({ type: "x ala x", word }))
-      ),
+      .then(alaXLongGlyph),
     specificToken("x ala x")
       .map(({ word }) => ({ type: "x ala x", word })),
     word
-      .then((word) =>
-        sequence(specificWord("ala"), specificWord(word))
-          .map(() => ({ type: "x ala x", word }))
-      ),
+      .then(alaX),
   );
 }
+const reduplicateRest = memoize((word: string) =>
+  count(manyAtLeastOnce(specificWord(word)))
+    .map((count) =>
+      ({
+        type: "reduplication",
+        word,
+        count: count + 1,
+      }) as const
+    )
+);
 function simpleWordUnit(
   word: Set<string>,
   description: string,
 ): Parser<SimpleHeadedWordUnit> {
   return choice<SimpleHeadedWordUnit>(
     wordFrom(word, description)
-      .then((word) =>
-        count(manyAtLeastOnce(specificWord(word)))
-          .map((count) => ({
-            type: "reduplication",
-            word,
-            count: count + 1,
-          }))
-      ),
+      .then(reduplicateRest),
     xAlaX(word, description),
     wordFrom(word, description)
       .map((word) => ({ type: "default", word })),