Reject invalid moshscript characters (#5)

Author: aiirvizionz

package.json

@@ -8,7 +8,8 @@
   },
   "scripts": {
     "start": "node bin/moshcode.mjs",
-    "demo": "node bin/moshcode.mjs run examples/alive.mosh"
+    "demo": "node bin/moshcode.mjs run examples/alive.mosh",
+    "test": "node --test"
   },
   "files": ["bin", "src", "examples", "install.sh", "README.md"],
   "license": "MIT"

src/interpreter.mjs

@@ -15,14 +15,22 @@ export function tokenize(src) {
   const re = /("(?:[^"\\]|\\.)*")|(\/\/[^\n]*)|(\d+(?:\.\d+)?)|([A-Za-z_]\w*)|([(){};,])|(\s+)/g;
   const tokens = [];
   let m;
+  let lastIndex = 0;
   while ((m = re.exec(src)) !== null) {
+    if (m.index !== lastIndex) {
+      throw new Error(`moshscript: unexpected character near "${src.slice(lastIndex, lastIndex + 12)}"`);
+    }
+    lastIndex = re.lastIndex;
     if (m[2] || m[6]) continue; // comment or whitespace
     if (m[1]) tokens.push({ t: "str", v: JSON.parse(m[1]) });
     else if (m[3]) tokens.push({ t: "num", v: Number(m[3]) });
     else if (m[4]) tokens.push({ t: "id", v: m[4] });
     else if (m[5]) tokens.push({ t: "punc", v: m[5] });
     else throw new Error(`moshscript: unexpected character near "${src.slice(m.index, m.index + 12)}"`);
   }
+  if (lastIndex !== src.length) {
+    throw new Error(`moshscript: unexpected character near "${src.slice(lastIndex, lastIndex + 12)}"`);
+  }
   return tokens;
 }
 

test/interpreter.test.mjs

@@ -0,0 +1,16 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { compile, tokenize } from "../src/interpreter.mjs";
+
+test("tokenize rejects unexpected characters between valid tokens", () => {
+  assert.throws(() => tokenize("say(@);"), /unexpected character/);
+  assert.throws(() => tokenize("say(\"ok\") @"), /unexpected character/);
+});
+
+test("compile preserves valid moshscript behavior", () => {
+  assert.deepEqual(compile("say(\"hi\");").body[0], {
+    type: "call",
+    name: "say",
+    args: ["hi"],
+  });
+});