Follow-up of moc.js API update (Serokell)

src/file.ts

@@ -14,6 +14,7 @@ function getValidPath(path: string): string {
 }
 
 export type MotokoFile = ReturnType<typeof file>;
+export type Scope = unknown;
 
 export const file = (mo: Motoko, path: string) => {
     path = getValidPath(path);
@@ -66,8 +67,11 @@ export const file = (mo: Motoko, path: string) => {
         parseMotoko() {
             return mo.parseMotoko(result.read());
         },
-        parseMotokoTyped() {
-            return mo.parseMotokoTyped(path);
+        parseMotokoWithDeps() {
+            return mo.parseMotokoWithDeps(path, result.read());
+        },
+        parseMotokoTyped(scopeCache: Map<string, Scope>) {
+            return mo.parseMotokoTyped(path, scopeCache);
         },
     };
     return result;

src/index.ts

@@ -1,5 +1,5 @@
 import { CompilerNode, Node, simplifyAST } from './ast';
-import { file } from './file';
+import { Scope, file } from './file';
 import {
     Package,
     PackageInfo,
@@ -72,23 +72,45 @@ export default function wrapMotoko(compiler: Compiler) {
     };
 
     // Function signatures for `mo.parseMotokoTyped()`
-    type ParseMotokoTypedResult = { ast: Node; type: Node };
-    function parseMotokoTyped(paths: string): ParseMotokoTypedResult;
-    function parseMotokoTyped(paths: string[]): ParseMotokoTypedResult[];
+    type ParseMotokoTypedResult = {
+        ast: Node;
+        type: Node;
+        immediateImports: string[];
+    };
+    function parseMotokoTyped(
+        paths: string,
+        scopeCache: Map<string, Scope>,
+    ): [ParseMotokoTypedResult, Map<string, Scope>];
+    function parseMotokoTyped(
+        paths: string[],
+        scopeCache: Map<string, Scope>,
+    ): [ParseMotokoTypedResult[], Map<string, Scope>];
     function parseMotokoTyped(
         paths: string | string[],
-    ): ParseMotokoTypedResult | ParseMotokoTypedResult[] {
+        scopeCache: Map<string, Scope>,
+    ): [ParseMotokoTypedResult | ParseMotokoTypedResult[], Map<string, Scope>] {
         if (typeof paths === 'string') {
-            return mo.parseMotokoTyped([paths])[0];
+            const [progs, outCache] = mo.parseMotokoTyped([paths], scopeCache);
+            return [progs[0], outCache];
         }
-        return invoke('parseMotokoTyped', true, [paths]).map(
-            ({ ast, typ }: { ast: CompilerNode; typ: CompilerNode }) => {
-                return {
-                    ast: simplifyAST(ast),
-                    type: simplifyAST(typ),
-                };
-            },
-        );
+        const [progs, outCache] =
+            invoke('parseMotokoTyped', true, [paths, scopeCache]);
+        return [
+            progs.map(
+                ({ ast, typ, immediateImports }: {
+                    ast: CompilerNode;
+                    typ: CompilerNode;
+                    immediateImports: string[];
+                }) => {
+                    return {
+                        ast: simplifyAST(ast),
+                        type: simplifyAST(typ),
+                        immediateImports,
+                    };
+                },
+            ),
+            outCache,
+        ];
     }
 
     const mo = {
@@ -188,6 +210,14 @@ export default function wrapMotoko(compiler: Compiler) {
             const ast = invoke('parseMotoko', true, [content]);
             return simplifyAST(ast);
         },
+        parseMotokoWithDeps(
+            path: string,
+            content: string,
+        ): { ast: Node, immediateImports: string[] } {
+            const { ast, immediateImports } =
+                invoke('parseMotokoWithDeps', true, [path, content]);
+            return { ast: simplifyAST(ast), immediateImports };
+        },
         parseMotokoTyped,
         resolveMain(directory: string = ''): string | undefined {
             return resolveMain(mo, directory);

tests/ast.test.ts

@@ -1,5 +1,8 @@
 import mo from '../src/versions/moc';
-import { Node, asNode, AST } from '../src/ast';
+import { Node, asNode } from '../src/ast';
+import { Scope } from '../src/file';
+import fs from 'fs';
+import path from 'path';
 
 const actorSource = `
 import { print } "mo:base/Debug";
@@ -11,7 +14,20 @@ actor Main {
 };
 `;
 
+function loadTopAndBottom() {
+    const root = path.join(__dirname, 'cache');
+    const bottom = mo.file('Bottom.mo');
+    bottom.write(fs.readFileSync(path.join(root, 'Bottom.mo'), 'utf-8'));
+    const top = mo.file('Top.mo');
+    top.write(fs.readFileSync(path.join(root, 'Top.mo'), 'utf-8'));
+    return [top, bottom];
+}
+
 describe('ast', () => {
+    beforeAll(() => {
+        mo.loadPackage(require('../packages/latest/base.json'));
+    });
+
     test('parent property in expression', async () => {
         const ast = mo.parseMotoko('let x = 0; x');
         const args = ast.args!.filter(
@@ -24,7 +40,6 @@ describe('ast', () => {
     });
 
     test('parent property in typed AST', async () => {
-        mo.loadPackage(require('../packages/latest/base.json'));
         const file = mo.file('AST.mo');
         file.write(actorSource);
 
@@ -37,8 +52,139 @@ describe('ast', () => {
                 }
             }
         };
-        const node = asNode(file.parseMotokoTyped().ast);
+        const node = asNode(file.parseMotokoTyped(new Map<string, Scope>())[0].ast);
         expect(node).toBeTruthy();
         check(node!);
     });
+
+    test('unchanged file should have equal caches', async () => {
+        const file = mo.file('AST.mo');
+        file.write(actorSource);
+        const [prog0, cache0] = file.parseMotokoTyped(new Map<string, Scope>());
+        expect(prog0.ast).toBeTruthy();
+        expect(prog0.immediateImports).toEqual(
+            ['.node-motoko/base/moc-0.14.1/Debug.mo'],
+        );
+        expect(Array.from(cache0.keys())).toEqual(
+            ['.node-motoko/base/moc-0.14.1/Debug.mo', '@prim'],
+        );
+
+        const [prog1, cache1] = file.parseMotokoTyped(cache0);
+        expect(cache1).toEqual(cache0);
+        expect(prog1.ast).toStrictEqual(prog0.ast);
+        expect(prog1.immediateImports).toEqual(prog0.immediateImports);
+    });
+
+    test('deleting from cache has equal new cache', async () => {
+        const file = mo.file('AST.mo');
+        file.write(actorSource);
+        const [prog0, cache0] = file.parseMotokoTyped(new Map<string, Scope>());
+
+        // Delete cache, ensure it works
+        const cache0Clone = new Map<string, Scope>(cache0);
+        cache0.delete('AST.mo');
+        const [prog1, cache1] = file.parseMotokoTyped(cache0);
+        expect(cache1).toEqual(cache0Clone);
+        expect(prog1.ast).toBeTruthy();
+        expect(prog1.immediateImports).toEqual(prog0.immediateImports);
+        expect(prog1.ast).toStrictEqual(prog0.ast);
+    });
+
+    test('changed file should have different caches', async () => {
+        const file = mo.file('AST.mo');
+        file.write(actorSource);
+        const [prog0, cache0] = file.parseMotokoTyped(new Map<string, Scope>());
+
+        // Remove import, ensure caches are different
+        const actorSource1 = actorSource.substring(1 + actorSource.indexOf(';'));
+        file.write(actorSource1);
+
+        const cache0Clone = new Map<string, Scope>(cache0);
+        cache0.clear();
+        const [prog1, cache1] = file.parseMotokoTyped(cache0);
+        expect(Array.from(cache1.keys())).toEqual([]);
+        expect(prog1.ast).toBeTruthy();
+        expect(prog1.immediateImports).toEqual([]);
+        expect(cache1).not.toStrictEqual(cache0Clone);
+        expect(prog1.ast).not.toStrictEqual(prog0.ast);
+    });
+
+    test('invalidating an unchanged reference without invalidating the file should be invalid', async () => {
+        const [top, _bottom] = loadTopAndBottom();
+
+        const [prog0, cache0] = top.parseMotokoTyped(new Map<string, Scope>());
+        expect(prog0.ast).toBeTruthy();
+        expect(prog0.immediateImports).toEqual(['Bottom.mo']);
+        expect(Array.from(cache0.keys())).toEqual(['Bottom.mo']);
+
+        // Remove bottom and ensure cache is NOT valid (because Top is
+        // invalidated)
+        const cache0Clone = new Map<string, Scope>(cache0);
+        cache0.delete('Bottom.mo');
+        const [prog1, cache1] = top.parseMotokoTyped(cache0);
+        expect(cache1).toStrictEqual(cache0Clone);
+        expect(Array.from(cache1.keys())).toEqual(['Bottom.mo']);
+        expect(prog1.ast).toBeTruthy();
+        expect(prog1.immediateImports).toEqual(prog0.immediateImports);
+        expect(prog1.ast).toStrictEqual(prog0.ast);
+    });
+
+    test('invalidating an unchanged file and reference should yield the same scope', async () => {
+        const [top, _bottom] = loadTopAndBottom();
+
+        // Remove top and bottom and ensure cache is valid
+        const [prog0, cache0] = top.parseMotokoTyped(new Map<string, Scope>());
+        const cache0Clone = new Map<string, Scope>(cache0);
+        cache0.delete('Bottom.mo');
+        cache0.delete('Top.mo');
+        const [prog1, cache1] = top.parseMotokoTyped(cache0);
+        expect(cache1).toStrictEqual(cache0Clone);
+        expect(Array.from(cache1.keys())).toEqual(['Bottom.mo']);
+        expect(prog1.ast).toBeTruthy();
+        expect(prog1.immediateImports).toEqual(prog0.immediateImports);
+        expect(prog1.ast).toStrictEqual(prog0.ast);
+    });
+
+    test('load bottom after top should work', async () => {
+        const [top, bottom] = loadTopAndBottom();
+
+        const [_progTop, cacheTop] = top.parseMotokoTyped(new Map<string, Scope>());
+        const [progBottom, cacheBottom] = bottom.parseMotokoTyped(cacheTop);
+        expect(Array.from(cacheBottom.keys())).toEqual(['Bottom.mo']);
+        expect(cacheBottom.get('Top.mo')).toStrictEqual(cacheTop.get('Top.mo'));
+        expect(cacheBottom.get('Bottom.mo')).toStrictEqual(cacheTop.get('Bottom.mo'));
+        expect(progBottom.ast).toBeTruthy();
+        expect(progBottom.immediateImports).toEqual([]);
+    });
+
+    test('load top after bottom should work', async () => {
+        const [top, bottom] = loadTopAndBottom();
+
+        const [progBottom, cacheBottom] = bottom.parseMotokoTyped(new Map<string, Scope>());
+        expect(Array.from(cacheBottom.keys())).toEqual([]);
+        expect(progBottom.ast).toBeTruthy();
+        expect(progBottom.immediateImports).toEqual([]);
+
+        const [progTop, cacheTop] = top.parseMotokoTyped(cacheBottom);
+        expect(Array.from(cacheTop.keys())).toEqual(['Bottom.mo']);
+        expect(cacheTop.get('Bottom.mo')).not.toStrictEqual(cacheBottom.get('Bottom.mo'));
+        expect(progTop.ast).toBeTruthy();
+        expect(progTop.immediateImports).toEqual(['Bottom.mo']);
+    });
+
+    test('immediate imports for top', async () => {
+        const [top, _bottom] = loadTopAndBottom();
+
+        const { ast, immediateImports } = top.parseMotokoWithDeps();
+        expect(ast).toBeTruthy();
+        expect(immediateImports).toEqual(['Bottom.mo']);
+    });
+
+    test('immediate imports for bottom', async () => {
+        const [_top, bottom] = loadTopAndBottom();
+
+        const { ast, immediateImports } = bottom.parseMotokoWithDeps();
+        expect(ast).toBeTruthy();
+        expect(immediateImports).toEqual([]);
+    });
 });

tests/cache/Bottom.mo

@@ -0,0 +1,5 @@
+module {
+    public object bottom {
+        public func bar() : () { return (); };
+    };
+}

tests/cache/Top.mo

@@ -0,0 +1,7 @@
+import Bottom "Bottom"
+
+module {
+    public object top {
+        public func foo() : () { return Bottom.bottom.bar(); };
+    };
+}