sync-fs and async-fs improvements (#217)

- Try to bump min measured iteration duration to ~10ms
- Use generator for randomFileContents
- Only measure setupDirectory in the faster sync-fs
- Increase NOF files and folders
- Add FSVisitor to get some more generator action  going

Author: camillobruni

JetStreamDriver.js

@@ -842,8 +842,9 @@ class Benchmark {
 
     get prerunCode() { return null; }
 
+
     get preIterationCode() {
-        let code = `benchmark.prepareForNextIteration?.();`;
+        let code = this.prepareForNextIterationCode ;
         if (this.plan.deterministicRandom)
             code += `Math.random.__resetSeed();`;
 
@@ -853,6 +854,10 @@ class Benchmark {
         return code;
     }
 
+    get prepareForNextIterationCode() {
+        return "benchmark.prepareForNextIteration?.();"
+    }
+
     get postIterationCode() {
         let code = "";
 
@@ -1519,6 +1524,10 @@ class AsyncBenchmark extends DefaultBenchmark {
         return str;
     }
 
+    get prepareForNextIterationCode() {
+        return "await benchmark.prepareForNextIteration?.();"
+    }
+
     get runnerCode() {
         return `
         async function doRun() {

generators/async-file-system.js

@@ -34,18 +34,21 @@ function computeIsLittleEndian() {
 }
 
 const isLittleEndian = computeIsLittleEndian();
-let globalCounter = 0;
-
-function randomFileContents() {
-    const numBytes = (globalCounter % 128)  + 2056;
-    globalCounter++;
-    let result = new ArrayBuffer(numBytes);
-    let view = new Uint8Array(result);
-    for (let i = 0; i < numBytes; ++i)
-        view[i] = (i + globalCounter) % 255;
-    return new DataView(result);
+
+async function *randomFileContents() {
+    let counter = 1;
+    while(true) {
+        const numBytes = ((counter * 1192.18851371) % 2056);
+        counter++;
+        let result = new ArrayBuffer(numBytes);
+        let view = new Uint8Array(result);
+        for (let i = 0; i < numBytes; ++i)
+            view[i] = (i + counter) % 255;
+        yield new DataView(result);
+    }
 }
 
+
 class File {
     constructor(dataView, permissions) {
         this._data = dataView;
@@ -55,6 +58,8 @@ class File {
 
     set data(dataView) { this._data = dataView; }
 
+    get byteLength() { return this._data.byteLength; }
+
     swapByteOrder() {
         let hash = 0x1a2b3c4d;
         for (let i = 0; i < Math.floor(this.data.byteLength / 8) * 8; i += 8) {
@@ -144,62 +149,152 @@ class Directory {
         return count;
     }
 
+    async totalFileSize() {
+        let size = 0;
+        for await (const {entry: file} of this.forEachFileRecursively()) {
+            size += file.byteLength;
+        }
+        return size;
+    }
+
     async rm(name) {
         return this.structure.delete(name);
     }
+
+    async visit(visitor) {
+        visitor.visitDir(undefined, this);
+        for await (const {name, entry, isDirectory} of this.ls()) {
+            if (isDirectory)
+                await entry.visit(visitor);
+            else
+                visitor.visitFile(name, entry);
+        }
+    }
+
 }
 
+const MAX_DIR_COUNT = 2500;
+const MAX_FILE_COUNT = 800;
+
 async function setupDirectory() {
     const fs = new Directory;
     let dirs = [fs];
     let counter = 0;
     for (let dir of dirs) {
-        for (let i = 0; i < 10; ++i) {
-            if (dirs.length < 400 && (counter % 3) <= 1) {
+        for (let i = 0; i < 15; ++i) {
+            if (dirs.length <= MAX_DIR_COUNT) {
                 dirs.push(await dir.addDirectory(`dir-${i}`));
             }
             counter++;
         }
     }
 
-    for (let dir of dirs) {
-        for (let i = 0; i < 5; ++i) {
-            if ((counter % 3) === 0) {
-                await dir.addFile(`file-${i}`, new File(randomFileContents()));
-            }
-            counter++;
-        }
+    let fileCounter = 0;
+    for await (const fileContents of randomFileContents()) {
+        const dirIndex = fileCounter * 107;
+        const dir = dirs[dirIndex % dirs.length];
+        await dir.addFile(`file-${fileCounter}`, new File(fileContents));
+        fileCounter++
+        if (fileCounter >= MAX_FILE_COUNT)
+            break;
     }
 
     return fs;
 }
 
+class FSVisitor {
+    visitFile(name, file) {
+    }
+
+    visitDir(name, dir) {
+    }
+}
+
+class CountVisitor extends FSVisitor {
+    fileCount = 0;
+    dirCount = 0;
+
+    visitFile() {
+        this.fileCount++;
+    }
+
+    visitDir() {
+        this.dirCount++;
+    }
+}
+
+class CountDracula extends FSVisitor {
+    bytes = 0;
+    visitFile(name, file) {
+        this.bytes += file.byteLength;
+    }
+}
+
+
 class Benchmark {
-    EXPECTED_FILE_COUNT = 666;
+    EXPECTED_FILE_COUNT = 739;
 
     totalFileCount = 0;
+    totalDirCount = 0;
     lastFileHash = undefined;
+    fs;
 
-    async runIteration() {
-        const fs = await setupDirectory();
+    async prepareForNextIteration() {
+        this.fs = await setupDirectory();
+    }
 
-        for await (let { entry: file } of fs.forEachFileRecursively()) {
+    async runIteration() {
+        for await (let { entry: file } of this.fs.forEachFileRecursively()) {
             this.lastFileHash = file.swapByteOrder();
         }
 
-        for await (let { name, entry: dir } of fs.forEachDirectoryRecursively()) {
-            if ((await dir.fileCount()) > 3) {
-                for await (let { name } of dir.forEachFile()) {
-                    let result = await dir.rm(name);
-                    if (!result)
-                        throw new Error("rm should have returned true");
-                }
+        let bytesDeleted = 0;
+        let counter = 0;
+        for await (const { name, entry: dir } of this.fs.forEachDirectoryRecursively()) {
+            const oldTotalSize = await dir.totalFileSize();
+            if (await dir.fileCount() === 0)
+                continue;
+            counter++;
+            if (counter % 13 !== 0)
+                continue;
+            for await (const { name } of dir.forEachFile()) {
+                const result = await dir.rm(name);
+                if (!result)
+                    throw new Error("rm should have returned true");
             }
+            const totalReducedSize = oldTotalSize - dir.totalFileSize();
+            bytesDeleted += totalReducedSize;
+        }
+        if (bytesDeleted === 0)
+            throw new Error("Did not delete any files");
+
+        const countVisitor = new CountVisitor();
+        await this.fs.visit(countVisitor);
+
+        const countDracula = new CountDracula();
+        await this.fs.visit(countDracula);
+
+        let fileCount = 0;
+        let fileBytes = 0;
+        for await (const {entry: file} of this.fs.forEachFileRecursively()) {
+            fileCount++
+            fileBytes += file.byteLength;
         }
+        this.totalFileCount += fileCount;
 
-        for await (let _ of fs.forEachFileRecursively()) {
-            this.totalFileCount++;
+        let dirCount = 1;
+        for await (let _ of this.fs.forEachDirectoryRecursively()) {
+            dirCount++;
         }
+        this.totalDirCount += dirCount;
+
+        if (countVisitor.fileCount !== fileCount)
+            throw new Error(`Invalid total file count ${countVisitor.fileCount}, expected ${fileCount}.`);
+        if (countDracula.bytes !== fileBytes)
+            throw new Error(`Invalid total file bytes ${countDracula.bytes}, expected ${fileBytes}.`);
+        if (countVisitor.dirCount !== dirCount)
+            throw new Error(`Invalid total dir count ${countVisitor.dirCount}, expected ${dirCount}.`);
+
     }
 
     validate(iterations) {