increase memory limit

_config.ts

@@ -38,9 +38,13 @@ function ensureReferenceDocsExist() {
   const requiredFiles = [
     "reference_gen/gen/deno.json",
     "reference_gen/gen/web.json",
-    "reference_gen/gen/node.json",
   ];
 
+  // Only require node.json if not skipping node docs
+  if (!Deno.env.get("SKIP_NODE_DOCS")) {
+    requiredFiles.push("reference_gen/gen/node.json");
+  }
+
   const missingFiles = [];
   for (const file of requiredFiles) {
     try {
@@ -58,6 +62,9 @@ function ensureReferenceDocsExist() {
       `   Run 'deno task generate:reference' to generate them before building`,
     );
     console.error(`   Or set SKIP_REFERENCE=1 to skip reference documentation`);
+    console.error(
+      `   Or set SKIP_NODE_DOCS=1 to skip Node.js documentation only`,
+    );
     Deno.exit(1);
   }
 }

reference_gen/deno.jsonc

@@ -22,7 +22,7 @@
     "doc:deno": "mkdir -p gen && deno run --allow-read --allow-write --allow-env --allow-net deno-doc.ts",
     "doc:web": "mkdir -p gen && deno run --allow-read --allow-write --allow-env --allow-net web-doc.ts",
     "doc:node": "mkdir -p gen && deno run --allow-read --allow-write --allow-env --allow-net node-doc.ts",
-    "doc": "mkdir -p gen && deno run --allow-read --allow-write --allow-env --allow-net --allow-run parallel-doc.ts",
+    "doc": "mkdir -p gen && deno run --allow-read --allow-write --allow-env --allow-net --allow-run sequential-doc.ts",
     "doc:sequential": "deno task doc:deno && deno task doc:web && deno task doc:node"
   },
   "exclude": [

reference_gen/node-doc.ts

@@ -48,40 +48,111 @@ class NodeDocGenerator {
       return;
     }
 
-    console.log("Generating doc nodes for all modules...");
-    const nodes = await doc(allFileNames);
-
-    console.log(`Generated doc nodes for ${Object.keys(nodes).length} modules`);
-    console.log("Generating JSON structure...");
-
-    const files = await generateHtmlAsJSON(nodes, {
-      packageName: "Node",
-      disableSearch: true,
-      symbolRedirectMap,
-      defaultSymbolMap,
-      rewriteMap: newRewriteMap,
-      hrefResolver,
-      usageComposer: {
-        singleMode: true,
-        compose(currentResolve, usageToMd) {
-          if ("file" in currentResolve) {
-            return new Map([[
-              {
-                name: "",
-              },
-              usageToMd(`node:${currentResolve.file.path}`, undefined),
-            ]]);
-          } else {
-            return new Map();
-          }
-        },
-      },
-      markdownRenderer: renderMarkdown,
-      markdownStripper: stripMarkdown,
-    });
-
-    console.log("Writing node.json...");
-    await Deno.writeTextFile("./gen/node.json", JSON.stringify(files));
+    console.log(
+      "Generating documentation one file at a time (ultra low memory mode)...",
+    );
+
+    // Process files individually to minimize memory usage
+    const processedModules: string[] = [];
+    const tempDir = "./temp_node_minimal";
+
+    // Create temporary directory for results
+    try {
+      await Deno.mkdir(tempDir, { recursive: true });
+    } catch (error) {
+      console.warn(`Warning creating temp directory:`, error);
+    }
+
+    // Process each file individually
+    for (let i = 0; i < allFileNames.length; i++) {
+      const fileName = allFileNames[i];
+      console.log(
+        `Processing file ${i + 1}/${allFileNames.length}: ${
+          fileName.split("/").pop()
+        }`,
+      );
+
+      try {
+        // Process single file
+        const nodes = await doc([fileName]);
+
+        // Generate JSON for this single file
+        const singleFileJson = await generateHtmlAsJSON(nodes, {
+          packageName: "Node",
+          disableSearch: true,
+          symbolRedirectMap,
+          defaultSymbolMap,
+          rewriteMap: newRewriteMap,
+          hrefResolver,
+          usageComposer: {
+            singleMode: true,
+            compose(currentResolve, usageToMd) {
+              if ("file" in currentResolve) {
+                return new Map([[
+                  {
+                    name: "",
+                  },
+                  usageToMd(`node:${currentResolve.file.path}`, undefined),
+                ]]);
+              } else {
+                return new Map();
+              }
+            },
+          },
+          markdownRenderer: renderMarkdown,
+          markdownStripper: stripMarkdown,
+        });
+
+        // Save individual result to disk
+        const resultFile = `${tempDir}/file_${i}.json`;
+        await Deno.writeTextFile(resultFile, JSON.stringify(singleFileJson));
+
+        processedModules.push(...Object.keys(nodes));
+      } catch (error) {
+        console.error(`Error processing ${fileName}:`, error);
+        throw error;
+      }
+
+      // Small delay between files
+      await new Promise((resolve) => setTimeout(resolve, 50));
+    }
+
+    console.log(`Processed ${processedModules.length} modules individually`);
+    console.log("Combining results into final JSON...");
+
+    // Combine all individual results into final structure
+    const finalResult: Record<string, unknown> = {};
+
+    for (let i = 0; i < allFileNames.length; i++) {
+      const resultFile = `${tempDir}/file_${i}.json`;
+      try {
+        const fileData = await Deno.readTextFile(resultFile);
+        const fileResult = JSON.parse(fileData);
+
+        // Merge into final result
+        Object.assign(finalResult, fileResult);
+
+        // Clean up individual file immediately
+        await Deno.remove(resultFile);
+      } catch (error) {
+        console.warn(`Could not load result ${i}:`, error);
+      }
+
+      // Progress indicator for large datasets
+      if ((i + 1) % 10 === 0) {
+        console.log(`Combined ${i + 1}/${allFileNames.length} results...`);
+      }
+    }
+
+    // Clean up temp directory
+    try {
+      await Deno.remove(tempDir, { recursive: true });
+    } catch {
+      // Ignore cleanup errors
+    }
+
+    console.log("Writing final node.json...");
+    await Deno.writeTextFile("./gen/node.json", JSON.stringify(finalResult));
     console.log("Node.js documentation generation completed");
   }
 }