diff --git a/_config.ts b/_config.ts index 308eb303f..a8c5f396b 100644 --- a/_config.ts +++ b/_config.ts @@ -38,9 +38,13 @@ function ensureReferenceDocsExist() { const requiredFiles = [ "reference_gen/gen/deno.json", "reference_gen/gen/web.json", - "reference_gen/gen/node.json", ]; + // Only require node.json if not skipping node docs + if (!Deno.env.get("SKIP_NODE_DOCS")) { + requiredFiles.push("reference_gen/gen/node.json"); + } + const missingFiles = []; for (const file of requiredFiles) { try { @@ -58,6 +62,9 @@ function ensureReferenceDocsExist() { ` Run 'deno task generate:reference' to generate them before building`, ); console.error(` Or set SKIP_REFERENCE=1 to skip reference documentation`); + console.error( + ` Or set SKIP_NODE_DOCS=1 to skip Node.js documentation only`, + ); Deno.exit(1); } } diff --git a/reference_gen/deno.jsonc b/reference_gen/deno.jsonc index 4ff8e2ee6..b0acc7141 100644 --- a/reference_gen/deno.jsonc +++ b/reference_gen/deno.jsonc @@ -22,7 +22,7 @@ "doc:deno": "mkdir -p gen && deno run --allow-read --allow-write --allow-env --allow-net deno-doc.ts", "doc:web": "mkdir -p gen && deno run --allow-read --allow-write --allow-env --allow-net web-doc.ts", "doc:node": "mkdir -p gen && deno run --allow-read --allow-write --allow-env --allow-net node-doc.ts", - "doc": "mkdir -p gen && deno run --allow-read --allow-write --allow-env --allow-net --allow-run parallel-doc.ts", + "doc": "mkdir -p gen && deno run --allow-read --allow-write --allow-env --allow-net --allow-run sequential-doc.ts", "doc:sequential": "deno task doc:deno && deno task doc:web && deno task doc:node" }, "exclude": [ diff --git a/reference_gen/node-doc.ts b/reference_gen/node-doc.ts index c2b5648bc..1e88253b4 100644 --- a/reference_gen/node-doc.ts +++ b/reference_gen/node-doc.ts @@ -48,40 +48,111 @@ class NodeDocGenerator { return; } - console.log("Generating doc nodes for all modules..."); - const nodes = await doc(allFileNames); - - console.log(`Generated doc nodes for ${Object.keys(nodes).length} modules`); - console.log("Generating JSON structure..."); - - const files = await generateHtmlAsJSON(nodes, { - packageName: "Node", - disableSearch: true, - symbolRedirectMap, - defaultSymbolMap, - rewriteMap: newRewriteMap, - hrefResolver, - usageComposer: { - singleMode: true, - compose(currentResolve, usageToMd) { - if ("file" in currentResolve) { - return new Map([[ - { - name: "", - }, - usageToMd(`node:${currentResolve.file.path}`, undefined), - ]]); - } else { - return new Map(); - } - }, - }, - markdownRenderer: renderMarkdown, - markdownStripper: stripMarkdown, - }); - - console.log("Writing node.json..."); - await Deno.writeTextFile("./gen/node.json", JSON.stringify(files)); + console.log( + "Generating documentation one file at a time (ultra low memory mode)...", + ); + + // Process files individually to minimize memory usage + const processedModules: string[] = []; + const tempDir = "./temp_node_minimal"; + + // Create temporary directory for results + try { + await Deno.mkdir(tempDir, { recursive: true }); + } catch (error) { + console.warn(`Warning creating temp directory:`, error); + } + + // Process each file individually + for (let i = 0; i < allFileNames.length; i++) { + const fileName = allFileNames[i]; + console.log( + `Processing file ${i + 1}/${allFileNames.length}: ${ + fileName.split("/").pop() + }`, + ); + + try { + // Process single file + const nodes = await doc([fileName]); + + // Generate JSON for this single file + const singleFileJson = await generateHtmlAsJSON(nodes, { + packageName: "Node", + disableSearch: true, + symbolRedirectMap, + defaultSymbolMap, + rewriteMap: newRewriteMap, + hrefResolver, + usageComposer: { + singleMode: true, + compose(currentResolve, usageToMd) { + if ("file" in currentResolve) { + return new Map([[ + { + name: "", + }, + usageToMd(`node:${currentResolve.file.path}`, undefined), + ]]); + } else { + return new Map(); + } + }, + }, + markdownRenderer: renderMarkdown, + markdownStripper: stripMarkdown, + }); + + // Save individual result to disk + const resultFile = `${tempDir}/file_${i}.json`; + await Deno.writeTextFile(resultFile, JSON.stringify(singleFileJson)); + + processedModules.push(...Object.keys(nodes)); + } catch (error) { + console.error(`Error processing ${fileName}:`, error); + throw error; + } + + // Small delay between files + await new Promise((resolve) => setTimeout(resolve, 50)); + } + + console.log(`Processed ${processedModules.length} modules individually`); + console.log("Combining results into final JSON..."); + + // Combine all individual results into final structure + const finalResult: Record = {}; + + for (let i = 0; i < allFileNames.length; i++) { + const resultFile = `${tempDir}/file_${i}.json`; + try { + const fileData = await Deno.readTextFile(resultFile); + const fileResult = JSON.parse(fileData); + + // Merge into final result + Object.assign(finalResult, fileResult); + + // Clean up individual file immediately + await Deno.remove(resultFile); + } catch (error) { + console.warn(`Could not load result ${i}:`, error); + } + + // Progress indicator for large datasets + if ((i + 1) % 10 === 0) { + console.log(`Combined ${i + 1}/${allFileNames.length} results...`); + } + } + + // Clean up temp directory + try { + await Deno.remove(tempDir, { recursive: true }); + } catch { + // Ignore cleanup errors + } + + console.log("Writing final node.json..."); + await Deno.writeTextFile("./gen/node.json", JSON.stringify(finalResult)); console.log("Node.js documentation generation completed"); } } diff --git a/reference_gen/parallel-doc.ts b/reference_gen/parallel-doc.ts deleted file mode 100644 index f95961ff7..000000000 --- a/reference_gen/parallel-doc.ts +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env -S deno run --allow-read --allow-write --allow-env --allow-net --allow-run - -/** - * Super-optimized parallel documentation generation script - * with intelligent caching and resource management - */ - -import { EnhancedGenerationCache } from "./cache.ts"; -import { existsSync } from "@std/fs"; - -interface TaskConfig { - name: string; - command: string[]; - shouldRun: boolean; - priority: number; // Lower numbers = higher priority - memoryIntensive: boolean; -} - -async function runOptimizedDocGeneration() { - console.log("šŸš€ Starting optimized parallel documentation generation..."); - - const startTime = performance.now(); - const cache = new EnhancedGenerationCache(); - - // Define all possible tasks with priorities and resource requirements - const allTasks: TaskConfig[] = [ - { - name: "Deno", - command: [ - "run", - "--allow-read", - "--allow-write", - "--allow-env", - "--allow-net", - "deno-doc.ts", - ], - shouldRun: await cache.shouldRegenerate("./types/deno.d.ts") || - !existsSync("./gen/deno.json"), - priority: 1, - memoryIntensive: false, - }, - { - name: "Web", - command: [ - "run", - "--allow-read", - "--allow-write", - "--allow-env", - "--allow-net", - "web-doc.ts", - ], - shouldRun: await cache.shouldRegenerate("./types/web.d.ts") || - !existsSync("./gen/web.json"), - priority: 2, - memoryIntensive: false, - }, - { - name: "Node", - command: [ - "run", - "--allow-read", - "--allow-write", - "--allow-env", - "--allow-net", - "node-doc.ts", - ], - shouldRun: await cache.shouldRegenerate("./types/node") || - !existsSync("./gen/node.json"), - priority: 3, - memoryIntensive: true, - }, - ]; - - // Filter tasks that need to run - const tasksToRun = allTasks.filter((task) => task.shouldRun); - - if (tasksToRun.length === 0) { - console.log("āœØ All documentation is up to date! No regeneration needed."); - return; - } - - console.log( - `šŸ“ Running ${tasksToRun.length} of ${allTasks.length} generation tasks...`, - ); - - // Sort by priority and memory requirements - tasksToRun.sort((a, b) => { - if (a.priority !== b.priority) { - return a.priority - b.priority; - } - // Run memory-intensive tasks last - return a.memoryIntensive ? 1 : -1; - }); - - // Run high-priority, low-memory tasks in parallel - const lightTasks = tasksToRun.filter((task) => !task.memoryIntensive); - const heavyTasks = tasksToRun.filter((task) => task.memoryIntensive); - - const allResults: Array<{ task: TaskConfig; result: Deno.CommandOutput }> = - []; - - // Process light tasks in parallel - if (lightTasks.length > 0) { - console.log( - `āš” Running ${lightTasks.length} lightweight tasks in parallel...`, - ); - - const lightPromises = lightTasks.map(async (task, index) => { - // Slight stagger to reduce initial resource spike - if (index > 0) { - await new Promise((resolve) => setTimeout(resolve, index * 200)); - } - - return new Deno.Command("deno", { - args: task.command, - stdout: "piped", - stderr: "piped", - env: { "DENO_V8_FLAGS": "--max-old-space-size=4096" }, // Increase memory limit - }).output().then((result) => ({ task, result })); - }); - - const lightResults = await Promise.all(lightPromises); - allResults.push(...lightResults); - - // Small delay to let memory settle - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - - // Process heavy tasks sequentially with memory management - if (heavyTasks.length > 0) { - console.log( - `šŸ‹ļø Running ${heavyTasks.length} memory-intensive tasks sequentially...`, - ); - - for (const task of heavyTasks) { - console.log(`šŸ“Š Starting ${task.name} generation...`); - - const result = await new Deno.Command("deno", { - args: task.command, - stdout: "piped", - stderr: "piped", - env: { - "DENO_V8_FLAGS": "--max-old-space-size=8192", - }, - }).output(); - - allResults.push({ task, result }); - - if (result.code === 0) { - console.log(`āœ… ${task.name} generation completed`); - } - - // Force garbage collection and memory cleanup - await new Promise((resolve) => setTimeout(resolve, 2000)); - } - } - - const endTime = performance.now(); - - // Process all results - let hasErrors = false; - for (const { task, result } of allResults) { - if (result.code !== 0) { - console.error(`āŒ ${task.name} generation failed:`); - console.error(new TextDecoder().decode(result.stderr)); - hasErrors = true; - } else { - const stdout = new TextDecoder().decode(result.stdout); - if (stdout.trim()) { - // Show key progress lines for completed tasks - const lines = stdout.split("\n").filter((line) => - line.includes("āœ…") || - line.includes("šŸ“") || - line.includes("šŸŽ‰") || - line.includes("completed") || - line.includes("Found") || - line.includes("Generated") - ); - if (lines.length > 0) { - console.log(` ${lines.slice(-3).join("\n ")}`); // Show last 3 key lines - } - } - } - } - - // Show skipped tasks - for (const task of allTasks) { - if (!task.shouldRun) { - console.log(`ā­ļø ${task.name} generation skipped (no changes detected)`); - } - } - - const totalTime = ((endTime - startTime) / 1000).toFixed(2); - const tasksRun = allResults.length; - const tasksSkipped = allTasks.length - tasksRun; - - if (hasErrors) { - console.log( - `\nāš ļø Documentation generation completed with errors in ${totalTime}s`, - ); - console.log(`šŸ“Š Tasks: ${tasksRun} run, ${tasksSkipped} skipped`); - Deno.exit(1); - } else { - console.log( - `\nšŸŽ‰ Documentation generation completed successfully in ${totalTime}s task.shouldRun); + + if (tasksToRun.length === 0) { + console.log("āœØ All documentation is up to date!"); + return; + } + + console.log(`šŸ“ Running ${tasksToRun.length} tasks sequentially...`); + + // Run each script as a subprocess + for (const task of tasksToRun) { + console.log(`šŸ“Š Starting ${task.name} generation...`); + + try { + const command = new Deno.Command("deno", { + args: [ + "run", + "--allow-read", + "--allow-write", + "--allow-env", + "--allow-net", + task.script, + ], + stdout: "inherit", + stderr: "inherit", + }); + + const { code } = await command.output(); + + if (code === 0) { + console.log(`āœ… ${task.name} generation completed`); + } else { + console.error( + `āŒ ${task.name} generation failed with exit code ${code}`, + ); + Deno.exit(1); + } + } catch (error) { + console.error(`āŒ ${task.name} generation failed:`, error); + Deno.exit(1); + } + + // Small delay between tasks + await new Promise((resolve) => setTimeout(resolve, 2000)); + } + + // Show skipped tasks + for (const task of tasks) { + if (!task.shouldRun) { + console.log(`ā­ļø ${task.name} generation skipped (no changes detected)`); + } + } + + const totalTime = ((performance.now() - startTime) / 1000).toFixed(2); + const tasksRun = tasksToRun.length; + const tasksSkipped = tasks.length - tasksRun; + + console.log( + `\nšŸŽ‰ Documentation generation completed successfully in ${totalTime}s`, + ); + console.log(`šŸ“Š Tasks: ${tasksRun} run, ${tasksSkipped} skipped`); +} + +if (import.meta.main) { + await runSequentialDocGeneration(); +}