feat: script for generating llm files

Author: luguellis

.gitignore

@@ -27,3 +27,5 @@ lib/
 
 src/**/*.mjs
 scripts/**/*.mjs
+public/llms/manual/**/*.txt
+public/llms/react/**/*.txt

package.json

@@ -58,10 +58,11 @@
   },
   "scripts": {
     "dev": "next",
-    "build": "rescript && npm run update-index && next build",
+    "build": "rescript && npm run generate-llms && npm run update-index && next build",
     "test": "node scripts/test-examples.mjs && node scripts/test-hrefs.mjs",
     "reanalyze": "reanalyze -all-cmt .",
-    "update-index": "node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml"
+    "update-index": "node scripts/extract-indices.mjs && node scripts/extract-tocs.mjs && node scripts/extract-syntax.mjs && node scripts/generate_feed.mjs > public/blog/feed.xml",
+    "generate-llms": "node scripts/generate_llms.mjs"
   },
   "devDependencies": {
     "@mdx-js/react": "^2.3.0",

scripts/generate_llms.res

@@ -0,0 +1,114 @@
+let readMarkdownFile = (filePath: string): string => {
+  let fileContent = Node.Fs.readFileSync2(filePath, "utf8")
+  fileContent
+}
+
+let rec collectFiles = (dirPath: string): array<string> => {
+  let entries = Node.Fs.readdirSync(dirPath)
+  entries->Array.reduce([], (acc, entry) => {
+    let fullPath = Node.Path.join([dirPath, entry])
+    let stats = Node.Fs.statSync(fullPath)
+    switch stats["isDirectory"]() {
+    | true => acc->Array.concat(collectFiles(fullPath))
+    | false => {
+        acc->Array.push(fullPath)
+        acc
+      }
+    }
+  })
+}
+
+let clearFile = (filePath: string): unit => {
+  Node.Fs.writeFileSync(filePath, "")
+}
+
+let createDirectoryIfNotExists = (dirPath: string): unit => {
+  if !Node.Fs.existsSync(dirPath) {
+    Node.Fs.mkdirSync(dirPath)
+  }
+}
+
+let removeCodeTabTags = (content: string): string => {
+  let regex = Js.Re.fromStringWithFlags("<CodeTab.*?>[\\s\\S]*?</CodeTab>", ~flags="g")
+  Js.String.replaceByRe(regex, "", content)
+}
+
+let removeCodeBlocks = (content: string): string => {
+  let regex = Js.Re.fromStringWithFlags("```[a-zA-Z]+\\s*[\\s\\S]*?```", ~flags="g")
+  Js.String.replaceByRe(regex, "", content)
+}
+
+let removeFileTitle = (content: string): string => {
+  let regex = Js.Re.fromStringWithFlags("---\ntitle[\\s\\S]*?---", ~flags="g")
+  Js.String.replaceByRe(regex, "", content)
+}
+
+let removeUnnecessaryBreaks = (content: string): string => {
+  let regex = Js.Re.fromStringWithFlags("^\n{2,}", ~flags="g")
+  Js.String.replaceByRe(regex, "", content)
+}
+
+let removeToDos = (content: string): string => {
+  let regex = Js.Re.fromStringWithFlags("<!-- TODO[\\s\\S]*?-->", ~flags="g")
+  Js.String.replaceByRe(regex, "", content)
+}
+
+let createFullFile = (content: string, filePath: string): unit => {
+  Node.Fs.appendFileSync(filePath, content ++ "\n", "utf8")
+}
+
+let createSmallFile = (content: string, filePath: string): unit => {
+  let smallContent =
+    content
+    ->removeCodeTabTags
+    ->removeFileTitle
+    ->removeToDos
+    ->removeCodeBlocks
+    ->removeUnnecessaryBreaks
+  Node.Fs.appendFileSync(filePath, smallContent, "utf8")
+}
+
+let processVersions = (
+  versions: array<string>,
+  docsDirectory: string,
+  llmsDirectory: string,
+): unit => {
+  let fullFileName = "llm-full.txt"
+  let smallFileName = "llm-small.txt"
+  let llmsFileName = "llms.txt"
+
+  versions->Array.forEach(version => {
+    let versionDir = docsDirectory->Node.Path.join2(version)
+    let llmsDir = llmsDirectory->Node.Path.join2(version)
+    let fullFilePath = llmsDir->Node.Path.join2(fullFileName)
+    let smallFilePath = llmsDir->Node.Path.join2(smallFileName)
+
+    createDirectoryIfNotExists(llmsDir)
+    clearFile(fullFilePath)
+    clearFile(smallFilePath)
+
+    versionDir
+    ->collectFiles
+    ->Array.forEach(filePath => {
+      if Js.String.endsWith(".mdx", filePath) {
+        let content = readMarkdownFile(filePath)
+
+        content->createFullFile(fullFilePath)
+
+        content->createSmallFile(smallFilePath)
+      }
+    })
+  })
+}
+
+let manualVersions = ["v12.0.0", "v11.0.0"]
+let reactManualVersions = ["latest", "v0.10.0", "v0.11.0"]
+
+let manualDocsDirectory = "pages/docs/manual"
+let reactDocsDirectory = "pages/docs/react"
+
+let manualLlmsDirectory = "public/llms/manual"
+let reactLlmsDirectory = "public/llms/react"
+
+processVersions(manualVersions, manualDocsDirectory, manualLlmsDirectory)
+processVersions(reactManualVersions, reactDocsDirectory, reactLlmsDirectory)

src/bindings/Node.res

@@ -20,10 +20,13 @@ module Process = {
 
 module Fs = {
   @module("fs") external readFileSync: string => string = "readFileSync"
+  @module("fs") external readFileSync2: (string, string) => string = "readFileSync"
   @module("fs") external readdirSync: string => array<string> = "readdirSync"
   @module("fs") external writeFileSync: (string, string) => unit = "writeFileSync"
+  @module("fs") external appendFileSync: (string, string, string) => unit = "appendFileSync"
   @module("fs") external existsSync: string => bool = "existsSync"
   @module("fs") external mkdirSync: string => unit = "mkdirSync"
+  @module("fs") external statSync: string => {.."isDirectory": unit => bool} = "statSync"
 }
 
 module Buffer = {
@@ -34,4 +37,4 @@ module Buffer = {
 module ChildProcess = {
   @module("child_process")
   external execSync: string => Buffer.t = "execSync"
-}
+}