chore: vendor deno modules used in integration test helpers (#2883)

Author: pieh

.github/workflows/run-tests.yml

@@ -151,6 +151,8 @@ jobs:
         run: npm ci
       - name: 'Build'
         run: npm run build
+      - name: 'Vendor deno helpers for integration tests'
+        run: node tools/vendor-deno-tools.js
       - name: Resolve Next.js version
         id: resolve-next-version
         shell: bash

.gitignore

@@ -2,6 +2,9 @@ node_modules/
 dist/
 .next
 edge-runtime/vendor
+# deno.json is ephemeral and generated for the purpose of vendoring remote modules in CI
+tools/deno/deno.json
+tools/deno/vendor
 
 # Local Netlify folder
 .netlify

tools/build-helpers.js

@@ -0,0 +1,70 @@
+import { createWriteStream } from 'node:fs'
+import { rm, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { Readable } from 'stream'
+import { finished } from 'stream/promises'
+
+import { execaCommand } from 'execa'
+
+/**
+ * @param {Object} options
+ * @param {string} options.vendorSource Path to the file to vendor
+ * @param {string} options.cwd Directory to run the command in
+ * @param {string[]} [options.wasmFilesToDownload] List of wasm files to download
+ * @param {boolean} [options.initEmptyDenoJson] If true, will create an empty deno.json file
+ */
+export async function vendorDeno({
+  vendorSource,
+  cwd,
+  wasmFilesToDownload = [],
+  initEmptyDenoJson = false,
+}) {
+  try {
+    await execaCommand('deno --version')
+  } catch {
+    throw new Error('Could not check the version of Deno. Is it installed on your system?')
+  }
+
+  const vendorDest = join(cwd, 'vendor')
+
+  console.log(`🧹 Deleting '${vendorDest}'...`)
+
+  await rm(vendorDest, { force: true, recursive: true })
+
+  if (initEmptyDenoJson) {
+    const denoJsonPath = join(cwd, 'deno.json')
+    console.log(`🧹 Generating clean '${denoJsonPath}`)
+    await writeFile(denoJsonPath, '{}')
+  }
+
+  console.log(`📦 Vendoring Deno modules for '${vendorSource}' into '${vendorDest}'...`)
+  // --output=${vendorDest}
+  await execaCommand(`deno vendor ${vendorSource}  --force`, {
+    cwd,
+  })
+
+  if (wasmFilesToDownload.length !== 0) {
+    console.log(`⬇️ Downloading wasm files...`)
+
+    // deno vendor doesn't work well with wasm files
+    // see https://github.com/denoland/deno/issues/14123
+    // to workaround this we copy the wasm files manually
+    // (note Deno 2 allows to vendor wasm files, but it also require modules to import them and not fetch and instantiate them
+    // se being able to drop downloading is dependent on implementation of wasm handling in external modules as well)
+    await Promise.all(
+      wasmFilesToDownload.map(async (urlString) => {
+        const url = new URL(urlString)
+
+        const destination = join(vendorDest, url.hostname, url.pathname)
+
+        const res = await fetch(url)
+        if (!res.ok)
+          throw new Error(`Failed to fetch .wasm file to vendor. Response status: ${res.status}`)
+        const fileStream = createWriteStream(destination, { flags: 'wx' })
+        await finished(Readable.fromWeb(res.body).pipe(fileStream))
+      }),
+    )
+  }
+
+  console.log(`✅ Vendored Deno modules for '${vendorSource}' into '${vendorDest}'`)
+}

tools/build.js

@@ -1,14 +1,12 @@
-import { createWriteStream } from 'node:fs'
 import { cp, readFile, rm } from 'node:fs/promises'
 import { dirname, join, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
-import { Readable } from 'stream'
-import { finished } from 'stream/promises'
 
 import { build, context } from 'esbuild'
-import { execaCommand } from 'execa'
 import glob from 'fast-glob'
 
+import { vendorDeno } from './build-helpers.js'
+
 const OUT_DIR = 'dist'
 await rm(OUT_DIR, { force: true, recursive: true })
 
@@ -83,47 +81,22 @@ async function bundle(entryPoints, format, watch) {
   })
 }
 
-async function vendorDeno() {
+async function vendorMiddlewareDenoModules() {
   const vendorSource = resolve('edge-runtime/vendor.ts')
-  const vendorDest = resolve('edge-runtime/vendor')
-
-  try {
-    await execaCommand('deno --version')
-  } catch {
-    throw new Error('Could not check the version of Deno. Is it installed on your system?')
-  }
-
-  console.log(`🧹 Deleting '${vendorDest}'...`)
-
-  await rm(vendorDest, { force: true, recursive: true })
+  const middlewareDir = resolve('edge-runtime')
 
-  console.log(`📦 Vendoring Deno modules into '${vendorDest}'...`)
-
-  await execaCommand(`deno vendor ${vendorSource} --output=${vendorDest} --force`)
-
-  // htmlrewriter contains wasm files and those don't currently work great with vendoring
-  // see https://github.com/denoland/deno/issues/14123
-  // to workaround this we copy the wasm files manually
-  const filesToDownload = ['https://deno.land/x/[email protected]/pkg/htmlrewriter_bg.wasm']
-  await Promise.all(
-    filesToDownload.map(async (urlString) => {
-      const url = new URL(urlString)
-
-      const destination = join(vendorDest, url.hostname, url.pathname)
-
-      const res = await fetch(url)
-      if (!res.ok) throw new Error('Failed to fetch .wasm file to vendor', { cause: err })
-      const fileStream = createWriteStream(destination, { flags: 'wx' })
-      await finished(Readable.fromWeb(res.body).pipe(fileStream))
-    }),
-  )
+  await vendorDeno({
+    vendorSource,
+    cwd: middlewareDir,
+    wasmFilesToDownload: ['https://deno.land/x/[email protected]/pkg/htmlrewriter_bg.wasm'],
+  })
 }
 
 const args = new Set(process.argv.slice(2))
 const watch = args.has('--watch') || args.has('-w')
 
 await Promise.all([
-  vendorDeno(),
+  vendorMiddlewareDenoModules(),
   bundle(entryPointsESM, 'esm', watch),
   ...entryPointsCJS.map((entry) => bundle([entry], 'cjs', watch)),
   cp('src/build/templates', join(OUT_DIR, 'build/templates'), { recursive: true, force: true }),

tools/vendor-deno-tools.js

@@ -0,0 +1,13 @@
+import { dirname, join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { vendorDeno } from './build-helpers.js'
+
+const denoToolsDirectory = join(dirname(fileURLToPath(import.meta.url)), 'deno')
+
+await vendorDeno({
+  vendorSource: join(denoToolsDirectory, 'eszip.ts'),
+  cwd: denoToolsDirectory,
+  wasmFilesToDownload: ['https://deno.land/x/[email protected]/eszip_wasm_bg.wasm'],
+  initEmptyDenoJson: true,
+})