Rewrite node module handling (npm plugin)

.changeset/strong-meals-hammer.md

@@ -0,0 +1,14 @@
+---
+'wmr': major
+---
+
+**tl;dr:** Auto-installation of npm packages is not enabled by default anymore and has to be opt-in to via `--autoInstall` on the CLI.
+
+The npm integration in WMR was rewritten from the ground up to support the following new features:
+
+- Reduce amount of requests by prebundling npm packages
+- Resolve the `browser` field in `package.json`
+- Resolve the `exports` field in `package.json`
+- Improve CommonJS handling by attempting to convert it to ESM
+- Ensure reproducible builds by moving auto installing of npm packages behind a CLI flag (`--autoInstall`)
+- Allow specifying the url of the npm registry to fetch from when `--autoInstall` is active. This can be done via `--registry URL_TO_MY_REGISTRY`

packages/wmr/package.json

@@ -95,7 +95,8 @@
 		"totalist": "^1.1.0",
 		"tsconfig-paths": "^3.11.0",
 		"utf-8-validate": "^5.0.2",
-		"ws": "^7.3.1"
+		"ws": "^7.3.1",
+		"zecorn": "^0.9.5"
 	},
 	"optionalDependencies": {
 		"fsevents": "^2.1.3"

packages/wmr/src/build.js

@@ -6,17 +6,13 @@ import { bundleProd } from './bundler.js';
 import { bundleStats } from './lib/output-utils.js';
 import { prerender } from './lib/prerender.js';
 import { normalizeOptions } from './lib/normalize-options.js';
-import { setCwd } from './plugins/npm-plugin/registry.js';
 
 /**
  * @param {Parameters<bundleProd>[0] & { prerender?: boolean }} options
  */
 export default async function build(options) {
 	options.out = options.out || 'dist';
 
-	// @todo remove this hack once registry.js is instantiable
-	setCwd(options.cwd);
-
 	options = await normalizeOptions(options, 'build');
 
 	// Clears out the output folder without deleting it -- useful

packages/wmr/src/cli.js

@@ -34,6 +34,8 @@ prog
 	.option('--sourcemap', 'Enable Source Maps')
 	.option('--visualize', 'Launch interactive bundle visualizer')
 	.option('--minify', 'Enable minification of generated code (default: true)', true)
+	.option('--autoInstall', 'Fetch missing npm packages from npm registry automatically (default: false')
+	.option('--registry', 'NPM registry url to fetch if "--autoInstall" is set (default: https://registry.npmjs.org)')
 	.action(opts => {
 		opts.minify = bool(opts.minify);
 		run(build(opts));
@@ -60,6 +62,8 @@ prog
 	.option('--compress', 'Enable compression (default: enabled)')
 	.option('--profile', 'Generate build statistics')
 	.option('--reload', 'Switch off hmr and reload on file saves')
+	.option('--autoInstall', 'Fetch missing npm packages from npm registry automatically (default: false')
+	.option('--registry', 'NPM registry url to fetch if "--autoInstall" is set (default: https://registry.npmjs.org)')
 	.action(opts => {
 		opts.optimize = !/false|0/.test(opts.compress);
 		opts.compress = bool(opts.compress);

packages/wmr/src/lib/acorn-traverse.js

@@ -834,6 +834,8 @@ const TYPES = {
 	importSpecifier: (local, imported) => ({ type: 'ImportSpecifier', local, imported }),
 	importDefaultSpecifier: local => ({ type: 'ImportDefaultSpecifier', local }),
 	importNamespaceSpecifier: local => ({ type: 'ImportNamespaceSpecifier', local }),
+	exportDefaultDeclaration: declaration => ({ type: 'ExportDefaultDeclaration', declaration }),
+	exportAllDeclaration: (source, exported = null) => ({ type: 'ExportAllDeclaration', source, exported }),
 	assignmentExpression: (operator, left, right) => ({ type: 'AssignmentExpression', operator, left, right }),
 	variableDeclaration: (kind, declarations) => ({ type: 'VariableDeclaration', kind, declarations }),
 	variableDeclarator: (id, init) => ({ type: 'VariableDeclarator', id, init }),

packages/wmr/src/lib/fs-utils.js

@@ -1,4 +1,5 @@
 import { promises as fs } from 'fs';
+import path from 'path';
 
 /**
  * Implementation of fs.rm() for Node 12+
@@ -47,3 +48,27 @@ export function hasCustomPrefix(id) {
 export function pathToUrl(p) {
 	return p.replace(/\\/g, '/');
 }
+
+/**
+ * Read a file as JSON
+ * @param {string} file
+ */
+export async function readJson(file) {
+	const raw = await fs.readFile(file, 'utf-8');
+	return JSON.parse(raw);
+}
+
+/**
+ * Write file and create directories automatically if necessary
+ * @param {string} file
+ * @param {Buffer | string} data
+ * @param {BufferEncoding} [encoding]
+ */
+export async function writeFile(file, data, encoding) {
+	await fs.mkdir(path.dirname(file), { recursive: true });
+	if (encoding) {
+		await fs.writeFile(file, data, encoding);
+	} else {
+		await fs.writeFile(file, data);
+	}
+}

packages/wmr/src/lib/normalize-options.js

@@ -25,6 +25,7 @@ export async function normalizeOptions(options, mode, configWatchFiles = []) {
 	options.features = { preact: true };
 	options.alias = options.alias || options.aliases || {};
 	options.customRoutes = options.customRoutes || [];
+	options.registry = options.registry || 'https://registry.npmjs.org';
 
 	// `wmr` / `wmr start` is a development command.
 	// `wmr build` / `wmr serve` are production commands.

packages/wmr/src/lib/npm-middleware.js

@@ -1,184 +1,36 @@
-import * as rollup from 'rollup';
-import commonjs from '@rollup/plugin-commonjs';
-import json from '@rollup/plugin-json';
-// import unpkgPlugin from '../plugins/unpkg-plugin.js';
-import npmPlugin, { normalizeSpecifier } from '../plugins/npm-plugin/index.js';
-import { resolvePackageVersion, loadPackageFile } from '../plugins/npm-plugin/registry.js';
-import { getCachedBundle, setCachedBundle, sendCachedBundle, enqueueCompress } from './npm-middleware-cache.js';
-import processGlobalPlugin from '../plugins/process-global-plugin.js';
-import aliasPlugin from '../plugins/aliases-plugin.js';
-import { getMimeType } from './mimetypes.js';
-import nodeBuiltinsPlugin from '../plugins/node-builtins-plugin.js';
-import * as kl from 'kolorist';
-import { hasDebugFlag, onWarn } from './output-utils.js';
+import path from 'path';
+import { getPackageInfo, isValidPackageName } from '../plugins/npm-plugin/utils.js';
 
-/**
- * Serve a "proxy module" that uses the WMR runtime to load CSS.
- * @param {ReturnType<typeof normalizeSpecifier>} meta
- * @param {import('http').ServerResponse} res
- * @param {boolean} [isModule]
- */
-async function handleAsset(meta, res, isModule) {
-	let code = '';
-	let type = null;
-
-	if (isModule) {
-		type = 'application/javascript;charset=utf-8';
-		const specifier = JSON.stringify('/@npm/' + meta.specifier + '?asset');
-		code = `import{style}from '/_wmr.js';\nstyle(${specifier});`;
-	} else {
-		type = getMimeType(meta.path);
-		code = await loadPackageFile(meta);
-	}
-	res.writeHead(200, {
-		'content-type': type || 'text/plain',
-		'content-length': Buffer.byteLength(code)
-	});
-	res.end(code);
-}
-
-/**
- * @param {object} [options]
- * @param {'npm'|'unpkg'} [options.source = 'npm'] How to fetch package files
- * @param {Record<string,string>} [options.alias]
- * @param {boolean} [options.optimize = true] Progressively minify and compress dependency bundles?
- * @param {string} [options.cwd] Virtual cwd
- * @returns {import('polka').Middleware}
- */
-export default function npmMiddleware({ source = 'npm', alias, optimize, cwd } = {}) {
+export function npmEtagCache() {
 	return async (req, res, next) => {
 		const url = new URL(req.url, 'https://localhost');
-		// @ts-ignore
-		const mod = url.pathname.replace(/^\//, '');
+		let id = path.posix.normalize(url.pathname);
 
-		const meta = normalizeSpecifier(mod);
+		if (!id.startsWith('/@npm/')) {
+			return next();
+		}
 
-		try {
-			await resolvePackageVersion(meta);
-		} catch (e) {
-			return next(e);
+		id = id.slice('/@npm/'.length);
+		if (!isValidPackageName(id)) {
+			return next();
 		}
 
+		const { name, version, pathname } = getPackageInfo(id);
+
 		try {
-			// The package name + path + version is a strong ETag since versions are immutable
-			const etag = Buffer.from(`${meta.specifier}${meta.version}`).toString('base64');
+			// The package name + version + pathname is a strong ETag since versions are immutablew
+			const etag = Buffer.from(`${name}${version}${pathname}`).toString('base64');
 			const ifNoneMatch = String(req.headers['if-none-match']).replace(/-(gz|br)$/g, '');
+
 			if (ifNoneMatch === etag) {
 				return res.writeHead(304).end();
 			}
-			res.setHeader('etag', etag);
-
-			// CSS files and proxy modules don't use Rollup.
-			if (/\.((css|s[ac]ss|less)|wasm|txt|json)$/.test(meta.path)) {
-				return handleAsset(meta, res, url.searchParams.has('module'));
-			}
-
-			res.setHeader('content-type', 'application/javascript;charset=utf-8');
-			if (hasDebugFlag()) {
-				// eslint-disable-next-line no-console
-				console.log(`  ${kl.dim('middleware:') + kl.bold(kl.magenta('npm'))}  ${JSON.stringify(meta.specifier)}`);
-			}
-			// serve from memory and disk caches:
-			const cached = await getCachedBundle(etag, meta, cwd);
-			if (cached) return sendCachedBundle(req, res, cached);
-
-			// const start = Date.now();
-			const code = await bundleNpmModule(mod, { source, alias, cwd });
-			// console.log(`Bundle dep: ${mod}: ${Date.now() - start}ms`);
-
-			// send it!
-			res.writeHead(200, { 'content-length': Buffer.byteLength(code) }).end(code);
 
-			// store the bundle in memory and disk caches
-			setCachedBundle(etag, code, meta, cwd);
-
-			// this is a new bundle, we'll compress it with terser and brotli shortly
-			if (optimize !== false) {
-				enqueueCompress(etag);
-			}
-		} catch (e) {
-			console.error(`Error bundling ${mod}: `, e);
-			next(e);
+			res.setHeader('etag', etag);
+		} catch (err) {
+			next(err);
 		}
-	};
-}
-
-let npmCache;
-
-/**
- * Bundle am npm module entry path into a single file
- * @param {string} mod The module to bundle, including subpackage/path
- * @param {object} options
- * @param {'npm'|'unpkg'} [options.source]
- * @param {Record<string,string>} [options.alias]
- * @param {string} [options.cwd]
- */
-async function bundleNpmModule(mod, { source, alias, cwd }) {
-	let npmProviderPlugin;
 
-	if (source === 'unpkg') {
-		throw Error('unpkg plugin is disabled');
-		// npmProviderPlugin = unpkgPlugin({
-		// 	publicPath: '/@npm',
-		// 	perPackage: true
-		// });
-	} else {
-		npmProviderPlugin = npmPlugin({
-			publicPath: '/@npm'
-		});
-	}
-
-	const bundle = await rollup.rollup({
-		input: mod,
-		onwarn: onWarn,
-		// input: '\0entry',
-		cache: npmCache,
-		shimMissingExports: true,
-		treeshake: false,
-		// inlineDynamicImports: true,
-		// shimMissingExports: true,
-		preserveEntrySignatures: 'allow-extension',
-		plugins: [
-			nodeBuiltinsPlugin({}),
-			aliasPlugin({ alias, cwd }),
-			npmProviderPlugin,
-			processGlobalPlugin({
-				sourcemap: false,
-				NODE_ENV: 'development'
-			}),
-			commonjs({
-				extensions: ['.js', '.cjs', ''],
-				sourceMap: false,
-				transformMixedEsModules: true
-			}),
-			json(),
-			{
-				name: 'no-builtins',
-				load(s) {
-					if (s === 'fs' || s === 'path') {
-						return 'export default {};';
-					}
-				}
-			},
-			{
-				name: 'never-disk',
-				load(s) {
-					throw Error('local access not allowed');
-				}
-			}
-		]
-	});
-
-	npmCache = bundle.cache;
-
-	const { output } = await bundle.generate({
-		format: 'es',
-		indent: false,
-		// entryFileNames: '[name].js',
-		// chunkFileNames: '[name].js',
-		// Don't transform paths at all:
-		paths: String
-	});
-
-	return output[0].code;
+		next();
+	};
 }

packages/wmr/src/lib/plugins.js

@@ -1,9 +1,9 @@
+import path from 'path';
 import htmPlugin from '../plugins/htm-plugin.js';
 import sucrasePlugin from '../plugins/sucrase-plugin.js';
 import wmrPlugin from '../plugins/wmr/plugin.js';
 import wmrStylesPlugin from '../plugins/wmr/styles/styles-plugin.js';
 import sassPlugin from '../plugins/sass-plugin.js';
-import npmPlugin from '../plugins/npm-plugin/index.js';
 import publicPathPlugin from '../plugins/public-path-plugin.js';
 import minifyCssPlugin from '../plugins/minify-css-plugin.js';
 import htmlEntriesPlugin from '../plugins/html-entries-plugin.js';
@@ -27,7 +27,9 @@ import { prefreshPlugin } from '../plugins/preact/prefresh.js';
 import { absolutePathPlugin } from '../plugins/absolute-path-plugin.js';
 import { lessPlugin } from '../plugins/less-plugin.js';
 import { workerPlugin } from '../plugins/worker-plugin.js';
+import { npmPlugin } from '../plugins/npm-plugin/index.js';
 import tsConfigPathsPlugin from '../plugins/tsconfig-paths-plugin.js';
+import { getNpmPlugins } from '../plugins/npm-plugin/npm-bundle.js';
 
 /**
  * @param {import("wmr").Options & { isIIFEWorker?: boolean}} options
@@ -38,16 +40,27 @@ export function getPlugins(options) {
 		plugins,
 		publicPath,
 		alias,
+		cwd,
 		root,
 		env,
 		minify,
 		mode,
 		isIIFEWorker = false,
 		sourcemap,
 		features,
-		visualize
+		visualize,
+		autoInstall,
+		registry
 	} = options;
 
+	const npmCacheDir = path.join(cwd, '.cache', '@npm');
+
+	/**
+	 * Map of package name to folder on disk
+	 * @type {Map<string, string>}
+	 */
+	const resolutionCache = new Map();
+
 	// Plugins are pre-sorted
 	let split = plugins.findIndex(p => p.enforce === 'post');
 	if (split === -1) split = plugins.length;
@@ -99,7 +112,20 @@ export function getPlugins(options) {
 			// Only transpile CommonJS in node_modules and explicit .cjs files:
 			include: /(^npm\/|[/\\]node_modules[/\\]|\.cjs$)/
 		}),
-		(production || isIIFEWorker) && npmPlugin({ external: false }),
+
+		...(production
+			? getNpmPlugins({
+					autoInstall,
+					production,
+					cacheDir: npmCacheDir,
+					cwd,
+					registryUrl: registry,
+					resolutionCache,
+					browserReplacement: new Map()
+			  })
+			: []),
+		!production &&
+			npmPlugin({ cwd, cacheDir: npmCacheDir, autoInstall, production, registryUrl: registry, resolutionCache, alias }),
 		resolveExtensionsPlugin({
 			extensions: ['.ts', '.tsx', '.js', '.cjs'],
 			index: true