Skip to content

Commit d077768

Browse files
mischnicmischnic
authored
Turbopack: Worker chunks (vercel#69734)
Closes PACK-2504 Closes PACK-3020 - Implemented `WorkerAssetReference` to handle `new Worker()` calls - Added `WorkerLoaderModule` and `WorkerLoaderChunkItem` to create separate chunk groups for worker scripts - Updated runtime code to support loading worker scripts - Modified the analysis process to detect and handle worker creation - Workers are loaded via blob urls (which then `importScripts`-s all required chunks), which required introducing a `TURBOPACK_WORKER_LOCATION` global variable to support the relative chunk urls in the isolated blob origin. - [x] `evaluated_chunk_group` is not implemented by `NodeJsChunkingContext`, so it currently fails to build in RSC/SSR contexts - I've made it ignore worker calls for `Rendering::Server` for now
1 parent b825912 commit d077768

File tree

109 files changed

+975
-426
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

109 files changed

+975
-426
lines changed

‎docs/04-architecture/turbopack.mdx

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,6 @@ These features are currently not supported:
4949
- This behavior is currently not supported since it changes input files, instead, an error will be shown for you to manually add a root layout in the desired location.
5050
- `@next/font` (legacy font support).
5151
- `@next/font` is deprecated in favor of `next/font`. [`next/font`](/docs/app/building-your-application/optimizing/fonts) is fully supported with Turbopack.
52-
- `new Worker('file', import.meta.url)`.
53-
- We are planning to implement this in the future.
5452
- [Relay transforms](/docs/architecture/nextjs-compiler#relay)
5553
- We are planning to implement this in the future.
5654
- Blocking `.css` imports in `pages/_document.tsx`

‎test/e2e/app-dir/app-external/app-external.test.ts

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -158,24 +158,20 @@ describe('app dir - external dependency', () => {
158158
)
159159
).toMatch(/^myFont, "myFont Fallback"$/)
160160
})
161-
// TODO: This test depends on `new Worker` which is not supported in Turbopack yet.
162-
;(process.env.TURBOPACK ? it.skip : it)(
163-
'should not apply swc optimizer transform for external packages in browser layer in web worker',
164-
async () => {
165-
const browser = await next.browser('/browser')
166-
// eslint-disable-next-line jest/no-standalone-expect
167-
expect(await browser.elementByCss('#worker-state').text()).toBe('default')
161+
it('should not apply swc optimizer transform for external packages in browser layer in web worker', async () => {
162+
const browser = await next.browser('/browser')
163+
// eslint-disable-next-line jest/no-standalone-expect
164+
expect(await browser.elementByCss('#worker-state').text()).toBe('default')
168165

169-
await browser.elementByCss('button').click()
166+
await browser.elementByCss('button').click()
170167

171-
await retry(async () => {
172-
// eslint-disable-next-line jest/no-standalone-expect
173-
expect(await browser.elementByCss('#worker-state').text()).toBe(
174-
'worker.js:browser-module/other'
175-
)
176-
})
177-
}
178-
)
168+
await retry(async () => {
169+
// eslint-disable-next-line jest/no-standalone-expect
170+
expect(await browser.elementByCss('#worker-state').text()).toBe(
171+
'worker.js:browser-module/other'
172+
)
173+
})
174+
})
179175

180176
describe('react in external esm packages', () => {
181177
it('should use the same react in client app', async () => {

‎test/e2e/app-dir/worker/app/layout.js

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
export default function AppLayout({ children }) {
2+
return (
3+
<html>
4+
<head></head>
5+
<body>{children}</body>
6+
</html>
7+
)
8+
}

‎test/e2e/app-dir/worker/app/page.js

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
'use client'
2+
import { useState } from 'react'
3+
4+
export default function Home() {
5+
const [state, setState] = useState('default')
6+
return (
7+
<div>
8+
<button
9+
onClick={() => {
10+
const worker = new Worker(new URL('./worker', import.meta.url))
11+
worker.addEventListener('message', (event) => {
12+
setState(event.data)
13+
})
14+
}}
15+
>
16+
Get web worker data
17+
</button>
18+
<p>Worker state: </p>
19+
<p id="worker-state">{state}</p>
20+
</div>
21+
)
22+
}

‎test/e2e/app-dir/worker/app/worker-dep.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export default 'worker-dep'

‎test/e2e/app-dir/worker/app/worker.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import('./worker-dep').then((mod) => {
2+
self.postMessage('worker.ts:' + mod.default)
3+
})

‎test/e2e/app-dir/worker/next.config.js

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/**
2+
* @type {import('next').NextConfig}
3+
*/
4+
const nextConfig = {}
5+
6+
module.exports = nextConfig

‎test/e2e/app-dir/worker/worker.test.ts

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import { nextTestSetup } from 'e2e-utils'
2+
import { check } from 'next-test-utils'
3+
4+
describe('app dir - workers', () => {
5+
const { next, skipped } = nextTestSetup({
6+
files: __dirname,
7+
skipDeployment: true,
8+
})
9+
10+
if (skipped) {
11+
return
12+
}
13+
14+
it('should support web workers with dynamic imports', async () => {
15+
const browser = await next.browser('/')
16+
expect(await browser.elementByCss('#worker-state').text()).toBe('default')
17+
18+
await browser.elementByCss('button').click()
19+
20+
await check(
21+
async () => browser.elementByCss('#worker-state').text(),
22+
'worker.ts:worker-dep'
23+
)
24+
})
25+
})

‎turbopack/crates/turbopack-core/src/reference_type.rs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,16 @@ pub enum TypeScriptReferenceSubType {
193193
Undefined,
194194
}
195195

196+
#[turbo_tasks::value(serialization = "auto_for_input")]
197+
#[derive(Debug, Clone, Hash)]
198+
pub enum WorkerReferenceSubType {
199+
WebWorker,
200+
SharedWorker,
201+
ServiceWorker,
202+
Custom(u8),
203+
Undefined,
204+
}
205+
196206
// TODO(sokra) this was next.js specific values. We want to solve this in a
197207
// different way.
198208
#[turbo_tasks::value(serialization = "auto_for_input")]
@@ -219,6 +229,7 @@ pub enum ReferenceType {
219229
Css(CssReferenceSubType),
220230
Url(UrlReferenceSubType),
221231
TypeScript(TypeScriptReferenceSubType),
232+
Worker(WorkerReferenceSubType),
222233
Entry(EntryReferenceSubType),
223234
Runtime,
224235
Internal(Vc<InnerAssets>),
@@ -238,6 +249,7 @@ impl Display for ReferenceType {
238249
ReferenceType::Css(_) => "css",
239250
ReferenceType::Url(_) => "url",
240251
ReferenceType::TypeScript(_) => "typescript",
252+
ReferenceType::Worker(_) => "worker",
241253
ReferenceType::Entry(_) => "entry",
242254
ReferenceType::Runtime => "runtime",
243255
ReferenceType::Internal(_) => "internal",
@@ -278,6 +290,10 @@ impl ReferenceType {
278290
matches!(other, ReferenceType::TypeScript(_))
279291
&& matches!(sub_type, TypeScriptReferenceSubType::Undefined)
280292
}
293+
ReferenceType::Worker(sub_type) => {
294+
matches!(other, ReferenceType::Worker(_))
295+
&& matches!(sub_type, WorkerReferenceSubType::Undefined)
296+
}
281297
ReferenceType::Entry(sub_type) => {
282298
matches!(other, ReferenceType::Entry(_))
283299
&& matches!(sub_type, EntryReferenceSubType::Undefined)

‎turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/dev/runtime/base/runtime-base.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
type RefreshRuntimeGlobals =
2020
import("@next/react-refresh-utils/dist/runtime").RefreshRuntimeGlobals;
2121

22+
// Workers are loaded via blob object urls and aren't relative to the main context, this gets
23+
// prefixed to chunk urls in the worker.
24+
declare var TURBOPACK_WORKER_LOCATION: string;
2225
declare var CHUNK_BASE_PATH: string;
2326
declare var $RefreshHelpers$: RefreshRuntimeGlobals["$RefreshHelpers$"];
2427
declare var $RefreshReg$: RefreshRuntimeGlobals["$RefreshReg$"];
@@ -376,6 +379,7 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
376379
U: relativeURL,
377380
k: refresh,
378381
R: createResolvePathFromModule(r),
382+
b: getWorkerBlobURL,
379383
__dirname: typeof module.id === "string" ? module.id.replace(/(^|\/)\/+$/, "") : module.id
380384
})
381385
);
@@ -402,6 +406,12 @@ function resolveAbsolutePath(modulePath?: string): string {
402406
return `/ROOT/${modulePath ?? ""}`;
403407
}
404408

409+
function getWorkerBlobURL(chunks: ChunkPath[]): string {
410+
let bootstrap = `TURBOPACK_WORKER_LOCATION = ${JSON.stringify(location.origin)};importScripts(${chunks.map(c => (`TURBOPACK_WORKER_LOCATION + ${JSON.stringify(getChunkRelativeUrl(c))}`)).join(", ")});`;
411+
let blob = new Blob([bootstrap], { type: "text/javascript" });
412+
return URL.createObjectURL(blob);
413+
}
414+
405415
/**
406416
* NOTE(alexkirsz) Webpack has a "module execution" interception hook that
407417
* Next.js' React Refresh runtime hooks into to add module context to the

‎turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/dev/runtime/dom/runtime-backend-dom.ts

Lines changed: 50 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -236,53 +236,64 @@ async function loadWebAssemblyModule(
236236
const chunkUrl = getChunkRelativeUrl(chunkPath);
237237
const decodedChunkUrl = decodeURI(chunkUrl);
238238

239-
if (chunkPath.endsWith(".css")) {
240-
const previousLinks = document.querySelectorAll(
241-
`link[rel=stylesheet][href="${chunkUrl}"],link[rel=stylesheet][href^="${chunkUrl}?"],link[rel=stylesheet][href="${decodedChunkUrl}"],link[rel=stylesheet][href^="${decodedChunkUrl}?"]`
242-
);
243-
if (previousLinks.length > 0) {
244-
// CSS chunks do not register themselves, and as such must be marked as
245-
// loaded instantly.
246-
resolver.resolve();
239+
if (typeof importScripts === "function") {
240+
// We're in a web worker
241+
if (chunkPath.endsWith(".css")) {
242+
// ignore
243+
} else if (chunkPath.endsWith(".js")) {
244+
importScripts(TURBOPACK_WORKER_LOCATION + chunkUrl);
247245
} else {
248-
const link = document.createElement("link");
249-
link.rel = "stylesheet";
250-
link.href = chunkUrl;
251-
link.onerror = () => {
252-
resolver.reject();
253-
};
254-
link.onload = () => {
246+
throw new Error(`can't infer type of chunk from path ${chunkPath} in worker`);
247+
}
248+
} else {
249+
if (chunkPath.endsWith(".css")) {
250+
const previousLinks = document.querySelectorAll(
251+
`link[rel=stylesheet][href="${chunkUrl}"],link[rel=stylesheet][href^="${chunkUrl}?"],link[rel=stylesheet][href="${decodedChunkUrl}"],link[rel=stylesheet][href^="${decodedChunkUrl}?"]`
252+
);
253+
if (previousLinks.length > 0) {
255254
// CSS chunks do not register themselves, and as such must be marked as
256255
// loaded instantly.
257256
resolver.resolve();
258-
};
259-
document.body.appendChild(link);
260-
}
261-
} else if (chunkPath.endsWith(".js")) {
262-
const previousScripts = document.querySelectorAll(
263-
`script[src="${chunkUrl}"],script[src^="${chunkUrl}?"],script[src="${decodedChunkUrl}"],script[src^="${decodedChunkUrl}?"]`
264-
);
265-
if (previousScripts.length > 0) {
266-
// There is this edge where the script already failed loading, but we
267-
// can't detect that. The Promise will never resolve in this case.
268-
for (const script of Array.from(previousScripts)) {
269-
script.addEventListener("error", () => {
257+
} else {
258+
const link = document.createElement("link");
259+
link.rel = "stylesheet";
260+
link.href = chunkUrl;
261+
link.onerror = () => {
270262
resolver.reject();
271-
});
263+
};
264+
link.onload = () => {
265+
// CSS chunks do not register themselves, and as such must be marked as
266+
// loaded instantly.
267+
resolver.resolve();
268+
};
269+
document.body.appendChild(link);
270+
}
271+
} else if (chunkPath.endsWith(".js")) {
272+
const previousScripts = document.querySelectorAll(
273+
`script[src="${chunkUrl}"],script[src^="${chunkUrl}?"],script[src="${decodedChunkUrl}"],script[src^="${decodedChunkUrl}?"]`
274+
);
275+
if (previousScripts.length > 0) {
276+
// There is this edge where the script already failed loading, but we
277+
// can't detect that. The Promise will never resolve in this case.
278+
for (const script of Array.from(previousScripts)) {
279+
script.addEventListener("error", () => {
280+
resolver.reject();
281+
});
282+
}
283+
} else {
284+
const script = document.createElement("script");
285+
script.src = chunkUrl;
286+
// We'll only mark the chunk as loaded once the script has been executed,
287+
// which happens in `registerChunk`. Hence the absence of `resolve()` in
288+
// this branch.
289+
script.onerror = () => {
290+
resolver.reject();
291+
};
292+
document.body.appendChild(script);
272293
}
273294
} else {
274-
const script = document.createElement("script");
275-
script.src = chunkUrl;
276-
// We'll only mark the chunk as loaded once the script has been executed,
277-
// which happens in `registerChunk`. Hence the absence of `resolve()` in
278-
// this branch.
279-
script.onerror = () => {
280-
resolver.reject();
281-
};
282-
document.body.appendChild(script);
295+
throw new Error(`can't infer type of chunk from path ${chunkPath}`);
283296
}
284-
} else {
285-
throw new Error(`can't infer type of chunk from path ${chunkPath}`);
286297
}
287298

288299
return resolver.promise;

‎turbopack/crates/turbopack-ecmascript-runtime/js/src/browser/dev/runtime/dom/tsconfig.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"extends": "../../../../tsconfig.base.json",
33
"compilerOptions": {
44
// environment
5-
"lib": ["ESNext", "DOM"]
5+
"lib": ["ESNext", "DOM", "WebWorker.ImportScripts"]
66
},
77
"include": ["*.ts"]
88
}

‎turbopack/crates/turbopack-ecmascript-runtime/js/src/nodejs/runtime.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,10 @@ function loadWebAssemblyModule(chunkPath: ChunkPath) {
177177
return compileWebAssemblyFromPath(resolved);
178178
}
179179

180+
function getWorkerBlobURL(_chunks: ChunkPath[]) {
181+
throw new Error("Worker blobs are not implemented yet for Node.js");
182+
}
183+
180184
function instantiateModule(id: ModuleId, source: SourceInfo): Module {
181185
const moduleFactory = moduleFactories[id];
182186
if (typeof moduleFactory !== "function") {
@@ -250,6 +254,7 @@ function instantiateModule(id: ModuleId, source: SourceInfo): Module {
250254
P: resolveAbsolutePath,
251255
U: relativeURL,
252256
R: createResolvePathFromModule(r),
257+
b: getWorkerBlobURL,
253258
__dirname: typeof module.id === "string" ? module.id.replace(/(^|\/)\/+$/, "") : module.id
254259
});
255260
} catch (error) {

‎turbopack/crates/turbopack-ecmascript-runtime/js/src/shared/runtime-types.d.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ type AsyncModule = (
5454
) => void;
5555

5656
type ResolveAbsolutePath = (modulePath?: string) => string;
57+
type GetWorkerBlobURL = (chunks: ChunkPath[]) => string;
5758

5859
interface TurbopackBaseContext {
5960
a: AsyncModule;
@@ -75,5 +76,6 @@ interface TurbopackBaseContext {
7576
g: typeof globalThis;
7677
P: ResolveAbsolutePath;
7778
U: RelativeURL;
79+
b: GetWorkerBlobURL,
7880
__dirname: string;
7981
}

‎turbopack/crates/turbopack-ecmascript/src/chunk/item.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ impl EcmascriptChunkItemContent {
9494
"P: __turbopack_resolve_absolute_path__",
9595
"U: __turbopack_relative_url__",
9696
"R: __turbopack_resolve_module_id_path__",
97+
"b: __turbopack_worker_blob_url__",
9798
"g: global",
9899
// HACK
99100
"__dirname",

‎turbopack/crates/turbopack-ecmascript/src/errors.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,6 @@ pub mod failed_to_analyse {
1616
pub const AMD_DEFINE: &str = "TP1200";
1717
pub const NEW_URL_IMPORT_META: &str = "TP1201";
1818
pub const FREE_VAR_REFERENCE: &str = "TP1202";
19+
pub const NEW_WORKER: &str = "TP1203";
1920
}
2021
}

‎turbopack/crates/turbopack-ecmascript/src/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ pub mod tree_shake;
3131
pub mod typescript;
3232
pub mod utils;
3333
pub mod webpack;
34+
pub mod worker_chunk;
3435

3536
use std::fmt::{Display, Formatter};
3637

0 commit comments

Comments
 (0)