Skip to content

Commit a2f3739

Browse files
chore(TextEncoderStream): better polyfill tests (QwikDev#6310)
Co-authored-by: Nick K. <[email protected]>
1 parent d79b934 commit a2f3739

File tree

7 files changed

+212
-69
lines changed

7 files changed

+212
-69
lines changed

packages/qwik-city/middleware/bun/index.ts

+4-32
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import type {
77
import {
88
mergeHeadersCookies,
99
requestHandler,
10+
_TextEncoderStream_polyfill,
1011
} from '@builder.io/qwik-city/middleware/request-handler';
1112
import { getNotFound } from '@qwik-city-not-found-paths';
1213
import { isStaticPath } from '@qwik-city-static-paths';
@@ -15,40 +16,11 @@ import { setServerPlatform } from '@builder.io/qwik/server';
1516
import { MIME_TYPES } from '../request-handler/mime-types';
1617
import { join, extname } from 'node:path';
1718

18-
// @builder.io/qwik-city/middleware/bun
19-
// still missing from bun: last check was bun version 1.1.8
20-
class TextEncoderStream_polyfill {
21-
private _encoder = new TextEncoder();
22-
private _reader: ReadableStreamDefaultController<any> | null = null;
23-
public ready = Promise.resolve();
24-
public closed = false;
25-
public readable = new ReadableStream({
26-
start: (controller) => {
27-
this._reader = controller;
28-
},
29-
});
30-
31-
public writable = new WritableStream({
32-
write: async (chunk) => {
33-
if (chunk != null && this._reader) {
34-
const encoded = this._encoder.encode(chunk);
35-
this._reader.enqueue(encoded);
36-
}
37-
},
38-
close: () => {
39-
this._reader?.close();
40-
this.closed = true;
41-
},
42-
abort: (reason) => {
43-
this._reader?.error(reason);
44-
this.closed = true;
45-
},
46-
});
47-
}
48-
4919
/** @public */
5020
export function createQwikCity(opts: QwikCityBunOptions) {
51-
globalThis.TextEncoderStream = TextEncoderStream || (TextEncoderStream_polyfill as any);
21+
// @builder.io/qwik-city/middleware/bun
22+
// still missing from bun: last check was bun version 1.1.8
23+
globalThis.TextEncoderStream ||= class TextEncoderStream extends _TextEncoderStream_polyfill {};
5224

5325
const qwikSerializer = {
5426
_deserializeData,

packages/qwik-city/middleware/cloudflare-pages/index.ts

+3-37
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import type {
55
import {
66
mergeHeadersCookies,
77
requestHandler,
8+
_TextEncoderStream_polyfill,
89
} from '@builder.io/qwik-city/middleware/request-handler';
910
import { getNotFound } from '@qwik-city-not-found-paths';
1011
import { isStaticPath } from '@qwik-city-static-paths';
@@ -20,7 +21,8 @@ export function createQwikCity(opts: QwikCityCloudflarePagesOptions) {
2021
// this will throw if CF compatibility_date < 2022-11-30
2122
new globalThis.TextEncoderStream();
2223
} catch (e) {
23-
globalThis.TextEncoderStream = TextEncoderStream as any;
24+
// @ts-ignore
25+
globalThis.TextEncoderStream = class TextEncoderStream extends _TextEncoderStream_polyfill {};
2426
}
2527
const qwikSerializer = {
2628
_deserializeData,
@@ -138,39 +140,3 @@ export interface PlatformCloudflarePages {
138140
env?: Record<string, any>;
139141
ctx: { waitUntil: (promise: Promise<any>) => void };
140142
}
141-
142-
const resolved = Promise.resolve();
143-
144-
class TextEncoderStream {
145-
// minimal polyfill implementation of TextEncoderStream
146-
// since Cloudflare Pages doesn't support readable.pipeTo()
147-
_writer: any;
148-
readable: any;
149-
writable: any;
150-
151-
constructor() {
152-
this._writer = null;
153-
this.readable = {
154-
pipeTo: (writableStream: any) => {
155-
this._writer = writableStream.getWriter();
156-
},
157-
};
158-
this.writable = {
159-
getWriter: () => {
160-
if (!this._writer) {
161-
throw new Error('No writable stream');
162-
}
163-
const encoder = new TextEncoder();
164-
return {
165-
write: async (chunk: any) => {
166-
if (chunk != null) {
167-
await this._writer.write(encoder.encode(chunk));
168-
}
169-
},
170-
close: () => this._writer.close(),
171-
ready: resolved,
172-
};
173-
},
174-
};
175-
}
176-
}

packages/qwik-city/middleware/node/node-fetch.ts

+1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { fetch, Headers, Request, Response, FormData } from 'undici';
88

99
import crypto from 'crypto';
1010

11+
// TODO: remove when undici is removed
1112
export function patchGlobalThis() {
1213
if (
1314
typeof global !== 'undefined' &&

packages/qwik-city/middleware/request-handler/api.md

+12
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,18 @@ export type ServerRequestMode = 'dev' | 'static' | 'server';
220220
// @public (undocumented)
221221
export type ServerResponseHandler<T = any> = (status: number, headers: Headers, cookies: Cookie, resolve: (response: T) => void, requestEv: RequestEventInternal) => WritableStream<Uint8Array>;
222222

223+
// @internal (undocumented)
224+
export class _TextEncoderStream_polyfill {
225+
// (undocumented)
226+
get [Symbol.toStringTag](): string;
227+
// (undocumented)
228+
get encoding(): string;
229+
// (undocumented)
230+
get readable(): ReadableStream<Uint8Array>;
231+
// (undocumented)
232+
get writable(): WritableStream<string>;
233+
}
234+
223235
// (No @packageDocumentation comment for this package)
224236

225237
```

packages/qwik-city/middleware/request-handler/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ export { getErrorHtml, ServerError } from './error-handler';
22
export { mergeHeadersCookies } from './cookie';
33
export { AbortMessage, RedirectMessage } from './redirect-handler';
44
export { requestHandler } from './request-handler';
5+
export { _TextEncoderStream_polyfill } from './polyfill';
56
export type {
67
CacheControl,
78
Cookie,
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Polyfill for TextEncoderStream
2+
3+
/**
4+
* TextEncoderStream polyfill based on Node.js' implementation
5+
* https://github.com/nodejs/node/blob/3f3226c8e363a5f06c1e6a37abd59b6b8c1923f1/lib/internal/webstreams/encoding.js#L38-L119
6+
* (MIT License)
7+
*/
8+
/** @internal */
9+
export class _TextEncoderStream_polyfill {
10+
#pendingHighSurrogate: string | null = null;
11+
12+
#handle = new TextEncoder();
13+
14+
#transform = new TransformStream<string, Uint8Array>({
15+
transform: (chunk, controller) => {
16+
// https://encoding.spec.whatwg.org/#encode-and-enqueue-a-chunk
17+
chunk = String(chunk);
18+
19+
let finalChunk = '';
20+
for (const item of chunk) {
21+
const codeUnit = item.charCodeAt(0);
22+
if (this.#pendingHighSurrogate !== null) {
23+
const highSurrogate = this.#pendingHighSurrogate;
24+
25+
this.#pendingHighSurrogate = null;
26+
if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) {
27+
finalChunk += highSurrogate + item;
28+
continue;
29+
}
30+
31+
finalChunk += '\uFFFD';
32+
}
33+
34+
if (codeUnit >= 0xd800 && codeUnit <= 0xdbff) {
35+
this.#pendingHighSurrogate = item;
36+
continue;
37+
}
38+
39+
if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) {
40+
finalChunk += '\uFFFD';
41+
continue;
42+
}
43+
44+
finalChunk += item;
45+
}
46+
47+
if (finalChunk) {
48+
controller.enqueue(this.#handle.encode(finalChunk));
49+
}
50+
},
51+
52+
flush: (controller) => {
53+
// https://encoding.spec.whatwg.org/#encode-and-flush
54+
if (this.#pendingHighSurrogate !== null) {
55+
controller.enqueue(new Uint8Array([0xef, 0xbf, 0xbd]));
56+
}
57+
},
58+
});
59+
60+
get encoding() {
61+
return this.#handle.encoding;
62+
}
63+
64+
get readable() {
65+
return this.#transform.readable;
66+
}
67+
68+
get writable() {
69+
return this.#transform.writable;
70+
}
71+
72+
get [Symbol.toStringTag]() {
73+
return 'TextEncoderStream';
74+
}
75+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
import { describe, it, expect } from 'vitest';
2+
import { _TextEncoderStream_polyfill } from './polyfill';
3+
4+
describe('_TextEncoderStream_polyfill tests', () => {
5+
it('should encode string to Uint8Array', async () => {
6+
const encoderStream = new _TextEncoderStream_polyfill();
7+
const reader = encoderStream.readable.getReader();
8+
9+
encoderStream.writable.getWriter().write('hello');
10+
const { value, done } = await reader.read();
11+
expect(value).toBeInstanceOf(Uint8Array);
12+
expect(new TextDecoder().decode(value)).toBe('hello');
13+
expect(done).toBeFalsy();
14+
});
15+
16+
it('should handle multiple chunks', async () => {
17+
const encoderStream = new _TextEncoderStream_polyfill();
18+
const encoderStream2 = new TextEncoderStream();
19+
const writer = encoderStream.writable.getWriter();
20+
const reader = encoderStream.readable.getReader();
21+
const writer2 = encoderStream2.writable.getWriter();
22+
const reader2 = encoderStream2.readable.getReader();
23+
24+
writer.write('hello');
25+
writer.write(' world');
26+
writer2.write('hello');
27+
writer2.write(' world');
28+
29+
const results1 = [await reader.read(), await reader.read()];
30+
const results2 = [await reader2.read(), await reader2.read()];
31+
await writer.close();
32+
await writer2.close();
33+
34+
expect(results1.length).toBe(results2.length);
35+
expect(new TextDecoder().decode(results1[0].value)).toBe(
36+
new TextDecoder().decode(results2[0].value)
37+
);
38+
expect(new TextDecoder().decode(results1[1].value)).toBe(
39+
new TextDecoder().decode(results2[1].value)
40+
);
41+
});
42+
43+
it('encoding consistency with native TextEncoderStream', async () => {
44+
const polyfillStream = new _TextEncoderStream_polyfill();
45+
const nativeStream = new TextEncoderStream();
46+
const testString = 'This is a test string.';
47+
48+
const polyReader = polyfillStream.readable.getReader();
49+
const nativeReader = nativeStream.readable.getReader();
50+
51+
polyfillStream.writable.getWriter().write(testString);
52+
nativeStream.writable.getWriter().write(testString);
53+
54+
const polyResult = await polyReader.read();
55+
const nativeResult = await nativeReader.read();
56+
57+
expect(polyResult.value).toEqual(nativeResult.value);
58+
expect(new TextDecoder().decode(polyResult.value)).toBe(testString);
59+
});
60+
61+
it('handles non-string inputs', async () => {
62+
const polyfillStream = new _TextEncoderStream_polyfill();
63+
const nativeStream = new TextEncoderStream();
64+
65+
const nativeWriter = nativeStream.writable.getWriter();
66+
const polyWriter = polyfillStream.writable.getWriter();
67+
68+
expect(polyWriter.write(123 as any)).toEqual(nativeWriter.write(123 as any));
69+
expect(polyWriter.write({} as any)).toEqual(nativeWriter.write({} as any));
70+
});
71+
72+
it('handles large input', async () => {
73+
const encoderStream = new _TextEncoderStream_polyfill();
74+
const writer = encoderStream.writable.getWriter();
75+
const reader = encoderStream.readable.getReader();
76+
const largeString = 'a'.repeat(10 ** 6); // 1 million characters
77+
78+
writer.write(largeString);
79+
const { value } = await reader.read();
80+
expect(value?.byteLength).toBe(largeString.length);
81+
});
82+
83+
it('sequential writes and reads', async () => {
84+
const encoderStream = new _TextEncoderStream_polyfill();
85+
const writer = encoderStream.writable.getWriter();
86+
const reader = encoderStream.readable.getReader();
87+
88+
writer.write('first');
89+
writer.write('second');
90+
91+
const firstResult = await reader.read();
92+
const secondResult = await reader.read();
93+
94+
await writer.close();
95+
96+
expect(new TextDecoder().decode(firstResult.value)).toBe('first');
97+
expect(new TextDecoder().decode(secondResult.value)).toBe('second');
98+
});
99+
100+
it('stream chaining', async () => {
101+
const encoderStream = new _TextEncoderStream_polyfill();
102+
const transformStream = new TransformStream({
103+
transform(chunk, controller) {
104+
controller.enqueue(chunk);
105+
},
106+
});
107+
108+
const writer = encoderStream.writable.getWriter();
109+
const chainedStream = encoderStream.readable.pipeThrough(transformStream);
110+
const reader = chainedStream.getReader();
111+
112+
writer.write('test chaining');
113+
const result = await reader.read();
114+
expect(new TextDecoder().decode(result.value)).toBe('test chaining');
115+
});
116+
});

0 commit comments

Comments
 (0)