Fix packaging very large projects as HTML files in Chrome (TurboWarp#862)

Actually fixes TurboWarp#528

TurboWarp#861 fixed running large projects
in Chrome, but packaging still used string concatenation so it remained broken.
Now the concatenation part is done using TextEncoder & Uint8Arrays and a
template tag function to keep it readable.

This time I have actually tested it with a 1.0GB sb3.

Breaking Node API change: The data property returned by Packager#package() is
now always a Uint8Array instead of sometimes string and sometimes ArrayBuffer.

Author: GarboMuffin

node-api-docs/README.md

@@ -126,7 +126,7 @@ const filename = result.filename;
 // MIME type of the packaged project. Either "text/html" or "application/zip"
 const type = result.type;
 
-// The packaged project's data. Will be either a string (for type text/html) or ArrayBuffer (for type application/zip).
+// The packaged project's data. Will always be a Uint8Array.
 const data = result.data;
 ```
 

src/packager/encode-big-string.js

@@ -0,0 +1,66 @@
+/**
+ * @template T
+ * @param {T[]} destination
+ * @param {T[]} newItems
+ */
+const concatInPlace = (destination, newItems) => {
+  for (const item of newItems) {
+    destination.push(item);
+  }
+};
+
+/**
+ * @param {unknown} value String, number, Uint8Array, etc. or a recursive array of them
+ * @returns {Uint8Array[]} UTF-8 arrays, in order
+ */
+const encodeComponent = (value) => {
+  if (typeof value === 'string') {
+    return [
+      new TextEncoder().encode(value)
+    ];
+  } else if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'undefined' || value === null) {
+    return [
+      new TextEncoder().encode(String(value))
+    ];
+  } else if (Array.isArray(value)) {
+    const result = [];
+    for (const i of value) {
+      concatInPlace(result, encodeComponent(i));
+    }
+    return result;
+  } else {
+    throw new Error(`Unknown value in encodeComponent: ${value}`);
+  }
+};
+
+/**
+ * Tagged template function to generate encoded UTF-8 without string concatenation as Chrome cannot handle
+ * strings that are longer than 0x1fffffe8 characters.
+ * @param {TemplateStringsArray} strings
+ * @param {unknown[]} values
+ * @returns {Uint8Array}
+ */
+const encodeBigString = (strings, ...values) => {
+  /** @type {Uint8Array[]} */
+  const encodedChunks = [];
+
+  for (let i = 0; i < strings.length - 1; i++) {
+    concatInPlace(encodedChunks, encodeComponent(strings[i]));
+    concatInPlace(encodedChunks, encodeComponent(values[i]));
+  }
+  concatInPlace(encodedChunks, encodeComponent(strings[strings.length - 1]));
+
+  let totalByteLength = 0;
+  for (let i = 0; i < encodedChunks.length; i++) {
+    totalByteLength += encodedChunks[i].byteLength;
+  }
+
+  const resultBuffer = new Uint8Array(totalByteLength);
+  for (let i = 0, j = 0; i < encodedChunks.length; i++) {
+    resultBuffer.set(encodedChunks[i], j);
+    j += encodedChunks[i].byteLength;
+  }
+  return resultBuffer;
+};
+
+export default encodeBigString;

src/packager/packager.js

@@ -11,6 +11,7 @@ import {APP_NAME, WEBSITE, COPYRIGHT_NOTICE, ACCENT_COLOR} from './brand';
 import {OutdatedPackagerError} from '../common/errors';
 import {darken} from './colors';
 import {Adapter} from './adapter';
+import encodeBigString from './encode-big-string';
 
 const PROGRESS_LOADED_SCRIPTS = 0.1;
 
@@ -881,7 +882,7 @@ cd "$(dirname "$0")"
   }
 
   async generateGetProjectData () {
-    let result = '';
+    const result = [];
     let getProjectDataFunction = '';
     let isZip = false;
     let storageProgressStart;
@@ -895,7 +896,7 @@ cd "$(dirname "$0")"
       const projectData = new Uint8Array(this.project.arrayBuffer);
 
       // keep this up-to-date with base85.js
-      result += `
+      result.push(`
       <script>
       const getBase85DecodeValue = (code) => {
         if (code === 0x28) code = 0x3c;
@@ -926,15 +927,15 @@ cd "$(dirname "$0")"
           handleError(e);
         }
       };
-      </script>`;
+      </script>`);
 
       // To avoid unnecessary padding, this should be a multiple of 4.
       const CHUNK_SIZE = 1024 * 64;
 
       for (let i = 0; i < projectData.length; i += CHUNK_SIZE) {
         const projectChunk = projectData.subarray(i, i + CHUNK_SIZE);
         const base85 = encode(projectChunk);
-        result += `<script data="${base85}">decodeChunk(${projectChunk.length})</script>\n`;
+        result.push(`<script data="${base85}">decodeChunk(${projectChunk.length})</script>\n`);
       }
 
       getProjectDataFunction = `() => {
@@ -978,7 +979,7 @@ cd "$(dirname "$0")"
       })`;
     }
 
-    result += `
+    result.push(`
     <script>
       const getProjectData = (function() {
         const storage = scaffolding.storage;
@@ -1024,7 +1025,8 @@ cd "$(dirname "$0")"
         );
         return ${getProjectDataFunction};`}
       })();
-    </script>`;
+    </script>`);
+
     return result;
   }
 
@@ -1107,7 +1109,7 @@ cd "$(dirname "$0")"
     this.ensureNotAborted();
     await this.loadResources();
     this.ensureNotAborted();
-    const html = `<!DOCTYPE html>
+    const html = encodeBigString`<!DOCTYPE html>
 <!-- Created with ${WEBSITE} -->
 <html>
 <head>
@@ -1565,7 +1567,7 @@ cd "$(dirname "$0")"
       this.ensureNotAborted();
       return {
         data: await zip.generateAsync({
-          type: 'arraybuffer',
+          type: 'uint8array',
           compression: 'DEFLATE',
           // Use UNIX permissions so that executable bits are properly set for macOS and Linux
           platform: 'UNIX'

test/p4/encode-big-string.test.js

@@ -0,0 +1,41 @@
+import encodeBigString from "../../src/packager/encode-big-string";
+
+test('simple behavior', () => {
+  expect(encodeBigString``).toEqual(new Uint8Array([]));
+  expect(encodeBigString`abc`).toEqual(new Uint8Array([97, 98, 99]));
+  expect(encodeBigString`a${'bc'}`).toEqual(new Uint8Array([97, 98, 99]));
+  expect(encodeBigString`${'ab'}c`).toEqual(new Uint8Array([97, 98, 99]));
+  expect(encodeBigString`${'abc'}`).toEqual(new Uint8Array([97, 98, 99]));
+  expect(encodeBigString`1${'a'}2${'b'}3${'c'}4`).toEqual(new Uint8Array([49, 97, 50, 98, 51, 99, 52]));
+  expect(encodeBigString`${''}`).toEqual(new Uint8Array([]));
+});
+
+test('non-string primitives', () => {
+  expect(encodeBigString`${1}`).toEqual(new Uint8Array([49]));
+  expect(encodeBigString`${false}`).toEqual(new Uint8Array([102, 97, 108, 115, 101]));
+  expect(encodeBigString`${true}`).toEqual(new Uint8Array([116, 114, 117, 101]));
+  expect(encodeBigString`${null}`).toEqual(new Uint8Array([110, 117, 108, 108]));
+  expect(encodeBigString`${undefined}`).toEqual(new Uint8Array([117, 110, 100, 101, 102, 105, 110, 101, 100]));
+});
+
+test('array', () => {
+  expect(encodeBigString`${[]}`).toEqual(new Uint8Array([]));
+  expect(encodeBigString`${['a', 'b', 'c']}`).toEqual(new Uint8Array([97, 98, 99]));
+  expect(encodeBigString`${[[[['a'], [['b']], 'c']]]}`).toEqual(new Uint8Array([97, 98, 99]));
+});
+
+// skipping for now because very slow
+test.skip('very big string', () => {
+  const MAX_LENGTH = 0x1fffffe8;
+  const maxLength = 'a'.repeat(MAX_LENGTH);
+  expect(() => maxLength + 'a').toThrow(/Invalid string length/);
+  const encoded = encodeBigString`${maxLength}aaaaa`;
+  expect(encoded.byteLength).toBe(MAX_LENGTH + 5);
+
+  // very hot loop, don't call into expect if we don't need to
+  for (let i = 0; i < encoded.length; i++) {
+    if (encoded[i] !== 97) {
+      throw new Error(`Wrong encoding at ${i}`);
+    }
+  }
+});