Remove compression on the case cache, split files to reduce transitive deps (#3197)

* Remove compression on the case cache

Compression is not stable between Node versions, nor between operating systems.

This inflates the cache size 10x (from 4.3M to 49M), however these files should not be frequently changing, and Git should be performing compressions with packfiles and during network operations.

* Split cache generation from spec.ts files

* Breakup monolithic imports

Move commonly used code out of monolithic imports to reduce the amount of code imported by the `.cache.ts` files.

Add a new `MUST_NOT_BE_IMPORTED_BY_DATA_CACHE` token that can be used to figure out why the cache wants to regenerate for a given file. Add this to device_pool.ts, as this really shouldn't be depended on for the cache.

Author: ben-clayton

docs/case_cache.md

@@ -187,11 +187,11 @@ export interface Cacheable<Data> {
   /**
    * serialize() encodes `data` to a binary representation so that it can be stored in a cache file.
    */
-  serialize(data: Data): Promise<Uint8Array>;
+  serialize(data: Data): Uint8Array;
 
   /**
    * deserialize() is the inverse of serialize(), decoding the binary representation back to a Data
    * object.
    */
-  deserialize(binary: Uint8Array): Promise<Data>;
+  deserialize(binary: Uint8Array): Data;
 }

src/common/framework/data_cache.ts

@@ -89,7 +89,7 @@ dataCache.setStore({
 });
 setIsBuildingDataCache();
 
-const specFileSuffix = __filename.endsWith('.ts') ? '.spec.ts' : '.spec.js';
+const cacheFileSuffix = __filename.endsWith('.ts') ? '.cache.ts' : '.cache.js';
 
 /**
  * @returns a list of all the files under 'dir' that has the given extension
@@ -105,13 +105,27 @@ function glob(dir: string, ext: string) {
         files.push(`${file}/${child}`);
       }
     }
+
     if (path.endsWith(ext) && fs.statSync(path).isFile()) {
       files.push(file);
     }
   }
   return files;
 }
 
+/**
+ * Exception type thrown by SourceHasher.hashFile() when a file annotated with
+ * MUST_NOT_BE_IMPORTED_BY_DATA_CACHE is transitively imported by a .cache.ts file.
+ */
+class InvalidImportException {
+  constructor(path: string) {
+    this.stack = [path];
+  }
+  toString(): string {
+    return `invalid transitive import for cache:\n  ${this.stack.join('\n  ')}`;
+  }
+  readonly stack: string[];
+}
 /**
  * SourceHasher is a utility for producing a hash of a source .ts file and its imported source files.
  */
@@ -141,8 +155,19 @@ class SourceHasher {
     const normalized = content.replace('\r\n', '\n');
     let hash = crc32(normalized);
     for (const importPath of parseImports(path, normalized)) {
-      const importHash = this.hashFile(importPath);
-      hash = this.hashCombine(hash, importHash);
+      try {
+        const importHash = this.hashFile(importPath);
+        hash = this.hashCombine(hash, importHash);
+      } catch (ex) {
+        if (ex instanceof InvalidImportException) {
+          ex.stack.push(path);
+          throw ex;
+        }
+      }
+    }
+
+    if (content.includes('MUST_NOT_BE_IMPORTED_BY_DATA_CACHE')) {
+      throw new InvalidImportException(path);
     }
 
     this.hashes.set(path, hash);
@@ -180,7 +205,7 @@ async function build(suiteDir: string) {
   }
 
   // Crawl files and convert paths to be POSIX-style, relative to suiteDir.
-  const filesToEnumerate = glob(suiteDir, specFileSuffix)
+  const filesToEnumerate = glob(suiteDir, cacheFileSuffix)
     .map(p => `${suiteDir}/${p}`)
     .sort();
 
@@ -221,7 +246,7 @@ and
           }
           console.log(`building '${outPath}'`);
           const data = await cacheable.build();
-          const serialized = await cacheable.serialize(data);
+          const serialized = cacheable.serialize(data);
           fs.mkdirSync(path.dirname(outPath), { recursive: true });
           fs.writeFileSync(outPath, serialized, 'binary');
           fileHashes[cacheable.path] = fileHash;

src/common/tools/gen_cache.ts

@@ -1,106 +1,106 @@
 {
-  "webgpu/shader/execution/binary/af_addition.bin": "8e47fc86",
-  "webgpu/shader/execution/binary/af_logical.bin": "348bc279",
-  "webgpu/shader/execution/binary/af_division.bin": "82029c55",
-  "webgpu/shader/execution/binary/af_matrix_addition.bin": "6deae35e",
-  "webgpu/shader/execution/binary/af_matrix_subtraction.bin": "221a913c",
-  "webgpu/shader/execution/binary/af_multiplication.bin": "53c87c5b",
-  "webgpu/shader/execution/binary/af_remainder.bin": "dadb3ceb",
-  "webgpu/shader/execution/binary/af_subtraction.bin": "a0a71e32",
-  "webgpu/shader/execution/binary/f16_addition.bin": "acfee8",
-  "webgpu/shader/execution/binary/f16_logical.bin": "78e0eec8",
-  "webgpu/shader/execution/binary/f16_division.bin": "289b2aba",
-  "webgpu/shader/execution/binary/f16_matrix_addition.bin": "bada3cbe",
-  "webgpu/shader/execution/binary/f16_matrix_matrix_multiplication.bin": "bce23630",
-  "webgpu/shader/execution/binary/f16_matrix_scalar_multiplication.bin": "2d6bcf91",
-  "webgpu/shader/execution/binary/f16_matrix_subtraction.bin": "c3db5537",
-  "webgpu/shader/execution/binary/f16_matrix_vector_multiplication.bin": "402931c",
-  "webgpu/shader/execution/binary/f16_multiplication.bin": "ce381066",
-  "webgpu/shader/execution/binary/f16_remainder.bin": "b2bd9246",
-  "webgpu/shader/execution/binary/f16_subtraction.bin": "7f7e9abc",
-  "webgpu/shader/execution/binary/f32_addition.bin": "71302f1a",
-  "webgpu/shader/execution/binary/f32_logical.bin": "ad34883d",
-  "webgpu/shader/execution/binary/f32_division.bin": "b35602bb",
-  "webgpu/shader/execution/binary/f32_matrix_addition.bin": "82339c4",
-  "webgpu/shader/execution/binary/f32_matrix_matrix_multiplication.bin": "33c2bd83",
-  "webgpu/shader/execution/binary/f32_matrix_scalar_multiplication.bin": "6b6dd016",
-  "webgpu/shader/execution/binary/f32_matrix_subtraction.bin": "7733299f",
-  "webgpu/shader/execution/binary/f32_matrix_vector_multiplication.bin": "d99b4cc3",
-  "webgpu/shader/execution/binary/f32_multiplication.bin": "ff22107c",
-  "webgpu/shader/execution/binary/f32_remainder.bin": "44133c4c",
-  "webgpu/shader/execution/binary/f32_subtraction.bin": "a9a5a319",
-  "webgpu/shader/execution/binary/i32_arithmetic.bin": "eb21bd8f",
-  "webgpu/shader/execution/binary/i32_comparison.bin": "83121ec4",
-  "webgpu/shader/execution/binary/u32_arithmetic.bin": "8d1429d1",
-  "webgpu/shader/execution/binary/u32_comparison.bin": "3491ba55",
-  "webgpu/shader/execution/abs.bin": "a5dcabd0",
-  "webgpu/shader/execution/acos.bin": "adbccc79",
-  "webgpu/shader/execution/acosh.bin": "7c07ba41",
-  "webgpu/shader/execution/asin.bin": "8fc62434",
-  "webgpu/shader/execution/asinh.bin": "a00937c0",
-  "webgpu/shader/execution/atan.bin": "a237eaa6",
-  "webgpu/shader/execution/atan2.bin": "b41d74be",
-  "webgpu/shader/execution/atanh.bin": "2b6934d5",
-  "webgpu/shader/execution/bitcast.bin": "86e79d18",
-  "webgpu/shader/execution/ceil.bin": "42a99291",
-  "webgpu/shader/execution/clamp.bin": "b3fc754f",
-  "webgpu/shader/execution/cos.bin": "13305519",
-  "webgpu/shader/execution/cosh.bin": "20965093",
-  "webgpu/shader/execution/cross.bin": "3cfeb2a1",
-  "webgpu/shader/execution/degrees.bin": "6d9b3197",
-  "webgpu/shader/execution/determinant.bin": "d09e35b7",
-  "webgpu/shader/execution/distance.bin": "790d45ea",
-  "webgpu/shader/execution/dot.bin": "5f7aa2bc",
-  "webgpu/shader/execution/exp.bin": "932473",
-  "webgpu/shader/execution/exp2.bin": "1ada59d6",
-  "webgpu/shader/execution/faceForward.bin": "dc44d720",
-  "webgpu/shader/execution/floor.bin": "cbbe2d21",
-  "webgpu/shader/execution/fma.bin": "b5f22cac",
-  "webgpu/shader/execution/fract.bin": "b9b72ee2",
-  "webgpu/shader/execution/frexp.bin": "cc49a118",
-  "webgpu/shader/execution/inverseSqrt.bin": "ab8d39ed",
-  "webgpu/shader/execution/ldexp.bin": "cca23df2",
-  "webgpu/shader/execution/length.bin": "4693ba28",
-  "webgpu/shader/execution/log.bin": "75d7c534",
-  "webgpu/shader/execution/log2.bin": "c0c65cb0",
-  "webgpu/shader/execution/max.bin": "11d13c22",
-  "webgpu/shader/execution/min.bin": "5c701467",
-  "webgpu/shader/execution/mix.bin": "af998787",
-  "webgpu/shader/execution/modf.bin": "3a3b367a",
-  "webgpu/shader/execution/normalize.bin": "ce31e122",
-  "webgpu/shader/execution/pack2x16float.bin": "d3198def",
-  "webgpu/shader/execution/pow.bin": "847bf3ff",
-  "webgpu/shader/execution/quantizeToF16.bin": "e2f5c3f9",
-  "webgpu/shader/execution/radians.bin": "8869a385",
-  "webgpu/shader/execution/reflect.bin": "2be62e10",
-  "webgpu/shader/execution/refract.bin": "a95cf2a",
-  "webgpu/shader/execution/round.bin": "cdc5ae01",
-  "webgpu/shader/execution/saturate.bin": "bb20b07c",
-  "webgpu/shader/execution/sign.bin": "a59f6a2b",
-  "webgpu/shader/execution/sin.bin": "b36c7552",
-  "webgpu/shader/execution/sinh.bin": "9385405",
-  "webgpu/shader/execution/smoothstep.bin": "5d740ae7",
-  "webgpu/shader/execution/sqrt.bin": "5e42a89b",
-  "webgpu/shader/execution/step.bin": "4447b38c",
-  "webgpu/shader/execution/tan.bin": "b47dfc33",
-  "webgpu/shader/execution/tanh.bin": "8bd0ae74",
-  "webgpu/shader/execution/transpose.bin": "afbbaef0",
-  "webgpu/shader/execution/trunc.bin": "4d15f320",
-  "webgpu/shader/execution/unpack2x16float.bin": "40ca16cc",
-  "webgpu/shader/execution/unpack2x16snorm.bin": "39ece93e",
-  "webgpu/shader/execution/unpack2x16unorm.bin": "dbdc06b8",
-  "webgpu/shader/execution/unpack4x8snorm.bin": "c83bc6c2",
-  "webgpu/shader/execution/unpack4x8unorm.bin": "678c5245",
-  "webgpu/shader/execution/unary/af_arithmetic.bin": "1679466",
-  "webgpu/shader/execution/unary/af_assignment.bin": "202b8019",
-  "webgpu/shader/execution/unary/bool_conversion.bin": "6cb7e11a",
-  "webgpu/shader/execution/unary/f16_arithmetic.bin": "d0409431",
-  "webgpu/shader/execution/unary/f16_conversion.bin": "884688a",
-  "webgpu/shader/execution/unary/f32_arithmetic.bin": "8925bac7",
-  "webgpu/shader/execution/unary/f32_conversion.bin": "798aaba2",
-  "webgpu/shader/execution/unary/i32_arithmetic.bin": "c21ece3",
-  "webgpu/shader/execution/unary/i32_complement.bin": "f65f0bec",
-  "webgpu/shader/execution/unary/i32_conversion.bin": "7ce468a1",
-  "webgpu/shader/execution/unary/u32_complement.bin": "d6408865",
-  "webgpu/shader/execution/unary/u32_conversion.bin": "aa09fe0"
+  "webgpu/shader/execution/binary/af_addition.bin": "b5345cca",
+  "webgpu/shader/execution/binary/af_logical.bin": "9ec85311",
+  "webgpu/shader/execution/binary/af_division.bin": "23dd840e",
+  "webgpu/shader/execution/binary/af_matrix_addition.bin": "ee95539e",
+  "webgpu/shader/execution/binary/af_matrix_subtraction.bin": "a2e64d57",
+  "webgpu/shader/execution/binary/af_multiplication.bin": "94d7a574",
+  "webgpu/shader/execution/binary/af_remainder.bin": "79f5abc4",
+  "webgpu/shader/execution/binary/af_subtraction.bin": "62dce5bf",
+  "webgpu/shader/execution/binary/f16_addition.bin": "ab3b4d9c",
+  "webgpu/shader/execution/binary/f16_logical.bin": "1edd08ec",
+  "webgpu/shader/execution/binary/f16_division.bin": "849eed26",
+  "webgpu/shader/execution/binary/f16_matrix_addition.bin": "54afc3f8",
+  "webgpu/shader/execution/binary/f16_matrix_matrix_multiplication.bin": "df5d3ccc",
+  "webgpu/shader/execution/binary/f16_matrix_scalar_multiplication.bin": "3be6032e",
+  "webgpu/shader/execution/binary/f16_matrix_subtraction.bin": "98a29837",
+  "webgpu/shader/execution/binary/f16_matrix_vector_multiplication.bin": "c463d7e0",
+  "webgpu/shader/execution/binary/f16_multiplication.bin": "cdf0775d",
+  "webgpu/shader/execution/binary/f16_remainder.bin": "e82ef89f",
+  "webgpu/shader/execution/binary/f16_subtraction.bin": "5125b5b0",
+  "webgpu/shader/execution/binary/f32_addition.bin": "d3b8004",
+  "webgpu/shader/execution/binary/f32_logical.bin": "fab7cfc5",
+  "webgpu/shader/execution/binary/f32_division.bin": "e9fdb0a9",
+  "webgpu/shader/execution/binary/f32_matrix_addition.bin": "d1165469",
+  "webgpu/shader/execution/binary/f32_matrix_matrix_multiplication.bin": "45a79521",
+  "webgpu/shader/execution/binary/f32_matrix_scalar_multiplication.bin": "bf8da7d0",
+  "webgpu/shader/execution/binary/f32_matrix_subtraction.bin": "82d7fa9",
+  "webgpu/shader/execution/binary/f32_matrix_vector_multiplication.bin": "57eebe84",
+  "webgpu/shader/execution/binary/f32_multiplication.bin": "eb0baa56",
+  "webgpu/shader/execution/binary/f32_remainder.bin": "e6059990",
+  "webgpu/shader/execution/binary/f32_subtraction.bin": "d57cfa1",
+  "webgpu/shader/execution/binary/i32_arithmetic.bin": "de93ee2a",
+  "webgpu/shader/execution/binary/i32_comparison.bin": "aaa1f21b",
+  "webgpu/shader/execution/binary/u32_arithmetic.bin": "d79a1011",
+  "webgpu/shader/execution/binary/u32_comparison.bin": "31764c75",
+  "webgpu/shader/execution/abs.bin": "9278901c",
+  "webgpu/shader/execution/acos.bin": "9c77ebf9",
+  "webgpu/shader/execution/acosh.bin": "2baaf7cc",
+  "webgpu/shader/execution/asin.bin": "8fd7a678",
+  "webgpu/shader/execution/asinh.bin": "321a7294",
+  "webgpu/shader/execution/atan.bin": "d3dada7e",
+  "webgpu/shader/execution/atan2.bin": "2871596c",
+  "webgpu/shader/execution/atanh.bin": "e7158ef",
+  "webgpu/shader/execution/bitcast.bin": "aa7d6f0e",
+  "webgpu/shader/execution/ceil.bin": "3c1d91ad",
+  "webgpu/shader/execution/clamp.bin": "a53fdbda",
+  "webgpu/shader/execution/cos.bin": "73b83288",
+  "webgpu/shader/execution/cosh.bin": "39ca32a4",
+  "webgpu/shader/execution/cross.bin": "409c3e26",
+  "webgpu/shader/execution/degrees.bin": "a44bb2b1",
+  "webgpu/shader/execution/determinant.bin": "41a06528",
+  "webgpu/shader/execution/distance.bin": "fe3ea84f",
+  "webgpu/shader/execution/dot.bin": "f70abfec",
+  "webgpu/shader/execution/exp.bin": "7587e11b",
+  "webgpu/shader/execution/exp2.bin": "c6f34d2b",
+  "webgpu/shader/execution/faceForward.bin": "8eb57a7d",
+  "webgpu/shader/execution/floor.bin": "b9ea647b",
+  "webgpu/shader/execution/fma.bin": "b9f657b2",
+  "webgpu/shader/execution/fract.bin": "73a18e4f",
+  "webgpu/shader/execution/frexp.bin": "857fe9b7",
+  "webgpu/shader/execution/inverseSqrt.bin": "a595983a",
+  "webgpu/shader/execution/ldexp.bin": "a04aaeca",
+  "webgpu/shader/execution/length.bin": "695740fb",
+  "webgpu/shader/execution/log.bin": "6703ec1a",
+  "webgpu/shader/execution/log2.bin": "8339559d",
+  "webgpu/shader/execution/max.bin": "ec000a56",
+  "webgpu/shader/execution/min.bin": "156cf7cc",
+  "webgpu/shader/execution/mix.bin": "32064d21",
+  "webgpu/shader/execution/modf.bin": "a5003ce0",
+  "webgpu/shader/execution/normalize.bin": "c12bac96",
+  "webgpu/shader/execution/pack2x16float.bin": "b2cb12ea",
+  "webgpu/shader/execution/pow.bin": "ee87eccb",
+  "webgpu/shader/execution/quantizeToF16.bin": "f77ae7e3",
+  "webgpu/shader/execution/radians.bin": "7ecbe5be",
+  "webgpu/shader/execution/reflect.bin": "d37034bd",
+  "webgpu/shader/execution/refract.bin": "6dc9adcf",
+  "webgpu/shader/execution/round.bin": "d91faa0f",
+  "webgpu/shader/execution/saturate.bin": "93230980",
+  "webgpu/shader/execution/sign.bin": "fb1071b8",
+  "webgpu/shader/execution/sin.bin": "a9ed8361",
+  "webgpu/shader/execution/sinh.bin": "65ba80fc",
+  "webgpu/shader/execution/smoothstep.bin": "17e1e103",
+  "webgpu/shader/execution/sqrt.bin": "4c29a5d4",
+  "webgpu/shader/execution/step.bin": "aed08458",
+  "webgpu/shader/execution/tan.bin": "5f671594",
+  "webgpu/shader/execution/tanh.bin": "991e903a",
+  "webgpu/shader/execution/transpose.bin": "9d388797",
+  "webgpu/shader/execution/trunc.bin": "959fe8bc",
+  "webgpu/shader/execution/unpack2x16float.bin": "26e5b05e",
+  "webgpu/shader/execution/unpack2x16snorm.bin": "c756d8a3",
+  "webgpu/shader/execution/unpack2x16unorm.bin": "5257591a",
+  "webgpu/shader/execution/unpack4x8snorm.bin": "83fbd41a",
+  "webgpu/shader/execution/unpack4x8unorm.bin": "77d46acb",
+  "webgpu/shader/execution/unary/af_arithmetic.bin": "2edb2dc5",
+  "webgpu/shader/execution/unary/af_assignment.bin": "36c04bba",
+  "webgpu/shader/execution/unary/bool_conversion.bin": "cb53bf65",
+  "webgpu/shader/execution/unary/f16_arithmetic.bin": "9f459fc4",
+  "webgpu/shader/execution/unary/f16_conversion.bin": "bf055ca9",
+  "webgpu/shader/execution/unary/f32_arithmetic.bin": "e15c49e7",
+  "webgpu/shader/execution/unary/f32_conversion.bin": "92e5069f",
+  "webgpu/shader/execution/unary/i32_arithmetic.bin": "d322b73d",
+  "webgpu/shader/execution/unary/i32_complement.bin": "c4e6cbb",
+  "webgpu/shader/execution/unary/i32_conversion.bin": "d6905a0f",
+  "webgpu/shader/execution/unary/u32_complement.bin": "d0e4aa97",
+  "webgpu/shader/execution/unary/u32_conversion.bin": "5a96b263"
 }