fix(#6990): reduce likelihood of libzip OOM in larger dependency graphs

.yarn/versions/93e6f510.yml

@@ -0,0 +1,39 @@
+releases:
+  "@yarnpkg/builder": patch
+  "@yarnpkg/cli": patch
+  "@yarnpkg/core": patch
+  "@yarnpkg/doctor": patch
+  "@yarnpkg/extensions": patch
+  "@yarnpkg/fslib": patch
+  "@yarnpkg/libzip": patch
+  "@yarnpkg/nm": patch
+  "@yarnpkg/plugin-catalog": patch
+  "@yarnpkg/plugin-compat": patch
+  "@yarnpkg/plugin-constraints": patch
+  "@yarnpkg/plugin-dlx": patch
+  "@yarnpkg/plugin-essentials": patch
+  "@yarnpkg/plugin-exec": patch
+  "@yarnpkg/plugin-file": patch
+  "@yarnpkg/plugin-git": patch
+  "@yarnpkg/plugin-github": patch
+  "@yarnpkg/plugin-http": patch
+  "@yarnpkg/plugin-init": patch
+  "@yarnpkg/plugin-interactive-tools": patch
+  "@yarnpkg/plugin-jsr": patch
+  "@yarnpkg/plugin-link": patch
+  "@yarnpkg/plugin-nm": patch
+  "@yarnpkg/plugin-npm": patch
+  "@yarnpkg/plugin-npm-cli": patch
+  "@yarnpkg/plugin-pack": patch
+  "@yarnpkg/plugin-patch": patch
+  "@yarnpkg/plugin-pnp": patch
+  "@yarnpkg/plugin-pnpm": patch
+  "@yarnpkg/plugin-stage": patch
+  "@yarnpkg/plugin-typescript": patch
+  "@yarnpkg/plugin-version": patch
+  "@yarnpkg/plugin-workspace-tools": patch
+  "@yarnpkg/pnp": patch
+  "@yarnpkg/pnpify": patch
+  "@yarnpkg/sdks": patch
+  "@yarnpkg/shell": patch
+  vscode-zipfs: patch

packages/yarnpkg-libzip/package.json

@@ -15,7 +15,8 @@
     "build:libzip:wasm": "cd ./artifacts && ./build.sh",
     "postpack": "rm -rf lib",
     "prepack": "run build:compile \"$(pwd)\"",
-    "release": "yarn npm publish"
+    "release": "yarn npm publish",
+    "test": "run test:unit \"$(pwd)\""
   },
   "publishConfig": {
     "main": "./lib/sync.js",

packages/yarnpkg-libzip/sources/instance.ts

@@ -6,6 +6,10 @@ let registeredFactory: () => Libzip = () => {
   throw new Error(`Assertion failed: No libzip instance is available, and no factory was configured`);
 };
 
+export function newInstance() {
+  return registeredFactory();
+}
+
 export function setFactory(factory: () => Libzip) {
   registeredFactory = factory;
 }

packages/yarnpkg-libzip/sources/libzipImpl.ts

@@ -2,7 +2,7 @@ import {PortablePath}                                                from '@yarn
 import {Libzip}                                                      from '@yarnpkg/libzip';
 
 import {ZipImplInput, type CompressionData, type Stat, type ZipImpl} from './ZipFS';
-import {getInstance}                                                 from './instance';
+import {newInstance}                                                 from './instance';
 
 
 export class LibzipError extends Error {
@@ -16,12 +16,93 @@ export class LibzipError extends Error {
   }
 }
 
+
+type LibzipInstance = {instance: Libzip | null, active: boolean, reserved: number, highWaterMark: number};
+type LibzipReservation = {byteLength: number, instanceIndex: number};
+/**
+ * Tracks the estimate of WASM memory usage by libzip to reduce the risk
+ * of OOM errors.
+ *
+ * Internally, favors the oldest WASM instances to minimize fragmentation.
+ * Cleans up instances when older instances have space to accomodate new zips.
+ */
+class ElasticLibzipFactory {
+  private static readonly LIBZIP_METADATA = 512 * 1024; // 500KB
+  private static readonly WASM_MEM_MAX = 2 * 1024 * 1024 * 1024 - (100 * 1024 * 1024); // 1.9GB
+  private static KEY = 1;
+
+  /**
+   * The WASM instances, their currently reserved memory, and the high water mark, since
+   * WASM memory isn't usually shrinkable.
+   */
+  private readonly instances: Array<LibzipInstance> = [];
+  /**
+   * The reservations by unique ID, and the index into the {@link instances} array.
+   */
+  private readonly reservations = new Map<number, LibzipReservation>();
+
+  /**
+   * Provide (and possibly build new) a libzip WASM for the given ZIP byte length
+   *
+   * @param byteLength The size of the ZIP file
+   * @returns [unique ID, Libzip instance]
+   */
+  getInstance(byteLength: number): [number, Libzip] {
+    const size = byteLength + ElasticLibzipFactory.LIBZIP_METADATA;
+    let index = this.instances.findIndex(i => i.active && (i.reserved + size) < ElasticLibzipFactory.WASM_MEM_MAX);
+    let instance;
+
+    if (index >= 0) {
+      instance = this.instances[index];
+      instance.reserved += size;
+      instance.highWaterMark = Math.max(instance.highWaterMark, instance.reserved);
+    } else {
+      index = this.instances.length;
+      instance = {instance: newInstance(), reserved: size, highWaterMark: size, active: true};
+      this.instances.push(instance);
+    }
+    ElasticLibzipFactory.KEY += 1;
+    this.reservations.set(ElasticLibzipFactory.KEY, {byteLength: size, instanceIndex: index});
+    return [ElasticLibzipFactory.KEY, instance.instance!];
+  }
+
+  remove(key: number) {
+    const reservation = this.reservations.get(key);
+    if (!reservation)
+      return;
+
+    this.reservations.delete(key);
+
+    const instance = this.instances[reservation.instanceIndex];
+    instance.reserved -= reservation.byteLength;
+    this.cleanup(reservation);
+  }
+
+  /**
+   * Remove the reservation's instance if the previous one has enough space,
+   * or if the reservations instance is nearly out of memory.
+   *
+   * @param reservation
+   */
+  private cleanup(reservation: LibzipReservation) {
+    const instance = this.instances[reservation.instanceIndex];
+
+    if (instance.reserved <= 0) {
+      instance.active = false;
+      this.instances[reservation.instanceIndex].instance = null;
+    }
+  }
+}
+
+const libzipFactory = new ElasticLibzipFactory();
+
 export class LibZipImpl implements ZipImpl {
   private readonly libzip: Libzip;
   private readonly lzSource: number;
   private readonly zip: number;
   private readonly listings: Array<string>;
   private readonly symlinkCount: number;
+  private readonly key: number;
 
   public filesShouldBeCached = true;
 
@@ -30,7 +111,7 @@ export class LibZipImpl implements ZipImpl {
       ? opts.buffer
       : opts.baseFs.readFileSync(opts.path);
 
-    this.libzip = getInstance();
+    [this.key, this.libzip] = libzipFactory.getInstance(buffer.byteLength);
 
     const errPtr = this.libzip.malloc(4);
     try {
@@ -50,20 +131,20 @@ export class LibZipImpl implements ZipImpl {
       if (this.zip === 0) {
         const error = this.libzip.struct.errorS();
         this.libzip.error.initWithCode(error, this.libzip.getValue(errPtr, `i32`));
-
         throw this.makeLibzipError(error);
       }
+    } catch(error) {
+      libzipFactory.remove(this.key);
+      throw error;
     } finally {
       this.libzip.free(errPtr);
     }
 
     const entryCount = this.libzip.getNumEntries(this.zip, 0);
 
-    const listings = new Array<string>(entryCount);
+    this.listings = new Array<string>(entryCount);
     for (let t = 0; t < entryCount; ++t)
-      listings[t] = this.libzip.getName(this.zip, t, 0);
-
-    this.listings = listings;
+      this.listings[t] = this.libzip.getName(this.zip, t, 0);
 
     this.symlinkCount = this.libzip.ext.countSymlinks(this.zip);
     if (this.symlinkCount === -1) {
@@ -121,6 +202,7 @@ export class LibZipImpl implements ZipImpl {
           throw this.makeLibzipError(this.libzip.getError(this.zip));
         }
       }
+
       return newIndex;
     } catch (error) {
       this.libzip.source.free(lzSource);
@@ -261,6 +343,7 @@ export class LibZipImpl implements ZipImpl {
     } finally {
       this.libzip.source.close(this.lzSource);
       this.libzip.source.free(this.lzSource);
+      libzipFactory.remove(this.key);
     }
   }
 
@@ -307,5 +390,6 @@ export class LibZipImpl implements ZipImpl {
 
   public discard(): void {
     this.libzip.discard(this.zip);
+    libzipFactory.remove(this.key);
   }
 }