Limit parallel Scala.js linking jobs to avoid high memory pressure

Fix: #6226

I just hardcoded the limit to `2`.

Author: lefou

libs/scalajslib/src/mill/scalajslib/worker/ParallelismLimiter.scala

@@ -0,0 +1,22 @@
+package mill.scalajslib.worker
+
+import java.util.concurrent.Semaphore
+
+/**
+ * Limit the parallelism of jobs run via [[runLimited]].
+ * @param maxJobs The maximal parallelism
+ */
+class ParallelismLimiter(maxJobs: Int) {
+
+  private val linkerJobsSemaphore = Semaphore(maxJobs)
+
+  def runLimited[T](thunk: => T): T = {
+    linkerJobsSemaphore.acquire()
+    try {
+      thunk
+    } finally {
+      linkerJobsSemaphore.release()
+    }
+  }
+
+}

libs/scalajslib/src/mill/scalajslib/worker/ScalaJSWorker.scala

@@ -2,7 +2,7 @@ package mill.scalajslib.worker
 
 import mill.*
 import mill.scalajslib.api
-import mill.scalajslib.worker.{api => workerApi}
+import mill.scalajslib.worker.api as workerApi
 import mill.api.TaskCtx
 import mill.api.Result
 import mill.api.daemon.internal.internal
@@ -11,9 +11,10 @@ import mill.util.CachedFactory
 
 import java.io.File
 import java.net.URLClassLoader
+import java.util.concurrent.Semaphore
 
 @internal
-private[scalajslib] class ScalaJSWorker(jobs: Int)
+private[scalajslib] class ScalaJSWorker(jobs: Int, linkerJobs: Int)
     extends CachedFactory[Seq[mill.PathRef], (URLClassLoader, workerApi.ScalaJSWorkerApi)] {
 
   override def setup(key: Seq[PathRef]) = {
@@ -190,6 +191,8 @@ private[scalajslib] class ScalaJSWorker(jobs: Int)
     }
   }
 
+  private val linkerJobLimiter = ParallelismLimiter(linkerJobs)
+
   def link(
       toolsClasspath: Seq[mill.PathRef],
       runClasspath: Seq[mill.PathRef],
@@ -207,7 +210,7 @@ private[scalajslib] class ScalaJSWorker(jobs: Int)
       minify: Boolean,
       importMap: Seq[api.ESModuleImportMapping],
       experimentalUseWebAssembly: Boolean
-  ): Result[api.Report] = {
+  ): Result[api.Report] = linkerJobLimiter.runLimited {
     withValue(toolsClasspath) { case (_, bridge) =>
       bridge.link(
         runClasspath = runClasspath.iterator.map(_.path.toNIO).toSeq,
@@ -258,7 +261,11 @@ private[scalajslib] class ScalaJSWorker(jobs: Int)
 @internal
 private[scalajslib] object ScalaJSWorkerExternalModule extends mill.api.ExternalModule {
 
-  def scalaJSWorker: Worker[ScalaJSWorker] =
-    Task.Worker { new ScalaJSWorker(Task.ctx().jobs) }
+  def scalaJSWorker: Worker[ScalaJSWorker] = Task.Worker {
+    new ScalaJSWorker(
+      jobs = Task.ctx().jobs,
+      linkerJobs = 2
+    )
+  }
   lazy val millDiscover = Discover[this.type]
 }

libs/scalajslib/test/src/mill/scalajslib/worker/ParallelismLimiterTests.scala

@@ -0,0 +1,44 @@
+package mill.scalajslib.worker
+
+import utest.*
+
+import java.util.concurrent.atomic.AtomicInteger
+
+class ParallelismLimiterTests extends TestSuite {
+
+  override def tests = Tests {
+    test("limitedoJobs") {
+
+      val maxJobs = 3
+      val limiter = ParallelismLimiter(maxJobs)
+
+      val concurrentCount = new AtomicInteger(0)
+      val maxObserved = new AtomicInteger(0)
+
+      def work(i: Int, workTimeMs: Int): Unit = {
+        val before = concurrentCount.incrementAndGet()
+        maxObserved.updateAndGet(v => Math.max(v, before))
+
+        Thread.sleep(workTimeMs)
+
+        val after = concurrentCount.decrementAndGet()
+        assert(after >= 0)
+      }
+
+      val tasks = (1 to 10).map { i =>
+        new Thread(() =>
+          limiter.runLimited {
+            work(i, 50)
+          }
+        )
+      }
+
+      tasks.foreach(_.start())
+      tasks.foreach(_.join())
+
+      assert(maxObserved.get() <= maxJobs)
+    }
+
+  }
+
+}