diff --git a/main/util/src/mill/util/PromptLoggerUtil.scala b/main/util/src/mill/util/PromptLoggerUtil.scala index 3e6734f7d9c..469589a600a 100644 --- a/main/util/src/mill/util/PromptLoggerUtil.scala +++ b/main/util/src/mill/util/PromptLoggerUtil.scala @@ -60,10 +60,6 @@ private object PromptLoggerUtil { (AnsiNav.clearScreen(0) + AnsiNav.up(1) + "\n").getBytes def spaceNonEmpty(s: String): String = if (s.isEmpty) "" else s" $s" - private def renderSecondsSuffix(millis: Long) = (millis / 1000).toInt match { - case 0 => "" - case n => s" ${n}s" - } def readTerminalDims(terminfoPath: os.Path): Option[(Option[Int], Option[Int])] = { try { @@ -98,7 +94,7 @@ private object PromptLoggerUtil { val maxWidth = consoleWidth - 1 // -1 to account for header val maxHeight = math.max(1, consoleHeight / 3 - 1) - val headerSuffix = renderSecondsSuffix(now - startTimeMillis) + val headerSuffix = mill.util.Util.renderSecondsSuffix(now - startTimeMillis) val header = renderHeader(headerPrefix, titleText, headerSuffix, maxWidth) @@ -120,7 +116,7 @@ private object PromptLoggerUtil { status.next else status.prev textOpt.map { t => - val seconds = renderSecondsSuffix(now - t.startTimeMillis) + val seconds = mill.util.Util.renderSecondsSuffix(now - t.startTimeMillis) val mainText = splitShorten(t.text + seconds, maxWidth) val detail = splitShorten(spaceNonEmpty(t.detail), maxWidth - mainText.length) diff --git a/main/util/src/mill/util/Util.scala b/main/util/src/mill/util/Util.scala index 897e83e74d9..77845840bc1 100644 --- a/main/util/src/mill/util/Util.scala +++ b/main/util/src/mill/util/Util.scala @@ -105,4 +105,9 @@ object Util { def leftPad(s: String, targetLength: Int, char: Char): String = { char.toString * (targetLength - s.length) + s } + + def renderSecondsSuffix(millis: Long): String = (millis / 1000).toInt match { + case 0 => "" + case n => s" ${n}s" + } } diff --git a/scalalib/src/mill/scalalib/TestModule.scala b/scalalib/src/mill/scalalib/TestModule.scala index 5e14481ccff..195b5f4263d 100644 --- a/scalalib/src/mill/scalalib/TestModule.scala +++ b/scalalib/src/mill/scalalib/TestModule.scala @@ -155,6 +155,7 @@ trait TestModule Task.Anon { val mainClass = "mill.testrunner.entrypoint.TestRunnerMain" val outputPath = Task.dest / "out.json" + val resultPath = Task.dest / "results.log" val selectors = Seq.empty val testArgs = TestArgs( @@ -163,6 +164,7 @@ trait TestModule arguments = args(), sysProps = Map.empty, outputPath = outputPath, + resultPath = resultPath, colored = Task.log.colored, testCp = testClasspath().map(_.path), home = Task.home, diff --git a/scalalib/src/mill/scalalib/TestModuleUtil.scala b/scalalib/src/mill/scalalib/TestModuleUtil.scala index a55cc38eec1..5801062b2b2 100644 --- a/scalalib/src/mill/scalalib/TestModuleUtil.scala +++ b/scalalib/src/mill/scalalib/TestModuleUtil.scala @@ -118,6 +118,7 @@ private final class TestModuleUtil( private def callTestRunnerSubprocess( baseFolder: os.Path, + resultPath: os.Path, // either: // - Left(selectors): // - list of glob selectors to feed to the test runner directly. @@ -125,7 +126,7 @@ private final class TestModuleUtil( // - first test class to run, folder containing test classes for test runner to claim from, and the worker's base folder. selector: Either[Seq[String], (Option[String], os.Path, os.Path)] )(implicit ctx: mill.api.Ctx) = { - os.makeDir.all(baseFolder) + if (!os.exists(baseFolder)) os.makeDir.all(baseFolder) val outputPath = baseFolder / "out.json" val testArgs = TestArgs( @@ -134,6 +135,7 @@ private final class TestModuleUtil( arguments = args, sysProps = props, outputPath = outputPath, + resultPath = resultPath, colored = Task.log.colored, testCp = testClasspath.map(_.path), home = Task.home, @@ -169,59 +171,87 @@ private final class TestModuleUtil( filteredClassLists: Seq[Seq[String]] )(implicit ctx: mill.api.Ctx) = { - val subprocessResult = filteredClassLists match { - // When no tests at all are discovered, run at least one test JVM - // process to go through the test framework setup/teardown logic - case Nil => callTestRunnerSubprocess(Task.dest, Left(Nil)) - case Seq(singleTestClassList) => - callTestRunnerSubprocess(Task.dest, Left(singleTestClassList)) - case multipleTestClassLists => - val maxLength = multipleTestClassLists.length.toString.length - val futures = multipleTestClassLists.zipWithIndex.map { case (testClassList, i) => - val groupPromptMessage = testClassList match { - case Seq(single) => single - case multiple => - TestModuleUtil.collapseTestClassNames( - multiple - ).mkString(", ") + s", ${multiple.length} suites" - } - - val paddedIndex = mill.util.Util.leftPad(i.toString, maxLength, '0') - val folderName = testClassList match { - case Seq(single) => single - case multiple => - s"group-$paddedIndex-${multiple.head}" - } + def runTestRunnerSubprocess( + base: os.Path, + testClassList: Seq[String], + workerResultSet: java.util.concurrent.ConcurrentMap[os.Path, Unit] + ) = { + os.makeDir.all(base) - // set priority = -1 to always prioritize test subprocesses over normal Mill - // tasks. This minimizes the number of blocked tasks since Mill tasks can be - // blocked on test subprocesses, but not vice versa, so better to schedule - // the test subprocesses first - Task.fork.async(Task.dest / folderName, paddedIndex, groupPromptMessage, priority = -1) { - logger => - (folderName, callTestRunnerSubprocess(Task.dest / folderName, Left(testClassList))) - } - } + // test runner will log success/failure test class counter here while running + val resultPath = base / s"result.log" + os.write.over(resultPath, upickle.default.write((0L, 0L))) + workerResultSet.put(resultPath, ()) - val outputs = Task.fork.awaitAll(futures) + callTestRunnerSubprocess( + base, + resultPath, + Left(testClassList) + ) + } - val (lefts, rights) = outputs.partitionMap { - case (name, Left(v)) => Left(name + " " + v) - case (name, Right((msg, results))) => Right((name + " " + msg, results)) + TestModuleUtil.withTestProgressTickerThread(filteredClassLists.map(_.size).sum) { + case (_, workerResultSet) => + filteredClassLists match { + // When no tests at all are discovered, run at least one test JVM + // process to go through the test framework setup/teardown logic + case Nil => runTestRunnerSubprocess(Task.dest, Nil, workerResultSet) + case Seq(singleTestClassList) => + runTestRunnerSubprocess(Task.dest, singleTestClassList, workerResultSet) + case multipleTestClassLists => + val maxLength = multipleTestClassLists.length.toString.length + val futures = multipleTestClassLists.zipWithIndex.map { case (testClassList, i) => + val groupPromptMessage = testClassList match { + case Seq(single) => single + case multiple => + TestModuleUtil.collapseTestClassNames( + multiple + ).mkString(", ") + s", ${multiple.length} suites" + } + + val paddedIndex = mill.util.Util.leftPad(i.toString, maxLength, '0') + val folderName = testClassList match { + case Seq(single) => single + case multiple => + s"group-$paddedIndex-${multiple.head}" + } + + // set priority = -1 to always prioritize test subprocesses over normal Mill + // tasks. This minimizes the number of blocked tasks since Mill tasks can be + // blocked on test subprocesses, but not vice versa, so better to schedule + // the test subprocesses first + Task.fork.async( + Task.dest / folderName, + paddedIndex, + groupPromptMessage, + priority = -1 + ) { + logger => + ( + folderName, + runTestRunnerSubprocess(Task.dest / folderName, testClassList, workerResultSet) + ) + } + } + + val outputs = Task.fork.awaitAll(futures) + + val (lefts, rights) = outputs.partitionMap { + case (name, Left(v)) => Left(name + " " + v) + case (name, Right((msg, results))) => Right((name + " " + msg, results)) + } + + if (lefts.nonEmpty) Left(lefts.mkString("\n")) + else Right((rights.map(_._1).mkString("\n"), rights.flatMap(_._2))) } - - if (lefts.nonEmpty) Left(lefts.mkString("\n")) - else Right((rights.map(_._1).mkString("\n"), rights.flatMap(_._2))) } - - subprocessResult } private def runTestQueueScheduler( filteredClassLists: Seq[Seq[String]] )(implicit ctx: mill.api.Ctx) = { - val workerStatusMap = new java.util.concurrent.ConcurrentHashMap[os.Path, String => Unit]() + val filteredClassCount = filteredClassLists.map(_.size).sum def preparetestClassQueueFolder(selectors2: Seq[String], base: os.Path): os.Path = { // test-classes folder is used to store the test classes for the children test runners to claim from @@ -237,10 +267,13 @@ private final class TestModuleUtil( base: os.Path, testClassQueueFolder: os.Path, force: Boolean, - logger: Logger + logger: Logger, + workerStatusMap: java.util.concurrent.ConcurrentMap[os.Path, String => Unit], + workerResultSet: java.util.concurrent.ConcurrentMap[os.Path, Unit] ) = { val claimFolder = base / "claim" os.makeDir.all(claimFolder) + val startingTestClass = try { os @@ -259,10 +292,17 @@ private final class TestModuleUtil( val claimLog = claimFolder / os.up / s"${claimFolder.last}.log" os.write.over(claimLog, Array.empty[Byte]) workerStatusMap.put(claimLog, logger.ticker) + // test runner will log success/failure test class counter here while running + val resultPath = base / s"result.log" + os.write.over(resultPath, upickle.default.write((0L, 0L))) + workerResultSet.put(resultPath, ()) + val result = callTestRunnerSubprocess( base, + resultPath, Right((startingTestClass, testClassQueueFolder, claimFolder)) ) + workerStatusMap.remove(claimLog) Some(result) } else { @@ -270,6 +310,16 @@ private final class TestModuleUtil( } } + def jobsProcessLength(numTests: Int) = { + val jobs = Task.ctx() match { + case j: Ctx.Jobs => j.jobs + case _ => 1 + } + + val cappedJobs = Math.max(Math.min(jobs, numTests), 1) + (cappedJobs, cappedJobs.toString.length) + } + val groupFolderData = filteredClassLists match { case Nil => Seq((Task.dest, preparetestClassQueueFolder(Nil, Task.dest), 0)) case Seq(singleTestClassList) => @@ -296,94 +346,73 @@ private final class TestModuleUtil( } } - def jobsProcessLength(numTests: Int) = { - val jobs = Task.ctx() match { - case j: Ctx.Jobs => j.jobs - case _ => 1 - } - - val cappedJobs = Math.max(Math.min(jobs, numTests), 1) - (cappedJobs, cappedJobs.toString.length) - } - val groupLength = groupFolderData.length val maxGroupLength = groupLength.toString.length - // We got "--jobs" threads, and "groupLength" test groups, so we will spawn at most jobs * groupLength runners here - // In most case, this is more than necessary, and runner creation is expensive, - // but we have a check for non-empty test-classes folder before really spawning a new runner, so in practice the overhead is low - val subprocessFutures = for { - ((groupFolder, testClassesFolder, numTests), groupIndex) <- groupFolderData.zipWithIndex - // Don't have re-calculate for every processes - groupName = groupFolder.last - (jobs, maxProcessLength) = jobsProcessLength(numTests) - paddedGroupIndex = mill.util.Util.leftPad(groupIndex.toString, maxGroupLength, '0') - processIndex <- 0 until Math.max(Math.min(jobs, numTests), 1) - } yield { - - val paddedProcessIndex = - mill.util.Util.leftPad(processIndex.toString, maxProcessLength, '0') - - val processFolder = groupFolder / s"worker-$paddedProcessIndex" - - val label = - if (groupFolderData.size == 1) paddedProcessIndex - else s"$paddedGroupIndex-$paddedProcessIndex" - - Task.fork.async( - processFolder, - label, - "", - // With the test queue scheduler, prioritize the *first* test subprocess - // over other Mill tasks via `priority = -1`, but de-prioritize the others - // increasingly according to their processIndex. This should help Mill - // use fewer longer-lived test subprocesses, minimizing JVM startup overhead - priority = if (processIndex == 0) -1 else processIndex - ) { logger => - val result = runTestRunnerSubprocess( - processFolder, - testClassesFolder, - // force run when processIndex == 0 (first subprocess), even if there are no tests to run - // to force the process to go through the test framework setup/teardown logic - force = processIndex == 0, - logger - ) - - val claimedClasses = - if (os.exists(processFolder / "claim")) os.list(processFolder / "claim").size else 0 + val outputs = TestModuleUtil.withTestProgressTickerThread(filteredClassCount) { + case (workerStatusMap, workerResultSet) => + // We got "--jobs" threads, and "groupLength" test groups, so we will spawn at most jobs * groupLength runners here + // In most case, this is more than necessary, and runner creation is expensive, + // but we have a check for non-empty test-classes folder before really spawning a new runner, so in practice the overhead is low + val subprocessFutures = for { + ((groupFolder, testClassesFolder, numTests), groupIndex) <- groupFolderData.zipWithIndex + // Don't re-calculate for every processes + groupName = groupFolder.last + (jobs, maxProcessLength) = jobsProcessLength(numTests) + paddedGroupIndex = mill.util.Util.leftPad(groupIndex.toString, maxGroupLength, '0') + processIndex <- 0 until Math.max(Math.min(jobs, numTests), 1) + } yield { + + val paddedProcessIndex = + mill.util.Util.leftPad(processIndex.toString, maxProcessLength, '0') + + val processFolder = groupFolder / s"worker-$paddedProcessIndex" + + val label = + if (groupFolderData.size == 1) paddedProcessIndex + else s"$paddedGroupIndex-$paddedProcessIndex" + + Task.fork.async( + processFolder, + label, + "", + // With the test queue scheduler, prioritize the *first* test subprocess + // over other Mill tasks via `priority = -1`, but de-prioritize the others + // increasingly according to their processIndex. This should help Mill + // use fewer longer-lived test subprocesses, minimizing JVM startup overhead + priority = if (processIndex == 0) -1 else processIndex + ) { logger => + val result = runTestRunnerSubprocess( + processFolder, + testClassesFolder, + // force run when processIndex == 0 (first subprocess), even if there are no tests to run + // to force the process to go through the test framework setup/teardown logic + force = processIndex == 0, + logger, + workerStatusMap, + workerResultSet + ) - (claimedClasses, groupName, result) - } - } + val claimedClasses = + if (os.exists(processFolder / "claim")) os.list(processFolder / "claim").size else 0 - val executor = Executors.newScheduledThreadPool(1) - val outputs = - try { - // Periodically check the claimLog file of every runner, and tick the executing test name - executor.scheduleWithFixedDelay( - () => - workerStatusMap.forEach { (claimLog, callback) => - // the last one is always the latest - os.read.lines(claimLog).lastOption.foreach(callback) - }, - 0, - 20, - java.util.concurrent.TimeUnit.MILLISECONDS - ) + (claimedClasses, groupName, result) + } + } Task.fork.blocking { // We special-case this to avoid while ({ val claimedCounts = subprocessFutures.flatMap(_.value).flatMap(_.toOption).map(_._1) - val expectedCounts = filteredClassLists.map(_.size) !( - (claimedCounts.sum == expectedCounts.sum && subprocessFutures.head.isCompleted) || + (claimedCounts.sum == filteredClassCount && subprocessFutures.head.isCompleted) || subprocessFutures.forall(_.isCompleted) ) }) Thread.sleep(1) } + subprocessFutures.flatMap(_.value).map(_.get) - } finally executor.shutdown() + } val subprocessResult = { val failMap = mutable.Map.empty[String, String] @@ -415,6 +444,53 @@ private final class TestModuleUtil( private[scalalib] object TestModuleUtil { + private def withTestProgressTickerThread[T](totalClassCount: Long)( + body: ( + java.util.concurrent.ConcurrentMap[os.Path, String => Unit], + java.util.concurrent.ConcurrentMap[os.Path, Unit] + ) => T + )(implicit ctx: mill.api.Ctx): T = { + val workerStatusMap = new java.util.concurrent.ConcurrentHashMap[os.Path, String => Unit]() + val workerResultSet = new java.util.concurrent.ConcurrentHashMap[os.Path, Unit]() + + val testClassTimeMap = new java.util.concurrent.ConcurrentHashMap[String, Long]() + + val executor = Executors.newScheduledThreadPool(1) + try { + // Periodically check the result log file and tick the relevant infos + executor.scheduleWithFixedDelay( + () => { + // reuse to reduce syscall, may not be as accurate as running `System.currentTimeMillis()` inside each entry + val now = System.currentTimeMillis() + workerStatusMap.forEach { (claimLog, callback) => + // the last one is always the latest + os.read.lines(claimLog).lastOption.foreach { currentTestClass => + testClassTimeMap.putIfAbsent(currentTestClass, now) + val last = testClassTimeMap.get(currentTestClass) + callback(s"$currentTestClass${mill.util.Util.renderSecondsSuffix(now - last)}") + } + } + var totalSuccess = 0L + var totalFailure = 0L + workerResultSet.forEach { (resultLog, _) => + val (success, failure) = upickle.default.read[(Long, Long)](os.read.stream(resultLog)) + totalSuccess += success + totalFailure += failure + } + ctx.log.ticker( + s"${totalSuccess + totalFailure}/${totalClassCount} completed${if (totalFailure > 0) { + s", ${totalFailure} failures." + } else { "." }}" + ) + }, + 0, + 20, + java.util.concurrent.TimeUnit.MILLISECONDS + ) + body(workerStatusMap, workerResultSet) + } finally executor.shutdown() + } + private[scalalib] def loadArgsAndProps(useArgsFile: Boolean, forkArgs: Seq[String]) = { if (useArgsFile) { val (props, jvmArgs) = forkArgs.partition(_.startsWith("-D")) diff --git a/scalalib/test/src/mill/scalalib/TestRunnerScalatestTests.scala b/scalalib/test/src/mill/scalalib/TestRunnerScalatestTests.scala index 3970ab83ca9..4fdaa76e179 100644 --- a/scalalib/test/src/mill/scalalib/TestRunnerScalatestTests.scala +++ b/scalalib/test/src/mill/scalalib/TestRunnerScalatestTests.scala @@ -32,8 +32,20 @@ object TestRunnerScalatestTests extends TestSuite { 3, Map( // No test grouping is triggered because we only run one test class - testrunner.scalatest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), - testrunnerGrouping.scalatest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), + testrunner.scalatest -> Set( + "out.json", + "result.log", + "sandbox", + "test-report.xml", + "testargs" + ), + testrunnerGrouping.scalatest -> Set( + "out.json", + "result.log", + "sandbox", + "test-report.xml", + "testargs" + ), testrunnerParallel.scalatest -> Set("worker-0", "test-classes", "test-report.xml") ) ) @@ -43,7 +55,13 @@ object TestRunnerScalatestTests extends TestSuite { Seq("*"), 9, Map( - testrunner.scalatest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), + testrunner.scalatest -> Set( + "out.json", + "result.log", + "sandbox", + "test-report.xml", + "testargs" + ), testrunnerGrouping.scalatest -> Set( "group-0-mill.scalalib.ScalaTestSpec", "mill.scalalib.ScalaTestSpec3", @@ -76,7 +94,13 @@ object TestRunnerScalatestTests extends TestSuite { Seq("*", "--", "-z", "A Set 2"), 3, Map( - testrunner.scalatest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), + testrunner.scalatest -> Set( + "out.json", + "result.log", + "sandbox", + "test-report.xml", + "testargs" + ), testrunnerGrouping.scalatest -> Set( "group-0-mill.scalalib.ScalaTestSpec", "mill.scalalib.ScalaTestSpec3", diff --git a/scalalib/test/src/mill/scalalib/TestRunnerUtestTests.scala b/scalalib/test/src/mill/scalalib/TestRunnerUtestTests.scala index 5053e5c94f4..52a0a7a6654 100644 --- a/scalalib/test/src/mill/scalalib/TestRunnerUtestTests.scala +++ b/scalalib/test/src/mill/scalalib/TestRunnerUtestTests.scala @@ -36,9 +36,21 @@ object TestRunnerUtestTests extends TestSuite { Seq("mill.scalalib.FooTests"), 1, Map( - testrunner.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), + testrunner.utest -> Set( + "out.json", + "result.log", + "sandbox", + "test-report.xml", + "testargs" + ), // When there is only one test group with test classes, we do not put it in a subfolder - testrunnerGrouping.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), + testrunnerGrouping.utest -> Set( + "out.json", + "result.log", + "sandbox", + "test-report.xml", + "testargs" + ), testrunnerParallel.utest -> Set("worker-0", "test-classes", "test-report.xml") ) ) @@ -46,7 +58,13 @@ object TestRunnerUtestTests extends TestSuite { Seq("*Bar*", "*bar*"), 2, Map( - testrunner.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), + testrunner.utest -> Set( + "out.json", + "result.log", + "sandbox", + "test-report.xml", + "testargs" + ), // When there are multiple test groups with one test class each, we // put each test group in a subfolder with the number of the class testrunnerGrouping.utest -> Set( @@ -61,7 +79,13 @@ object TestRunnerUtestTests extends TestSuite { Seq("*"), 3, Map( - testrunner.utest -> Set("out.json", "sandbox", "test-report.xml", "testargs"), + testrunner.utest -> Set( + "out.json", + "result.log", + "sandbox", + "test-report.xml", + "testargs" + ), // When there are multiple test groups some with multiple test classes, we put each // test group in a subfolder with the index of the group, and for any test groups // with only one test class we append the name of the class diff --git a/testrunner/src/mill/testrunner/Model.scala b/testrunner/src/mill/testrunner/Model.scala index aeccd6bea2c..91238b71aaf 100644 --- a/testrunner/src/mill/testrunner/Model.scala +++ b/testrunner/src/mill/testrunner/Model.scala @@ -9,6 +9,7 @@ import mill.api.internal arguments: Seq[String], sysProps: Map[String, String], outputPath: os.Path, + resultPath: os.Path, colored: Boolean, testCp: Seq[os.Path], home: os.Path, diff --git a/testrunner/src/mill/testrunner/TestRunnerMain0.scala b/testrunner/src/mill/testrunner/TestRunnerMain0.scala index 1330211508b..16941aa2980 100644 --- a/testrunner/src/mill/testrunner/TestRunnerMain0.scala +++ b/testrunner/src/mill/testrunner/TestRunnerMain0.scala @@ -35,7 +35,8 @@ import mill.util.PrintLogger args = testArgs.arguments, classFilter = cls => filter(cls.getName), cl = classLoader, - testReporter = DummyTestReporter + testReporter = DummyTestReporter, + resultPathOpt = Some(testArgs.resultPath) )(ctx) case Right((startingTestClass, testClassQueueFolder, claimFolder)) => TestRunnerUtils.queueTestFramework0( @@ -46,7 +47,8 @@ import mill.util.PrintLogger testClassQueueFolder = testClassQueueFolder, claimFolder = claimFolder, cl = classLoader, - testReporter = DummyTestReporter + testReporter = DummyTestReporter, + resultPath = testArgs.resultPath )(ctx) } diff --git a/testrunner/src/mill/testrunner/TestRunnerUtils.scala b/testrunner/src/mill/testrunner/TestRunnerUtils.scala index c06eca2dda9..931e811de9e 100644 --- a/testrunner/src/mill/testrunner/TestRunnerUtils.scala +++ b/testrunner/src/mill/testrunner/TestRunnerUtils.scala @@ -12,6 +12,7 @@ import java.util.regex.Pattern import java.util.zip.ZipInputStream import scala.collection.mutable import scala.jdk.CollectionConverters.IteratorHasAsScala +import java.util.concurrent.atomic.AtomicBoolean @internal object TestRunnerUtils { @@ -113,30 +114,41 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala classFilter: Class[_] => Boolean, cl: ClassLoader, testClassfilePath: Loose.Agg[Path] - ): (Runner, Array[Task]) = { + ): (Runner, Array[Array[Task]]) = { val runner = framework.runner(args.toArray, Array[String](), cl) val testClasses = discoverTests(cl, framework, testClassfilePath) - val tasks = runner.tasks( - for ((cls, fingerprint) <- testClasses.iterator.toArray if classFilter(cls)) - yield new TaskDef( - cls.getName.stripSuffix("$"), - fingerprint, - false, - Array(new SuiteSelector) - ) - ) + val filteredTestClasses = testClasses.iterator.filter { case (cls, _) => + classFilter(cls) + }.toArray + + val tasksArr: Array[Array[Task]] = // each test class can have multiple test tasks ==> array of test classes will have this signature + if (filteredTestClasses.isEmpty) { + // We still need to run runner's tasks on empty array + Array(runner.tasks(Array.empty)) + } else { + filteredTestClasses.map { case (cls, fingerprint) => + runner.tasks( + Array(new TaskDef( + cls.getName.stripSuffix("$"), + fingerprint, + false, + Array(new SuiteSelector) + )) + ) + } + } - (runner, tasks) + (runner, tasksArr) } private def executeTasks( tasks: Seq[Task], testReporter: TestReporter, - runner: Runner, events: ConcurrentLinkedQueue[Event] - )(implicit ctx: Ctx.Log with Ctx.Home): Unit = { + )(implicit ctx: Ctx.Log with Ctx.Home): Boolean = { + val taskStatus = new AtomicBoolean(true) val taskQueue = tasks.to(mutable.Queue) while (taskQueue.nonEmpty) { val next = taskQueue.dequeue().execute( @@ -144,6 +156,11 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala def handle(event: Event) = { testReporter.logStart(event) events.add(event) + event.status match { + case Status.Error => taskStatus.set(false) + case Status.Failure => taskStatus.set(false) + case _ => () + } testReporter.logFinish(event) } }, @@ -159,6 +176,7 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala taskQueue.enqueueAll(next) } + taskStatus.get() } def parseRunTaskResults(events: Iterator[Event]): Iterator[TestResult] = { @@ -200,13 +218,39 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala (doneMessage, results) } - def runTasks(tasks: Seq[Task], testReporter: TestReporter, runner: Runner)(implicit + def runTasks( + tasksSeq: Seq[Seq[Task]], + testReporter: TestReporter, + runner: Runner, + resultPathOpt: Option[os.Path] + )(implicit ctx: Ctx.Log with Ctx.Home ): (String, Iterator[TestResult]) = { // Capture this value outside of the task event handler so it // isn't affected by a test framework's stream redirects val events = new ConcurrentLinkedQueue[Event]() - executeTasks(tasks, testReporter, runner, events) + + var successCounter = 0L + var failureCounter = 0L + + val resultLog: () => Unit = resultPathOpt match { + case Some(resultPath) => + () => os.write.over(resultPath, upickle.default.write((successCounter, failureCounter))) + case None => () => + ctx.log.outputStream.println( + s"Test result: ${successCounter + failureCounter} completed${if (failureCounter > 0) { + s", ${failureCounter} failures." + } else { "." }}" + ) + } + + tasksSeq.foreach { tasks => + val taskResult = executeTasks(tasks, testReporter, events) + if (taskResult) { successCounter += 1 } + else { failureCounter += 1 } + resultLog() + } + handleRunnerDone(runner, events) } @@ -216,14 +260,16 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala args: Seq[String], classFilter: Class[_] => Boolean, cl: ClassLoader, - testReporter: TestReporter + testReporter: TestReporter, + resultPathOpt: Option[os.Path] = None )(implicit ctx: Ctx.Log with Ctx.Home): (String, Seq[TestResult]) = { val framework = frameworkInstances(cl) - val (runner, tasks) = getTestTasks(framework, args, classFilter, cl, testClassfilePath) + val (runner, tasksArr) = getTestTasks(framework, args, classFilter, cl, testClassfilePath) - val (doneMessage, results) = runTasks(tasks, testReporter, runner) + val (doneMessage, results) = + runTasks(tasksArr.view.map(_.toSeq).toSeq, testReporter, runner, resultPathOpt) (doneMessage, results.toSeq) } @@ -234,7 +280,8 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala testReporter: TestReporter, runner: Runner, claimFolder: os.Path, - testClassQueueFolder: os.Path + testClassQueueFolder: os.Path, + resultPath: os.Path )(implicit ctx: Ctx.Log with Ctx.Home): (String, Iterator[TestResult]) = { // Capture this value outside of the task event handler so it // isn't affected by a test framework's stream redirects @@ -242,9 +289,8 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala val globSelectorCache = testClasses.view .map { case (cls, fingerprint) => cls.getName.stripSuffix("$") -> (cls, fingerprint) } .toMap - - // append only log, used to communicate with parent about what test is being claimed - // so that the parent can log the claimed test's name to its logger + var successCounter = 0L + var failureCounter = 0L def runClaimedTestClass(testClassName: String) = { @@ -257,10 +303,16 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala } val tasks = runner.tasks(taskDefs.toArray) - executeTasks(tasks, testReporter, runner, events) + val taskResult = executeTasks(tasks, testReporter, events) + if (taskResult) { successCounter += 1 } + else { failureCounter += 1 } + os.write.over(resultPath, upickle.default.write((successCounter, failureCounter))) } - startingTestClass.foreach(runClaimedTestClass) + startingTestClass.foreach { testClass => + os.write.append(claimFolder / os.up / s"${claimFolder.last}.log", s"$testClass\n") + runClaimedTestClass(testClass) + } for (file <- os.list(testClassQueueFolder)) { for (claimedTestClass <- claimFile(file, claimFolder)) runClaimedTestClass(claimedTestClass) @@ -273,8 +325,10 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala os.exists(file) && scala.util.Try(os.move(file, claimFolder / file.last, atomicMove = true)).isSuccess ) { - val queueLog = claimFolder / os.up / s"${claimFolder.last}.log" - os.write.append(queueLog, s"${file.last}\n") + // append only log, used to communicate with parent about what test is being claimed + // so that the parent can log the claimed test's name to its logger + val claimLog = claimFolder / os.up / s"${claimFolder.last}.log" + os.write.append(claimLog, s"${file.last}\n") file.last } } @@ -287,7 +341,8 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala testClassQueueFolder: os.Path, claimFolder: os.Path, cl: ClassLoader, - testReporter: TestReporter + testReporter: TestReporter, + resultPath: os.Path )(implicit ctx: Ctx.Log with Ctx.Home): (String, Seq[TestResult]) = { val framework = frameworkInstances(cl) @@ -302,7 +357,8 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala testReporter, runner, claimFolder, - testClassQueueFolder + testClassQueueFolder, + resultPath ) (doneMessage, results.toSeq) @@ -316,8 +372,8 @@ import scala.jdk.CollectionConverters.IteratorHasAsScala cl: ClassLoader ): Array[String] = { val framework = frameworkInstances(cl) - val (runner, tasks) = getTestTasks(framework, args, classFilter, cl, testClassfilePath) - tasks.map(_.taskDef().fullyQualifiedName()) + val (runner, tasksArr) = getTestTasks(framework, args, classFilter, cl, testClassfilePath) + tasksArr.flatten.map(_.taskDef().fullyQualifiedName()) } def globFilter(selectors: Seq[String]): String => Boolean = {