diff --git a/.github/img/cover.jpg b/.github/img/cover.jpg new file mode 100644 index 0000000..7c9a28a Binary files /dev/null and b/.github/img/cover.jpg differ diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000..5a6dd6d --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,43 @@ +name: Unit Tests + +on: [ push, pull_request ] + +jobs: + unit-tests: + runs-on: ubuntu-latest + strategy: + matrix: + suite: [ clitool ] + java-version: [ 11, 17, 21 ] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.java-version }} + - name: Cache for Scala Dependencies + uses: actions/cache@v4 + with: + path: | + ~/.mill/download + ~/.m2/repository + ~/.cache/coursier + key: ${{ runner.os }}-java-mill-${{ matrix.java-version }}-${{ hashFiles('**/build.sc') }} + restore-keys: ${{ runner.os }}-java-mill- + - name: Compile Scala Code + run: | + ./mill --no-server clean + ./mill --no-server --disable-ticker ${{ matrix.suite }}.compile + - name: Test Scala Code + run: | + ./mill --no-server --disable-ticker ${{ matrix.suite }}.test + - name: Create Code Coverage Report + if: matrix.java-version == '11' + run: | + ./mill --no-server --disable-ticker ${{ matrix.suite }}.scoverage.htmlReport + - name: Upload Code Coverage Report + uses: actions/upload-artifact@v4 + if: matrix.java-version == '11' + with: + name: code-coverage + path: out/clitool/scoverage/htmlReport.dest/ diff --git a/.gitignore b/.gitignore index 7169cab..b7fe05a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,13 @@ +.bsp/* +.DS_Store +.idea/* +.java-version +.metals/* +.vscode/* *.class *.log - -# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +**/*.iml +bin/* hs_err_pid* +out/* +target/* diff --git a/.mill-version b/.mill-version new file mode 100644 index 0000000..aa22d3c --- /dev/null +++ b/.mill-version @@ -0,0 +1 @@ +0.12.3 diff --git a/LICENSE b/LICENSE index 74adfb0..6d6ccda 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 Clint Valentine +Copyright © 2024 Clint Valentine Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index fcf6d5b..d4f9a7f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,34 @@ # clitool -A small set of Scala traits to help execute command line tools + +[![Unit Tests](https://github.com/clintval/clitool/actions/workflows/unit-tests.yml/badge.svg?branch=main)](https://github.com/clintval/clitool/actions/workflows/unit-tests.yml?query=branch%3Amain) +[![Java Version](https://img.shields.io/badge/java-11,17,21-c22d40.svg)](https://github.com/AdoptOpenJDK/homebrew-openjdk) +[![Language](https://img.shields.io/badge/language-scala-c22d40.svg)](https://www.scala-lang.org/) +[![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/clintval/clitool/blob/master/LICENSE) + +A small set of Scala traits to help execute command line tools. + +![Cisco and the Grand Canyon](.github/img/cover.jpg) + +```scala +val ScriptResource = "io/cvbio/io/CliToolTest.py" + +Python3.execScript( + scriptResource = ScriptResource, + args = Seq.empty, + logger = Some(logger), + stdoutRedirect = logger.info, + stderrRedirect = logger.warning, +) +``` + +#### If Mill is your build tool + +```scala +ivyDeps ++ Agg(ivy"io.cvbio.io::clitool::0.1.0") +``` + +#### If SBT is your build tool + +```scala +libraryDependencies += "io.cvbio.io" %% "clitool" % "0.1.0" +``` \ No newline at end of file diff --git a/build.sc b/build.sc new file mode 100644 index 0000000..1280693 --- /dev/null +++ b/build.sc @@ -0,0 +1,110 @@ +import $ivy.`com.lihaoyi::mill-contrib-scoverage:$MILL_VERSION` +import coursier.maven.MavenRepository +import mill._ +import mill.api.JarManifest +import mill.contrib.scoverage.ScoverageModule +import mill.define.{Target, Task} +import mill.scalalib._ +import mill.scalalib.publish._ + +import java.util.jar.Attributes.Name.{IMPLEMENTATION_VERSION => ImplementationVersion} + +/** The official package version. */ +private val packageVersion = "0.1.0" + +/** A base trait for all test targets. */ +trait ScalaTest extends TestModule { + + /** The dependencies needed only for tests. */ + override def ivyDeps = Agg( + ivy"ch.qos.logback:logback-classic:1.5.12", + ivy"org.scalatest::scalatest::3.2.19".excludeOrg(organizations = "org.junit"), + ) + + /** The test framework to use for this project. */ + override def testFramework: Target[String] = T { "org.scalatest.tools.Framework" } +} + +/** The clitool Scala package package. */ +object clitool extends ScalaModule with PublishModule with ScoverageModule { + object test extends ScalaTests with ScalaTest with ScoverageTests + + def scalaVersion = "2.13.14" + def scoverageVersion = "2.1.1" + def publishVersion = T { packageVersion } + + /** POM publishing settings for this package. */ + def pomSettings: Target[PomSettings] = PomSettings( + description = "A small set of Scala traits to help execute command line tools.", + organization = "io.cvbio.io", + url = "https://github.com/clintval/clitool", + licenses = Seq(License.MIT), + versionControl = VersionControl.github(owner = "clintval", repo = "clitool", tag = Some(packageVersion)), + developers = Seq( + Developer(id = "clintval", name = "Clint Valentine", url = "https://github.com/clintval"), + Developer(id = "fangylo", name = "Fang Yin Lo", url = "https://github.com/fangylo"), + ) + ) + + /** The artifact name, fully resolved within the coordinate. */ + override def artifactName: T[String] = T { "clitool" } + + /** The JAR manifest. */ + override def manifest: T[JarManifest] = super.manifest().add(ImplementationVersion.toString -> packageVersion) + + /** The dependencies of the project. */ + override def ivyDeps = Agg( + ivy"org.slf4j:slf4j-nop:1.7.6", + ivy"org.scala-lang.modules::scala-parallel-collections::1.0.0", + ) + + /** All the repositories we want to pull from. */ + override def repositoriesTask: Task[Seq[coursier.Repository]] = T.task { + super.repositoriesTask() ++ Seq(MavenRepository("https://oss.sonatype.org/content/repositories/public")) + } + + /** All Scala compiler options for this package. */ + override def scalacOptions: T[Seq[String]] = T { + Seq( + "-opt:inline:io.cvbio.**", // Turn on the inliner. + "-opt-inline-from:io.cvbio.**", // Tells the inliner that it is allowed to inline things from these classes. + "-Yopt-log-inline", "_", // Optional, logs the inliner activity so you know it is doing something. + "-Yopt-inline-heuristics:at-inline-annotated", // Tells the inliner to use your `@inliner` tags. + "-opt-warnings:at-inline-failed", // Tells you if methods marked with `@inline` cannot be inlined, so you can remove the tag. + // The following are sourced from https://nathankleyn.com/2019/05/13/recommended-scalac-flags-for-2-13/ + "-deprecation", // Emit warning and location for usages of deprecated APIs. + "-explaintypes", // Explain type errors in more detail. + "-feature", // Emit warning and location for usages of features that should be imported explicitly. + "-unchecked", // Enable additional warnings where generated code depends on assumptions. + "-Xcheckinit", // Wrap field accessors to throw an exception on uninitialized access. + "-Xfatal-warnings", // Fail the compilation if there are any warnings. + "-Xlint:adapted-args", // Warn if an argument list is modified to match the receiver. + "-Xlint:constant", // Evaluation of a constant arithmetic expression results in an error. + "-Xlint:delayedinit-select", // Selecting member of DelayedInit. + "-Xlint:doc-detached", // A Scaladoc comment appears to be detached from its element. + "-Xlint:inaccessible", // Warn about inaccessible types in method signatures. + "-Xlint:infer-any", // Warn when a type argument is inferred to be `Any`. + "-Xlint:missing-interpolator", // A string literal appears to be missing an interpolator id. + "-Xlint:nullary-unit", // Warn when nullary methods return Unit. + "-Xlint:option-implicit", // Option.apply used implicit view. + "-Xlint:package-object-classes", // Class or object defined in package object. + "-Xlint:poly-implicit-overload", // Parameterized overloaded implicit methods are not visible as view bounds. + "-Xlint:private-shadow", // A private field (or class parameter) shadows a superclass field. + "-Xlint:stars-align", // Pattern sequence wildcard must align with sequence component. + "-Xlint:type-parameter-shadow", // A local type parameter shadows a type already in scope. + "-Ywarn-dead-code", // Warn when dead code is identified. + "-Ywarn-extra-implicit", // Warn when more than one implicit parameter section is defined. + "-Ywarn-numeric-widen", // Warn when numerics are widened. + "-Ywarn-unused:implicits", // Warn if an implicit parameter is unused. + "-Ywarn-unused:imports", // Warn if an import selector is not referenced. + "-Ywarn-unused:locals", // Warn if a local definition is unused. + "-Ywarn-unused:params", // Warn if a value parameter is unused. + "-Ywarn-value-discard", // Warn when non-Unit expression results are unused. + "-Ywarn-unused:patvars", // Warn if a variable bound in a pattern is unused. + "-Ywarn-unused:privates", // Warn if a private member is unused. + "-Ybackend-parallelism", Math.min(Runtime.getRuntime.availableProcessors(), 8).toString, // Enable parallelization — scalac max is 16. + "-Ycache-plugin-class-loader:last-modified", // Enables caching of classloaders for compiler plugins + "-Ycache-macro-class-loader:last-modified", // and macro definitions. This can lead to performance improvements. + ) + } +} diff --git a/clitool/src/io/cvbio/io/AsyncStreamSink.scala b/clitool/src/io/cvbio/io/AsyncStreamSink.scala new file mode 100644 index 0000000..d35fce5 --- /dev/null +++ b/clitool/src/io/cvbio/io/AsyncStreamSink.scala @@ -0,0 +1,30 @@ +package io.cvbio.io + +import java.io.{Closeable, InputStream} +import java.util.concurrent.atomic.AtomicInteger + +import scala.io.Source + +/** Companion object for [[AsyncStreamSink]]. */ +private[io] object AsyncStreamSink { + private val n: AtomicInteger = new AtomicInteger(1) + private def nextName: String = s"AsyncStreamSinkThread-${n.getAndIncrement}" +} + +/** Non-blocking class that will read output from a stream and pass it to a sink. */ +private[io] class AsyncStreamSink(in: InputStream, private val sink: String => Unit) extends Closeable { + private val source = Source.fromInputStream(in).withClose(() => in.close()) + private val thread = new Thread(() => source.getLines().foreach(sink)) + this.thread.setName(AsyncStreamSink.nextName) + this.thread.setDaemon(true) + this.thread.start() + + /** Give the thread 500 seconds to wrap up what it's doing and then interrupt it. */ + def close() : Unit = close(500) + + /** Give the thread `millis` milliseconds to finish what it's doing, then interrupt it. */ + def close(millis: Long) : Unit = { + this.thread.join(millis) + this.thread.interrupt() + } +} diff --git a/clitool/src/io/cvbio/io/CliTool.scala b/clitool/src/io/cvbio/io/CliTool.scala new file mode 100644 index 0000000..2e228b6 --- /dev/null +++ b/clitool/src/io/cvbio/io/CliTool.scala @@ -0,0 +1,306 @@ +package io.cvbio.io + +import org.slf4j.Logger + +import java.io.BufferedInputStream +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} +import scala.collection.mutable +import scala.collection.mutable.ListBuffer +import scala.collection.parallel.CollectionConverters._ +import scala.io.Source +import scala.jdk.CollectionConverters._ +import scala.util.Properties.lineSeparator +import scala.util.Try + +/** A base trait for a commandline tool that defines values associated with the tool. + * For example, the name of the executable and if the executable is available to be executed. + */ +trait CliTool { + + /** The name of the executable. */ + val executable: String + + /** The arguments that are used to test if the executable is available. */ + val argsToTestAvailability: Seq[String] + + /** True if the tool is available and false otherwise. */ + lazy val available: Boolean = Try(CliTool.execCommand(Seq(executable) ++ argsToTestAvailability)).isSuccess +} + +/** Indicates the executable can run scripts. */ +trait ScriptRunner { + self: CliTool => + + /** Suffix (file extension) of scripts that can be run by this tool. */ + val scriptSuffix: String + + /** Executes a script from the classpath, raise an exception otherwise. + * + * @throws Exception when we are unable to execute to this script on the classpath with the given arguments. + * @throws ToolException when the exit code from the called process is not zero. + * @param scriptResource the name of the script resource on the classpath + * @param args a variable list of arguments to pass to the script + * @param logger an optional logger to use for emitting status updates + * @param stdoutRedirect an optional function that will capture or sink away the stdout of the underlying process + * @param stderrRedirect an optional function that will capture or sink away the stderr of the underlying process + * @param environment key value pairs to set in the process's environment before execution + */ + def execScript( + scriptResource: String, + args: Seq[String] = Seq.empty, + logger: Option[Logger] = None, + stdoutRedirect: String => Unit = _ => (), + stderrRedirect: String => Unit = _ => (), + environment: Map[String, String] = Map.empty, + ): Unit = { + execScript( + CliTool.writeResourceToTempFile(scriptResource), + args = args, + logger = logger, + stdoutRedirect = stdoutRedirect, + stderrRedirect = stderrRedirect, + environment = environment, + ) + } + + /** Executes a script from the filesystem path. + * + * @throws Exception when we are unable to execute the script with the given arguments + * @throws ToolException when the exit code from the called process is not zero + * @param scriptPath Path to the script to be executed + * @param args a variable list of arguments to pass to the script + * @param logger an optional logger to use for emitting status updates + * @param stdoutRedirect an optional function that will capture or sink away the stdout of the underlying process + * @param stderrRedirect an optional function that will capture or sink away the stderr of the underlying process + * @param environment key value pairs to set in the process's environment before execution + */ + def execScript( + scriptPath: Path, + args: Seq[String], + logger: Option[Logger], + stdoutRedirect: String => Unit, + stderrRedirect: String => Unit, + environment: Map[String, String], + ): Unit = { + + val basename = scriptPath.getFileName.toString + val command = Seq(executable, scriptPath.toAbsolutePath.toString) ++ args + + logger.foreach { log => + log.info(s"Executing script: $basename with $executable using the arguments: '${args.mkString(" ")}'") + } + + try { + CliTool.execCommand( + command, + logger = logger, + stdoutRedirect = stdoutRedirect, + stderrRedirect = stderrRedirect, + environment = environment, + ) + } catch { case exception: Throwable => + logger.foreach(_.error(s"Failed to execute $basename using $executable with arguments: '${args.mkString(" ")}'" )) + throw exception + } + } +} + +/** Defines values used to get the version of the executable. */ +private[io] trait Versioned { + self: CliTool => + + /** The default version flag. */ + val versionFlag: String = "--version" + + /** Use the version flag to test the successful install of the executable. */ + lazy val argsToTestAvailability: Seq[String] = Seq(versionFlag) // Must be lazy in case versionFlag is overridden + + /** Version of this executable. */ + val version: String +} + +/** Defines values used to get the version of the executable from both stdout and stderr. */ +trait VersionOnStream extends Versioned { + self: CliTool => + + /** Version of this executable. */ + lazy val version: String = { + val versionBuffer = ListBuffer[String]() + def redirect(s: String): Unit = { val _ = versionBuffer.synchronized(versionBuffer.addOne(s)) } + CliTool.execCommand(Seq(executable, versionFlag), stdoutRedirect = redirect, stderrRedirect = redirect) + versionBuffer.mkString(lineSeparator) + } +} + +/** Defines values used to get the version from stdout. */ +trait VersionOnStdOut extends Versioned { + self: CliTool => + + /** Version of this executable. */ + lazy val version: String = { + val version_buffer = ListBuffer[String]() + CliTool.execCommand(Seq(executable, versionFlag), stdoutRedirect = version_buffer.addOne) + version_buffer.mkString(lineSeparator) + } +} + +/** Defines values used to get the version from stderr. */ +trait VersionOnStdErr extends Versioned { + self: CliTool => + + /** Version of this executable. */ + lazy val version: String = { + val version_buffer = ListBuffer[String]() + CliTool.execCommand(Seq(executable, versionFlag), stderrRedirect = version_buffer.addOne) + version_buffer.mkString(lineSeparator) + } +} + +/** Defines methods used to check if specific modules are installed with the executable . */ +trait Modular { + self: CliTool => + + /** A cache for remembering which modules are available to speedup successive calls. */ + private lazy val cache: mutable.Map[String, Boolean] = mutable.Map.empty + + /** The command to use to test the existence of a module with the executable. */ + def testModuleCommand(module: String): Seq[String] + + /** Returns true if the tested module exists with the tested executable. */ + def moduleAvailable(module: String): Boolean = { + this.cache synchronized { + this.available && cache.getOrElseUpdate(module, Try(CliTool.execCommand(testModuleCommand(module))).isSuccess) + } + } + + /** Returns true if all tested modules exist with the tested executable. + * + * For example: + * {{ + * scala> import io.cvbio.io._ + * scala> Rscript.ModuleAvailable(Seq("stats", "stats4")) + * res1: Boolean = true + * }} + */ + def moduleAvailable(modules: Seq[String]): Boolean = { + modules.toList.par.map(moduleAvailable).forall(_ == true) + } + + /** Clear the internal cache that remembers which modules are available. */ + def clearModuleAvailableCache(): Unit = cache synchronized { cache.clear() } +} + +/** Companion object for [[CliTool]]. */ +object CliTool { + + /** The characters that are illegal to have in file paths. */ + private[io] val IllegalPathCharacters: Set[Char] = "[!\"#$%&'()*/:;<=>?@\\^`{|}~] ".toSet + + /** The maximum number of characters allowed in a filename. */ + private[io] val MaxFileNameSize: Int = 254 + + /** Exception class that holds onto the exit/status code and command execution. + * + * @param status The exist/status code of the executed command. + * @param command The command that triggered the exception. + */ + case class ToolException(status: Int, command: Seq[String]) extends RuntimeException { + override def getMessage: String = s"Command failed with exit code $status: ${command.mkString(" ")}" + } + + /** Execute a command while redirecting stdout or stderr streams elsewhere. + * + * @param command the command to run + * @param logger an optional logger to use for emitting status updates + * @param stdoutRedirect an optional function that will capture or sink away the stdout of the underlying process + * @param stderrRedirect an optional function that will capture or sink away the stderr of the underlying process + * @throws ToolException if command has a non-zero exit code + */ + def execCommand( + command: Seq[String], + logger: Option[Logger] = None, + stdoutRedirect: String => Unit = _ => (), + stderrRedirect: String => Unit = _ => (), + environment: Map[String, String] = Map.empty, + ): Unit = { + logger.foreach(log => log.info(s"Executing command: ${command.mkString(" ")}")) + val builder = new ProcessBuilder(command: _*) + + builder.environment.putAll(environment.asJava) + + val process = builder.start() + val pipeOut = new AsyncStreamSink(process.getInputStream, stdoutRedirect) // Get stdout + val pipeErr = new AsyncStreamSink(process.getErrorStream, stderrRedirect) // Get stderr + val exitCode = process.waitFor() + + pipeOut.close() + pipeErr.close() + + if (exitCode != 0) throw ToolException(exitCode, command) + } + + /** Extracts a resource from the classpath and writes it to a temp file on disk. + * + * @param resource a given name on the classpath + * @return path to the temporary file + */ + private[io] def writeResourceToTempFile(resource: String): Path = { + val stream = Seq(getClass.getResourceAsStream _, getClass.getClassLoader.getResourceAsStream _) + .flatMap(get => Option(get(resource))) + .headOption + .getOrElse(throw new IllegalArgumentException(s"Resource does not exist at path: $resource")) + + val source = Source.fromInputStream(new BufferedInputStream(stream, 32 * 1024)) + val lines = try source.getLines().toList finally source.close() + val name = this.getClass.getSimpleName.map(char => if (IllegalPathCharacters.contains(char)) "_" else char) + val dir = Files.createTempDirectory(name.mkString.substring(0, Math.min(name.length, MaxFileNameSize))) + val path = Paths.get(dir.toString, Paths.get(resource).getFileName.toString).toAbsolutePath.normalize + + dir.toFile.deleteOnExit() + Files.write(path, lines.mkString(lineSeparator).getBytes(StandardCharsets.UTF_8)) + path + } +} + +/** A collection of values and methods specific for the Rscript executable */ +object Rscript extends CliTool with VersionOnStdErr with Modular with ScriptRunner { + + /** Rscript executable. */ + val executable: String = "Rscript" + + /** The file extension for Rscript files. */ + val scriptSuffix: String = ".R" + + /** Command to test if a R module exists. + * + * @param module name of the module to be tested + * @return command used to test if a given R module is installed + */ + def testModuleCommand(module: String): Seq[String] = Seq(executable, "-e", s"stopifnot(require('$module'))") + + /** True if both Rscript exists and the library ggplot2 is installed. */ + lazy val ggplot2Available: Boolean = available && moduleAvailable("ggplot2") +} + +/** Defines tools to test various version of Python executables */ +trait Python extends CliTool with Versioned with Modular with ScriptRunner { + + /** Python executable. */ + val executable: String = "python" + + /** The file extension for Python files. */ + val scriptSuffix: String = ".py" + + /** The command to use to test the existence of a Python module. */ + def testModuleCommand(module: String): Seq[String] = Seq(executable, "-c", s"import $module") +} + +/** The system Python version. */ +object Python extends Python with VersionOnStream + +/** The system Python 2. */ +object Python2 extends Python with VersionOnStdErr { override val executable: String = "python2" } + +/** The system Python3. */ +object Python3 extends Python with VersionOnStdOut { override val executable: String = "python3" } diff --git a/clitool/test/resources/io/cvbio/io/CliToolFailureTest.R b/clitool/test/resources/io/cvbio/io/CliToolFailureTest.R new file mode 100644 index 0000000..512c69b --- /dev/null +++ b/clitool/test/resources/io/cvbio/io/CliToolFailureTest.R @@ -0,0 +1 @@ +stopifnot(require("thisPackageDoesNotExist")) diff --git a/clitool/test/resources/io/cvbio/io/CliToolFailureTest.py b/clitool/test/resources/io/cvbio/io/CliToolFailureTest.py new file mode 100644 index 0000000..b5580fe --- /dev/null +++ b/clitool/test/resources/io/cvbio/io/CliToolFailureTest.py @@ -0,0 +1 @@ +import thisPackageDoesNotExist diff --git a/clitool/test/resources/io/cvbio/io/CliToolTest.R b/clitool/test/resources/io/cvbio/io/CliToolTest.R new file mode 100644 index 0000000..27086fa --- /dev/null +++ b/clitool/test/resources/io/cvbio/io/CliToolTest.R @@ -0,0 +1 @@ +stopifnot(require("stats4")) diff --git a/clitool/test/resources/io/cvbio/io/CliToolTest.py b/clitool/test/resources/io/cvbio/io/CliToolTest.py new file mode 100644 index 0000000..21b405d --- /dev/null +++ b/clitool/test/resources/io/cvbio/io/CliToolTest.py @@ -0,0 +1 @@ +import os diff --git a/clitool/test/resources/logback.xml b/clitool/test/resources/logback.xml new file mode 100644 index 0000000..8313843 --- /dev/null +++ b/clitool/test/resources/logback.xml @@ -0,0 +1,2 @@ + + diff --git a/clitool/test/src/io/cvbio/io/CliToolTest.scala b/clitool/test/src/io/cvbio/io/CliToolTest.scala new file mode 100644 index 0000000..2dec024 --- /dev/null +++ b/clitool/test/src/io/cvbio/io/CliToolTest.scala @@ -0,0 +1,271 @@ +package io.cvbio.io + +import io.cvbio.io.CliTool.ToolException +import io.cvbio.io.testing.UnitSpec + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Paths} +import scala.annotation.nowarn +import scala.collection.mutable.ListBuffer +import scala.util.Try + +/** Unit tests for [[CliTool]]. */ +class CliToolTest extends UnitSpec { + + "CliToolTest.execCommand" should "execute a command and return stdout and stderr successfully" in { + val stdout = ListBuffer[String]() + val stderr = ListBuffer[String]() + + CliTool.execCommand( + Seq("echo", "hi"), stdoutRedirect = stdout.append, stderrRedirect = stderr.append + ) + + stdout.mkString shouldBe "hi" + stderr.mkString shouldBe empty + } + + it should "throw a ToolException with correct exit code when executing an invalid command and return stderr" in { + val stdout = ListBuffer[String]() + val stderr = ListBuffer[String]() + val invalidCommand = Seq("cut", "-invalidArgs") + + intercept[ToolException]{ + CliTool.execCommand( + invalidCommand, + stdoutRedirect = s => stdout synchronized { stdout.append(s) }: @nowarn("msg=discarded non-Unit value"), + stderrRedirect = s => stderr synchronized { stderr.append(s) }: @nowarn("msg=discarded non-Unit value"), + ) + }.getMessage should include("Command failed with exit code 1: " + invalidCommand.mkString(" ")) + + stdout.mkString shouldBe empty + stderr.mkString should include regex "(invalid|illegal) option" + } + + it should "fail to execute an invalid command" in { + val invalidCommand = Seq("cut", "-invalidArgs") + val attempt = Try(CliTool.execCommand(invalidCommand)) + attempt.failure.exception.getMessage should include ("invalid") + } + + it should "execute a java command successfully and return stdout and stderr " in { + val streams = new ListBuffer[String]() + val javaCommand = Seq("java", "-version") + + CliTool.execCommand( + javaCommand, + stdoutRedirect = s => streams synchronized { streams.append(s) }: @nowarn("msg=discarded non-Unit value"), + stderrRedirect = s => streams synchronized { streams.append(s) }: @nowarn("msg=discarded non-Unit value"), + ) + + streams.mkString should include ("version") + } + + it should "emit a status update on the logger if provided" in { + val command = Seq("echo", "hi") + CliTool.execCommand(command, logger = Some(logger)) + logs.mkString should include (s"Executing command: ${command.mkString(" ")}") + } + + it should "emit no logging when no logger is provided" in { + val command = Seq("echo", "hi") + CliTool.execCommand(command, logger = None) + logs.mkString shouldBe empty + } + + "CliTool.execIfAvailable" should "throw a ToolException when running invalid R script if Rscript is available" in { + val scriptResource = "io/cvbio/io/CliToolFailureTest.R" + + if (Rscript.available) { + val stdout = ListBuffer[String]() + val stderr = ListBuffer[String]() + + intercept[ToolException] { + Rscript.execScript( + scriptResource = scriptResource, + args = Seq.empty, + logger = None, + stdoutRedirect = stdout.append, + stderrRedirect = stderr.append, + ) + }.getMessage should include("Command failed with exit code 1: Rscript" ) + } + } + + it should "emit status update on the logger if a logger is provided " in { + val scriptResource = "io/cvbio/io/CliToolTest.R" + + if (Rscript.available) { + Rscript.execScript( + scriptResource = scriptResource, + args = Seq.empty, + logger = Some(logger), + stdoutRedirect = logger.info, + stderrRedirect = logger.error, + ) + logs.mkString should include ("Executing script:") + logs.mkString should include ("Executing command:") + logs.mkString should include ("Loading required package: stats4") + } + } + + it should "emit no logging when no logger is provided " in { + val scriptResource = "io/cvbio/io/CliToolTest.R" + + if (Rscript.available) { + Rscript.execScript( + scriptResource = scriptResource, + args = Seq.empty, + logger = None, + stdoutRedirect = _ => (), + stderrRedirect = _ => (), + ) + logs.mkString shouldBe empty + } + } + + "CliTool.ToolException" should "wrap the exit code and command in the exception message" in { + val invalidCommand = Seq("invalidCommand") + val code = 2 + ToolException(code, invalidCommand).getMessage shouldBe "Command failed with exit code 2: " + invalidCommand.mkString("") + } + + "CliTool.ScriptRunner" should "execute a script from resource and emits status update to logger correctly if the " + + "executable is available and logger is provided" in { + if (Rscript.available){ + Rscript.execScript( + scriptResource = "io/cvbio/io/CliToolTest.R", + args = Seq.empty, + logger = Some(logger), + stdoutRedirect = logger.info, + stderrRedirect = logger.info, + ) + + logs.mkString should include ("Loading required package") + } + } + + it should "execute a R script from a given path if the Rscript is available" in { + if (Rscript.available){ + val tempFile = Files.createTempFile("CliToolTest.", ".R") + + Files.write(tempFile, "stopifnot(require(\"stats4\"))".getBytes(StandardCharsets.UTF_8)) + tempFile.toFile.deleteOnExit() + + noException should be thrownBy { + Rscript.execScript( + scriptPath = tempFile, + args = Seq.empty, + logger = None, + stdoutRedirect = _ => (), + stderrRedirect = _ => (), + environment = Map.empty, + ) + } + } + } + + it should "execute a script from script resource and correctly return stdout and stderr if the executable is available" in { + if (Rscript.available){ + val stdout = ListBuffer[String]() + val stderr = ListBuffer[String]() + + Rscript.execScript( + scriptResource = "io/cvbio/io/CliToolTest.R", + args = Seq.empty, + logger = None, + stdoutRedirect = stdout.append, + stderrRedirect = stderr.append, + ) + stdout.mkString shouldBe empty + stderr.mkString should include ("Loading required package") + } + } + + it should "execute a script from script resource if the executable is available" in { + if (Python.available){ + noException shouldBe thrownBy{ + Python.execScript( + scriptResource = "io/cvbio/io/CliToolTest.py", + args = Seq.empty, + logger = None, + stdoutRedirect = _ => (), + stderrRedirect = _ => (), + ) + } + } + } + + it should "throw ToolException and correct exit code if trying to run a script from a given path does not exist" in { + if (Python.available){ + intercept[ToolException]{ + val path = Paths.get("nowhere.py").toAbsolutePath.normalize + Python.execScript( + scriptPath = path, + args = Seq.empty, + logger = None, + stdoutRedirect = _ => (), + stderrRedirect = _ => (), + environment = Map.empty, + ) + }.getMessage should include("Command failed with exit code 2: python") + } + } + + it should "throw ToolException and correct exit code when running an script with invalid command" in { + val stdout = ListBuffer[String]() + val stderr = ListBuffer[String]() + + if (Python.available){ + intercept[ToolException]{ + Python.execScript( + scriptResource = "io/cvbio/io/CliToolFailureTest.py", + args = Seq.empty, + logger = None, + stdoutRedirect = stdout.append, + stderrRedirect = stderr.append, + ) + }.getMessage should include("Command failed with exit code 1: python") + + // Check stderr is written correctly + stdout.mkString("") shouldBe empty + stderr.mkString("") should include ("No module named") + } + } + + "CliTool.Modular" should "test that generic builtins are available in Python if Python is available" in { + if (Python.available){ + Python.moduleAvailable(Seq("sys")) shouldBe true + Python.moduleAvailable(Seq("sys", "os")) shouldBe true + } + } + + it should "test that generic builtins packages are available in R if Rscript is available" in { + if (Rscript.available){ + Rscript.moduleAvailable(Seq("stats")) shouldBe true + Rscript.moduleAvailable(Seq("stats", "stats4")) shouldBe true + } + } + + it should "allow for repeated query of the same module and then let us clear the internal cache" in { + if (Rscript.available) { + Rscript.moduleAvailable("stats") shouldBe true + Rscript.moduleAvailable("stats") shouldBe true + noException shouldBe thrownBy { Rscript.clearModuleAvailableCache() } + } + } + + "CliTool.Versioned" should "emit the current version of Python if Python is available" in { + if (Python3.available){ + Python3.version should include ("Python") + } + + if (Python2.available){ + Python2.version should include ("Python") + } + } + + it should "use its version for availability arguments" in { + object TestPython extends Python with VersionOnStdOut { override lazy val version: String = "--version" } + TestPython.argsToTestAvailability should contain theSameElementsInOrderAs Seq("--version") + } +} diff --git a/clitool/test/src/io/cvbio/io/testing/UnitSpec.scala b/clitool/test/src/io/cvbio/io/testing/UnitSpec.scala new file mode 100644 index 0000000..e2f532e --- /dev/null +++ b/clitool/test/src/io/cvbio/io/testing/UnitSpec.scala @@ -0,0 +1,63 @@ +package io.cvbio.io.testing + +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.read.ListAppender +import org.scalatest._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.Logger.{ROOT_LOGGER_NAME => RootLoggerName} +import org.slf4j.LoggerFactory + +import java.nio.file.Path +import scala.io.Source +import scala.jdk.CollectionConverters.CollectionHasAsScala + + +/** A trait for creating a single logger for all tests which clears the loggers state in between test executions. */ +trait LoggerPerTest extends BeforeAndAfterEach with BeforeAndAfterAll { self: Suite => + lazy val appender: ListAppender[ILoggingEvent] = new ListAppender() + lazy val logger: Logger = LoggerFactory.getLogger(RootLoggerName).asInstanceOf[Logger] + + /** Before we run all tests, silence the internal verbosity of the logging system. */ + override protected def beforeAll(): Unit = { + super.beforeAll() + val _ = System.setProperty("slf4j.internal.verbosity", "WARN") + } + + /** Before each test, add a new empty event appender to the logger. */ + override def beforeEach(): Unit = { + logger.addAppender(appender) + appender.start() + super.beforeEach() + } + + /** After each test, stop the appender, clear it, and detach it. */ + override def afterEach(): Unit = { + try super.afterEach() + finally { + appender.stop() + appender.list.clear() + logger.detachAndStopAllAppenders() + } + } + + /** Get all formatted log messages as strings. */ + def logs: Seq[String] = appender.list.asScala.toSeq.map(_.getFormattedMessage) +} + +/** Base class for unit testing. */ +trait UnitSpec extends AnyFlatSpec with Matchers with OptionValues with TryValues with LoggerPerTest { + + /** Asserts the length and content of the two files are the same. */ + protected def assertFilesEqual(actual: Path, expected: Path): Unit = { + val source1 = Source.fromFile(actual.toFile) + val actualLines = try source1.getLines().toList finally source1.close() + + val source2 = Source.fromFile(expected.toFile) + val expectedLines = try source2.getLines().toList finally source2.close() + + actualLines.length shouldBe expectedLines.length + actualLines.zip(expectedLines).foreach { case (actualLine, expectedLine) => actualLine shouldBe expectedLine } + } +} diff --git a/mill b/mill new file mode 100755 index 0000000..57b8776 --- /dev/null +++ b/mill @@ -0,0 +1,26 @@ +#!/usr/bin/env sh +# This is a wrapper script that automatically downloads Mill from GitHub. +set -e + +if [ -z "$MILL_VERSION" ] ; then + MILL_VERSION="$(head -n 1 .mill-version 2> /dev/null)" +fi + +MILL_DOWNLOAD_PATH="$HOME/.mill/download" +MILL_EXEC_PATH="${MILL_DOWNLOAD_PATH}/$MILL_VERSION" + +if [ ! -x "$MILL_EXEC_PATH" ] ; then + mkdir -p "${MILL_DOWNLOAD_PATH}" + DOWNLOAD_FILE=$MILL_EXEC_PATH-tmp-download + MILL_DOWNLOAD_URL="https://github.com/lihaoyi/mill/releases/download/${MILL_VERSION%%-*}/$MILL_VERSION-assembly" + curl --fail -L -o "$DOWNLOAD_FILE" "$MILL_DOWNLOAD_URL" + chmod +x "$DOWNLOAD_FILE" + mv "$DOWNLOAD_FILE" "$MILL_EXEC_PATH" + unset DOWNLOAD_FILE + unset MILL_DOWNLOAD_URL +fi + +unset MILL_DOWNLOAD_PATH +unset MILL_VERSION + +exec "${MILL_EXEC_PATH}" "$@"