From 78525066df84f05cd0f7aef6fb4b9d8924153e68 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 12 Dec 2023 12:21:41 -0500 Subject: [PATCH 01/16] Compiles --- .github/set_up_cromwell_action/action.yml | 3 +-- .github/workflows/chart_update_on_merge.yml | 2 +- .sdkmanrc | 2 +- .../scala/centaur/api/DaemonizedDefaultThreadFactory.scala | 3 +-- .../cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala | 2 +- project/Publishing.scala | 3 +-- publish/docker-setup.sh | 2 +- 7 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index 13cb11601db..1a20507b80b 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -43,5 +43,4 @@ runs: uses: actions/setup-java@v3 with: distribution: temurin - java-version: 11 - + java-version: 17 diff --git a/.github/workflows/chart_update_on_merge.yml b/.github/workflows/chart_update_on_merge.yml index a2b14f2ec65..62910862545 100644 --- a/.github/workflows/chart_update_on_merge.yml +++ b/.github/workflows/chart_update_on_merge.yml @@ -25,7 +25,7 @@ jobs: path: cromwell - uses: olafurpg/setup-scala@v10 with: - java-version: adopt@1.11 + java-version: adopt@1.17 - name: Clone Cromwhelm uses: actions/checkout@v2 with: diff --git a/.sdkmanrc b/.sdkmanrc index 0262a261019..f3b37566860 100644 --- a/.sdkmanrc +++ b/.sdkmanrc @@ -1,3 +1,3 @@ # Enable auto-env through the sdkman_auto_env config # Add key=value pairs of SDKs to use below -java=11.0.11.hs-adpt +java=17.0.9-tem diff --git a/centaur/src/main/scala/centaur/api/DaemonizedDefaultThreadFactory.scala b/centaur/src/main/scala/centaur/api/DaemonizedDefaultThreadFactory.scala index db9eb07d79b..d75299aa590 100644 --- a/centaur/src/main/scala/centaur/api/DaemonizedDefaultThreadFactory.scala +++ b/centaur/src/main/scala/centaur/api/DaemonizedDefaultThreadFactory.scala @@ -14,8 +14,7 @@ import java.util.concurrent.atomic.AtomicInteger * > main thread). */ object DaemonizedDefaultThreadFactory extends ThreadFactory { - private val s = System.getSecurityManager - private val group = if (s != null) s.getThreadGroup else Thread.currentThread.getThreadGroup + private val group = Thread.currentThread.getThreadGroup private val threadNumber = new AtomicInteger(1) private val namePrefix = "daemonpool-thread-" diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala index b7d9ed02035..0ccb83eaea9 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala @@ -67,7 +67,7 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { def isAbsolute: Boolean = UnixPath.isAbsolute(path) - def isEmpty: Boolean = path.isEmpty + override def isEmpty: Boolean = path.isEmpty def hasTrailingSeparator: Boolean = UnixPath.hasTrailingSeparator(path) diff --git a/project/Publishing.scala b/project/Publishing.scala index 0af27e77147..812af1751f8 100644 --- a/project/Publishing.scala +++ b/project/Publishing.scala @@ -163,8 +163,7 @@ object Publishing { val additionalResolvers = List( broadArtifactoryResolver, broadArtifactoryResolverSnap, - Resolver.sonatypeRepo("releases") - ) + ) ++ Resolver.sonatypeOssRepos("releases") private val artifactoryCredentialsFile = file("target/ci/resources/artifactory_credentials.properties").getAbsoluteFile diff --git a/publish/docker-setup.sh b/publish/docker-setup.sh index 613d251df96..bc39f31d10a 100755 --- a/publish/docker-setup.sh +++ b/publish/docker-setup.sh @@ -20,7 +20,7 @@ mkdir -p /etc/apt/keyrings wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list apt update -apt install -y temurin-11-jdk +apt install -y temurin-17-jdk # Install jq 1.6 to ensure --rawfile is supported curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/bin/jq From 7ec33706de030fa9f7171f08c036ea6f8f72605c Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 12 Dec 2023 12:48:35 -0500 Subject: [PATCH 02/16] Javaism --- .../src/main/scala/cloud/nio/spi/CloudNioPath.scala | 2 +- .../cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala index b6803670d39..530ba5526ef 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala @@ -176,7 +176,7 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un ): WatchKey = throw new UnsupportedOperationException override def iterator(): java.util.Iterator[Path] = - if (unixPath.isEmpty || unixPath.isRoot) { + if (unixPath.izEmpty || unixPath.isRoot) { java.util.Collections.emptyIterator() } else { unixPath.split().to(LazyList).map(part => newPath(UnixPath.getPath(part)).asInstanceOf[Path]).iterator.asJava diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala index 0ccb83eaea9..7d929d148ab 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala @@ -67,7 +67,7 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { def isAbsolute: Boolean = UnixPath.isAbsolute(path) - override def isEmpty: Boolean = path.isEmpty + def izEmpty: Boolean = path.isEmpty def hasTrailingSeparator: Boolean = UnixPath.hasTrailingSeparator(path) From 6261dbae8a7ecd7cab551ce53451b801899394ff Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 12 Dec 2023 12:50:12 -0500 Subject: [PATCH 03/16] Fix --- .github/workflows/docker_build_test.yml | 2 +- .github/workflows/trivy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml index 01c2ea502c9..1095240c1cf 100644 --- a/.github/workflows/docker_build_test.yml +++ b/.github/workflows/docker_build_test.yml @@ -27,7 +27,7 @@ jobs: path: cromwell - uses: olafurpg/setup-scala@v10 with: - java-version: adopt@1.11 + java-version: adopt@1.17 # The following invocation should be as similar as possible to the one in chart_update_on_merge.yml # To state the obvious: This test should not publish anything. It should simply verify that the build completes. - name: Build Cromwell Docker diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index b005da65041..7baad425549 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -22,7 +22,7 @@ jobs: # fetch SBT package - uses: olafurpg/setup-scala@v10 with: - java-version: adopt@1.11 + java-version: adopt@1.17 # set up SBT cache - uses: actions/cache@v2 From 5482aadfa5f5811643d086c37360dd847d8b4158 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 12 Dec 2023 13:03:52 -0500 Subject: [PATCH 04/16] Github Actions fix --- .github/workflows/chart_update_on_merge.yml | 5 +++-- .github/workflows/docker_build_test.yml | 5 +++-- .github/workflows/trivy.yml | 5 +++-- docs/tutorials/FiveMinuteIntro.md | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/chart_update_on_merge.yml b/.github/workflows/chart_update_on_merge.yml index 62910862545..b3f7199ab35 100644 --- a/.github/workflows/chart_update_on_merge.yml +++ b/.github/workflows/chart_update_on_merge.yml @@ -23,9 +23,10 @@ jobs: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work path: cromwell - - uses: olafurpg/setup-scala@v10 + - uses: actions/setup-java@v4 with: - java-version: adopt@1.17 + distribution: 'temurin' + java-version: '17' - name: Clone Cromwhelm uses: actions/checkout@v2 with: diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml index 1095240c1cf..a50e39a380f 100644 --- a/.github/workflows/docker_build_test.yml +++ b/.github/workflows/docker_build_test.yml @@ -25,9 +25,10 @@ jobs: repository: broadinstitute/cromwell token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} path: cromwell - - uses: olafurpg/setup-scala@v10 + - uses: actions/setup-java@v4 with: - java-version: adopt@1.17 + distribution: 'temurin' + java-version: '17' # The following invocation should be as similar as possible to the one in chart_update_on_merge.yml # To state the obvious: This test should not publish anything. It should simply verify that the build completes. - name: Build Cromwell Docker diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 7baad425549..69a90937671 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -20,9 +20,10 @@ jobs: - uses: actions/checkout@v2 # fetch SBT package - - uses: olafurpg/setup-scala@v10 + - uses: actions/setup-java@v4 with: - java-version: adopt@1.17 + distribution: 'temurin' + java-version: '17' # set up SBT cache - uses: actions/cache@v2 diff --git a/docs/tutorials/FiveMinuteIntro.md b/docs/tutorials/FiveMinuteIntro.md index 1c16a56c606..21093853428 100644 --- a/docs/tutorials/FiveMinuteIntro.md +++ b/docs/tutorials/FiveMinuteIntro.md @@ -6,7 +6,7 @@ * A Java 11 runtime environment * You can see what you have by running `$ java -version` on a terminal. * If not, consider installing via conda or brew [as explained here](../Releases.md). - * We recommend [SDKMAN](https://sdkman.io/install) to install the latest 11 build of [Temurin](https://adoptium.net/temurin/releases/?version=11) + * We recommend [SDKMAN](https://sdkman.io/install) to install the latest 17 build of [Temurin](https://adoptium.net/temurin/releases/?version=17) * `sdk install java 11.0.16-tem` as of the time of this writing * You might need to update the `export JAVA_HOME` in your bash profile to point to your JAVA install location. * A sense of adventure! From 7a8c5b3bf58d1e7bf6343a0a1a188da12157ae3a Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 12 Dec 2023 13:40:43 -0500 Subject: [PATCH 05/16] Work around slightly different errors in tests --- docs/Configuring.md | 2 +- wom/src/main/resources/reference.conf | 2 +- wom/src/test/scala/wom/util/YamlUtilsSpec.scala | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/Configuring.md b/docs/Configuring.md index aa810d19a0c..288ec9723de 100644 --- a/docs/Configuring.md +++ b/docs/Configuring.md @@ -689,7 +689,7 @@ This limit may be configured via the configuration value: ```hocon yaml { - max-depth = 1000 + max-depth = 100 } ``` diff --git a/wom/src/main/resources/reference.conf b/wom/src/main/resources/reference.conf index 1907a23eac6..fce1a8e0283 100644 --- a/wom/src/main/resources/reference.conf +++ b/wom/src/main/resources/reference.conf @@ -2,7 +2,7 @@ yaml { # The maximum number of nodes (scalars + sequences + mappings) that will be parsed in a YAML max-nodes = 1000000 # The maximum nested depth for nodes (sequences + mappings) that will be parsed in a YAML - max-depth = 1000 + max-depth = 100 } wom-parse { diff --git a/wom/src/test/scala/wom/util/YamlUtilsSpec.scala b/wom/src/test/scala/wom/util/YamlUtilsSpec.scala index 986201a2213..8f5ddd22c95 100644 --- a/wom/src/test/scala/wom/util/YamlUtilsSpec.scala +++ b/wom/src/test/scala/wom/util/YamlUtilsSpec.scala @@ -80,7 +80,7 @@ class YamlUtilsSpec "a null yaml", null, refineMV[NonNegative](0), - null + "Cannot invoke \"String.length()\" because \"s\" is null" ), ( "an empty yaml mapping when limited to zero nodes", @@ -179,14 +179,14 @@ class YamlUtilsSpec val nesting = 1000000 val yaml = ("[" * nesting) + ("]" * nesting) val exception: Exception = YamlUtils.parse(yaml).left.value - exception should have message "Parsing halted at node depth 1000" // <-- Updating here? Also get the docs too! + exception should have message "Parsing halted at node depth 100" // <-- Updating here? Also get the docs too! } it should "fail to parse deeply nested yaml mapping with the default configuration" in { val nesting = 1000000 val yaml = ("{a:" * nesting) + "b" + ("}" * nesting) val exception: Exception = YamlUtils.parse(yaml).left.value - exception should have message "Parsing halted at node depth 1000" // <-- Updating here? Also get the docs too! + exception should have message "Parsing halted at node depth 100" // <-- Updating here? Also get the docs too! } it should "not parse a config with a negative value" in { From 48266e22e26f01cbf33473dc22859c598c409534 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 12 Dec 2023 13:57:05 -0500 Subject: [PATCH 06/16] Remove test whose reflection broke (and is redundant to Centaur) --- ...haredFileSystemJobExecutionActorSpec.scala | 430 ------------------ 1 file changed, 430 deletions(-) delete mode 100644 supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala deleted file mode 100644 index 5d1b1e02c12..00000000000 --- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemJobExecutionActorSpec.scala +++ /dev/null @@ -1,430 +0,0 @@ -package cromwell.backend.sfs - -import _root_.wdl.draft2.model.LocallyQualifiedName -import akka.testkit.{TestDuration, TestProbe} -import com.typesafe.config.ConfigFactory -import common.collections.EnhancedCollections._ -import cromwell.backend.BackendJobExecutionActor.{ - JobAbortedResponse, - JobFailedNonRetryableResponse, - JobSucceededResponse, - RunOnBackend -} -import cromwell.backend.BackendLifecycleActor.AbortJobCommand -import cromwell.backend._ -import cromwell.backend.async.WrongReturnCode -import cromwell.backend.io.JobPathsSpecHelper._ -import cromwell.backend.io.TestWorkflows._ -import cromwell.backend.io.{JobPathsWithDocker, TestWorkflows} -import cromwell.backend.sfs.TestLocalAsyncJobExecutionActor._ -import cromwell.backend.standard.StandardValidatedRuntimeAttributesBuilder -import cromwell.core.CromwellGraphNode._ -import cromwell.core.Tags._ -import cromwell.core._ -import cromwell.core.callcaching.NoDocker -import cromwell.core.path.{DefaultPathBuilder, Path} -import cromwell.services.keyvalue.KeyValueServiceActor._ -import cromwell.util.WomMocks -import org.scalatest.concurrent.PatienceConfiguration.Timeout -import org.scalatest.flatspec.AnyFlatSpecLike -import org.scalatest.prop.TableDrivenPropertyChecks -import org.scalatest.{Assertion, OptionValues} -import wom.expression.NoIoFunctionSet -import wom.graph.{CommandCallNode, WomIdentifier} -import wom.types._ -import wom.values._ - -import java.io.FileNotFoundException -import scala.concurrent.duration._ -import scala.sys.process._ - -class SharedFileSystemJobExecutionActorSpec - extends TestKitSuite - with AnyFlatSpecLike - with BackendSpec - with TableDrivenPropertyChecks - with OptionValues { - - behavior of "SharedFileSystemJobExecutionActor" - - lazy val runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] = - StandardValidatedRuntimeAttributesBuilder.default(Some(TestConfig.optionalRuntimeConfig)).definitions.toSet - - private val call = CommandCallNode(WomIdentifier("SfsJEASpec_call"), null, null, null, Set.empty, null, None) - private val mockBackendJobDescriptorKey = BackendJobDescriptorKey(call, None, 1) - - def executeSpec(docker: Boolean): Any = { - val expectedOutputs: CallOutputs = - WomMocks.mockOutputExpectations(Map("hello.salutation" -> WomString("Hello you !"))) - - val expectedResponse = JobSucceededResponse(mockBackendJobDescriptorKey, - Some(0), - expectedOutputs, - None, - Seq.empty, - None, - resultGenerationMode = RunOnBackend - ) - val runtime = if (docker) s"""runtime { docker: "$dockerImageUbuntu" }""" else "" - val workflowDescriptor = buildWdlWorkflowDescriptor(HelloWorld, runtime = runtime) - val workflow = TestWorkflow(workflowDescriptor, TestConfig.backendRuntimeConfigDescriptor, expectedResponse) - val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, - Map.empty, - WorkflowOptions.empty, - runtimeAttributeDefinitions - ), - workflow.config - ) - testWorkflow(workflow, backend) - } - - private val dockerImageUbuntu = "ubuntu:latest" - - it should "docker pull images now so that 'DockerTest' specs below do not timeout" taggedAs DockerTest in { - /* - This is a poor substitute for `beforeAll()`. - If one wants to parallelize the test runs consider using a separate tagged-Suite with a `.beforeAll()`. - - https://medium.com/@linda0511ny/tag-whole-test-class-in-scalatest-94cd67fa85a4 - */ - List("docker", "pull", dockerImageUbuntu).! should be(0) - } - - it should "execute an hello world workflow" in { - executeSpec(docker = false) - } - - it should "execute an hello world workflow on Docker" taggedAs DockerTest in { - executeSpec(docker = true) - } - - it should "send back an execution failure if the task fails" in { - val expectedResponse = - JobFailedNonRetryableResponse(mockBackendJobDescriptorKey, - WrongReturnCode("wf_goodbye.goodbye:NA:1", 1, None), - Option(1) - ) - val workflow = TestWorkflow(buildWdlWorkflowDescriptor(GoodbyeWorld), - TestConfig.backendRuntimeConfigDescriptor, - expectedResponse - ) - val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, - Map.empty, - WorkflowOptions.empty, - runtimeAttributeDefinitions - ), - workflow.config - ) - testWorkflow(workflow, backend) - } - - def localizationSpec(docker: Boolean): Assertion = { - def templateConf(localizers: String) = BackendConfigurationDescriptor( - ConfigFactory.parseString(s"""|{ - | root = "local-cromwell-executions" - | filesystems { - | local { - | localization = [ - | $localizers - | ] - | } - | } - | default-runtime-attributes { - | cpu: 1 - | failOnStderr: false - | continueOnReturnCode: 0 - | } - |} - |""".stripMargin), - ConfigFactory.parseString("{}") - ) - - val hardConf = templateConf("hard-link") - val symConf = templateConf("soft-link") - val copyConf = templateConf("copy") - - val jsonInputFile = - createCannedFile("localize", "content from json inputs", Option(DefaultPathBuilder.build(".").get)) - val callInputFile = - createCannedFile("localize", "content from call inputs", Option(DefaultPathBuilder.build(".").get)) - val inputs = Option(s"""{ - "wf_localize.workflowFile": "${callInputFile.pathAsString}", - "wf_localize.localize.inputFileFromJson": "${jsonInputFile.pathAsString}" - }""") - - val expectedOutputs: CallOutputs = WomMocks.mockOutputExpectations( - Map( - "localize.out" -> WomArray(WomArrayType(WomStringType), - List(WomString("content from json inputs"), WomString("content from call inputs")) - ) - ) - ) - - val confs = List( - (hardConf, false), - (copyConf, false) - ) ++ (if (!docker) List((symConf, true)) else List.empty) - - val localizers = Table( - ("conf", "isSymLink"), - confs: _* - ) - - forAll(localizers) { (conf, isSymlink) => - val runtime = if (docker) s"""runtime { docker: "$dockerImageUbuntu" } """ else "" - val workflowDescriptor = buildWdlWorkflowDescriptor(InputFiles, inputs, runtime = runtime) - val callInputs = Map( - "inputFileFromCallInputs" -> workflowDescriptor.knownValues.collectFirst { - case (outputPort, resolvedValue) if outputPort.fullyQualifiedName == "wf_localize.workflowFile" => - resolvedValue - }.get, - "inputFileFromJson" -> workflowDescriptor.knownValues.collectFirst { - case (outputPort, resolvedValue) - if outputPort.fullyQualifiedName == "wf_localize.localize.inputFileFromJson" => - resolvedValue - }.get - ) - - val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflowDescriptor, - callInputs, - WorkflowOptions.empty, - runtimeAttributeDefinitions - ), - conf - ) - val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, - callInputs, - WorkflowOptions.empty, - runtimeAttributeDefinitions - ) - val expectedResponse = JobSucceededResponse(jobDescriptor.key, - Some(0), - expectedOutputs, - None, - Seq.empty, - None, - resultGenerationMode = RunOnBackend - ) - - val jobPaths = JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, conf.backendConfig) - - whenReady(backend.execute) { executionResponse => - assertResponse(executionResponse, expectedResponse) - val localizedJsonInputFile = - DefaultPathBuilder.get(jobPaths.callInputsRoot.pathAsString, - jsonInputFile.parent.pathAsString.hashCode.toString + "/" + jsonInputFile.name - ) - val localizedCallInputFile = - DefaultPathBuilder.get(jobPaths.callInputsRoot.pathAsString, - callInputFile.parent.pathAsString.hashCode.toString + "/" + callInputFile.name - ) - - localizedJsonInputFile.isSymbolicLink shouldBe isSymlink - val realJsonInputFile = - if (isSymlink) localizedJsonInputFile.symbolicLink.get else localizedJsonInputFile - realJsonInputFile.toFile should exist - - localizedCallInputFile.isSymbolicLink shouldBe isSymlink - val realCallInputFile = - if (isSymlink) localizedCallInputFile.symbolicLink.get else localizedCallInputFile - realCallInputFile.toFile should exist - } - } - } - - it should "execute calls with input files and localize them appropriately" in { - localizationSpec(docker = false) - } - - it should "execute calls with input files and localize them appropriately (in Docker)" taggedAs DockerTest in { - localizationSpec(docker = true) - } - - it should "abort a job and kill a process" in { - val workflowDescriptor = buildWdlWorkflowDescriptor(Sleep20) - val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, - Map.empty, - WorkflowOptions.empty, - runtimeAttributeDefinitions - ) - val backendRef = createBackendRef(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor) - val backend = backendRef.underlyingActor - - val execute = backend.execute - backendRef ! AbortJobCommand - - whenReady(execute) { executionResponse => - executionResponse shouldBe a[JobAbortedResponse] - } - } - - def recoverSpec(completed: Boolean, writeReturnCode: Boolean = true): Assertion = { - val workflowDescriptor = buildWdlWorkflowDescriptor(HelloWorld) - val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, - Map.empty, - WorkflowOptions.empty, - runtimeAttributeDefinitions - ) - val backendRef = createBackendRef(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor) - val backend = backendRef.underlyingActor - - val jobPaths = JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, ConfigFactory.empty) - jobPaths.callExecutionRoot.createPermissionedDirectories() - jobPaths.stdout.write("Hello stubby ! ") - jobPaths.stderr.touch() - - val pid = - if (completed) { - if (writeReturnCode) - jobPaths.returnCode.write("0") - "0" - } else { - import sys.process._ - val proc = Seq("bash", "-c", s"sleep 2; echo 0 > ${jobPaths.returnCode}").run() - val pField = proc.getClass.getDeclaredField("p") - pField.setAccessible(true) - val p = pField.get(proc) - val pidField = p.getClass.getDeclaredField("pid") - pidField.setAccessible(true) - pidField.get(p).toString - } - - def execute = backend.recover - - val kvJobKey = - KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt) - val scopedKey = ScopedKey(workflowDescriptor.id, kvJobKey, SharedFileSystemAsyncJobExecutionActor.JobIdKey) - val kvPair = KvPair(scopedKey, pid) - - val previousKvPutter = TestProbe("previousKvPutter") - val kvPutReq = KvPut(kvPair) - backendRef.underlyingActor.serviceRegistryActor.tell(msg = kvPutReq, sender = previousKvPutter.ref) - previousKvPutter.expectMsg(KvPutSuccess(kvPutReq)) - val outputPort = jobDescriptor.taskCall.outputPorts.head - whenReady(execute, Timeout(10.seconds.dilated)) { executionResponse => - if (writeReturnCode) { - executionResponse should be(a[JobSucceededResponse]) - val succeededResponse = executionResponse.asInstanceOf[JobSucceededResponse] - succeededResponse.returnCode.value should be(0) - succeededResponse.jobOutputs should be(CallOutputs(Map(outputPort -> WomString("Hello stubby !")))) - } else { - executionResponse should be(a[JobFailedNonRetryableResponse]) - val failedResponse = executionResponse.asInstanceOf[JobFailedNonRetryableResponse] - failedResponse.returnCode should be(empty) - failedResponse.throwable should be(a[RuntimeException]) - failedResponse.throwable.getMessage should startWith("Unable to determine that 0 is alive, and") - failedResponse.throwable.getMessage should endWith("call-hello/execution/rc does not exist.") - } - } - } - - it should "recover a job in progress" in { - recoverSpec(completed = false) - } - - it should "recover a job that already completed" in { - recoverSpec(completed = true) - } - - it should "not recover a job for a non-existent pid" in { - recoverSpec(completed = true, writeReturnCode = false) - } - - it should "execute shards from a scatter" in { - val workflowDescriptor = buildWdlWorkflowDescriptor(TestWorkflows.Scatter) - - val call: CommandCallNode = workflowDescriptor.callable.taskCallNodes.head - - 0 to 2 foreach { shard => - // This assumes that engine will give us the evaluated value of the scatter item at the correct index - // If this is not the case, more context/logic will need to be moved to the backend so it can figure it out by itself - val symbolMaps: Map[LocallyQualifiedName, WomInteger] = Map("intNumber" -> WomInteger(shard)) - - val evaluatedAttributes = call.callable.runtimeAttributes.attributes - .safeMapValues(_.evaluateValue(Map.empty, NoIoFunctionSet).getOrElse(fail("Can't evaluate runtime attribute"))) - val runtimeAttributes: Map[LocallyQualifiedName, WomValue] = - RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, WorkflowOptions.empty)( - evaluatedAttributes - ) - - val jobDescriptor: BackendJobDescriptor = - BackendJobDescriptor(workflowDescriptor, - BackendJobDescriptorKey(call, Option(shard), 1), - runtimeAttributes, - fqnWdlMapToDeclarationMap(symbolMaps), - NoDocker, - None, - Map.empty - ) - val backend = createBackend(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor) - val response = - JobSucceededResponse( - mockBackendJobDescriptorKey, - Some(0), - WomMocks.mockOutputExpectations(Map("scattering.out" -> WomInteger(shard))), - None, - Seq.empty, - None, - resultGenerationMode = RunOnBackend - ) - executeJobAndAssertOutputs(backend, response) - } - } - - it should "post process outputs" in { - val inputFile = createCannedFile("localize", "content from json inputs").pathAsString - val inputs = Option(s"""{ - "wf_localize.localize.inputFile": "$inputFile" - }""") - val workflowDescriptor = buildWdlWorkflowDescriptor(OutputProcess, inputs) - val jobDescriptor: BackendJobDescriptor = jobDescriptorFromSingleCallWorkflow(workflowDescriptor, - Map.empty, - WorkflowOptions.empty, - runtimeAttributeDefinitions - ) - val backend = createBackend(jobDescriptor, TestConfig.backendRuntimeConfigDescriptor) - val jobPaths = - JobPathsWithDocker(jobDescriptor.key, workflowDescriptor, TestConfig.backendRuntimeConfigDescriptor.backendConfig) - val expectedA = WomSingleFile(jobPaths.callExecutionRoot.resolve("a").toAbsolutePath.pathAsString) - val expectedB = WomSingleFile(jobPaths.callExecutionRoot.resolve("dir").toAbsolutePath.resolve("b").pathAsString) - val expectedOutputs = WomMocks.mockOutputExpectations( - Map( - "localize.o1" -> expectedA, - "localize.o2" -> WomArray(WomArrayType(WomSingleFileType), Seq(expectedA, expectedB)), - "localize.o3" -> WomSingleFile(inputFile) - ) - ) - val expectedResponse = JobSucceededResponse(jobDescriptor.key, - Some(0), - expectedOutputs, - None, - Seq.empty, - None, - resultGenerationMode = RunOnBackend - ) - - executeJobAndAssertOutputs(backend, expectedResponse) - } - - it should "fail post processing if an output file is not found" in { - val expectedResponse = - JobFailedNonRetryableResponse(mockBackendJobDescriptorKey, - new FileNotFoundException("Could not process output, file not found:"), - Option(0) - ) - val workflow = TestWorkflow(buildWdlWorkflowDescriptor(MissingOutputProcess), - TestConfig.backendRuntimeConfigDescriptor, - expectedResponse - ) - val backend = createBackend(jobDescriptorFromSingleCallWorkflow(workflow.workflowDescriptor, - Map.empty, - WorkflowOptions.empty, - runtimeAttributeDefinitions - ), - workflow.config - ) - testWorkflow(workflow, backend) - } - - def createCannedFile(prefix: String, contents: String, parent: Option[Path] = None): Path = - DefaultPathBuilder.createTempFile(prefix, ".out", parent).write(contents) -} From e6e1ebdca6cb0377349b5818326144cebe5f26f4 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 12 Dec 2023 14:41:56 -0500 Subject: [PATCH 07/16] Try disabling cache. Remove invalid option. --- .github/set_up_cromwell_action/action.yml | 8 ++++---- runConfigurations/renderCiResources.run.xml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml index 1a20507b80b..330baf560be 100644 --- a/.github/set_up_cromwell_action/action.yml +++ b/.github/set_up_cromwell_action/action.yml @@ -10,10 +10,10 @@ runs: using: "composite" # <-- this allows these steps to be used by other workflows. steps: #Allows this github action to use a cache to store stuff like Java and sbt files between runs. - - uses: actions/checkout@v3 - name: Checkout Coursier Cache - - uses: coursier/cache-action@v6 - name: Enable Coursier Cache + # - uses: actions/checkout@v3 + # name: Checkout Coursier Cache + # - uses: coursier/cache-action@v6 + # name: Enable Coursier Cache #Cromwell requires git-secrets be setup. Here, we set up secrets and verify success with a script. - name: Git secrets setup diff --git a/runConfigurations/renderCiResources.run.xml b/runConfigurations/renderCiResources.run.xml index 2a5ab5e29b8..ef56bf5e4fd 100644 --- a/runConfigurations/renderCiResources.run.xml +++ b/runConfigurations/renderCiResources.run.xml @@ -7,8 +7,8 @@