diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index ebafe51064c..61ed3c5af41 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -32,6 +32,9 @@ jobs:
- build_type: centaurPapiV2beta
build_mysql: 5.7
friendly_name: Centaur Papi V2 Beta with MySQL 5.7
+ - build_type: centaurPapiV2betaRestart
+ build_mysql: 5.7
+ friendly_name: Centaur Papi V2 Beta (restart)
- build_type: dbms
friendly_name: DBMS
- build_type: centaurTes
diff --git a/.gitignore b/.gitignore
index 250b6aa3c16..94accae9038 100644
--- a/.gitignore
+++ b/.gitignore
@@ -58,6 +58,10 @@ tesk_application.conf
**/venv/
exome_germline_single_sample_v1.3/
**/*.pyc
+src/ci/resources/*.temp
# GHA credentials
gha-creds-*.json
+
+# jenv
+.java-version
diff --git a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test
index 1ac12322f4f..182a833fbdb 100644
--- a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test
+++ b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test
@@ -2,6 +2,7 @@ name: failures.restart_while_failing_jes
testFormat: WorkflowFailureRestartWithRecover
callMark: restart_while_failing.B1
backends: [Papi]
+tags: [restart]
files {
workflow: failures/restart_while_failing/restart_while_failing.wdl
diff --git a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test
index f686b16c817..58abbf78245 100644
--- a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test
+++ b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test
@@ -3,7 +3,7 @@ testFormat: WorkflowFailureRestartWithRecover
callMark: restart_while_failing.B1
backendsMode: "only"
backends: [Local, LocalNoDocker]
-tags: [localdockertest]
+tags: [localdockertest, restart]
files {
workflow: failures/restart_while_failing/restart_while_failing.wdl
diff --git a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test
index 64ea5fd8906..9bb9c1a55d6 100644
--- a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test
+++ b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test
@@ -2,7 +2,7 @@ name: failures.restart_while_failing_tes
testFormat: WorkflowFailureRestartWithoutRecover
callMark: restart_while_failing.B1
backends: [TES]
-tags: [localdockertest]
+tags: [localdockertest, restart]
files {
workflow: failures/restart_while_failing/restart_while_failing.wdl
diff --git a/project/ContinuousIntegration.scala b/project/ContinuousIntegration.scala
index d5dd3262f87..f0516150cf5 100644
--- a/project/ContinuousIntegration.scala
+++ b/project/ContinuousIntegration.scala
@@ -9,6 +9,8 @@ object ContinuousIntegration {
lazy val ciSettings: Seq[Setting[_]] = List(
srcCiResources := sourceDirectory.value / "ci" / "resources",
targetCiResources := target.value / "ci" / "resources",
+ envFile := srcCiResources.value / "env.temp", //generated by resources/acquire_b2c_token.sh
+
vaultToken := userHome / ".vault-token",
copyCiResources := {
IO.copyDirectory(srcCiResources.value, targetCiResources.value)
@@ -26,7 +28,12 @@ object ContinuousIntegration {
if (vaultToken.value.isDirectory) {
sys.error(s"""The vault token file "${vaultToken.value}" should not be a directory.""")
}
- val cmd = List(
+
+ // Only include the local file argument if the file exists (local development w/ acquire_b2c_token.sh)
+ // Don't include it otherwise (CI/CD and other development)
+ val localEnvFileArgs = if(envFile.value.exists()) List("-e", s"ENV_FILE=${envFile.value}") else List()
+
+ val cmd: List[String] = List.concat(List(
"docker",
"run",
"--rm",
@@ -35,7 +42,9 @@ object ContinuousIntegration {
"-v",
s"${srcCiResources.value}:${srcCiResources.value}",
"-v",
- s"${targetCiResources.value}:${targetCiResources.value}",
+ s"${targetCiResources.value}:${targetCiResources.value}"),
+ localEnvFileArgs,
+ List(
"-e",
"ENVIRONMENT=not_used",
"-e",
@@ -44,7 +53,7 @@ object ContinuousIntegration {
s"OUT_PATH=${targetCiResources.value}",
"broadinstitute/dsde-toolbox:dev",
"render-templates.sh"
- )
+ ))
val result = cmd ! log
if (result != 0) {
sys.error(
@@ -70,6 +79,7 @@ object ContinuousIntegration {
private val srcCiResources: SettingKey[File] = settingKey[File]("Source directory for CI resources")
private val targetCiResources: SettingKey[File] = settingKey[File]("Target directory for CI resources")
private val vaultToken: SettingKey[File] = settingKey[File]("File with the vault token")
+ private val envFile: SettingKey[File] = settingKey[File]("File with the environment variables needed to render CI resources.")
/**
* For "reasons" these projects are excluded from the root aggregation in build.sbt.
diff --git a/runConfigurations/Repo template_ Cromwell server TES.run.xml b/runConfigurations/Repo template_ Cromwell server TES.run.xml
index 32127027ecc..192b73735b8 100644
--- a/runConfigurations/Repo template_ Cromwell server TES.run.xml
+++ b/runConfigurations/Repo template_ Cromwell server TES.run.xml
@@ -19,4 +19,24 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/runConfigurations/Repo template_ Cromwell server Terra TES.run.xml b/runConfigurations/Repo template_ Cromwell server Terra TES.run.xml
new file mode 100644
index 00000000000..d5d7ef07b98
--- /dev/null
+++ b/runConfigurations/Repo template_ Cromwell server Terra TES.run.xml
@@ -0,0 +1,23 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh
index 36b215a2f52..c894909cd3f 100755
--- a/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh
+++ b/src/ci/bin/testCentaurHoricromtalPapiV2beta.sh
@@ -25,6 +25,7 @@ cromwell::build::run_centaur \
-e relative_output_paths_colliding \
-e standard_output_paths_colliding_prevented \
-e papi_v2alpha1_gcsa \
+ -e restart \
cromwell::build::generate_code_coverage
diff --git a/src/ci/bin/testCentaurPapiV2beta.sh b/src/ci/bin/testCentaurPapiV2beta.sh
index 43d5a7d62aa..d66d2a14334 100755
--- a/src/ci/bin/testCentaurPapiV2beta.sh
+++ b/src/ci/bin/testCentaurPapiV2beta.sh
@@ -23,6 +23,7 @@ cromwell::build::run_centaur \
-e relative_output_paths_colliding \
-e standard_output_paths_colliding_prevented \
-e papi_v2alpha1_gcsa \
+ -e restart \
cromwell::build::generate_code_coverage
diff --git a/src/ci/bin/testCentaurPapiV2betaRestart.sh b/src/ci/bin/testCentaurPapiV2betaRestart.sh
new file mode 100755
index 00000000000..e0d50bb18c6
--- /dev/null
+++ b/src/ci/bin/testCentaurPapiV2betaRestart.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail
+export CROMWELL_BUILD_REQUIRES_SECURE=true
+# import in shellcheck / CI / IntelliJ compatible ways
+# shellcheck source=/dev/null
+source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
+# shellcheck source=/dev/null
+source "${BASH_SOURCE%/*}/test_papi.inc.sh" || source test_papi.inc.sh
+
+cromwell::build::setup_common_environment
+
+cromwell::build::setup_centaur_environment
+
+cromwell::build::papi::setup_papi_centaur_environment
+
+cromwell::build::assemble_jars
+
+cromwell::build::run_centaur \
+ -p 100 \
+ -i restart \
+
+cromwell::build::generate_code_coverage
+
+cromwell::build::print_workflow_statistics
diff --git a/src/ci/resources/acquire_b2c_token.sh b/src/ci/resources/acquire_b2c_token.sh
new file mode 100755
index 00000000000..23eb25461c6
--- /dev/null
+++ b/src/ci/resources/acquire_b2c_token.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# This script acquires a b2c token using gcloud auth and writes it to a file.
+# This file is used by renderCiResources to populate the CROMWELL_B2C_TOKEN in tes_application.conf.
+# It should be run as a "Before Launch" task in IntelliJ so the token is refreshed regularly.
+
+### Setting up this script for the first time in IntelliJ?
+### 1. Open the "Run/Debug Configurations" dialog for the TES repo template.
+### 2. Add a new "Before Launch" task, selecting "Run External Tool".
+### This task should run BEFORE the renderCiResources task.
+### 3. Configure the new task:
+### - Name: "Acquire b2c token"
+### - Program: /full/path/to/src/ci/resources/acquire_b2c_token.sh
+### - Arguments: ./env.temp
+### - Working directory: /full/path/to/src/ci/resources
+### - Advanced Options > Uncheck "Synchronize files after execution"
+### 4. Click "OK" to save the new task. Apply and close the "Run/Debug Configurations" dialog.
+
+# User must provide an output file path as an argument.
+# This can be provided by IntelliJ as a program argument in the run configuration.
+if [ -z "$1" ]; then
+ echo "Error: Must specify an output file path for the environment file."
+ exit 1
+fi
+
+# Acquire a b2c token using gcloud auth.
+# Send stderr to /dev/null so that any gcloud auth errors/update prompts don't pollute the token output.
+echo "Using local gcloud auth to acquire a b2c token..."
+B2C_TOKEN=$(gcloud auth print-access-token 2>/dev/null)
+if [ $? -eq 0 ]; then
+ echo "Acquired b2c token: ${B2C_TOKEN:0:4}****"
+else
+ echo "Failed to acquire b2c token. Is your local shell logged into gcloud?"
+fi
+
+# Create the output directory if it doesn't exist
+output_dir=$(dirname "$1")
+mkdir -p "$output_dir"
+
+# Write the token to the output file. dsde-toolbox will call `source` on this file that we're writing,
+# so it's expected that this file contains valid bash syntax for setting an environment variable.
+echo "export CROMWELL_B2C_TOKEN=${B2C_TOKEN}" > "$1"
diff --git a/src/ci/resources/papi_v2beta_restart_application.conf b/src/ci/resources/papi_v2beta_restart_application.conf
new file mode 100644
index 00000000000..2f0bb75860b
--- /dev/null
+++ b/src/ci/resources/papi_v2beta_restart_application.conf
@@ -0,0 +1 @@
+include "papi_v2beta_application.conf"
diff --git a/src/ci/resources/terra_tes_application.conf.ctmpl b/src/ci/resources/terra_tes_application.conf.ctmpl
new file mode 100644
index 00000000000..92ae72f9e7a
--- /dev/null
+++ b/src/ci/resources/terra_tes_application.conf.ctmpl
@@ -0,0 +1,80 @@
+include required(classpath("application.conf"))
+include "build_application.inc.conf"
+
+filesystems {
+ blob {
+ class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+ global {
+ class = "cromwell.filesystems.blob.BlobFileSystemManager"
+ config {
+ workspace-manager: {
+ # static: Where is WSM Endpoint
+ url: "https://workspace.dsde-dev.broadinstitute.org"
+ # See comment in src/ci/resources/acquire_b2c_token.sh for the one-time setup required to automatically acquire this.
+ # This token can also be manually acquired with gcloud auth print-access-token myAccount@gmail.com
+ b2cToken: "{{ env "CROMWELL_B2C_TOKEN" }}"
+ }
+ }
+ }
+ }
+}
+
+engine {
+ filesystems {
+ local {
+ enabled: true
+ }
+ http {
+ enabled: true
+ }
+ blob {
+ enabled: true
+ }
+ }
+}
+
+
+backend {
+ default = "TES"
+ providers {
+ TES {
+ actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
+ config {
+ # Root of a blob storage container to use
+ # Might look something like: https://lz813a3d637adefec2c6e88f.blob.core.windows.net/sc-d8143fd8-aa07-446d-9ba0-af72203f1794/some/path/"
+ root = ""
+ dockerRoot = "/cromwell-executions"
+ # TES Endpoint for cromwell to use. Might look something like: "https://lz7388ada396994bb48ea5c87a02eed673689c82c2af423d03.servicebus.windows.net/something/tes/v1/tasks
+ endpoint = ""
+ concurrent-job-limit = 1000
+ # Identity to execute the workflow as. Might look something like: "pet-2676571657071603a9eab"
+ workflow-execution-identity = ""
+ # See comment in src/ci/resources/acquire_b2c_token.sh for the one-time setup required to automatically acquire this.
+ # This token can also be manually acquired with gcloud auth print-access-token myAccount@gmail.com
+ bearer-token="Bearer {{ env "CROMWELL_B2C_TOKEN" }}"
+ # Override default polling to make it faster for speedy tests
+ poll-backoff {
+ min: "10 seconds"
+ max: "5 minutes"
+ multiplier: 1.1
+ randomization-factor: 0.5
+ }
+
+ filesystems {
+ blob {
+ enabled: true
+ }
+ local {
+ enabled: false
+ }
+ http {
+ enabled: true
+ }
+ }
+ }
+
+ # Have the engine (maybe) authenticate to docker.io. See BT-141 for more info.
+ include "dockerhub_provider_config_v1.inc.conf"
+ }
+ }
+}
diff --git a/src/ci/resources/tes_application.conf b/src/ci/resources/tes_application.conf
index 5eb812a52d4..29982575eab 100644
--- a/src/ci/resources/tes_application.conf
+++ b/src/ci/resources/tes_application.conf
@@ -2,26 +2,12 @@ include required(classpath("application.conf"))
include "build_application.inc.conf"
filesystems {
- blob {
- class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
- global {
- # One BFSM is shared across all BlobPathBuilders
- class = "cromwell.filesystems.blob.BlobFileSystemManager"
- config {
- container: "cromwell"
- endpoint: "https://.blob.core.windows.net"
- subscription: "00001111-2222-3333-aaaa-bbbbccccdddd"
- # WSM config is needed for accessing WSM-managed blob containers
- # created in Terra workspaces.
- workspace-manager {
- url: "https://workspace.dsde-dev.broadinstitute.org"
- workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
- container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
- b2cToken: "Zardoz"
+ blob {
+ class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+ global {
+ class = "cromwell.filesystems.blob.BlobFileSystemManager"
}
- }
}
- }
}
engine {
@@ -33,7 +19,7 @@ engine {
enabled: true
}
blob {
- enabled: false
+ enabled: true
}
}
}
@@ -45,27 +31,29 @@ backend {
TES {
actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
config {
- # Use for running on blob storage
- #root = "https://.blob.core.windows.net/cromwell/cromwell-executions"
- root = "cromwell-executions"
+ # Root of a blob storage container to use
+ # Might look something like: https://lz813a3d637adefec2c6e88f.blob.core.windows.net/sc-d8143fd8-aa07-446d-9ba0-af72203f1794/some/path/"
+ root = ""
dockerRoot = "/cromwell-executions"
+ # TES Endpoint for cromwell to use. Might look something like: "https://lz7388ada396994bb48ea5c87a02eed673689c82c2af423d03.servicebus.windows.net/something/tes/v1/tasks
endpoint = "http://127.0.0.1:9000/v1/tasks"
concurrent-job-limit = 1000
-
+ # Identity to execute the workflow as. Might look something like: "pet-2676571657071603a9eab"
+ workflow-execution-identity = ""
# Override default polling to make it faster for speedy tests
poll-backoff {
- min: "1 seconds"
+ min: "10 seconds"
max: "5 minutes"
multiplier: 1.1
randomization-factor: 0.5
}
-
+
filesystems {
blob {
- enabled: false
+ enabled: true
}
local {
- enabled: true
+ enabled: false
}
http {
enabled: true