diff --git a/centaur/src/main/resources/standardTestCases/drs_tests/gcpbatch_drs_usa_jdr.wdl b/centaur/src/main/resources/standardTestCases/drs_tests/gcpbatch_drs_usa_jdr.wdl new file mode 100644 index 00000000000..1cf35d219fc --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/drs_tests/gcpbatch_drs_usa_jdr.wdl @@ -0,0 +1,115 @@ +version 1.0 + +workflow drs_usa_jdr { + input { + File file1 + File file2 + } + + call localize_jdr_drs_with_usa { + input: + file1 = file1, + file2 = file2 + } + + call skip_localize_jdr_drs_with_usa { + input: + file1 = file1, + file2 = file2 + } + + call read_drs_with_usa { + input: + file1 = file1, + file2 = file2 + } + + output { + String path1 = localize_jdr_drs_with_usa.path1 + String path2 = localize_jdr_drs_with_usa.path2 + String hash1 = localize_jdr_drs_with_usa.hash1 + String hash2 = localize_jdr_drs_with_usa.hash2 + Float size1 = localize_jdr_drs_with_usa.size1 + Float size2 = localize_jdr_drs_with_usa.size2 + String cloud1 = skip_localize_jdr_drs_with_usa.path1 + String cloud2 = skip_localize_jdr_drs_with_usa.path2 + Map[String, String] map1 = read_drs_with_usa.map1 + Map[String, String] map2 = read_drs_with_usa.map2 + } +} + +task localize_jdr_drs_with_usa { + input { + File file1 + File file2 + } + + command <<< + echo ~{file1} > path1 + echo ~{file2} > path2 + md5sum ~{file1} | cut -c1-32 > hash1 + md5sum ~{file2} | cut -c1-32 > hash2 + >>> + + output { + String path1 = read_string("path1") + String path2 = read_string("path2") + String hash1 = read_string("hash1") + String hash2 = read_string("hash2") + Float size1 = size(file1) + Float size2 = size(file2) + } + + runtime { + docker: "ubuntu:latest" + backend: "GCPBATCH-usa" + } +} + +task skip_localize_jdr_drs_with_usa { + input { + File file1 + File file2 + } + + parameter_meta { + file1: { localization_optional: true } + file2: { localization_optional: true } + } + + command <<< + echo ~{file1} > path1 + echo ~{file2} > path2 + >>> + + output { + String path1 = read_string("path1") + String path2 = read_string("path2") + } + + runtime { + docker: "ubuntu:latest" + backend: "GCPBATCH-usa" + } +} + +task read_drs_with_usa { + input { + File file1 + File file2 + } + + command <<< + echo file is read by the engine + >>> + + output { + Map[String, String] map1 = read_json(file1) + Map[String, String] map2 = read_json(file2) + } + + runtime { + docker: "ubuntu:latest" + backend: "GCPBATCH-usa" + } +} diff --git a/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test b/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test index 58a9977679c..4404d97a73e 100644 --- a/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test +++ b/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test @@ -1,6 +1,6 @@ name: drs_usa_jdr testFormat: WorkflowSuccess -backends: ["papi-v2-usa", "GCPBATCH_NEEDS_ALT"] +backends: ["papi-v2-usa", "GCPBATCH_ALT"] tags: [ drs ] skipDescribeEndpointValidation: true diff --git a/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test b/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test index db2780566a1..8ba0547b38d 100644 --- a/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test +++ b/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test @@ -1,6 +1,6 @@ name: drs_usa_jdr_preresolve testFormat: WorkflowSuccess -backends: ["papi-v2-usa", GCPBATCH_NEEDS_ALT] +backends: ["papi-v2-usa", GCPBATCH_ALT] tags: [ drs ] skipDescribeEndpointValidation: true diff --git a/centaur/src/main/resources/standardTestCases/gcpbatch_drs_usa_jdr.test b/centaur/src/main/resources/standardTestCases/gcpbatch_drs_usa_jdr.test new file mode 100644 index 00000000000..cbba3a7c22a --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/gcpbatch_drs_usa_jdr.test @@ -0,0 +1,36 @@ +name: gcpbatch_drs_usa_jdr +testFormat: WorkflowSuccess +backends: ["GCPBATCH-usa"] +tags: [ drs ] +skipDescribeEndpointValidation: true + +files { + workflow: drs_tests/gcpbatch_drs_usa_jdr.wdl + options-dir: "Error: BA-6546 The environment variable CROMWELL_BUILD_RESOURCES_DIRECTORY must be set/export pointing to a valid path such as '${YOUR_CROMWELL_DIR}/target/ci/resources'" + options-dir: ${?CROMWELL_BUILD_RESOURCES_DIRECTORY} + options: ${files.options-dir}/papi_v2_usa.options.json + inputs: drs_tests/drs_usa_jdr.inputs +} + +metadata { + workflowName: drs_usa_jdr + status: Succeeded + + "outputs.drs_usa_jdr.path1" = + "/mnt/disks/cromwell_root/drs_localization_paths/CromwellSimpleWithFilerefs/hello_jade.json" + "outputs.drs_usa_jdr.path2" = + "/mnt/disks/cromwell_root/drs_localization_paths/CromwellSimpleWithFilerefs2/hello_jade_2.json" + "outputs.drs_usa_jdr.hash1" = "faf12e94c25bef7df62e4a5eb62573f5" + "outputs.drs_usa_jdr.hash2" = "19e1b021628130fda04c79ee9a056b67" + "outputs.drs_usa_jdr.size1" = 18.0 + "outputs.drs_usa_jdr.size2" = 38.0 + # This JDR file has a gsUri that doesn't end in /fileName so it must be downloaded with the DRS localizer + "outputs.drs_usa_jdr.cloud1" = + "/mnt/disks/cromwell_root/drs_localization_paths/CromwellSimpleWithFilerefs/hello_jade.json" + # This JDR file has a gsUri that can skip localization + "outputs.drs_usa_jdr.cloud2" = + "gs://broad-jade-dev-data-bucket/e1941fb9-6537-4e1a-b70d-34352a3a7817/ad783b60-aeba-4055-8f7b-194880f37259/hello_jade_2.json" + "outputs.drs_usa_jdr.map1.hello" = "jade" + "outputs.drs_usa_jdr.map2.hello" = "jade" + "outputs.drs_usa_jdr.map2.attempt" = "2" +} diff --git a/centaur/src/main/resources/standardTestCases/gcpbatch_drs_usa_jdr_preresolve.test b/centaur/src/main/resources/standardTestCases/gcpbatch_drs_usa_jdr_preresolve.test new file mode 100644 index 00000000000..9ca9f7ceaf3 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/gcpbatch_drs_usa_jdr_preresolve.test @@ -0,0 +1,40 @@ +name: gcpbatch_drs_usa_jdr_preresolve +testFormat: WorkflowSuccess +backends: ["GCPBATCH-usa"] +tags: [ drs ] +skipDescribeEndpointValidation: true + +files { + workflow: drs_tests/gcpbatch_drs_usa_jdr.wdl + options-dir: "Error: BA-6546 The environment variable CROMWELL_BUILD_RESOURCES_DIRECTORY must be set/export pointing to a valid path such as '${YOUR_CROMWELL_DIR}/target/ci/resources'" + options-dir: ${?CROMWELL_BUILD_RESOURCES_DIRECTORY} + options: ${files.options-dir}/gcpbatch_papi_v2_usa_preresolve.options.json + inputs: drs_tests/drs_usa_jdr.inputs +} + +metadata { + workflowName: drs_usa_jdr + status: Succeeded + + "outputs.drs_usa_jdr.path1" = + "/mnt/disks/cromwell_root/drs_localization_paths/CromwellSimpleWithFilerefs/hello_jade.json" + # This JDR file has a gsUri that can be preresolved to a regular GCS file for improved localization performance. + # However this means that the file's container path is determined by the GCS localization logic and not the + # `localizationPath`-aware DRS localization logic. The GCS localization logic always uses a containerized version + # of the GCS path, which is what this expectation represents. + "outputs.drs_usa_jdr.path2" = + "/mnt/disks/cromwell_root/broad-jade-dev-data-bucket/e1941fb9-6537-4e1a-b70d-34352a3a7817/ad783b60-aeba-4055-8f7b-194880f37259/hello_jade_2.json" + "outputs.drs_usa_jdr.hash1" = "faf12e94c25bef7df62e4a5eb62573f5" + "outputs.drs_usa_jdr.hash2" = "19e1b021628130fda04c79ee9a056b67" + "outputs.drs_usa_jdr.size1" = 18.0 + "outputs.drs_usa_jdr.size2" = 38.0 + # This JDR file has a gsUri that doesn't end in /fileName so it must be downloaded with the DRS localizer + "outputs.drs_usa_jdr.cloud1" = + "/mnt/disks/cromwell_root/drs_localization_paths/CromwellSimpleWithFilerefs/hello_jade.json" + # This JDR file has a gsUri that can skip localization + "outputs.drs_usa_jdr.cloud2" = + "gs://broad-jade-dev-data-bucket/e1941fb9-6537-4e1a-b70d-34352a3a7817/ad783b60-aeba-4055-8f7b-194880f37259/hello_jade_2.json" + "outputs.drs_usa_jdr.map1.hello" = "jade" + "outputs.drs_usa_jdr.map2.hello" = "jade" + "outputs.drs_usa_jdr.map2.attempt" = "2" +} diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala index 64f36239db5..14ccf0bff8d 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala @@ -233,7 +233,9 @@ class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]], ) } else { IO.raiseError( - new RuntimeException(s"Exhausted $resolutionRetries resolution retries to resolve $drsUrlToResolve.drsUrl", t) + new RuntimeException(s"Exhausted $resolutionRetries resolution retries to resolve ${drsUrlToResolve.drsUrl}", + t + ) ) } diff --git a/src/ci/resources/gcp_batch_application.conf b/src/ci/resources/gcp_batch_application.conf index 5e2843290e2..ee1650f14e6 100644 --- a/src/ci/resources/gcp_batch_application.conf +++ b/src/ci/resources/gcp_batch_application.conf @@ -41,6 +41,12 @@ backend { include "gcp_batch_provider_config.inc.conf" } } + GCPBATCH-usa { + actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory" + config { + include "gcp_batch_provider_config.inc.conf" + } + } GCPBATCHParallelCompositeUploads { actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory" config { diff --git a/src/ci/resources/gcp_batch_shared_application.inc.conf b/src/ci/resources/gcp_batch_shared_application.inc.conf index 1944a2a50b0..2202a744901 100644 --- a/src/ci/resources/gcp_batch_shared_application.inc.conf +++ b/src/ci/resources/gcp_batch_shared_application.inc.conf @@ -60,6 +60,24 @@ backend { filesystems.http {} } } + GCPBATCH-usa { + actor-factory = "REPLACEME!" + config { + # When importing: Remember to also include an appropriate provider_config.inc.conf here. + + project = "user_error: google_project must be set in workflow options http://cromwell.readthedocs.io/en/develop/wf_options/Google/" + root = "user_error: jes_gcs_root must be set in workflow options http://cromwell.readthedocs.io/en/develop/wf_options/Google/" + batch.compute-service-account = "user_error: google_compute_service_account must be set in workflow options http://cromwell.readthedocs.io/en/develop/wf_options/Google/" + batch.auth = "user_service_account" + filesystems.http {} + filesystems.drs.auth = "user_service_account" + filesystems.gcs.auth = "user_service_account" + filesystems.gcs.project = "user_error: user_service_account must be set in workflow options http://cromwell.readthedocs.io/en/develop/wf_options/Google/" + + # Have the engine authenticate to docker.io. See BT-141 for more info. + include "dockerhub_provider_config_v1.inc.conf" + } + } GCPBATCHParallelCompositeUploads { actor-factory = "REPLACEME!" config { diff --git a/src/ci/resources/gcpbatch_papi_v2_usa_preresolve.options.json.ctmpl b/src/ci/resources/gcpbatch_papi_v2_usa_preresolve.options.json.ctmpl new file mode 100644 index 00000000000..09abfcb111f --- /dev/null +++ b/src/ci/resources/gcpbatch_papi_v2_usa_preresolve.options.json.ctmpl @@ -0,0 +1,12 @@ +{{with $cromwellServiceAccount := secret (printf "secret/dsde/cromwell/common/cromwell-service-account.json")}} +{ + "google_compute_service_account": "{{$cromwellServiceAccount.Data.client_email}}", + "user_service_account_json": "{{$cromwellServiceAccount.Data | toJSON | replaceAll "\\" "\\\\" | replaceAll "\"" "\\\"" }}", + "google_project": "broad-dsde-cromwell-dev", + "jes_gcs_root": "gs://cloud-cromwell-dev-self-cleaning/cromwell_execution/ci", +{{/* We're re-using the WDL with the same inputs in multiple tests, make sure we don't accidentally cache hit */}} + "read_from_cache": false, +{{/* Test pre-resolving DrsPath to GcsPath */}} + "override_preresolve_for_test": true +} +{{end}} diff --git a/src/ci/resources/papi_v2_usa_preresolve.options.json.ctmpl b/src/ci/resources/papi_v2_usa_preresolve.options.json.ctmpl index 09abfcb111f..12706637ac6 100644 --- a/src/ci/resources/papi_v2_usa_preresolve.options.json.ctmpl +++ b/src/ci/resources/papi_v2_usa_preresolve.options.json.ctmpl @@ -3,7 +3,7 @@ "google_compute_service_account": "{{$cromwellServiceAccount.Data.client_email}}", "user_service_account_json": "{{$cromwellServiceAccount.Data | toJSON | replaceAll "\\" "\\\\" | replaceAll "\"" "\\\"" }}", "google_project": "broad-dsde-cromwell-dev", - "jes_gcs_root": "gs://cloud-cromwell-dev-self-cleaning/cromwell_execution/ci", + "gcp_batch_gcs_root": "gs://cloud-cromwell-dev-self-cleaning/cromwell_execution/ci", {{/* We're re-using the WDL with the same inputs in multiple tests, make sure we don't accidentally cache hit */}} "read_from_cache": false, {{/* Test pre-resolving DrsPath to GcsPath */}}