Merge branch 'develop' into wx_927_moar_labels

centaur/src/main/resources/standardTestCases/gcpbatch_requester_pays_engine_functions.test

@@ -0,0 +1,16 @@
+name: gcpbatch_requester_pays_engine_functions
+testFormat: workflowsuccess
+backends: [GCPBATCHRequesterPays]
+workflowType: WDL
+workflowTypeVersion: 1.0
+tags: ["wdl_1.0"]
+
+files {
+  workflow: requester_pays_gcs/engine_functions.wdl
+  options: requester_pays_gcs/gcpbatch_engine_functions.options
+}
+
+metadata {
+  workflowName: requester_pays_engine_functions
+  status: Succeeded
+}

centaur/src/main/resources/standardTestCases/gcpbatch_requester_pays_localization.test

@@ -0,0 +1,16 @@
+name: gcpbatch_requester_pays_localization
+testFormat: workflowsuccess
+backends: [GCPBATCHRequesterPays]
+workflowType: WDL
+workflowTypeVersion: 1.0
+tags: ["wdl_1.0"]
+
+files {
+  workflow: requester_pays_gcs/localization.wdl
+  options: requester_pays_gcs/gcpbatch_localization.options
+}
+
+metadata {
+  workflowName: requester_pays_localization
+  status: Succeeded
+}

centaur/src/main/resources/standardTestCases/gcpbatch_requester_pays_localization_negative.test

@@ -0,0 +1,18 @@
+name: gcpbatch_requester_pays_localization_negative
+testFormat: workflowfailure
+# GCPBATCH backend configured SA does not have permission to bill the project, so this workflow should fail
+backends: [GCPBATCH]
+workflowType: WDL
+workflowTypeVersion: 1.0
+tags: ["wdl_1.0"]
+
+files {
+  workflow: requester_pays_gcs/localization.wdl
+}
+
+metadata {
+  workflowName: requester_pays_localization
+  status: Failed
+  "failures.0.message": "Workflow failed"
+  "failures.0.causedBy.0.message": ~~"Job failed due to task failure. Specifically, task with index 0 failed due to the following task event: \"Task state is updated from RUNNING to FAILED"
+}

centaur/src/main/resources/standardTestCases/hello_docker_image_format_v1.test

@@ -0,0 +1,31 @@
+# 'Hello world' for a public image in Docker v1 manifest format.
+name: hello_docker_image_format_v1
+testFormat: workflowsuccess
+backends: [Papiv2, GCPBATCH]
+
+# Attempting to pull locally I get an exit code 1, no image pulled, and the following output:
+# % docker pull "ubuntu:utopic-20150319"
+#  utopic-20150319: Pulling from library/ubuntu
+#
+#  What's next:
+#      View a summary of image vulnerabilities and recommendations → docker scout quickview ubuntu:utopic-20150319
+#  [DEPRECATION NOTICE] Docker Image Format v1 and Docker Image manifest version 2, schema 1 support is disabled by default and will be removed in an upcoming release. Suggest the author of docker.io/library/ubuntu:utopic-20150319 to upgrade the image to the OCI Format or Docker Image manifest v2, schema 2. More information at https://docs.docker.com/go/deprecated-image-specs/
+#
+# Also:
+# % docker scout quickview ubuntu:utopic-20150319
+#     ✗ Pull failed
+# ERROR   Status: failed to get image ubuntu:utopic-20150319: failed to pull image ubuntu:utopic-20150319: unsupported MediaType: "application/vnd.docker.distribution.manifest.v1+prettyjws", see https://github.com/google/go-containerregistry/issues/377, Code: 1
+
+files {
+  workflow: hello_docker_image_format_v1/hello_docker_image_format_v1.wdl
+  inputs: hello/hello.inputs
+}
+
+metadata {
+  workflowName: wf_hello
+  status: Succeeded
+  "calls.wf_hello.hello.executionStatus": Done
+  "calls.wf_hello.hello.runtimeAttributes.docker": "ubuntu:utopic-20150319"
+  "outputs.wf_hello.hello.salutation": "Hello m'Lord!"
+  "inputs.wf_hello.hello.addressee": "m'Lord"
+}

centaur/src/main/resources/standardTestCases/hello_docker_image_format_v1/hello_docker_image_format_v1.wdl

@@ -0,0 +1,19 @@
+task hello {
+  String addressee
+  command {
+    echo "Hello ${addressee}!"
+  }
+  output {
+    String salutation = read_string(stdout())
+  }
+  runtime {
+    docker: "ubuntu:utopic-20150319"
+  }
+}
+
+workflow wf_hello {
+  call hello
+  output {
+     hello.salutation
+  }
+}

centaur/src/main/resources/standardTestCases/requester_pays_engine_functions.test

@@ -1,6 +1,6 @@
 name: requester_pays_engine_functions
 testFormat: workflowsuccess
-backends: [Papiv2RequesterPays, GCPBATCH_NEEDS_ALT]
+backends: [Papiv2RequesterPays, GCPBATCH_ALT]
 workflowType: WDL
 workflowTypeVersion: 1.0
 tags: ["wdl_1.0"]

centaur/src/main/resources/standardTestCases/requester_pays_gcs/gcpbatch_engine_functions.options

@@ -0,0 +1,4 @@
+{
+    "jes_gcs_root": "gs://cromwell_bucket_with_requester_pays/cromwell-execution",
+    "backend": "GCPBATCHRequesterPays"
+}

centaur/src/main/resources/standardTestCases/requester_pays_gcs/gcpbatch_localization.options

@@ -0,0 +1,3 @@
+{
+  "backend": "GCPBATCHRequesterPays"
+}

centaur/src/main/resources/standardTestCases/requester_pays_localization.test

@@ -1,6 +1,6 @@
 name: requester_pays_localization
 testFormat: workflowsuccess
-backends: [Papiv2RequesterPays, GCPBATCH_NEEDS_ALT]
+backends: [Papiv2RequesterPays, GCPBATCH_ALT]
 workflowType: WDL
 workflowTypeVersion: 1.0
 tags: ["wdl_1.0"]

centaur/src/main/resources/standardTestCases/requester_pays_localization_negative.test

@@ -1,8 +1,7 @@
 name: requester_pays_localization_negative
 testFormat: workflowfailure
 # Papiv2 backend configured SA does not have permission to bill the project, so this workflow should fail
-# not sure
-backends: [Papiv2, GCPBATCH_FAIL]
+backends: [Papiv2, GCPBATCH_ALT]
 workflowType: WDL
 workflowTypeVersion: 1.0
 tags: ["wdl_1.0"]

src/ci/resources/gcp_batch_application.conf

@@ -8,5 +8,14 @@ backend {
         include "gcp_batch_provider_config.inc.conf"
       }
     }
+    GCPBATCHRequesterPays {
+      actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory"
+      config {
+        include "gcp_batch_provider_config.inc.conf"
+
+        # Need this again (even though it's in the "gcp_batch_shared" file because the provider_config include overrides it.
+        filesystems.gcs.auth = "requester_pays_service_account"
+      }
+    }
   }
 }

src/ci/resources/gcp_batch_shared_application.inc.conf

@@ -17,11 +17,11 @@ backend {
   # GCPBATCH_NEEDS_ALT: a GCPBATCH alt of a PAPI v2 test is needed but does not yet exist.
   # GCPBATCH_FAIL: a test is failing on GCPBATCH, reasons may or may not be understood yet.
   # GCPBATCH_SKIP: test is not going to be run on GCPBATCH for reasons explained in test comments.
-  enabled = ["GCPBATCH"]
+  enabled = ["GCPBATCH", "GCPBATCHRequesterPays"]
   providers {
     # Default gcp batch backend
     GCPBATCH {
-      actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory"
+      actor-factory = "REPLACEME!"
       config {
         # When importing: Remember to also include an appropriate provider_config.inc.conf here.
 
@@ -35,5 +35,18 @@ backend {
         filesystems.http {}
       }
     }
+    GCPBATCHRequesterPays {
+      actor-factory = "REPLACEME!"
+      config {
+        # When importing: Remember to also include an appropriate provider_config.inc.conf here.
+
+        include "dockerhub_provider_config_v2.inc.conf"
+        filesystems.gcs.auth = "requester_pays_service_account"
+        # In order to function as the compute service account in GCP Batch, the service account specified here must have
+        # Batch Agent Reporter role.  See WX-1576 and the tickets / docs linked from there for more info.
+        batch.compute-service-account = "requester-pays-authorized@broad-dsde-cromwell-dev.iam.gserviceaccount.com"
+        filesystems.http {}
+      }
+    }
   }
 }