Merge branch 'develop' into wx_927_google_compute_service_account

broadinstitute · Sep 25, 2024 · 5911489 · 5911489
2 parents af31a35 + 69e7e90
commit 5911489
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 3 deletions.
diff --git a/centaur/src/main/resources/standardTestCases/gcpbatch_monitoring_log_papiv2.test b/centaur/src/main/resources/standardTestCases/gcpbatch_monitoring_log_papiv2.test
@@ -0,0 +1,18 @@
+# Test to confirm the output of the monitoring script will be written to a monitoring.log file
+
+name: gcpbatch_monitoring_log
+testFormat: workflowsuccess
+backends: [GCPBATCH]
+
+files {
+  workflow: monitoring_log/monitoring_log.wdl
+  options: monitoring_log/monitoring_log.options
+}
+
+metadata {
+  "calls.monitoring_log.get_stats.gcpBatch.monitoringScript": "gs://cloud-cromwell-dev/some/rounding_script_int.sh"
+  "calls.monitoring_log.get_stats.monitoringLog": "gs://cloud-cromwell-dev-self-cleaning/cromwell_execution/ci/monitoring_log/<<UUID>>/call-get_stats/monitoring.log"
+  "outputs.monitoring_log.get_stats.stats.0": "CPU: 1"
+  "outputs.monitoring_log.get_stats.stats.1": "Total Memory: 2G"
+  "outputs.monitoring_log.get_stats.stats.2": "Total Disk space: 9G"
+}
diff --git a/centaur/src/main/resources/standardTestCases/monitoring_log_papiv2.test b/centaur/src/main/resources/standardTestCases/monitoring_log_papiv2.test
@@ -2,8 +2,7 @@
 
 name: monitoring_log
 testFormat: workflowsuccess
-# see doc 2.14
-backends: [Papiv2, GCPBATCH_FAIL]
+backends: [Papiv2, GCPBATCH_ALT]
 
 files {
   workflow: monitoring_log/monitoring_log.wdl

diff --git a/...s/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchJobPaths.scala b/...s/google/batch/src/main/scala/cromwell/backend/google/batch/models/GcpBatchJobPaths.scala
@@ -1,6 +1,7 @@
 package cromwell.backend.google.batch.models
 
 import cromwell.backend.BackendJobDescriptorKey
+import cromwell.backend.google.batch.runnable.GcpBatchMetadataKeys
 import cromwell.backend.io.JobPaths
 import cromwell.core.path.Path
 
@@ -36,5 +37,19 @@ case class GcpBatchJobPaths(override val workflowPaths: GcpBatchWorkflowPaths,
   val batchMonitoringScriptFilename: String = s"${GcpBatchJobPaths.BatchMonitoringKey}.sh"
   val batchMonitoringImageScriptFilename: String = s"${GcpBatchJobPaths.BatchMonitoringImageKey}.sh"
 
+  val jesMonitoringLogFilename: String = s"${GcpBatchJobPaths.BatchMonitoringKey}.log"
+  lazy val jesMonitoringLogPath: Path = callExecutionRoot.resolve(jesMonitoringLogFilename)
+
+  override lazy val customMetadataPaths =
+    workflowPaths.monitoringScriptPath map { p =>
+      Map(GcpBatchMetadataKeys.MonitoringScript -> p, GcpBatchMetadataKeys.MonitoringLog -> jesMonitoringLogPath)
+    } getOrElse Map.empty
+
+  override def standardOutputAndErrorPaths: Map[String, Path] =
+    super.standardOutputAndErrorPaths map { case (k, v) =>
+      val updated = workflowPaths.standardStreamNameToFileNameMetadataMapper(this, k)
+      k -> v.parent.resolve(updated)
+    }
+
   override def forCallCacheCopyAttempts: JobPaths = this.copy(isCallCacheCopyAttempt = true)
 }
diff --git a/...le/batch/src/main/scala/cromwell/backend/google/batch/runnable/GcpBatchMetadataKeys.scala b/...le/batch/src/main/scala/cromwell/backend/google/batch/runnable/GcpBatchMetadataKeys.scala
@@ -4,8 +4,9 @@ object GcpBatchMetadataKeys {
   val GoogleProject = "gcpBatch:googleProject"
   val ExecutionBucket = "gcpBatch:executionBucket"
   val MonitoringScript = "gcpBatch:monitoringScript"
+  val MonitoringLog = "monitoringLog"
+  // TODO unused metadata keys below, this is probably not right and there may not be tests to cover these
   val MachineType = "gcpBatch:machineType"
   val Zone = "gcpBatch:zone"
   val InstanceName = "gcpBatch:instanceName"
-  val MonitoringLog = "monitoringLog"
 }