From ef8745af3ead6bed4f822fc9a2dddb922728e9a9 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Mon, 9 Dec 2024 17:59:03 -0500 Subject: [PATCH] AN-311 Remove BigQuery Centaur error reporting (#7668) --- .gitignore | 1 - .../centaur/reporting/BigQueryReporter.scala | 231 ------------------ project/Dependencies.scala | 2 - src/ci/resources/centaur_application.conf | 1 - .../resources/centaur_secure.inc.conf.ctmpl | 13 - 5 files changed, 248 deletions(-) delete mode 100644 centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala delete mode 100644 src/ci/resources/centaur_secure.inc.conf.ctmpl diff --git a/.gitignore b/.gitignore index 5d82610b828..e355705cf37 100644 --- a/.gitignore +++ b/.gitignore @@ -40,7 +40,6 @@ scripts/docker-compose-mysql/compose/mysql/data # Vault rendered resources artifactory_credentials.properties aws_credentials -centaur_secure.inc.conf cromwell-centaur-requester-pays-service-account.json cromwell-centaur-service-account.json cromwell-service-account.json diff --git a/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala b/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala deleted file mode 100644 index b2335e4a794..00000000000 --- a/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala +++ /dev/null @@ -1,231 +0,0 @@ -package centaur.reporting - -import java.time.OffsetDateTime -import java.util - -import cats.effect.IO -import cats.instances.list._ -import cats.syntax.apply._ -import cats.syntax.traverse._ -import centaur.reporting.BigQueryReporter._ -import centaur.test.CentaurTestException -import centaur.test.metadata.CallAttemptFailure -import com.google.api.gax.retrying.RetrySettings -import com.google.api.services.bigquery.BigqueryScopes -import com.google.auth.Credentials -import com.google.cloud.bigquery.InsertAllRequest.RowToInsert -import com.google.cloud.bigquery.{ - BigQuery, - BigQueryError, - BigQueryOptions, - InsertAllRequest, - InsertAllResponse, - TableId -} -import common.util.TimeUtil._ -import common.validation.Validation._ -import cromwell.cloudsupport.gcp.GoogleConfiguration -import cromwell.database.sql.SqlConverters._ -import cromwell.database.sql.tables.{JobKeyValueEntry, MetadataEntry} -import net.ceedubs.ficus.Ficus._ -import org.apache.commons.lang3.exception.ExceptionUtils -import org.threeten.bp.Duration - -import scala.jdk.CollectionConverters._ -import scala.concurrent.ExecutionContext - -class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorReporter { - - private val bigQueryAuth: String = params.reporterConfig.getString("auth") - private val bigQueryProjectOption: Option[String] = params.reporterConfig.getAs[String]("project") - private val bigQueryDataset: String = params.reporterConfig.getString("dataset") - - override lazy val destination: String = bigQueryProjectOption.map(_ + "/").getOrElse("") + bigQueryDataset - - private val retrySettings: RetrySettings = - RetrySettings - .newBuilder() - .setMaxAttempts(3) - .setTotalTimeout(Duration.ofSeconds(30)) - .setInitialRetryDelay(Duration.ofMillis(100)) - .setRetryDelayMultiplier(1.1) - .setMaxRetryDelay(Duration.ofSeconds(1)) - .setInitialRpcTimeout(Duration.ofMillis(100)) - .setRpcTimeoutMultiplier(1.1) - .setMaxRpcTimeout(Duration.ofSeconds(5)) - .build() - - private val bigQueryCredentials: Credentials = GoogleConfiguration - .apply(params.rootConfig) - .auth(bigQueryAuth) - .unsafe - .credentials(Set(BigqueryScopes.BIGQUERY_INSERTDATA)) - - private val bigQuery: BigQuery = BigQueryOptions - .newBuilder() - .setRetrySettings(retrySettings) - .setCredentials(bigQueryCredentials) - .build() - .getService - - private val testFailureTableId = bigQueryTable("test_failure") - private val callAttemptFailureTableId = bigQueryTable("call_attempt_failure") - private val jobKeyValueTableId = bigQueryTable("job_key_value") - private val metadataTableId = bigQueryTable("metadata") - - def bigQueryTable(table: String): TableId = - bigQueryProjectOption match { - case Some(project) => TableId.of(project, bigQueryDataset, table) - case None => TableId.of(bigQueryDataset, table) - } - - /** - * In this ErrorReporter implementation this method will send information about exceptions of type - * CentaurTestException to BigQuery. Exceptions of other types will be ignored. - */ - override def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit - executionContext: ExecutionContext - ): IO[Unit] = - throwable match { - case centaurTestException: CentaurTestException => - for { - callAttemptFailures <- CallAttemptFailure.buildFailures(centaurTestException.metadataJsonOption) - jobKeyValueEntries <- params.database.jobKeyValueEntriesIo(centaurTestException.workflowIdOption) - metadataEntries <- params.database.metadataEntriesIo(centaurTestException.workflowIdOption) - _ <- sendBigQueryFailure( - testEnvironment, - ciEnvironment, - centaurTestException, - callAttemptFailures, - jobKeyValueEntries, - metadataEntries - ) - } yield () - case _ => - IO.unit // this ErrorReporter only supports exceptions of CentaurTestException type - } - - private def sendBigQueryFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - centaurTestException: CentaurTestException, - callAttemptFailures: Vector[CallAttemptFailure], - jobKeyValueEntries: Seq[JobKeyValueEntry], - metadataEntries: Seq[MetadataEntry] - ): IO[Unit] = { - - val metadata: IO[List[BigQueryError]] = { - val metadataRows: List[util.List[RowToInsert]] = - metadataEntries.map(toMetadataRow).grouped(10000).map(_.asJava).toList - val metadataRequest: List[InsertAllRequest] = metadataRows.map(InsertAllRequest.of(metadataTableId, _)) - - if (metadataEntries.nonEmpty) - metadataRequest - .traverse[IO, List[BigQueryError]](req => IO(bigQuery.insertAll(req)).map(_.getErrors)) - .map(_.flatten) - else - IO(Nil) - } - (IO { - val testFailureRow = toTestFailureRow(testEnvironment, ciEnvironment, centaurTestException) - val callAttemptFailureRows = callAttemptFailures.map(toCallAttemptFailureRow).asJava - val jobKeyValueRows = jobKeyValueEntries.map(toJobKeyValueRow).asJava - - val testFailureRequest = InsertAllRequest.of(testFailureTableId, testFailureRow) - val callAttemptFailuresRequest = InsertAllRequest.of(callAttemptFailureTableId, callAttemptFailureRows) - val jobKeyValuesRequest = InsertAllRequest.of(jobKeyValueTableId, jobKeyValueRows) - val testFailureErrors = bigQuery.insertAll(testFailureRequest).getErrors - val callAttemptFailuresErrors = - if (callAttemptFailures.nonEmpty) bigQuery.insertAll(callAttemptFailuresRequest).getErrors else Nil - val jobKeyValuesErrors = - if (jobKeyValueEntries.nonEmpty) bigQuery.insertAll(jobKeyValuesRequest).getErrors else Nil - - testFailureErrors ++ callAttemptFailuresErrors ++ jobKeyValuesErrors - }, - metadata - ).mapN(_ ++ _).flatMap { - case errors if errors.isEmpty => IO.unit - case errors => - IO.raiseError { - val errorCount = errors.size - val threeErrors = errors.map(String.valueOf).distinct.sorted.take(3) - val continued = if (errorCount > 3) "\n..." else "" - val message = - threeErrors.mkString(s"$errorCount error(s) occurred uploading to BigQuery: \n", "\n", continued) - new RuntimeException(message) - } - } - } - - private def toTestFailureRow(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - centaurTestException: CentaurTestException - ): RowToInsert = - RowToInsert of Map( - "ci_env_branch" -> ciEnvironment.branch, - "ci_env_event" -> ciEnvironment.event, - "ci_env_is_ci" -> ciEnvironment.isCi, - "ci_env_number" -> ciEnvironment.number, - "ci_env_os" -> ciEnvironment.os, - "ci_env_provider" -> ciEnvironment.provider, - "ci_env_tag" -> ciEnvironment.tag, - "ci_env_type" -> ciEnvironment.`type`, - "ci_env_url" -> ciEnvironment.url, - "ci_env_centaur_type" -> ciEnvironment.centaurType, - "test_attempt" -> Option(testEnvironment.attempt + 1), - "test_message" -> Option(centaurTestException.message), - "test_name" -> Option(testEnvironment.testCase.name), - "test_stack_trace" -> Option(ExceptionUtils.getStackTrace(centaurTestException)), - "test_timestamp" -> Option(OffsetDateTime.now.toUtcMilliString), - "test_workflow_id" -> centaurTestException.workflowIdOption - ).collect { case (key, Some(value)) => - (key, value) - }.asJava - - private def toCallAttemptFailureRow(callAttemptFailure: CallAttemptFailure): RowToInsert = - RowToInsert of Map( - "call_fully_qualified_name" -> Option(callAttemptFailure.callFullyQualifiedName), - "call_root" -> callAttemptFailure.callRootOption, - "end" -> callAttemptFailure.endOption.map(_.toUtcMilliString), - "job_attempt" -> Option(callAttemptFailure.jobAttempt), - "job_index" -> Option(callAttemptFailure.jobIndex), - "message" -> Option(callAttemptFailure.message), - "start" -> callAttemptFailure.startOption.map(_.toUtcMilliString), - "stderr" -> callAttemptFailure.stderrOption, - "stdout" -> callAttemptFailure.stdoutOption, - "workflow_id" -> Option(callAttemptFailure.workflowId) - ).collect { case (key, Some(value)) => - (key, value) - }.asJava - - private def toJobKeyValueRow(jobKeyValueEntry: JobKeyValueEntry): RowToInsert = - RowToInsert of Map[String, Any]( - "call_fully_qualified_name" -> jobKeyValueEntry.callFullyQualifiedName, - "job_attempt" -> jobKeyValueEntry.jobAttempt, - "job_index" -> jobKeyValueEntry.jobIndex, - "store_key" -> jobKeyValueEntry.storeKey, - "store_value" -> jobKeyValueEntry.storeValue, - "workflow_execution_uuid" -> jobKeyValueEntry.workflowExecutionUuid - ).asJava - - private def toMetadataRow(metadataEntry: MetadataEntry): RowToInsert = - RowToInsert of Map( - "call_fully_qualified_name" -> metadataEntry.callFullyQualifiedName, - "job_attempt" -> metadataEntry.jobAttempt, - "job_index" -> metadataEntry.jobIndex, - "metadata_key" -> Option(metadataEntry.metadataKey), - "metadata_timestamp" -> Option(metadataEntry.metadataTimestamp.toSystemOffsetDateTime.toUtcMilliString), - "metadata_value" -> metadataEntry.metadataValue.map(_.toRawString), - "metadata_value_type" -> metadataEntry.metadataValueType, - "workflow_execution_uuid" -> Option(metadataEntry.workflowExecutionUuid) - ).collect { case (key, Some(value)) => - (key, value) - }.asJava -} - -object BigQueryReporter { - - implicit class EnhancedInsertAllResponse(val response: InsertAllResponse) extends AnyVal { - def getErrors: List[BigQueryError] = response.getInsertErrors.asScala.values.flatMap(_.asScala).toList - } - -} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 253d917d0aa..009684522bf 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -46,7 +46,6 @@ object Dependencies { private val ficusV = "1.5.2" private val fs2V = "2.5.9" // scala-steward:off (CROM-6564) private val googleApiClientV = "2.1.4" - private val googleCloudBigQueryV = "2.25.0" // latest date via: https://github.com/googleapis/google-api-java-client-services/blob/main/clients/google-api-services-cloudkms/v1.metadata.json private val googleCloudKmsV = "v1-rev20230421-2.0.0" private val googleCloudMonitoringV = "3.2.5" @@ -558,7 +557,6 @@ object Dependencies { val centaurDependencies: List[ModuleID] = List( "org.apache.commons" % "commons-math3" % commonsMathV, "com.github.kxbmap" %% "configs" % configsV, - "com.google.cloud" % "google-cloud-bigquery" % googleCloudBigQueryV % IntegrationTest, "org.gnieh" %% "diffson-spray-json" % diffsonSprayJsonV ) ++ circeDependencies ++ slf4jBindingDependencies ++ cloudSupportDependencies ++ http4sDependencies diff --git a/src/ci/resources/centaur_application.conf b/src/ci/resources/centaur_application.conf index 5c37796e5d4..9494f8b8607 100644 --- a/src/ci/resources/centaur_application.conf +++ b/src/ci/resources/centaur_application.conf @@ -40,5 +40,4 @@ centaur { include "cromwell_database.inc.conf" } - include "centaur_secure.inc.conf" } diff --git a/src/ci/resources/centaur_secure.inc.conf.ctmpl b/src/ci/resources/centaur_secure.inc.conf.ctmpl deleted file mode 100644 index ff0bc7b1c53..00000000000 --- a/src/ci/resources/centaur_secure.inc.conf.ctmpl +++ /dev/null @@ -1,13 +0,0 @@ -# NOTE: This file will only be included if secrets can be rendered. -error-reporter { - providers { - big-query { - class: "centaur.reporting.BigQueryReporter" - config { - auth: "service-account" - project: "broad-dsde-cromwell-dev" - dataset: "cromwell_ci_errors" - } - } - } -}