From fd5a6b59dfb228a7c1855ae2a0beb7e71904bbb0 Mon Sep 17 00:00:00 2001 From: Adam Nichols Date: Tue, 19 Jul 2022 11:12:09 -0400 Subject: [PATCH] BW-1320 Finite retry `User project specified in the request is invalid` (#6807) --- CHANGELOG.md | 3 +++ .../cromwell/engine/io/RetryableRequestSupport.scala | 8 +++++--- .../src/test/scala/cromwell/engine/io/IoActorSpec.scala | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1307ad6cb53..0c40f47b651 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ * Restored missing example configuration file * Upgraded to latest version of the Google Cloud Storage NIO library (0.124.8) + * Cromwell will now finitely retry the following Google Cloud Storage I/O error. + * Response code `400` bad request, message `User project specified in the request is invalid` + * The default retry count is `5` and may be customized with `system.io.number-of-attempts`. ## 81 Release Notes diff --git a/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala b/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala index 03ef9224b7d..d2aa427a567 100644 --- a/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala +++ b/engine/src/main/scala/cromwell/engine/io/RetryableRequestSupport.scala @@ -11,8 +11,8 @@ import javax.net.ssl.SSLException object RetryableRequestSupport { /** - * Failures that are considered retryable. - * Retrying them should increase the "retry counter" + * Failures that are considered retryable. Retrying them increases the retry counter. + * The default count is `5` and may be customized with `system.io.number-of-attempts`. */ def isRetryable(failure: Throwable): Boolean = failure match { case gcs: StorageException => gcs.isRetryable || @@ -47,7 +47,9 @@ object RetryableRequestSupport { // Error messages not included in the list of built-in GCS retryable errors (com.google.cloud.storage.StorageException) but that we still want to retry private val AdditionalRetryableErrorMessages = List( - "Connection closed prematurely" + "Connection closed prematurely", + // This is a 400 "non-retryable" error that is nevertheless sporadic and succeeds on subsequent re-runs with identical parameters [BW-1320] + "User project specified in the request is invalid" ).map(_.toLowerCase) /** diff --git a/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala b/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala index 518788c26b0..e6209ff5958 100644 --- a/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala +++ b/engine/src/test/scala/cromwell/engine/io/IoActorSpec.scala @@ -253,6 +253,7 @@ class IoActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with I new StorageException(504, "message"), new StorageException(408, "message"), new StorageException(429, "message"), + new StorageException(400, "User project specified in the request is invalid"), BatchFailedException(new Exception), ChecksumFailedException("message"), new SocketException(),