Merge pull request #32 from emilyzheng/retry-cosign

Update run command

requirements-test.txt

@@ -9,3 +9,4 @@ mypy
 types-requests
 types-pyOpenSSL
 types-PyYAML
+pubtools

src/pubtools/sign/signers/cosignsigner.py

@@ -232,7 +232,7 @@ def container_sign(self: CosignSigner, operation: ContainerSignOperation) -> Sig
         )
 
         outputs = {}
-        processes = {}
+        ref_args = {}
         common_args = [
             self.cosign_bin,
             "-t",
@@ -256,16 +256,12 @@ def container_sign(self: CosignSigner, operation: ContainerSignOperation) -> Sig
         if operation.references:
             for ref, digest in zip(operation.references, operation.digests):
                 repo, tag = ref.rsplit(":", 1)
-                processes[f"{repo}:{digest}"] = run_command(
-                    common_args + ["-a", f"tag={tag}", f"{repo}@{digest}"],
-                    env=env_vars,
-                )
+                ref_args[f"{repo}@{digest}"] = ["-a", f"tag={tag}", f"{repo}@{digest}"]
         else:
             for ref_digest in operation.digests:
-                processes[f"{ref_digest}"] = run_command(common_args + [ref_digest], env=env_vars)
-        for ref, process in processes.items():
-            stdout, stderr = process.communicate()
-            outputs[ref] = (stdout, stderr, process.returncode)
+                ref_args[ref_digest] = [ref_digest]
+        for ref, args in ref_args.items():
+            outputs[ref] = run_command(common_args + args, env=env_vars, tries=self.retries)
 
         for ref, (stdout, stderr, returncode) in outputs.items():
             if returncode != 0:
@@ -300,12 +296,11 @@ def existing_signatures(self, reference: str) -> Tuple[bool, str]:
             common_args += ["--registry-password", self.registry_password]
         env_vars = os.environ.copy()
         env_vars.update(self.env_variables)
-        process = run_command(
+        stdout, stderr, returncode = run_command(
             common_args + [reference],
             env=env_vars,
         )
-        stdout, stderr = process.communicate()
-        if process.returncode != 0:
+        if returncode != 0:
             return False, stderr
         else:
             ret, err_msg = self.container_registry_client.check_container_image_exists(

src/pubtools/sign/utils.py

@@ -5,13 +5,15 @@
 import subprocess
 import os
 import logging
-from typing import Any, Dict, List, Union, Callable, cast, Iterable
+import time
+from typing import Any, Dict, List, Union, Callable, cast, Iterable, Tuple
 
 from .conf.conf import CONFIG_PATHS
 
 from pubtools.tracing import get_trace_wrapper
 
 tw = get_trace_wrapper()
+LOG = logging.getLogger("pubtools.sign.utils")
 
 
 def set_log_level(logger: logging.Logger, level: str) -> None:
@@ -50,12 +52,33 @@ def isodate_now() -> str:
 
 
 @tw.instrument_func(args_to_attr=True)
-def run_command(cmd: List[str], env: Union[Dict[str, Any], None] = None) -> Any:
-    """Run external command and return Process instance."""
-    process = subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env
-    )
-    return process
+def run_command(
+    cmd: List[str], env: Union[Dict[str, Any], None] = None, tries: int = 3
+) -> Tuple[str, str, int]:
+    """Run external command and return stdout, stderr and returncode."""
+
+    def _run_command(
+        cmd: List[str], env: Union[Dict[str, Any], None] = None
+    ) -> Tuple[str, str, int]:
+        process = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env
+        )
+        stdout, stderr = process.communicate()
+        return (stdout, stderr, process.returncode)
+
+    for i in range(tries):
+        stdout, stderr, returncode = _run_command(cmd, env)
+        if returncode != 0:
+            wait_time = i * 10
+            LOG.warning(
+                "Run command failed. Will retry in %d seconds [try %s/%s]: %s"
+                % (wait_time, i + 1, tries, stderr)
+            )
+            time.sleep(wait_time)
+            stdout, stderr, returncode = _run_command(cmd, env)
+        else:
+            break
+    return (stdout, stderr, returncode)
 
 
 def _get_config_file(config_candidate: str) -> str:

tests/conftest_cosignsig.py

@@ -20,6 +20,7 @@ def f_config_cosign_signer_ok(f_client_certificate):
   rekor_url: https://rekor.sigstore.dev
   registry_user: some-user
   registry_password: some-password
+  retries: 1
   log_level: debug
         """.encode(
                 "utf-8"