move s3 argparser to helpers

Author: strtgbb

alter/stress/regression.py

@@ -7,7 +7,11 @@
 
 from helpers.cluster import create_cluster
 from helpers.common import check_clickhouse_version, experimental_analyzer
-from s3.regression import argparser as argparser_base, CaptureClusterArgs
+from helpers.argparser import (
+    argparser_s3 as argparser_base,
+    CaptureClusterArgs,
+    CaptureS3Args,
+)
 from s3.tests.common import start_minio
 
 xfails = {
@@ -246,22 +250,13 @@ def gcs(
 @XFails(xfails)
 @FFails(ffails)
 @CaptureClusterArgs
+@CaptureS3Args
 def regression(
     self,
-    cluster_args,
-    clickhouse_version,
-    storages,
-    minio_uri,
-    gcs_uri,
-    aws_s3_region,
-    aws_s3_bucket,
-    minio_root_user,
-    minio_root_password,
-    aws_s3_access_key,
-    aws_s3_key_id,
-    gcs_key_secret,
-    gcs_key_id,
-    stress,
+    cluster_args: dict,
+    s3_args: dict,
+    clickhouse_version: str,
+    stress: bool,
     with_analyzer=False,
     unsafe=False,
 ):
@@ -272,6 +267,7 @@ def regression(
     self.context.stress = stress
     self.context.unsafe = unsafe
 
+    storages = s3_args.pop("storages", None)
     if storages is None:
         storages = ["minio"]
 
@@ -280,30 +276,29 @@ def regression(
         with_analyzer=with_analyzer,
     )
 
-    if "aws_s3" in storages:
-        Module(test=aws_s3)(
-            bucket=aws_s3_bucket,
-            region=aws_s3_region,
-            key_id=aws_s3_key_id,
-            access_key=aws_s3_access_key,
-            **module_args,
-        )
-
-    if "gcs" in storages:
-        Module(test=gcs)(
-            uri=gcs_uri,
-            key_id=gcs_key_id,
-            access_key=gcs_key_secret,
-            **module_args,
-        )
-
-    if "minio" in storages:
-        Module(test=minio)(
-            uri=minio_uri,
-            root_user=minio_root_user,
-            root_password=minio_root_password,
-            **module_args,
-        )
+    for storage in storages:
+        if storage == "aws_s3":
+            Module(test=aws_s3)(
+                bucket=s3_args["aws_s3_bucket"],
+                region=s3_args["aws_s3_region"],
+                key_id=s3_args["aws_s3_key_id"],
+                access_key=s3_args["aws_s3_access_key"],
+                **module_args,
+            )
+        elif storage == "gcs":
+            Module(test=gcs)(
+                uri=s3_args["gcs_uri"],
+                key_id=s3_args["gcs_key_id"],
+                access_key=s3_args["gcs_key_secret"],
+                **module_args,
+            )
+        elif storage == "minio":
+            Module(test=minio)(
+                uri=s3_args["minio_uri"],
+                root_user=s3_args["minio_root_user"],
+                root_password=s3_args["minio_root_password"],
+                **module_args,
+            )
 
     if "local" in storages:
         Module(test=local_storage)(

helpers/argparser.py

@@ -1,8 +1,10 @@
 import os
 
+from testflows.core import Secret
+
 
 def argparser(parser):
-    """Default argument parser for regressions."""
+    """Default argument parser for regression."""
     parser.add_argument(
         "--local",
         action="store_true",
@@ -169,3 +171,153 @@ def capture_cluster_args(
         return func(self, cluster_args=cluster_args, **kwargs)
 
     return capture_cluster_args
+
+
+def argparser_s3(parser):
+    """Extended argument parser for suites with S3 storage."""
+    argparser(parser)
+
+    parser.add_argument(
+        "--storage",
+        action="append",
+        help="select which storage types to run tests with",
+        choices=["minio", "aws_s3", "gcs", "local"],
+        default=None,
+        dest="storages",
+    )
+
+    parser.add_argument(
+        "--minio-uri",
+        action="store",
+        help="set url for the minio connection",
+        type=Secret(name="minio_uri"),
+        default="http://minio1:9001",
+    )
+
+    parser.add_argument(
+        "--minio-root-user",
+        action="store",
+        help="minio root user name (access key id)",
+        type=Secret(name="minio_root_user"),
+        default="minio_user",
+    )
+
+    parser.add_argument(
+        "--minio-root-password",
+        action="store",
+        help="minio root user password (secret access key)",
+        type=Secret(name="minio_root_password"),
+        default="minio123",
+    )
+
+    parser.add_argument(
+        "--aws-s3-bucket",
+        action="store",
+        help="set bucket for the aws connection",
+        type=Secret(name="aws_s3_bucket"),
+        default=os.getenv("S3_AMAZON_BUCKET"),
+    )
+
+    parser.add_argument(
+        "--aws-s3-region",
+        action="store",
+        help="set aws region for the aws connection",
+        type=Secret(name="aws_s3_region"),
+        default=os.getenv("AWS_DEFAULT_REGION"),
+    )
+
+    parser.add_argument(
+        "--aws-s3-key-id",
+        action="store",
+        help="aws s3 key id",
+        type=Secret(name="aws_s3_key_id"),
+        default=os.getenv("AWS_ACCESS_KEY_ID"),
+    )
+
+    parser.add_argument(
+        "--aws-s3-access-key",
+        action="store",
+        help="aws s3 access key",
+        type=Secret(name="aws_s3_access_key"),
+        default=os.getenv("AWS_SECRET_ACCESS_KEY"),
+    )
+
+    parser.add_argument(
+        "--gcs-uri",
+        action="store",
+        help="set url for the gcs connection",
+        type=Secret(name="gcs_uri"),
+        default=os.getenv("GCS_URI"),
+    )
+
+    parser.add_argument(
+        "--gcs-key-id",
+        action="store",
+        help="gcs key id",
+        type=Secret(name="gcs_key_id"),
+        default=os.getenv("GCS_KEY_ID"),
+    )
+
+    parser.add_argument(
+        "--gcs-key-secret",
+        action="store",
+        help="gcs key secret",
+        type=Secret(name="gcs_key_secret"),
+        default=os.getenv("GCS_KEY_SECRET"),
+    )
+
+
+def CaptureS3Args(func):
+    """
+    Collect S3 arguments from argparser into s3_args.
+
+    Usage:
+
+        @TestModule
+        @ArgumentParser(argparser_s3)
+        @...  # other decorators
+        @CaptureClusterArgs
+        @CaptureS3Args
+        def regression(
+            self,
+            cluster_args,
+            s3_args,
+            clickhouse_version,
+            stress=None,
+            with_analyzer=False,
+        ):
+            ...
+
+    """
+
+    def capture_s3_args(
+        self,
+        storages,
+        minio_uri,
+        minio_root_user,
+        minio_root_password,
+        aws_s3_bucket,
+        aws_s3_region,
+        aws_s3_key_id,
+        aws_s3_access_key,
+        gcs_uri,
+        gcs_key_id,
+        gcs_key_secret,
+        **kwargs
+    ):
+        s3_args = {
+            "storages": storages,
+            "minio_uri": minio_uri,
+            "minio_root_user": minio_root_user,
+            "minio_root_password": minio_root_password,
+            "aws_s3_bucket": aws_s3_bucket,
+            "aws_s3_region": aws_s3_region,
+            "aws_s3_key_id": aws_s3_key_id,
+            "aws_s3_access_key": aws_s3_access_key,
+            "gcs_uri": gcs_uri,
+            "gcs_key_id": gcs_key_id,
+            "gcs_key_secret": gcs_key_secret,
+        }
+        return func(self, s3_args=s3_args, **kwargs)
+
+    return capture_s3_args

ontime_benchmark/benchmark.py

@@ -8,7 +8,11 @@
 
 from helpers.cluster import Cluster, create_cluster
 from helpers.common import check_clickhouse_version
-from s3.regression import argparser as argparser_base, CaptureClusterArgs
+from helpers.argparser import (
+    argparser_s3 as argparser_base,
+    CaptureClusterArgs,
+    CaptureS3Args,
+)
 
 from s3.tests.common import *
 
@@ -25,7 +29,7 @@
         (
             Fail,
             "strange and rare error on 23.8",
-            lambda test: check_clickhouse_version(">23.8")(test)
+            lambda test: check_clickhouse_version(">=23.8")(test)
             and check_clickhouse_version("<24")(test),
             ".*Cannot assign requested address.*",
         )
@@ -55,23 +59,14 @@ def argparser(parser):
 @XFails(xfails)
 @FFails(ffails)
 @CaptureClusterArgs
+@CaptureS3Args
 def regression(
     self,
-    cluster_args,
-    clickhouse_version,
-    storages,
-    stress,
-    minio_uri,
-    gcs_uri,
-    aws_s3_region,
-    aws_s3_bucket,
-    minio_root_user,
-    minio_root_password,
-    aws_s3_access_key,
-    aws_s3_key_id,
-    gcs_key_secret,
-    gcs_key_id,
-    format,
+    cluster_args: dict,
+    s3_args: dict,
+    clickhouse_version: str,
+    stress: bool,
+    format: str,
     with_analyzer=False,
     node="clickhouse1",
 ):
@@ -87,9 +82,18 @@ def regression(
     self.context.clickhouse_version = clickhouse_version
     self.context.stress = stress
 
+    storages = s3_args.pop("storages", None)
     if storages is None:
         storages = ["minio"]
 
+    aws_s3_access_key = s3_args.get("aws_s3_access_key")
+    aws_s3_key_id = s3_args.get("aws_s3_key_id")
+    aws_s3_bucket = s3_args.get("aws_s3_bucket")
+    aws_s3_region = s3_args.get("aws_s3_region")
+    gcs_uri = s3_args.get("gcs_uri")
+    gcs_key_id = s3_args.get("gcs_key_id")
+    gcs_key_secret = s3_args.get("gcs_key_secret")
+
     for storage in storages:
         environ = {}
         nodes = {"clickhouse": ("clickhouse1", "clickhouse2", "clickhouse3")}