[add] Using strings to store the binaries produced by tooling (#50)

* [add] Using strings to store the binaries produced by tooling

* Bumping version from 0.1.12 to 0.1.13

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.12"
+version = "0.1.13"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__api__/app.py

@@ -97,7 +97,13 @@ def base():
                 event_type = "Git pushes to repo"
 
             if use_event is True:
-                fields = {"git_hash": sha, "ref_label": ref_label, "ref": ref}
+                fields = {
+                    "git_hash": sha,
+                    "ref_label": ref_label,
+                    "ref": ref,
+                    "gh_repo": gh_repo,
+                    "gh_org": gh_org,
+                }
                 app.logger.info(
                     "Using event {} to trigger benchmark. final fields: {}".format(
                         event_type, fields

redis_benchmarks_specification/__builder__/builder.py

@@ -178,10 +178,18 @@ def builder_process_stream(builders_folder, conn, different_build_specs, previou
         if b"git_hash" in testDetails:
             git_hash = testDetails[b"git_hash"]
             logging.info("Received commit hash specifier {}.".format(git_hash))
-            buffer = testDetails[b"zip_archive"]
+            binary_zip_key = testDetails[b"zip_archive_key"]
+            logging.info(
+                "Retriving zipped source from key {}.".format(
+                    testDetails[b"zip_archive_key"]
+                )
+            )
+            buffer = conn.get(binary_zip_key)
             git_branch = None
             if b"git_branch" in testDetails:
                 git_branch = testDetails[b"git_branch"]
+            if b"ref_label" in testDetails:
+                git_branch = testDetails[b"ref_label"]
             git_timestamp_ms = None
             use_git_timestamp = False
             if b"use_git_timestamp" in testDetails:
@@ -274,10 +282,13 @@ def builder_process_stream(builders_folder, conn, different_build_specs, previou
                 if git_timestamp_ms is not None:
                     build_stream_fields["git_timestamp_ms"] = git_timestamp_ms
                 for artifact in build_artifacts:
+                    bin_key = "zipped:artifacts:{}:{}.zip".format(id, artifact)
                     bin_artifact = open(
                         "{}src/{}".format(redis_temporary_dir, artifact), "rb"
                     ).read()
-                    build_stream_fields[artifact] = bytes(bin_artifact)
+                    ttl = 24 * 7 * 60 * 60
+                    conn.set(bin_key, bytes(bin_artifact), ex=ttl)
+                    build_stream_fields[artifact] = bin_key
                     build_stream_fields["{}_len_bytes".format(artifact)] = len(
                         bytes(bin_artifact)
                     )

redis_benchmarks_specification/__cli__/cli.py

@@ -121,12 +121,20 @@ def main():
         )
         for rep in range(0, 1):
             for commit in Commits:
-                result, error_msg, commit_dict, _ = get_commit_dict_from_sha(
+                (
+                    result,
+                    error_msg,
+                    commit_dict,
+                    _,
+                    binary_key,
+                    binary_value,
+                ) = get_commit_dict_from_sha(
                     commit.hexsha, "redis", "redis", {}, True, args.gh_token
                 )
+                binary_exp_secs = 24 * 7 * 60 * 60
                 if result is True:
                     result, reply_fields, error_msg = request_build_from_commit_info(
-                        conn, commit_dict, {}
+                        conn, commit_dict, {}, binary_key, binary_value, binary_exp_secs
                     )
                     logging.info(
                         "Successfully requested a build for commit: {}. Request stream id: {}. Commit summary: {}".format(

redis_benchmarks_specification/__common__/builder_schema.py

@@ -33,16 +33,25 @@ def commit_schema_to_stream(
     if "git_hash" not in fields:
         error_msg = "Missing required 'git_hash' field"
     else:
-        result, error_msg, fields, _ = get_commit_dict_from_sha(
+        (
+            result,
+            error_msg,
+            fields,
+            _,
+            binary_key,
+            binary_value,
+        ) = get_commit_dict_from_sha(
             fields["git_hash"], gh_org, gh_repo, fields, use_git_timestamp, gh_token
         )
         reply_fields["use_git_timestamp"] = fields["use_git_timestamp"]
         if "git_timestamp_ms" in fields:
             reply_fields["git_timestamp_ms"] = fields["git_timestamp_ms"]
         reply_fields["archived_zip"] = True
     if result is True:
+        # 7 days expire
+        binary_exp_secs = 24 * 60 * 60 * 7
         result, reply_fields, error_msg = request_build_from_commit_info(
-            conn, fields, reply_fields
+            conn, fields, reply_fields, binary_key, binary_value, binary_exp_secs
         )
 
     return result, reply_fields, error_msg
@@ -51,22 +60,25 @@ def commit_schema_to_stream(
 def get_archive_zip_from_hash(gh_org, gh_repo, git_hash, fields):
     error_msg = None
     result = False
+    binary_value = None
+    bin_key = "zipped:source:{}/{}/archive/{}.zip".format(gh_org, gh_repo, git_hash)
     github_url = "https://github.com/{}/{}/archive/{}.zip".format(
         gh_org, gh_repo, git_hash
     )
     try:
         response = urlopen(github_url, timeout=5)
         content = response.read()
-        fields["zip_archive"] = bytes(content)
+        fields["zip_archive_key"] = bin_key
         fields["zip_archive_len"] = len(bytes(content))
+        binary_value = bytes(content)
         result = True
     except URLError as e:
         error_msg = "Catched URLError while fetching {} content. Error {}".format(
             github_url, e.__str__()
         )
         logging.error(error_msg)
         result = False
-    return result, error_msg
+    return result, bin_key, binary_value, error_msg
 
 
 def get_commit_dict_from_sha(
@@ -91,13 +103,15 @@ def get_commit_dict_from_sha(
     commit_dict["use_git_timestamp"] = str(use_git_timestamp)
     commit_dict["git_hash"] = git_hash
 
-    result, error_msg = get_archive_zip_from_hash(
+    result, binary_key, binary_value, error_msg = get_archive_zip_from_hash(
         gh_org, gh_repo, git_hash, commit_dict
     )
-    return result, error_msg, commit_dict, commit
+    return result, error_msg, commit_dict, commit, binary_key, binary_value
 
 
-def request_build_from_commit_info(conn, fields, reply_fields):
+def request_build_from_commit_info(
+    conn, fields, reply_fields, binary_key, binary_value, binary_exp_secs
+):
     """Generates a build event from the commit dictionary
     It expected the fields dictionary to contain at least the following keys:
         - "git_branch": reference to the branch that the commit refers to
@@ -118,6 +132,7 @@ def request_build_from_commit_info(conn, fields, reply_fields):
     """
     result = True
     error_msg = None
+    conn.set(binary_key, binary_value, ex=binary_exp_secs)
     for k, v in fields.items():
         if type(v) not in [str, int, float, bytes]:
             raise Exception(

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -233,6 +233,7 @@ def self_contained_coordinator_blocking_read(
         stream_id = ">"
     else:
         stream_id, overall_result = process_self_contained_coordinator_stream(
+            conn,
             datasink_push_results_redistimeseries,
             docker_client,
             home,
@@ -267,6 +268,7 @@ def self_contained_coordinator_blocking_read(
 
 
 def process_self_contained_coordinator_stream(
+    conn,
     datasink_push_results_redistimeseries,
     docker_client,
     home,
@@ -327,9 +329,15 @@ def process_self_contained_coordinator_stream(
 
                         benchmark_tool = "redis-benchmark"
                         for build_artifact in build_artifacts:
-                            buffer = testDetails[
-                                bytes("{}".format(build_artifact).encode())
+                            buffer_key = testDetails[
+                                "{}".format(build_artifact).encode()
                             ]
+                            logging.info(
+                                "Reading artifact binary {} from key {}".format(
+                                    build_artifact, buffer_key
+                                )
+                            )
+                            buffer = bytes(conn.get(buffer_key))
                             artifact_fname = "{}/{}".format(
                                 temporary_dir, build_artifact
                             )

utils/tests/test_api.py

@@ -54,7 +54,14 @@ def test_commit_schema_to_stream():
 
 
 def test_get_commit_dict_from_sha():
-    result, error_msg, commit_dict, _ = get_commit_dict_from_sha(
+    (
+        result,
+        error_msg,
+        commit_dict,
+        _,
+        binary_key,
+        binary_value,
+    ) = get_commit_dict_from_sha(
         "492d8d09613cff88f15dcef98732392b8d509eb1", "redis", "redis", {}, True, GH_TOKEN
     )
     if GH_TOKEN is not None:

utils/tests/test_data/api_builder_common.py

@@ -16,7 +16,11 @@ def flow_1_and_2_api_builder_checks(conn):
     builder_consumer_group_create(conn)
     assert conn.xlen(STREAM_KEYNAME_GH_EVENTS_COMMIT) == 0
     result, reply_fields, error_msg = commit_schema_to_stream(
-        '{"git_hash":"0cf2df84d4b27af4bffd2bf3543838f09e10f874", "git_branch":"unstable", "use_git_timestamp":true }',
+        {
+            "git_hash": "0cf2df84d4b27af4bffd2bf3543838f09e10f874",
+            "git_branch": "unstable",
+            "use_git_timestamp": True,
+        },
         conn,
         "redis",
         "redis",

utils/tests/test_data/dump.rdb

@@ -1,5 +1,5 @@
 version: 0.1
-id: icc-2021.3.0-amd64-ubuntu18.04-monotonic-clock
+id: icc-2021.3.0-amd64-ubuntu18.04-libc
 os: ubuntu18.04
 arch: amd64
 compiler: "icc"
@@ -16,5 +16,5 @@ metadata:
   arch: amd64
 
 env:
-  CFLAGS: "-DUSE_PROCESSOR_CLOCK"
+  MALLOC: "libc"
 

redis_benchmarks_specification/setups/builders/icc-2021.3.0-amd64-ubuntu18.04-monotonic-clock.yml renamed to utils/tests/test_data/icc-2021.3.0-amd64-ubuntu18.04-libc.yml

@@ -6,17 +6,17 @@
 
 def test_get_build_config():
     config_files = [
-        "./utils/tests/test_data/icc-2021.3.0-amd64-ubuntu18.04-monotonic-clock.yml",
+        "./utils/tests/test_data/icc-2021.3.0-amd64-ubuntu18.04-libc.yml",
     ]
     for filename in config_files:
         build_config, id = get_build_config(filename)
-        assert id == "icc-2021.3.0-amd64-ubuntu18.04-monotonic-clock"
-        assert build_config["env"]["CC"] == "icc"
+        assert id == "icc-2021.3.0-amd64-ubuntu18.04-libc"
+        assert build_config["env"]["MALLOC"] == "libc"
 
 
 def test_get_build_config_metadata():
     build_config, _ = get_build_config(
-        "./utils/tests/test_data/icc-2021.3.0-amd64-ubuntu18.04-monotonic-clock.yml"
+        "./utils/tests/test_data/icc-2021.3.0-amd64-ubuntu18.04-libc.yml"
     )
     build_config_metadata = get_build_config_metadata(build_config)
     for k in ["arch", "compiler", "compiler_version", "os"]: