Moved from gh action API to gh webhook API (#49)

* [add] Moved from gh action API to gh webhook API

* [add] Further logging on API. Using PULL_REQUEST_TRIGGER_LABEL env var to decide uppon which PR label to use for trigger automation

Author: filipecosta90

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.11"
+version = "0.1.12"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__api__/Readme.md

@@ -5,12 +5,3 @@ Within root project folder
 ```
 poetry run redis-benchmarks-spec-api
 ```
-
-## Testing triggering a new commit spec via local api
-
-```
-curl -u <USER>:<PASS> \
-    -X POST -H "Content-Type: application/json" \
-    --data '{"git_hash":"0cf2df84d4b27af4bffd2bf3543838f09e10f874"}' \
-    http://localhost:5000/api/gh/redis/redis/commits
-```

redis_benchmarks_specification/__api__/__init__.py

@@ -0,0 +1,5 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs
+#  All rights reserved.
+#

redis_benchmarks_specification/__api__/api.py

@@ -15,13 +15,12 @@
     GH_REDIS_SERVER_HOST,
     GH_REDIS_SERVER_PORT,
     GH_REDIS_SERVER_AUTH,
-    REDIS_AUTH_SERVER_HOST,
-    REDIS_AUTH_SERVER_PORT,
     LOG_FORMAT,
     LOG_DATEFMT,
     LOG_LEVEL,
     REDIS_HEALTH_CHECK_INTERVAL,
     REDIS_SOCKET_TIMEOUT,
+    GH_REDIS_SERVER_USER,
 )
 from redis_benchmarks_specification.__common__.package import (
     populate_with_poetry_data,
@@ -47,11 +46,6 @@ def main():
             GH_REDIS_SERVER_HOST, GH_REDIS_SERVER_PORT
         )
     )
-    print(
-        "Using redis available at: {}:{} as auth server.".format(
-            REDIS_AUTH_SERVER_HOST, REDIS_AUTH_SERVER_PORT
-        )
-    )
     conn = redis.StrictRedis(
         host=GH_REDIS_SERVER_HOST,
         port=GH_REDIS_SERVER_PORT,
@@ -61,7 +55,7 @@ def main():
         socket_connect_timeout=REDIS_SOCKET_TIMEOUT,
         socket_keepalive=True,
     )
-    app = create_app(conn)
+    app = create_app(conn, GH_REDIS_SERVER_USER)
     if args.logname is not None:
         print("Writting log to {}".format(args.logname))
         handler = logging.handlers.RotatingFileHandler(

redis_benchmarks_specification/__api__/app.py

@@ -1,71 +1,126 @@
-from flask import Flask, jsonify, request
-from marshmallow import ValidationError
-from json import dumps
+import json
+
+from flask import jsonify
 import redis
-from flask_httpauth import HTTPBasicAuth
+from flask import Flask, request
+from hmac import HMAC, compare_digest
+from hashlib import sha1
 
-from redis_benchmarks_specification.__api__.schema import (
-    CommitSchema,
-)
 from redis_benchmarks_specification.__common__.builder_schema import (
     commit_schema_to_stream,
 )
-from redis_benchmarks_specification.__common__.env import (
-    REDIS_AUTH_SERVER_HOST,
-    REDIS_AUTH_SERVER_PORT,
-)
+from redis_benchmarks_specification.__common__.env import PULL_REQUEST_TRIGGER_LABEL
 
+SIG_HEADER = "X-Hub-Signature"
 
-def create_app(conn, test_config=None):
+
+def create_app(conn, user, test_config=None):
     app = Flask(__name__)
-    auth = HTTPBasicAuth()
+
     conn = conn
 
-    @auth.verify_password
-    def verify_password(username, password):
+    # GH Token Authentication
+    def verify_signature(req):
         result = False
         try:
-            auth_server_conn = redis.StrictRedis(
-                host=REDIS_AUTH_SERVER_HOST,
-                port=REDIS_AUTH_SERVER_PORT,
-                decode_responses=True,
-                username=username,
-                password=password,
-            )
-            auth_server_conn.ping()
-            result = True
+            secret = conn.get("{}:auth_token".format(user))
+            sig_header = req.headers.get(SIG_HEADER)
+            if secret is not None and sig_header is not None:
+                if type(secret) == str:
+                    secret = secret.encode()
+                if "sha1=" in sig_header:
+                    received_sign = sig_header.split("sha1=")[-1].strip()
+                    expected_sign = HMAC(
+                        key=secret, msg=req.data, digestmod=sha1
+                    ).hexdigest()
+                    result = compare_digest(received_sign, expected_sign)
         except redis.exceptions.ResponseError:
-            result = False
+            pass
         except redis.exceptions.AuthenticationError:
-            result = False
+            pass
         return result
 
     @app.route("/api/gh/redis/redis/commits", methods=["POST"])
-    @auth.login_required
     def base():
-        # Get Request body from JSON
-        request_data = request.json
-        gh_org = "redis"
-        gh_repo = "redis"
-        schema = CommitSchema()
-        response_data = {}
-        err_message = ""
-        try:
-            # Validate request body against schema data types
-            result = schema.load(request_data)
-        except ValidationError as err:
-            err_message = err.messages
-        if result is True:
-            # Convert request body back to JSON str
-            data_now_json_str = dumps(result)
+        if verify_signature(request):
+            print(request)
+            # Get Request body from JSON
+            request_data = request.json
+            if type(request_data) is str:
+                request_data = json.loads(request_data)
+            if type(request_data) is bytes:
+                request_data = json.loads(request_data.decode())
+
+            gh_org = "redis"
+            gh_repo = "redis"
+            ref = None
+            ref_label = None
+            sha = None
+
+            event_type = "Ignored event from webhook"
+            use_event = False
+            # Pull request labeled
+            trigger_label = PULL_REQUEST_TRIGGER_LABEL
+            if "pull_request" in request_data:
+                action = request_data["action"]
+                if "labeled" == action:
+                    pull_request_dict = request_data["pull_request"]
+                    head_dict = pull_request_dict["head"]
+                    repo_dict = head_dict["repo"]
+                    labels = []
+                    if "labels" in pull_request_dict:
+                        labels = pull_request_dict["labels"]
+                    ref = head_dict["ref"]
+                    ref_label = head_dict["label"]
+                    sha = head_dict["sha"]
+                    html_url = repo_dict["html_url"].split("/")
+                    gh_repo = html_url[-1]
+                    gh_org = html_url[-2]
+                    for label in labels:
+                        label_name = label["name"]
+                        if trigger_label == label_name:
+                            use_event = True
+                            event_type = "Pull request labeled with '{}'".format(
+                                trigger_label
+                            )
+
+            # Git pushes to repo
+            if "ref" in request_data:
+                repo_dict = request_data["repository"]
+                html_url = repo_dict["html_url"].split("/")
+                gh_repo = html_url[-1]
+                gh_org = html_url[-2]
+                ref = request_data["ref"].split("/")[-1]
+                ref_label = request_data["ref"]
+                sha = request_data["after"]
+                use_event = True
+                event_type = "Git pushes to repo"
+
+            if use_event is True:
+                fields = {"git_hash": sha, "ref_label": ref_label, "ref": ref}
+                app.logger.info(
+                    "Using event {} to trigger benchmark. final fields: {}".format(
+                        event_type, fields
+                    )
+                )
+                result, response_data, err_message = commit_schema_to_stream(
+                    fields, conn, gh_org, gh_repo
+                )
+                app.logger.info(
+                    "Using event {} to trigger benchmark. final fields: {}".format(
+                        event_type, response_data
+                    )
+                )
 
-            result, response_data, err_message = commit_schema_to_stream(
-                data_now_json_str, conn, gh_org, gh_repo
-            )
-        if result is False:
-            return jsonify(err_message), 400
+            else:
+                app.logger.info(
+                    "{}. input json was: {}".format(event_type, request_data)
+                )
+                response_data = {"message": event_type}
 
-        # Send data back as JSON
-        return jsonify(response_data), 200
+            # Send data back as JSON
+            return jsonify(response_data), 200
+        else:
+            return "Forbidden", 403
 
     return app

redis_benchmarks_specification/__api__/schema.py

@@ -4,7 +4,6 @@
 #  All rights reserved.
 #
 import logging
-from json import loads
 from urllib.error import URLError
 from urllib.request import urlopen
 from github import Github
@@ -17,15 +16,15 @@
 
 
 def commit_schema_to_stream(
-    json_str: str,
+    fields: dict,
     conn: redis.StrictRedis,
-    gh_org="redis",
-    gh_repo="redis",
+    gh_org,
+    gh_repo,
     gh_token=None,
 ):
     """ uses to the provided JSON dict of fields and pushes that info to the corresponding stream  """
-    fields = loads(json_str)
-    reply_fields = loads(json_str)
+    fields = fields
+    reply_fields = dict(fields)
     result = False
     error_msg = None
     use_git_timestamp = False
@@ -72,8 +71,8 @@ def get_archive_zip_from_hash(gh_org, gh_repo, git_hash, fields):
 
 def get_commit_dict_from_sha(
     git_hash,
-    gh_org="redis",
-    gh_repo="redis",
+    gh_org,
+    gh_repo,
     commit_dict={},
     use_git_timestamp=False,
     gh_token=None,
@@ -119,6 +118,13 @@ def request_build_from_commit_info(conn, fields, reply_fields):
     """
     result = True
     error_msg = None
+    for k, v in fields.items():
+        if type(v) not in [str, int, float, bytes]:
+            raise Exception(
+                "Type of field {} is not bytes, string, int or float. Type ({}). Value={}".format(
+                    k, type(v), v
+                )
+            )
     id = conn.xadd(STREAM_KEYNAME_GH_EVENTS_COMMIT.encode(), fields)
     reply_fields["id"] = id
     return result, reply_fields, error_msg

redis_benchmarks_specification/__common__/builder_schema.py

@@ -50,6 +50,9 @@
 REDIS_SOCKET_TIMEOUT = int(os.getenv("REDIS_SOCKET_TIMEOUT", "300"))
 
 # environment variables
+PULL_REQUEST_TRIGGER_LABEL = os.getenv(
+    "PULL_REQUEST_TRIGGER_LABEL", "trigger-benchmark"
+)
 DATASINK_RTS_PUSH = bool(os.getenv("DATASINK_PUSH_RTS", False))
 DATASINK_RTS_AUTH = os.getenv("DATASINK_RTS_AUTH", None)
 DATASINK_RTS_USER = os.getenv("DATASINK_RTS_USER", None)

redis_benchmarks_specification/__common__/env.py

@@ -3,8 +3,6 @@
 #  Copyright (c) 2021., Redis Labs
 #  All rights reserved.
 #
-import redis_benchmarks_specification
-from redis_benchmarks_specification.__api__.app import create_app
 
 from redis_benchmarks_specification.__common__.builder_schema import (
     commit_schema_to_stream,
@@ -26,7 +24,7 @@
 
 def test_commit_schema_to_stream():
     result, reply_fields, error_msg = commit_schema_to_stream(
-        '{"git_hashss":"0cf2df84d4b27af4bffd2bf3543838f09e10f874"}',
+        {"git_hashss": "0cf2df84d4b27af4bffd2bf3543838f09e10f874"},
         None,
         "redis",
         "redis",
@@ -38,7 +36,7 @@ def test_commit_schema_to_stream():
         conn.ping()
         conn.flushall()
         result, reply_fields, error_msg = commit_schema_to_stream(
-            '{"git_hash":"0cf2df84d4b27af4bffd2bf3543838f09e10f874"}',
+            {"git_hash": "0cf2df84d4b27af4bffd2bf3543838f09e10f874"},
             conn,
             "redis",
             "redis",