Skip to content

Commit 5cdebfd

Browse files
filipecosta90filipecosta90
authored
[fix] fixed xack builder check on str vs int (#26)
1 parent 29d7227 commit 5cdebfd

File tree

10 files changed

+141
-113
lines changed

10 files changed

+141
-113
lines changed

‎pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "redis-benchmarks-specification"
3-
version = "0.1.5"
3+
version = "0.1.6"
44
description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
55
authors = ["filipecosta90 <[email protected]>"]
66
readme = "Readme.md"

‎redis_benchmarks_specification/__api__/api.py

+8-105
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,13 @@
55
#
66

77
import logging
8-
from urllib.request import urlopen
9-
from urllib.error import URLError
108
import logging.handlers
11-
from flask import Flask, jsonify, request
12-
from marshmallow import Schema, fields, ValidationError
139
from redis_benchmarks_specification import __version__
14-
from json import dumps, loads
1510
import redis
1611
import argparse
17-
from flask_httpauth import HTTPBasicAuth
1812

13+
from redis_benchmarks_specification.__api__.app import create_app
1914
from redis_benchmarks_specification.__common__.env import (
20-
STREAM_KEYNAME_GH_EVENTS_COMMIT,
2115
GH_REDIS_SERVER_HOST,
2216
GH_REDIS_SERVER_PORT,
2317
GH_REDIS_SERVER_AUTH,
@@ -32,96 +26,6 @@
3226
get_version_string,
3327
)
3428

35-
auth = HTTPBasicAuth()
36-
37-
app = Flask(__name__)
38-
conn = None
39-
40-
41-
@auth.verify_password
42-
def verify_password(username, password):
43-
result = False
44-
try:
45-
auth_server_conn = redis.StrictRedis(
46-
host=REDIS_AUTH_SERVER_HOST,
47-
port=REDIS_AUTH_SERVER_PORT,
48-
decode_responses=True,
49-
username=username,
50-
password=password,
51-
)
52-
auth_server_conn.ping()
53-
result = True
54-
except redis.exceptions.ResponseError:
55-
result = False
56-
except redis.exceptions.AuthenticationError:
57-
result = False
58-
return result
59-
60-
61-
def commit_schema_to_stream(json_str: str, conn: redis.StrictRedis):
62-
""" uses to the provided JSON dict of fields and pushes that info to the corresponding stream """
63-
fields = loads(json_str)
64-
reply_fields = loads(json_str)
65-
result = False
66-
error_msg = None
67-
if "git_hash" not in fields:
68-
error_msg = "Missing required 'git_hash' field"
69-
else:
70-
github_url = "https://github.com/redis/redis/archive/{}.zip".format(
71-
fields["git_hash"]
72-
)
73-
try:
74-
response = urlopen(github_url, timeout=5)
75-
content = response.read()
76-
fields["zip_archive"] = bytes(content)
77-
fields["zip_archive_len"] = len(bytes(content))
78-
reply_fields["archived_zip"] = True
79-
result = True
80-
except URLError as e:
81-
error_msg = "Catched URLError while fetching {} content. Error {}".format(
82-
github_url, e.__str__()
83-
)
84-
logging.error(error_msg)
85-
result = False
86-
87-
if result is True:
88-
id = conn.xadd(STREAM_KEYNAME_GH_EVENTS_COMMIT.encode(), fields)
89-
reply_fields["id"] = id
90-
91-
return result, reply_fields, error_msg
92-
93-
94-
class CommitSchema(Schema):
95-
git_branch = fields.String(required=False)
96-
git_tag = fields.String(required=False)
97-
git_hash = fields.String(required=True)
98-
99-
100-
@app.route("/api/gh/redis/redis/commits", methods=["POST"])
101-
@auth.login_required
102-
def base():
103-
# Get Request body from JSON
104-
request_data = request.json
105-
schema = CommitSchema()
106-
try:
107-
# Validate request body against schema data types
108-
result = schema.load(request_data)
109-
except ValidationError as err:
110-
# Return a nice message if validation fails
111-
return jsonify(err.messages), 400
112-
113-
# Convert request body back to JSON str
114-
data_now_json_str = dumps(result)
115-
116-
result, response_data, err_message = commit_schema_to_stream(
117-
data_now_json_str, conn
118-
)
119-
if result is False:
120-
return jsonify(err_message), 400
121-
122-
# Send data back as JSON
123-
return jsonify(response_data), 200
124-
12529

12630
def main():
12731
_, _, project_version = populate_with_poetry_data()
@@ -146,7 +50,13 @@ def main():
14650
REDIS_AUTH_SERVER_HOST, REDIS_AUTH_SERVER_PORT
14751
)
14852
)
149-
53+
conn = redis.StrictRedis(
54+
host=GH_REDIS_SERVER_HOST,
55+
port=GH_REDIS_SERVER_PORT,
56+
decode_responses=True,
57+
password=GH_REDIS_SERVER_AUTH,
58+
)
59+
app = create_app(conn)
15060
if args.logname is not None:
15161
print("Writting log to {}".format(args.logname))
15262
handler = logging.handlers.RotatingFileHandler(
@@ -169,13 +79,6 @@ def main():
16979
GH_REDIS_SERVER_HOST, GH_REDIS_SERVER_PORT
17080
)
17181
)
172-
conn = redis.StrictRedis(
173-
host=GH_REDIS_SERVER_HOST,
174-
port=GH_REDIS_SERVER_PORT,
175-
decode_responses=True,
176-
password=GH_REDIS_SERVER_AUTH,
177-
)
178-
17982
app.run(host="0.0.0.0")
18083

18184

‎redis_benchmarks_specification/__api__/app.py

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
from flask import Flask, jsonify, request
2+
from marshmallow import ValidationError
3+
from json import dumps
4+
import redis
5+
from flask_httpauth import HTTPBasicAuth
6+
7+
from redis_benchmarks_specification.__api__.schema import (
8+
commit_schema_to_stream,
9+
CommitSchema,
10+
)
11+
from redis_benchmarks_specification.__common__.env import (
12+
REDIS_AUTH_SERVER_HOST,
13+
REDIS_AUTH_SERVER_PORT,
14+
)
15+
16+
17+
def create_app(conn, test_config=None):
18+
app = Flask(__name__)
19+
auth = HTTPBasicAuth()
20+
conn = conn
21+
22+
@auth.verify_password
23+
def verify_password(username, password):
24+
result = False
25+
try:
26+
auth_server_conn = redis.StrictRedis(
27+
host=REDIS_AUTH_SERVER_HOST,
28+
port=REDIS_AUTH_SERVER_PORT,
29+
decode_responses=True,
30+
username=username,
31+
password=password,
32+
)
33+
auth_server_conn.ping()
34+
result = True
35+
except redis.exceptions.ResponseError:
36+
result = False
37+
except redis.exceptions.AuthenticationError:
38+
result = False
39+
return result
40+
41+
@app.route("/api/gh/redis/redis/commits", methods=["POST"])
42+
@auth.login_required
43+
def base():
44+
# Get Request body from JSON
45+
request_data = request.json
46+
schema = CommitSchema()
47+
try:
48+
# Validate request body against schema data types
49+
result = schema.load(request_data)
50+
except ValidationError as err:
51+
# Return a nice message if validation fails
52+
return jsonify(err.messages), 400
53+
54+
# Convert request body back to JSON str
55+
data_now_json_str = dumps(result)
56+
57+
result, response_data, err_message = commit_schema_to_stream(
58+
data_now_json_str, conn
59+
)
60+
if result is False:
61+
return jsonify(err_message), 400
62+
63+
# Send data back as JSON
64+
return jsonify(response_data), 200
65+
66+
return app

‎redis_benchmarks_specification/__api__/schema.py

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import logging
2+
from json import loads
3+
from urllib.error import URLError
4+
from urllib.request import urlopen
5+
6+
import redis
7+
from marshmallow import Schema, fields
8+
9+
from redis_benchmarks_specification.__common__.env import (
10+
STREAM_KEYNAME_GH_EVENTS_COMMIT,
11+
)
12+
13+
14+
def commit_schema_to_stream(json_str: str, conn: redis.StrictRedis):
15+
""" uses to the provided JSON dict of fields and pushes that info to the corresponding stream """
16+
fields = loads(json_str)
17+
reply_fields = loads(json_str)
18+
result = False
19+
error_msg = None
20+
if "git_hash" not in fields:
21+
error_msg = "Missing required 'git_hash' field"
22+
else:
23+
github_url = "https://github.com/redis/redis/archive/{}.zip".format(
24+
fields["git_hash"]
25+
)
26+
try:
27+
response = urlopen(github_url, timeout=5)
28+
content = response.read()
29+
fields["zip_archive"] = bytes(content)
30+
fields["zip_archive_len"] = len(bytes(content))
31+
reply_fields["archived_zip"] = True
32+
result = True
33+
except URLError as e:
34+
error_msg = "Catched URLError while fetching {} content. Error {}".format(
35+
github_url, e.__str__()
36+
)
37+
logging.error(error_msg)
38+
result = False
39+
40+
if result is True:
41+
id = conn.xadd(STREAM_KEYNAME_GH_EVENTS_COMMIT.encode(), fields)
42+
reply_fields["id"] = id
43+
44+
return result, reply_fields, error_msg
45+
46+
47+
class CommitSchema(Schema):
48+
git_branch = fields.String(required=False)
49+
git_tag = fields.String(required=False)
50+
git_hash = fields.String(required=True)

‎redis_benchmarks_specification/__builder__/builder.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ def builder_process_stream(builders_folder, conn, different_build_specs, previou
286286
)
287287
if type(ack_reply) == bytes:
288288
ack_reply = ack_reply.decode()
289-
if ack_reply == "1":
289+
if ack_reply == "1" or ack_reply == 1:
290290
logging.info(
291291
"Sucessfully acknowledge build variation stream with id {}.".format(
292292
streamId

‎redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ def self_contained_coordinator_blocking_read(
236236
)
237237
if type(ack_reply) == bytes:
238238
ack_reply = ack_reply.decode()
239-
if ack_reply == "1":
239+
if ack_reply == "1" or ack_reply == 1:
240240
logging.info(
241241
"Sucessfully acknowledge build variation stream with id {}.".format(
242242
stream_id

‎utils/tests/test_api.py

+11-2
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,22 @@
33
# Copyright (c) 2021., Redis Labs
44
# All rights reserved.
55
#
6-
from redis_benchmarks_specification.__api__.api import commit_schema_to_stream
6+
import redis_benchmarks_specification
7+
from redis_benchmarks_specification.__api__.app import create_app
8+
9+
from redis_benchmarks_specification.__api__.schema import commit_schema_to_stream
710
import redis
811

912
from redis_benchmarks_specification.__common__.env import (
1013
STREAM_KEYNAME_GH_EVENTS_COMMIT,
1114
)
1215

16+
import pytest
17+
18+
import pytest
19+
20+
# First party modules
21+
1322

1423
def test_commit_schema_to_stream():
1524
result, reply_fields, error_msg = commit_schema_to_stream(
@@ -18,7 +27,7 @@ def test_commit_schema_to_stream():
1827
assert result == False
1928
assert error_msg is not None
2029
try:
21-
conn = redis.StrictRedis()
30+
conn = redis.StrictRedis(port=16379)
2231
conn.ping()
2332
conn.flushall()
2433
result, reply_fields, error_msg = commit_schema_to_stream(

‎utils/tests/test_builder.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
#
66
import os
77

8-
from redis_benchmarks_specification.__api__.api import commit_schema_to_stream
8+
from redis_benchmarks_specification.__api__.schema import commit_schema_to_stream
99
import redis
1010

1111
from redis_benchmarks_specification.__builder__.builder import (

‎utils/tests/test_data/api_builder_common.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from redis_benchmarks_specification.__api__.api import commit_schema_to_stream
1+
from redis_benchmarks_specification.__api__.schema import commit_schema_to_stream
22
from redis_benchmarks_specification.__builder__.builder import (
33
builder_consumer_group_create,
44
builder_process_stream,

‎utils/tests/test_self_contained_coordinator.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
from redisbench_admin.utils.utils import get_ts_metric_name
1010

11-
from redis_benchmarks_specification.__api__.api import commit_schema_to_stream
11+
from redis_benchmarks_specification.__api__.schema import commit_schema_to_stream
1212
from redis_benchmarks_specification.__builder__.builder import (
1313
builder_consumer_group_create,
1414
builder_process_stream,

0 commit comments

Comments
 (0)