Skip to content

Commit

Permalink
Adding the -o option (#7)
Browse files Browse the repository at this point in the history 
* Adding the -o option

This option allows the user to run touchstone with JSON|YAML formatted
result data.

Also added dfs_list_dict func to build the tree and assign value to the leaf.

Co-authored-by: Aakarsh Gopi <[email protected]>
  • Loading branch information
@jtaleric @aakarshg
jtaleric and aakarshg authored Feb 5, 2020
1 parent d108d43 commit 49f1683
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 15 deletions.
1 change: 1 addition & 0 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pyyaml
24 changes: 12 additions & 12 deletions src/touchstone/benchmarks/ycsb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ def __init__(self, source_type=None, harness_type=None):
'compare': ['uuid', 'user','recordcount','operationcount','driver'],
'compute': [{
'filter': {
'workload_type': 'workloada',
'phase': 'run'
'phase': 'run',
'workload_type': 'workloada'
},
'buckets': ['iteration'],
'aggregations': {},
Expand All @@ -50,8 +50,8 @@ def __init__(self, source_type=None, harness_type=None):
},
{
'filter': {
'workload_type': 'workloadb',
'phase': 'run'
'phase': 'run',
'workload_type': 'workloadb'
},
'buckets': ['iteration'],
'aggregations': {},
Expand All @@ -61,8 +61,8 @@ def __init__(self, source_type=None, harness_type=None):
},
{
'filter': {
'workload_type': 'workloadc',
'phase': 'run'
'phase': 'run',
'workload_type': 'workloadc'
},
'buckets': ['iteration'],
'aggregations': {},
Expand All @@ -71,8 +71,8 @@ def __init__(self, source_type=None, harness_type=None):
},
{
'filter': {
'workload_type': 'workloadd',
'phase': 'run'
'phase': 'run',
'workload_type': 'workloadd'
},
'buckets': ['iteration'],
'aggregations': {},
Expand All @@ -82,17 +82,17 @@ def __init__(self, source_type=None, harness_type=None):
},
{
'filter': {
'workload_type': 'workloade',
'phase': 'run'
'phase': 'run',
'workload_type': 'workloade'
},
'buckets': ['iteration'],
'aggregations': {},
'collate': ['data.OVERALL.Throughput(ops/sec)']
},
{
'filter': {
'workload_type': 'workloadf',
'phase': 'run'
'phase': 'run',
'workload_type': 'workloadf'
},
'buckets': ['iteration'],
'aggregations': {},
Expand Down
32 changes: 29 additions & 3 deletions src/touchstone/compare.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
import sys
import logging
import json
import yaml

from touchstone import __version__
from . import benchmarks
from . import databases
from .utils.temp import compare_dict, dfs_dict_list, mergedicts
from .utils.temp import compare_dict, dfs_dict_list, mergedicts, dfs_list_dict

__author__ = "aakarshg"
__copyright__ = "aakarshg"
Expand Down Expand Up @@ -55,6 +56,12 @@ def parse_args(args):
help="2 uuids to compare",
type=str,
nargs='+')
parser.add_argument(
'-o', '--output',
dest="output",
help="How should touchstone output the result",
type=str,
choices=['json','yaml'])
parser.add_argument(
'-url', '--connection-url',
dest="conn_url",
Expand Down Expand Up @@ -88,7 +95,6 @@ def setup_logging(loglevel):
logging.basicConfig(level=loglevel, stream=sys.stdout,
format=logformat, datefmt="%Y-%m-%d %H:%M:%S")


def main(args):
"""Main entry point allowing external calls
Expand Down Expand Up @@ -116,6 +122,27 @@ def main(args):
compare_map=benchmark_instance.emit_compare_map(),
index=index,
input_dict=compare_uuid_dict) # noqa
if args.output :
compute_uuid_dict = {}
for compute in benchmark_instance.emit_compute_map()[index]:
current_compute_dict = {}
compute_aggs_set = []
for uuid_index, uuid in enumerate(args.uuid):
database_instance = databases.grab(args.database,
conn_url=args.conn_url[uuid_index])
catch = database_instance.emit_compute_dict(uuid=uuid,
compute_map=compute,
index=index,
input_dict=compare_uuid_dict) # noqa
if catch != {} :
current_compute_dict = dfs_list_dict(list(compute['filter'].items()),compute_uuid_dict,len(compute['filter']),catch)
compute_uuid_dict = dict(mergedicts(compute_uuid_dict, current_compute_dict))
if args.output == "json" :
print(json.dumps(compute_uuid_dict,indent=4))
if args.output == "yaml" :
print(yaml.dump(compute_uuid_dict,allow_unicode=True))
exit(0)

print("{} Key Metadata {}".format(("="*57),("="*57)))
for key in benchmark_instance.emit_compare_map()[index]:
_message = "{:40} |".format(key)
Expand Down Expand Up @@ -144,7 +171,6 @@ def main(args):
compute_uuid_dict = dict(mergedicts(compute_uuid_dict, _current_uuid_dict))
compute_aggs_set = set(compute_aggs_set)
compute_buckets = database_instance._bucket_list
# print(json.dumps(compute_uuid_dict,indent=4))
compare_dict(compute_uuid_dict, compute_aggs_set, _compute_value, compute_buckets, args.uuid, _compute_header, max_level=2*len(compute_buckets))

_logger.info("Script ends here")
Expand Down
15 changes: 15 additions & 0 deletions src/touchstone/utils/temp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,21 @@ def id_dict(obj):
def snake(input_str):
return input_str.replace('.','_')

def dfs_list_dict(input_list, input_dict, max_level, end_value, recurse_level=0):
# This function helps with building a dictionary in dfs
_recurse_level = recurse_level + 1
if _recurse_level <= max_level:
key, value = input_list[recurse_level]
if key not in input_dict:
input_dict[key] = {}
if value not in input_dict[key]:
input_dict[key][value] = {}
input_dict[key][value] = dict(mergedicts(dfs_list_dict(input_list, {}, max_level, end_value, _recurse_level), input_dict[key][value]))
return input_dict
else:
return end_value


def dfs_dict_list(input_list, input_dict, max_level, recurse_level=0):
_recurse_level = recurse_level + 1
if _recurse_level <= max_level:
Expand Down

0 comments on commit 49f1683

Please sign in to comment.