diff --git a/jenkins/helper/test_launch_controller.py b/jenkins/helper/test_launch_controller.py index 8c833e96..b2846db8 100644 --- a/jenkins/helper/test_launch_controller.py +++ b/jenkins/helper/test_launch_controller.py @@ -3,7 +3,9 @@ import argparse import copy import sys +import json from traceback import print_exc +import yaml from site_config import IS_ARM, IS_WINDOWS, IS_MAC, IS_COVERAGE @@ -81,6 +83,23 @@ def list_generator(cluster): return res_sg + res_cl return list_generator(args.cluster) +def filter_one_test(args, test): + """filter testcase by operations target Single/Cluster/full""" + if args.all: + return False + if IS_COVERAGE: + if 'coverage' in test: + return True + full = args.full + filters = [] + + if 'full' in test: + if full and not test['full']: + return True + if not full and test['full']: + return True + return False + formats = { "dump": generate_dump_output, "launch": launch, @@ -89,6 +108,7 @@ def list_generator(cluster): known_flags = { "cluster": "this test requires a cluster", "single": "this test requires a single server", + "mixed": "some buckets will run cluster, some not.", "full": "this test is only executed in full tests", "!full": "this test is only executed in non-full tests", "gtest": "only the testsuites starting with 'gtest' are to be executed", @@ -110,7 +130,10 @@ def list_generator(cluster): "buckets": "number of buckets to use for this test", "suffix": "suffix that is appended to the tests folder name", "priority": "priority that controls execution order. Testsuites with lower priority are executed later", - "parallelity": "parallelity how many resources will the job use in the SUT? Default: 1 in Single server, 4 in Clusters" + "parallelity": "parallelity how many resources will the job use in the SUT? Default: 1 in Single server, 4 in Clusters", + "type": "single or cluster flag", + "full": "whether to spare from a single or full run", + "sniff": "whether to enable sniffing", } @@ -160,8 +183,8 @@ def parse_number(value): raise Exception(f"invalid numeric value: {value}") from exc def parse_number_or_default(key, default_value=None): - """ check number """ - if key in params: + """check number""" + if key in params and not isinstance(params[key], int): if params[key][0] == '*': # factor the default params[key] = default_value * parse_number(params[key][1:]) else: @@ -177,7 +200,7 @@ def parse_number_or_default(key, default_value=None): def validate_flags(flags): - """ check whether target flags are valid """ + """check whether target flags are valid""" if "cluster" in flags and "single" in flags: raise Exception("`cluster` and `single` specified for the same test") if "full" in flags and "!full" in flags: @@ -185,7 +208,7 @@ def validate_flags(flags): def read_definition_line(line): - """ parse one test definition line """ + """parse one test definition line""" bits = line.split() if len(bits) < 1: raise Exception("expected at least one argument: ") @@ -221,33 +244,233 @@ def read_definition_line(line): validate_flags(flags) params = validate_params(params, 'cluster' in flags) + if len(arangosh_args) == 0: + arangosh_args = "" + run_job = 'run-linux-tests' return { "name": params.get("name", suites), - "suite": suites, + "suites": suites, "priority": params["priority"], "parallelity": params["parallelity"], "flags": flags, "args": args, "arangosh_args": arangosh_args, - "params": params + "params": params, + "testfile_definitions": testfile_definitions, + "run_job": run_job, + } + +def read_yaml_suite(name, suite, definition, testfile_definitions): + """ convert yaml representation into the internal one """ + if not 'options' in definition: + definition['options'] = {} + flags = [] + params = {} + arangosh_args = [] + args = [] + if 'args' in definition: + if not isinstance(definition['args'], dict): + raise Exception(f"expected args to be a key value list! have: {definition['args']}") + for key, val in definition['args'].items(): + if key == 'moreArgv': + args.append(val) + else: + args.append(f"--{key}") + if isinstance(val, bool): + args.append("true" if val else "false") + else: + args.append(val) + if 'arangosh_args' in definition: + if not isinstance(definition['arangosh_args'], dict): + raise Exception(f"expected arangosh_args to be a key value list! have: {definition['arangosh_args']}") + for key, val in definition['arangosh_args'].items(): + arangosh_args.append(f"--{key}") + if isinstance(val, bool): + arangosh_args.append("true" if val else "false") + else: + arangosh_args.append(val) + + medium_size = False + is_cluster = (definition['options'] and + 'type' in definition['options'] and + definition['options']['type'] == 'cluster') + params = validate_params(definition['options'], is_cluster) + if 'type' in params: + if params['type'] == "cluster": + medium_size = True + flags.append('cluster') + elif params['type'] == "mixed": + medium_size = True + flags.append('mixed') + else: + flags.append('single') + size = "medium" if medium_size else "small" + size = size if not "size" in params else params['size'] + + if 'full' in params: + flags.append("full" if params["full"] else "!full") + if 'coverage' in params: + flags.append("coverage" if params["coverage"] else "!coverage") + if 'sniff' in params: + flags.append("sniff" if params["sniff"] else "!sniff") + run_job = 'run-linux-tests' + return { + "name": name if not "name" in params else params['name'], + "suite": suite, + "size": size, + "flags": flags, + "args": args.copy(), + "priority": params["priority"], + "arangosh_args": arangosh_args.copy(), + "params": params.copy(), + "testfile_definitions": testfile_definitions, + "run_job": run_job, + "parallelity": params["parallelity"], } +def get_args(args): + """ serialize args into json similar to fromArgv in testing.js """ + sub_args = {} + for key in args.keys(): + value = args[key] + if ":" in key: + keyparts = key.split(":") + if not keyparts[0] in sub_args: + sub_args[keyparts[0]] = {} + sub_args[keyparts[0]][keyparts[1]] = value + elif key in sub_args: + if isinstance(sub_args[key], list): + sub_args[key].append(value) + else: + sub_args[key] = [value] + else: + sub_args[key] = value + return sub_args -def read_definitions(filename): - """ read test definitions txt """ + +def read_yaml_multi_suite(name, definition, testfile_definitions, cli_args): + """ convert yaml representation into the internal one """ + generated_definition = { + } + args = {} + if 'options' in definition: + generated_definition['options'] = definition['options'] + if 'args' in definition: + args = definition['args'].copy() + suite_strs = [] + options_json = [] + for suite in definition['suites']: + if isinstance(suite, str): + options_json.append({}) + suite_name = suite + else: + suite_name = list(suite.keys())[0] + if not isinstance(suite, dict): + raise Exception(f"suite should be a dict, it is {type(suite)}") + if 'options' in suite[suite_name]: + if filter_one_test(cli_args, suite[suite_name]['options']): + print(f"skipping {suite}") + continue + if 'args' in suite[suite_name]: + options_json.append(get_args(suite[suite_name]['args'])) + else: + options_json.append({}) + suite_strs.append(suite_name) + generated_name = ','.join(suite_strs) + args['optionsJson'] = json.dumps(options_json, separators=(',', ':')) + if args != {}: + generated_definition['args'] = args + return read_yaml_suite(name, generated_name, generated_definition, testfile_definitions) + +def read_yaml_bucket_suite(bucket_name, definition, testfile_definitions, cli_args): + """ convert yaml representation into the internal one """ + ret = [] + args = {} + options = [] + if 'options' in definition: + options = definition['options'] + if 'args' in definition: + args = definition['args'] + for suite in definition['suites']: + if isinstance(suite, str): + ret.append( + read_yaml_suite(suite, + suite, + { + 'options': definition['options'], + 'name': bucket_name, + 'args': args, + 'suite': suite + }, + testfile_definitions) + ) + else: + suite_name = list(suite.keys())[0] + local_options = options.copy() + if 'options' in suite[suite_name]: + local_options = local_options | suite[suite_name]['options'] + local_args = args.copy() + if 'args' in suite[suite_name]: + local_args = local_args | suite[suite_name]['args'] + ret.append( + read_yaml_suite(suite_name, + suite_name, + { + 'options': local_options, + 'name': bucket_name, + 'args': local_args, + 'suite': suite_name + }, + testfile_definitions) + ) + + return ret + +def read_definitions(filename, override_branch, args): + """read test definitions txt""" tests = [] has_error = False - with open(filename, "r", encoding="utf-8") as filep: - for line_no, line in enumerate(filep): - line = line.strip() - if line.startswith("#") or len(line) == 0: - continue # ignore comments - try: - test = read_definition_line(line) - tests.append(test) - except Exception as exc: - print(f"{filename}:{line_no + 1}: \n`{line}`\n {exc}", file=sys.stderr) - has_error = True + testfile_definitions = {} + yaml_text = "" + if filename.endswith(".yml"): + with open(filename, "r", encoding="utf-8") as filep: + config = yaml.safe_load(filep) + if isinstance(config, dict): + if "add-yaml" in config: + parsed_yaml = {"add-yaml": copy.deepcopy(config["add-yaml"])} + if "jobProperties" in config: + testfile_definitions = copy.deepcopy(config["jobProperties"]) + config = config['tests'] + for testcase in config: + suite_name = list(testcase.keys())[0] + try: + suite = testcase[suite_name] + if "suites" in suite: + if ('options' in suite and + 'buckets' in suite['options'] and + suite['options']['buckets'] == "auto"): + del suite['options']['buckets'] + tests += read_yaml_bucket_suite(suite_name, suite, testfile_definitions, args) + else: + tests.append(read_yaml_multi_suite(suite_name, suite, testfile_definitions, args)) + else: + tests.append(read_yaml_suite(suite_name, suite_name, + suite, testfile_definitions)) + except Exception as ex: + print(f"while parsing {suite_name} {testcase}") + raise ex + else: + with open(filename, "r", encoding="utf-8") as filep: + for line_no, line in enumerate(filep): + line = line.strip() + if line.startswith("#") or len(line) == 0: + continue # ignore comments + try: + test = read_definition_line(line) + tests.append(test) + except Exception as exc: + print(f"{filename}:{line_no + 1}: \n`{line}`\n {exc}", file=sys.stderr) + has_error = True if has_error: raise Exception("abort due to errors") return tests @@ -263,7 +486,7 @@ def main(): """ entrypoint """ try: args = parse_arguments() - tests = read_definitions(args.definitions) + tests = read_definitions(args.definitions, "", args) if args.validate_only: return # nothing left to do tests = filter_tests(args, tests) diff --git a/jenkins/runCoverageRocksDB.fish b/jenkins/runCoverageRocksDB.fish index 0ea9eac0..c4cfca3c 100755 --- a/jenkins/runCoverageRocksDB.fish +++ b/jenkins/runCoverageRocksDB.fish @@ -1,6 +1,10 @@ #!/usr/bin/env fish -set -xg TEST_DEFINITIONS test-definitions.txt +if test -f "$INNERWORKDIR/ArangoDB/tests/test-definitions.yml" + set -xg TEST_DEFINITIONS test-definitions.yml +else + set -xg TEST_DEFINITIONS test-definitions.txt +end if test (count $argv) -gt 0 set -xg TEST_DEFINITIONS $argv[1] end diff --git a/scripts/runTests.fish b/scripts/runTests.fish index ae83463e..48c88c6a 100755 --- a/scripts/runTests.fish +++ b/scripts/runTests.fish @@ -7,13 +7,20 @@ if test "$ENTERPRISEEDITION" = "On" set ENTERPRISE_ARG "--enterprise" end +if test -f "$INNERWORKDIR/ArangoDB/tests/test-definitions.yml" + echo yaml + set -xg TD_TYPE yml +else + set -xg TD_TYPE txt +end + ################################################################################ ## Single tests: runtime,command ################################################################################ function launchSingleTests echo "Using test definitions from arangodb repo" - python3 -u "$WORKSPACE/jenkins/helper/test_launch_controller.py" "$INNERWORKDIR/ArangoDB/tests/test-definitions.txt" -f launch "$ENTERPRISE_ARG" + python3 -u "$WORKSPACE/jenkins/helper/test_launch_controller.py" "$INNERWORKDIR/ArangoDB/tests/test-definitions.$TD_TYPE" -f launch "$ENTERPRISE_ARG" set x $status if test "$x" = "0" -a -f $INNERWORKDIR/testRuns.html set -xg result "GOOD" @@ -28,7 +35,7 @@ end ################################################################################ function launchGTest - python3 -u "$WORKSPACE/jenkins/helper/test_launch_controller.py" "$INNERWORKDIR/ArangoDB/tests/test-definitions.txt" -f launch --gtest "$ENTERPRISE_ARG" + python3 -u "$WORKSPACE/jenkins/helper/test_launch_controller.py" "$INNERWORKDIR/ArangoDB/tests/test-definitions.$TD_TYPE" -f launch --gtest "$ENTERPRISE_ARG" set x $status if test "$x" = "0" -a -f $INNERWORKDIR/testRuns.html set -xg result "GOOD" @@ -44,7 +51,7 @@ end function launchClusterTests echo "Using test definitions from arangodb repo" - python3 -u "$WORKSPACE/jenkins/helper/test_launch_controller.py" "$INNERWORKDIR/ArangoDB/tests/test-definitions.txt" -f launch --cluster "$ENTERPRISE_ARG" + python3 -u "$WORKSPACE/jenkins/helper/test_launch_controller.py" "$INNERWORKDIR/ArangoDB/tests/test-definitions.$TD_TYPE" -f launch --cluster "$ENTERPRISE_ARG" set x $status if test "$x" = "0" -a -f $INNERWORKDIR/testRuns.html set -xg result "GOOD" @@ -60,7 +67,7 @@ end function launchSingleClusterTests echo "Using test definitions from arangodb repo" - python3 -u "$WORKSPACE/jenkins/helper/test_launch_controller.py" "$INNERWORKDIR/ArangoDB/tests/test-definitions.txt" -f launch --single_cluster "$ENTERPRISE_ARG" + python3 -u "$WORKSPACE/jenkins/helper/test_launch_controller.py" "$INNERWORKDIR/ArangoDB/tests/test-definitions.$TD_TYPE" -f launch --single_cluster "$ENTERPRISE_ARG" set x $status if test "$x" = "0" -a -f $INNERWORKDIR/testRuns.html set -xg result "GOOD"