diff --git a/scripts/genensprod_or_ensemblestat.sh b/scripts/genensprod_or_ensemblestat.sh index 880acc19..37c35481 100755 --- a/scripts/genensprod_or_ensemblestat.sh +++ b/scripts/genensprod_or_ensemblestat.sh @@ -249,14 +249,16 @@ mkdir -p "${OUTPUT_DIR}" # #----------------------------------------------------------------------- # -# Check for existence of top-level OBS_DIR. +# Check for existence of top-level OBS_DIR, if necessary. # #----------------------------------------------------------------------- # -if [ ! -d "${OBS_DIR}" ]; then - print_err_msg_exit "\ -OBS_DIR does not exist or is not a directory: - OBS_DIR = \"${OBS_DIR}\"" +if [ "${MetplusToolName}" = "EnsembleStat" ]; then + if [ ! -d "${OBS_DIR}" ]; then + print_err_msg_exit "\ + OBS_DIR does not exist or is not a directory: + OBS_DIR = \"${OBS_DIR}\"" + fi fi # #----------------------------------------------------------------------- diff --git a/setup_conda.sh b/setup_conda.sh index fc1890d3..5a166399 100644 --- a/setup_conda.sh +++ b/setup_conda.sh @@ -88,16 +88,26 @@ fi conda activate -if ! conda env list | grep -q "^vx_workflow\s" ; then - echo "Creating vx_workflow environment..." - mamba env create -n vx_workflow --file "${SCRIPT_DIR}/environment.yml" --quiet +# if first argument is vx_diff, create that environment +# otherwise create vx_workflow +ENV_NAME=$1 +ENV_YAML=${SCRIPT_DIR}/environment.yml +if [ "${ENV_NAME}" == vx_diff ]; then + ENV_YAML=${SCRIPT_DIR}/tests/regression/environment.yml else - read -p "vx_workflow environment has already been built. Check for updates using environment.yml? (y/n) " -r + ENV_NAME=vx_workflow +fi + +if ! conda env list | grep -q "^${ENV_NAME}\s" ; then + echo "Creating ${ENV_NAME} environment..." + mamba env create -n ${ENV_NAME} --file "${ENV_YAML}" --quiet +else + read -p "${ENV_NAME} environment has already been built. Check for updates using environment.yml? (y/n) " -r echo if [[ $REPLY =~ ^[Yy]$ ]] ; then - echo "Updating vx_workflow environment..." - mamba env update -n vx_workflow --file "${SCRIPT_DIR}/environment.yml" --prune --quiet + echo "Updating ${ENV_NAME} environment..." + mamba env update -n ${ENV_NAME} --file "${ENV_YAML}" --prune --quiet fi fi -conda activate vx_workflow +conda activate ${ENV_NAME} diff --git a/tests/regression/README.md b/tests/regression/README.md new file mode 100644 index 00000000..6e888df6 --- /dev/null +++ b/tests/regression/README.md @@ -0,0 +1,177 @@ +# Regression Testing + +## Initializing the Test Environment + +1. Create regression directory + +Determine a directory to store the test output, e.g. `/scratch3/BMC/dtc/dtc-vx-workflow_testing`. +This directory will be used to store the output of the end-to-end test runs +and the METplus code that contains the diff utility script. + +Create the directory if it does not already exist. + +``` +regression_dir=/scratch3/BMC/dtc/dtc-vx-workflow_testing +mkdir -p ${regression_dir} +``` + +To enable other users to run the tests in this directory, +ensure that the **{regression_dir}** directory has group read/write permissions. + +``` +chmod g+w ${regression_dir} +``` + +2. Get METplus + +Navigate to the test directory and clone the METplus repository, +using the develop branch. + +``` +regression_dir=/scratch3/BMC/dtc/dtc-vx-workflow_testing +cd ${regression_dir} +git clone git@github.com:dtcenter/METplus --branch develop +``` + +3. Create the Baseline Output + +Run the regression test script on the develop branch of `dtc-vx-workflow` +to establish a baseline output dataset to compare to the output from other +branches and/or pull requests. +Follow the instructions under the **Generating/Updating the Baseline Output** section. + +## Running the Tests + +Tests can be run for a branch, pull request, or a specific commit, e.g. the baseline commit. +The `run_regression.py` script will submit a workflow for each end-to-end test case. +In the **{regression_dir}** directory, +a directory will be created named after the branch or pull request. +In that directory, the **dtc-vx-workflow** repository will be cloned using +the branch or merge commit of the pull request. The end-to-end test script +will be run from the **dtc-vx-workflow** directory. +The output will be written to a directory named **output.XXXXXXX** where +*XXXXXXX* is the latest commit has of the branch or pull request. +Subsequent runs will pull the latest changes from the branch or pull request +and rerun the tests, writing to a new **output.XXXXXXX** directory. + +```text +{regression_dir}/ +├── develop/ +│ ├── dtc-vx-workflow/ +│ ├── output.abcdef1/ +│ └── output.2345678/ +├── pr_10/ +│ ├── dtc-vx-workflow/ +│ ├── output.9abcdef/ +│ └── output.1234567/ +└── feature/add_HAFS_vx/ + ├── dtc-vx-workflow/ + └── output.b252ab7/ +``` + +The script starts the end-to-end test script using nohup. +The full path to the nohup.out is printed to the screen. +Run `tail -f` on it to see the progress of the tests. + +### Generating/Updating the Baseline Output + +Calling the `run_regression.py` script with the `--baseline` argument will +run the end-to-end tests for the baseline commit and create a symbolic link +named **output.baseline** in the regression directory. +The baseline commit is stored in the `regression_baseline.py` file in the +**BASELINE_COMMIT** variable. + +To update the baseline version, note the first 7 characters of the latest +commit on the develop branch after the changes that modify the output +have been merged. +Open the `regression_baseline.py` file and modify the value of the +**BASELINE_COMMIT** variable to the new commit hash. +**Be sure to commit the change to the develop branch of `dtc-vx-workflow`.** +If the output from the baseline commit has already been generated locally, +the script will skip the tests and update the symbolic link to the baseline commit. + +``` +account=dtc + +machine=ursa +regression_dir=/scratch3/BMC/dtc/dtc-vx-workflow_testing + +cd dtc-vx-workflow +python3 ./tests/regression/run_regression.py \ + --baseline \ + --account ${account} \ + --machine ${machine} \ + --regression_dir ${regression_dir} +``` + +### Running on a Branch + +``` +branch=develop +account=dtc + +machine=ursa +regression_dir=/scratch3/BMC/dtc/dtc-vx-workflow_testing + +cd dtc-vx-workflow +python3 ./tests/regression/run_regression.py \ + --branch ${branch} \ + --account ${account} \ + --machine ${machine} \ + --regression_dir ${regression_dir} +``` + +### Running on a Pull Request + +The instructions for running on a pull request are nearly the same as for running on a branch. +The only difference is that the pull request number is specified using the `--pr` argument instead +of specifying the branch name using the `--branch` argument. + +``` +pr_number=10 +account=dtc + +machine=ursa +regression_dir=/scratch3/BMC/dtc/dtc-vx-workflow_testing + +cd dtc-vx-workflow +python3 ./tests/regression/run_regression.py \ + --pr ${pr_number} \ + --account ${account} \ + --machine ${machine} \ + --regression_dir ${regression_dir} +``` + +### Running on a Subset of Tests + +The `--tests` argument can be provided to the `run_regression.py` script +to define a subset of tests to run. +The format of the argument is the same as the `--tests` argument to the +`tests/WE2E/run_we2e_tests.py` script. +The default behavior is to pass `--tests all` to the `run_we2e_tests.py` script. + +## Running the Diff Utility + +To run the METplus diff utility, call the `run_diff.py` script, +passing the path to the output directory of the end-to-end tests. +By default, the output.baseline directory is used as the baseline. +You can override this by passing the `--baseline` argument. +You can also override the location of METplus to use with the `--metplus` argument. + +The default behavior is to run the diff utility on all files in the dated subdirectories +under the output directory, because these are assumed to contain the actual MET output. +The `--diff_inputs` argument can be added to run the diff utility on each +output directory, which includes the input observation files. +A list of keywords to skip workflow files are defined in the `run_diff.py` +script in the **SKIP_KEYWORDS** variable. + +``` +regression_dir=/scratch3/BMC/dtc/dtc-vx-workflow_testing +test_dir=${regression_dir}/feature/my_branch_name/output.abcdef1 + +cd dtc-vx-workflow +source ./setup_conda.sh vx_diff +python3 ./tests/regression/run_diff.py \ + ${test_dir} \ + --regression_dir ${regression_dir} +``` diff --git a/tests/regression/environment.yml b/tests/regression/environment.yml new file mode 100644 index 00000000..fc1298b9 --- /dev/null +++ b/tests/regression/environment.yml @@ -0,0 +1,7 @@ +name: vx_diff +channels: + - conda-forge +dependencies: + - pandas~=3.0.3 + - netcdf4~=1.7.4 + - pillow~=12.2.0 diff --git a/tests/regression/regression_baseline.py b/tests/regression/regression_baseline.py new file mode 100644 index 00000000..9d289495 --- /dev/null +++ b/tests/regression/regression_baseline.py @@ -0,0 +1,2 @@ +BASELINE_COMMIT = "84b78ac3" +BASELINE_BRANCH = "develop" diff --git a/tests/regression/regression_common.py b/tests/regression/regression_common.py new file mode 100644 index 00000000..7b8c4bc1 --- /dev/null +++ b/tests/regression/regression_common.py @@ -0,0 +1,8 @@ +# default directory for regression test files (Ursa) +DEFAULT_REGRESSION_DIR = "/scratch3/BMC/dtc/dtc-vx-workflow_testing" + +# repository for dtc-vx-workflow git clone +WORKFLOW_REPO = "dtcenter/dtc-vx-workflow" + +# string to used in diff results to indicate a test was not run +NOT_RUN = "NOT RUN" \ No newline at end of file diff --git a/tests/regression/run_diff.py b/tests/regression/run_diff.py new file mode 100644 index 00000000..a2b759b8 --- /dev/null +++ b/tests/regression/run_diff.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 + +import sys +import argparse +from pathlib import Path +import io +from contextlib import redirect_stdout +from dataclasses import dataclass +import logging +from datetime import datetime + +from regression_common import DEFAULT_REGRESSION_DIR, NOT_RUN + +@dataclass +class TestResults: + status: str = NOT_RUN + details: str = "" + +SKIP_KEYWORDS = [ + "vx_wflow.xml", + "var_defns.yaml", + "vx_wflow_lock.db", + "rocoto_defns.yaml", + "vx_wflow.db", + "config.yaml", + "launch_vx_wflow.sh", + "log.generate_wflow", + ".conf", + "/log/", + "/stage/", +] + +def main(): + args = read_args() + + # set up logging + log_file = setup_logging(args) + + if not Path(args.metplus).exists(): + print(f"ERROR: METplus directory does not exist: {args.metplus}") + sys.exit(1) + + success = True + + sys.path.insert(0, str(args.metplus)) + from metplus.util import diff_util + diff_util.SKIP_KEYWORDS = SKIP_KEYWORDS + + msg = ( + "Running diff tests" + f"\nBASELINE: {Path(args.baseline).resolve()}" + f"\nNEW : {Path(args.test_dir).resolve()}" + ) + logging.info(msg) + if not args.log_to_terminal: + print(msg) + + tests_baseline = get_test_paths(args.baseline, not args.diff_inputs) + tests_new = get_test_paths(args.test_dir, not args.diff_inputs) + + all_tests = list(set(tests_baseline).union(tests_new)) + test_results = {x: TestResults() for x in all_tests} + + # set tests that are not found in either baseline or new output to failed + not_in_baseline = [x for x in tests_new if x not in tests_baseline] + not_in_new = [x for x in tests_baseline if x not in tests_new] + for test_name in not_in_baseline: + test_results[test_name].status = 'FAILED (not in baseline)' + success = False + for test_name in not_in_new: + test_results[test_name].status = 'FAILED (not in new output)' + success = False + + for test_name, test_result in test_results.items(): + if test_result.status != NOT_RUN: + continue + + print(f"Diffing {test_name}...") + baseline_test = Path(args.baseline) / test_name + new_test = Path(args.test_dir) / test_name + + # create text stream to capture diff output for each test + text_stream = io.StringIO() + with redirect_stdout(text_stream): + diff_files = diff_util.compare_dir(str(baseline_test), str(new_test), debug=args.debug) + + # save detailed report of diff test + test_result.details = text_stream.getvalue().strip() + + if not diff_files: + test_result.status = 'SUCCEEDED' + continue + + test_result.status = 'FAILED' + success = False + + logging.info("SUMMARY:") + for test_name, test_result in test_results.items(): + logging.info(f"{test_name.ljust(65)}: {test_result.status}") + + logging.info("DETAILS:") + for test_name, test_result in test_results.items(): + logging.info(f"{'-' * 80}\n{test_name}: {test_result.status}\n{'-' * 80}") + logging.info(test_result.details) + logging.info(f"{'-' * 80}\n\n") + + log_msg = '' if not log_file else f"\nSee log file for details: {log_file}" + if success: + msg = f"SUCCESS: No differences found!{log_msg}" + logging.info(msg) + else: + msg = f"ERROR: Differences were found!{log_msg}" + logging.error(msg) + + if not args.log_to_terminal: + print(msg) + + return success + +def read_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run difference tests") + parser.add_argument("test_dir", + help="Directory with test output data to compare to baseline") + parser.add_argument("--regression_dir", default=DEFAULT_REGRESSION_DIR, + help=f"Directory containing regression test output (default: {DEFAULT_REGRESSION_DIR})") + parser.add_argument("--metplus", + help="Directory of METplus repo to get diff_util.py (default: regression_dir/METplus)") + parser.add_argument("--baseline", + help="Directory with baseline data to use for comparison (default: regression_dir/output.baseline)") + parser.add_argument("--diff_inputs", action="store_true", + help="If set, run the diff utility on each output directory, which includes the input observation files.") + parser.add_argument("--log_dir", + help="Directory where the log file should be saved (default: regression_dir)") + parser.add_argument("--log_to_terminal", action="store_true", + help="If set, log to the terminal (standard output) instead of a file") + parser.add_argument("-d", "--debug", action="store_true", + help="Log information about files that were skipped or had no differences") + + args = parser.parse_args() + + if not args.metplus: + args.metplus = Path(args.regression_dir) / "METplus" + if not args.baseline: + args.baseline = Path(args.regression_dir) / "output.baseline" + if not args.log_dir: + args.log_dir = args.regression_dir + + return args + +def setup_logging(args): + log_file_path = None + log_config = { + "format": "%(message)s", + "level": logging.INFO, + } + if args.log_to_terminal: + # Route logs to stdout (terminal) + print("Logging to terminal because --log_to_terminal was set.") + log_config["stream"] = sys.stdout + else: + # Ensure the directory exists before writing to it + Path(args.log_dir).mkdir(parents=True, exist_ok=True) + log_file_path = Path(args.log_dir) / f"diff_WE2E_{datetime.now().strftime('%Y%m%d%H%M%S')}.txt" + print(f"Logging to file: {log_file_path}") + log_config["filename"] = str(log_file_path) + log_config["filemode"] = "w" + + logging.basicConfig(**log_config) + return log_file_path + +def get_test_paths(base_path, get_dated=True): + paths = [] + # Loop through the main test directories (e.g., TEST_NAME) + for test_dir in Path(base_path).iterdir(): + if not test_dir.is_dir(): + continue + + has_date_subdirs = False + + if get_dated: + # Look for numeric subdirectories inside the test directory + for sub_dir in test_dir.iterdir(): + if sub_dir.is_dir() and sub_dir.name.isdigit(): + # Store as 'TEST_NAME/202602010000' + paths.append(f"{test_dir.name}/{sub_dir.name}") + has_date_subdirs = True + + # Fallback if get_dated is False or no numeric subdirs were found + if not has_date_subdirs: + paths.append(test_dir.name) + + return paths + +if __name__ == "__main__": + status = main() + if not status: + sys.exit(1) diff --git a/tests/regression/run_regression.py b/tests/regression/run_regression.py new file mode 100644 index 00000000..e57eb04c --- /dev/null +++ b/tests/regression/run_regression.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 + +import argparse +import subprocess +import sys +import shlex +from pathlib import Path + +from regression_baseline import BASELINE_COMMIT, BASELINE_BRANCH +from regression_common import WORKFLOW_REPO, DEFAULT_REGRESSION_DIR + +def main(): + + args = read_args() + + print(f"Regression directory: {args.regression_dir}") + + branch_or_pr_dir = f"pr_{args.pr}" if args.pr else args.branch + branch_or_pr_dir = Path(args.regression_dir) / branch_or_pr_dir + workflow_repo_dir = Path(branch_or_pr_dir) / WORKFLOW_REPO.split('/')[-1] + + setup_repo_dir(args, workflow_repo_dir) + + # create directory for test output + # get explicit commit if specified, otherwise get latest commit + if args.commit: + commit = args.commit + else: + commit = run_command(f"git -C {workflow_repo_dir} rev-parse HEAD")[:7] + + output_path = Path(branch_or_pr_dir) / f"output.{commit}" + + # Error/exit if branch/commit directory already exists + if output_path.is_dir(): + print(f"WARNING: Test dir already exists: {output_path}") + print(" Remove it to run") + else: + + # Create test directory + print(f"Creating directory: {output_path}") + output_path.mkdir(parents=True, exist_ok=True) + + launch_tests(args, workflow_repo_dir, output_path) + + # if running baseline, update symbolic link for baseline output dir + if args.baseline: + baseline_dir = Path(args.regression_dir) / "output.baseline" + print(f"Updating symbolic link output.baseline to point to {output_path}") + + # remove link if it exists already + if baseline_dir.is_symlink(): + print(f"Removing existing symbolic link: {baseline_dir}") + baseline_dir.unlink() + + baseline_dir.symlink_to(output_path, target_is_directory=True) + +def read_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run regression tests") + parser.add_argument("--baseline", action="store_true", + help="Run tests for baseline commit. Ignore branch/pr/commit args.") + parser.add_argument("--branch", help="Branch to run tests for.") + parser.add_argument("--pr", help="Pull request to run tests for") + parser.add_argument("--commit", help="Specific commit to run tests for") + parser.add_argument("--account", required=True, help="Account to run jobs") + parser.add_argument("--machine", required=True, help="Machine to run jobs") + parser.add_argument("--regression_dir", default=DEFAULT_REGRESSION_DIR, + help=f"Directory to run regression tests (default: {DEFAULT_REGRESSION_DIR})") + parser.add_argument("--clone_https", action="store_true", + help="Clone via https instead of ssh", ) + parser.add_argument("--tests", default="all", + help="Defines tests to run (default: all)." + " Matches format expected by run_we2e_tests.py script.") + args = parser.parse_args() + + # if baseline is requested, set branch and commit to baseline values and unset pr arg + + if args.baseline: + args.branch = BASELINE_BRANCH + args.commit = BASELINE_COMMIT + args.pr = None + + # tests argument cannot be provided if baseline argument is requested + if args.tests != "all": + print("ERROR: Cannot specify --baseline and a subset of tests with --tests") + sys.exit(1) + + # error if pr is requested and a branch or commit is also requested + + if args.pr and (args.branch or args.commit): + print("ERROR: Cannot specify --branch or --commit with --pr") + sys.exit(1) + + return args + +def run_command(command): + """Run a command and return its output.""" + try: + print("Running command:", command, flush=True) + result = subprocess.run(shlex.split(command), capture_output=True, text=True, check=True) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + print(f"Error running command {command}: {e.stderr}") + sys.exit(1) + +def setup_repo_dir(args: argparse.Namespace, workflow_repo_dir: Path): + if not workflow_repo_dir.exists(): + # Clone workflow repo + repo_loc = f"https://github.com/{WORKFLOW_REPO}" if args.clone_https else f"git@github.com:{WORKFLOW_REPO}" + run_command(f"git clone {repo_loc} {workflow_repo_dir}") + else: + # get the latest changes if the repo has already been cloned + run_command(f"git -C {workflow_repo_dir} fetch origin") + + # check out the branch or PR merge commit + if args.pr: + merge_commit_id = f"refs/pull/{args.pr}/merge" + run_command(f"git -C {workflow_repo_dir} fetch origin {merge_commit_id}") + run_command(f"git -C {workflow_repo_dir} checkout {merge_commit_id}") + elif args.branch: + run_command(f"git -C {workflow_repo_dir} checkout {args.branch}") + + # check out specific commit if specified + if args.commit: + run_command(f"git -C {workflow_repo_dir} checkout {args.commit}") + # otherwise pull the latest changes if obtaining a branch + elif args.branch: + run_command(f"git -C {workflow_repo_dir} pull origin {args.branch}") + +def launch_tests(args: argparse.Namespace, workflow_repo_dir: Path, output_path: Path): + print(f"Running WE2E tests in {Path(output_path.parent.name) / output_path.name}: {args.tests}") + + # Determine paths relative to the workflow repo directory + + we2e_test_dir = workflow_repo_dir / "tests" / "WE2E" + test_script = we2e_test_dir / "run_we2e_tests.py" + + # Commands to set up conda and run the tests + + cmd = ( + f"source {workflow_repo_dir}/setup_conda.sh &&" + f" {test_script} --account {args.account} --machine {args.machine}" + f" --tests {args.tests} --expt_basedir {output_path}" + ) + + print(f"RUNNING: {cmd}") + print(f"CWD: {we2e_test_dir}") + print("Launching in background with nohup") + print(f"Follow {we2e_test_dir}/nohup.out for output", flush=True) + + try: + # run setup_conda.sh and end-to-end test script + + subprocess.Popen(f"nohup bash -c '{cmd}' &", shell=True, cwd=we2e_test_dir) + + except Exception as e: + print(f"Failed to launch test command: {e}") + sys.exit(1) + +if __name__ == "__main__": + main()