diff --git a/fre/__init__.py b/fre/__init__.py index 3a3702c89..a7a2fa10c 100644 --- a/fre/__init__.py +++ b/fre/__init__.py @@ -9,7 +9,7 @@ fre_logger = logging.getLogger(__name__) -FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s" +FORMAT = "[%(levelname)7s:%(filename)24s:%(funcName)24s] %(message)s" logging.basicConfig(level = logging.WARNING, format = FORMAT, filename = None, diff --git a/fre/fre.py b/fre/fre.py index 25b717504..2c60308d0 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -22,7 +22,8 @@ # click and lazy group loading @click.group( cls = LazyGroup, - lazy_subcommands = {"pp": ".pp.frepp.pp_cli", + lazy_subcommands = {"workflow": ".workflow.freworkflow.workflow_cli", + "pp": ".pp.frepp.pp_cli", "catalog": ".catalog.frecatalog.catalog_cli", "list": ".list_.frelist.list_cli", "check": ".check.frecheck.check_cli", diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py index bf12c33ff..c7c08113a 100644 --- a/fre/tests/test_fre_cmor_cli.py +++ b/fre/tests/test_fre_cmor_cli.py @@ -70,8 +70,8 @@ def test_cli_fre_cmor_help_and_debuglog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long - log_text_line_2='[DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_2='[ DEBUG: fre.py: fre] click entry-point function call done.\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] @@ -89,7 +89,7 @@ def test_cli_fre_cmor_help_and_infolog(): assert result.exit_code == 0 assert Path("TEST_FOO_LOG.log").exists() - log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long + log_text_line_1='[ INFO: fre.py: fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text: line_list=log_text.readlines() assert log_text_line_1 in line_list[0] diff --git a/fre/tests/test_fre_workflow_cli.py b/fre/tests/test_fre_workflow_cli.py new file mode 100644 index 000000000..db2a002ae --- /dev/null +++ b/fre/tests/test_fre_workflow_cli.py @@ -0,0 +1,76 @@ +""" +CLI Tests for fre workflow * + +Tests the command-line-interface commands for each tool in the fre workflow suite. + - successful invocation of fre workflow $tool + - successful invocation of fre workflow $tool --help + - expected failure for fre workflow $tool --optionDne (failure for undefined click option) +""" +from pathlib import Path +from click.testing import CliRunner +from fre import fre + +runner = CliRunner() + +#-- fre workflow +def test_cli_fre_workflow(): + ''' fre workflow ''' + result = runner.invoke(fre.fre, args=["workflow"]) + assert result.exit_code == 2 + +def test_cli_fre_workflow_help(): + ''' fre workflow --help ''' + result = runner.invoke(fre.fre, args=["workflow", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_workflow_opt_dne(): + ''' fre workflow optionDNE ''' + result = runner.invoke(fre.fre, args=["workflow", "optionDNE"]) + assert result.exit_code == 2 + +#-- fre workflow checkout +def test_cli_fre_workflow_checkout(): + ''' fre workflow checkout''' + result = runner.invoke(fre.fre, args=["workflow", "checkout"]) + assert result.exit_code == 2 + +def test_cli_fre_workflow_checkout_help(): + ''' fre workflow checkout --help ''' + result = runner.invoke(fre.fre, args=["workflow", "checkout", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_workflow_checkout_opt_dne(): + ''' fre workflow checkout optionDNE ''' + result = runner.invoke(fre.fre, args=["workflow", "checkout", "optionDNE"]) + assert result.exit_code == 2 + +def test_cli_fre_workflow_checkout_target_dir_set(tmp_path): + """ + Test checkout in target directory if --target-dir is explicitly set. + """ + experiment = "c96L65_am5f7b12r1_amip_TESTING" + result = runner.invoke(fre.fre, args=["workflow", "checkout", + "--yamlfile", "fre/workflow/tests/AM5_example/am5.yaml", + "--experiment", experiment, + "--application", "pp", + "--target-dir", tmp_path]) + assert result.exit_code == 0 + assert Path(f"{tmp_path}/cylc-src/{experiment}").exists() + +def test_cli_fre_workflow_checkout_default_dir(monkeypatch, tmp_path): + """ + Test workflow repository is cloned in the default location + if --target-dir is not set; default = ~./fre-workflows + """ + # Create and set a mock HOME + fake_home = f"{tmp_path}/fake_home" + Path(fake_home).mkdir(parents=True,exist_ok=True) + monkeypatch.setenv("HOME", f"{tmp_path}/fake_home") + + experiment = "c96L65_am5f7b12r1_amip_TESTING" + result = runner.invoke(fre.fre, args=["workflow", "checkout", + "-y", "fre/workflow/tests/AM5_example/am5.yaml", + "-e", experiment, + "-a", "pp"]) + assert result.exit_code == 0 + assert Path(f"{fake_home}/.fre-workflows/cylc-src/{experiment}").exists() diff --git a/fre/workflow/README.md b/fre/workflow/README.md new file mode 100644 index 000000000..75bc72dc3 --- /dev/null +++ b/fre/workflow/README.md @@ -0,0 +1,22 @@ +# FRE workflow + +The`fre workflow` toolset allows users to clone, install, and run a cylc workflow. + +The workflow repository and version are specified in `the setting.yaml`. + +## Quickstart +From the top-level dircetory of the fre-cli repository: +``` +# Checkout/clone the post-processing workflow repository +fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING --application pp +``` + +## Subtools +- `fre workflow checkout [options]` + - Purpose: Clone the specified workflow repository from the settings.yaml, associated with the application passed. + - Options: + - `-y, --yamlfile [model yaml] (str; required)` + - `-e, --experiment [experiment name] (str; required)` + - `-a, --application [ run | pp ] (str; required)` + - `--target-dir [target location where workflow will be cloned] (str; optional; default is ~/.fre-workflows` + - `--force-checkout (bool; optional)` diff --git a/fre/workflow/__init__.py b/fre/workflow/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/fre/workflow/checkout_script.py b/fre/workflow/checkout_script.py new file mode 100644 index 000000000..744c4bc0f --- /dev/null +++ b/fre/workflow/checkout_script.py @@ -0,0 +1,205 @@ +""" Workflow checkout """ +import os +import subprocess +import filecmp +from pathlib import Path +import logging +import shutil +from datetime import datetime +from typing import Optional +import json +from jsonschema import validate, SchemaError, ValidationError + +import fre.yamltools.combine_yamls_script as cy +from fre.app.helpers import change_directory + +fre_logger = logging.getLogger(__name__) + +######VALIDATE##### +def validate_yaml(yamlfile: dict, application: str): + """ + Validate the format of the yaml file against the + schema.json held in [gfdl_msd_schemas](https://github.com/NOAA-GFDL/gfdl_msd_schemas). + + :param yamlfile: Dictionary containing the combined model, + settings, pp, and analysis yaml content + :type yamlfile: dict + :param application: type of workflow to check out/clone + :type application: string + :raises ValueError: + - invalid gfdl_msd_schema path + - invalid combined yaml + - miscellaneous error in validation + """ + schema_dir = Path(__file__).resolve().parents[1] + schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', f'fre_{application}.json') + fre_logger.info("Using yaml schema '%s'", schema_path) + # Load the json schema: .load() (vs .loads()) reads and parses the json in one) + try: + with open(schema_path,'r', encoding='utf-8') as s: + schema = json.load(s) + except: + fre_logger.error("Schema '%s' is not valid. Contact the FRE team.", schema_path) + raise + + # Validate yaml + # If the yaml is not valid, the schema validation will raise errors and exit + try: + validate(instance = yamlfile,schema=schema) + fre_logger.info(" ** COMBINED YAML VALID ** ") + except SchemaError as exc: + raise ValueError(f"Schema '{schema_path}' is not valid. Contact the FRE team.") from exc + except ValidationError as exc: + raise ValueError("Combined yaml is not valid. Please fix the errors and try again.") from exc + except Exception as exc: + raise ValueError("Miscellaneous error from validation. Please try to find the error and try again.") from exc + +def workflow_checkout(target_dir: str = None, yamlfile: str = None, experiment: str = None, + application: str = None, force_checkout: Optional[bool] = False): + """ + Create a directory and clone the workflow template files from a specified repository. + + :param yamlfile: Model yaml configuration file + :type yamlfile: str + :param experiment: One of the experiment names listed in the model yaml file. + Note: the command "fre list exps -y [model_yamlfile]" can be used to + list the available experiment names + :type experiment: str + :param application: String used to specify the type of workflow to be cloned. + Ex.: run, postprocess + :type application: str + :param target_dir: Target location to create the cylc-src/ directory in + :type target_dir: str + :param force_checkout: If the workflow directory exists, move it to an archived location + (~/.fre-workflows/archived) and re-clone the workflow repository + :type force_checkout: bool + :raises OSError: if the checkout script cannot be created + :raises ValueError: + - if the repository and/or tag was not defined + - if the target directory does not exist or cannot be found + - if tag or branch does not match the git clone branch arg + """ + # Used in consolidate_yamls function for now + platform = None + target = None + + # Set the default target directory location + if target_dir is None: + target_dir = os.path.expanduser("~/.fre-workflows") + + if application in ["run", "pp"]: + fre_logger.info(" ** Configuring the resolved YAML for the %s **", application) + yaml = cy.consolidate_yamls(yamlfile=yamlfile, + experiment=experiment, + platform=platform, + target=target, + use=application, + output="config.yaml") + + validate_yaml(yamlfile = yaml, application = application) + + # Reset application for pp to make it discoverable in yaml config + if application == "pp": + application = "postprocess" + + workflow_info = yaml.get(application).get("workflow") + + yaml_filepath = f"{Path.cwd()}/config.yaml" + repo = workflow_info.get("repository") + tag = workflow_info.get("version") + fre_logger.info("Defined tag ==> '%s'", tag) + + if None in [repo, tag]: + raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}") + + fre_logger.info("(%s):(%s) check out for %s ==> REQUESTED", repo, tag, application) + + # Create src_dir if it does not exist + if not Path(target_dir).exists(): + Path(target_dir).mkdir(parents=True, exist_ok=True) + + # Define cylc-src directory + src_dir = f"{target_dir}/cylc-src" + # workflow name + workflow_name = experiment + + # create workflow in cylc-src + try: + Path(src_dir).mkdir(parents=True, exist_ok=True) + except Exception as exc: + raise OSError( + f"(checkoutScript) directory {src_dir} wasn't able to be created. exit!") from exc + + if Path(f"{src_dir}/{workflow_name}").is_dir(): + fre_logger.info(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name) + if force_checkout: + # Create archived workflows location + archived = f"{target_dir}/archived_workflows" + Path(archived).mkdir(parents=True, exist_ok=True) + + # Move previous workflow to archived location + fre_logger.warning(" *** Moving previous checkout to %s ***", archived) + shutil.move(f"{src_dir}/{workflow_name}", archived) + + # Rename previous workflow + move_timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + os.rename(f"{archived}/{workflow_name}", f"{archived}/{workflow_name}_{move_timestamp}") + +# # Keeping this here in case we want to switch force_checkout to removing instead of moving +# fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name) +# shutil.rmtree(f"{src_dir}/{workflow_name}") + else: + with change_directory(f"{src_dir}/{workflow_name}"): + ## Compare previous workflow directory + # capture the branch and tag + # if either match git_clone_branch_arg, then success. otherwise, fail. + current_tag = subprocess.run(["git","describe","--tags"], + capture_output = True, + text = True, check = True).stdout.strip() + current_branch = subprocess.run(["git", "branch", "--show-current"], + capture_output = True, + text = True, check = True).stdout.strip() + + if tag in (current_tag, current_branch): + fre_logger.info("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag) + else: + fre_logger.error( + "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag) + fre_logger.error( + "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag) + raise ValueError('Neither tag nor branch matches the git clone branch arg') + + + ## Compare content of current and previous configured, resolved yamls + if filecmp.cmp(yaml_filepath, "config.yaml", shallow=False): + fre_logger.info("Resolved yaml already exists and did not change.") + else: + fre_logger.error("") + fre_logger.error("ERROR: Checkout and resolved yaml already exist but resolved yaml files " + "are not identical!") + fre_logger.error("For troubleshooting:") + fre_logger.error(" - Current resolved yaml: %s", yaml_filepath) + fre_logger.error(" - Previous resolved yaml: %s", f"{src_dir}/{workflow_name}/config.yaml") + fre_logger.error("Try:") + fre_logger.error(" - resolving yaml differences if nothing else has changed") + fre_logger.error(f" - removing the {target_dir}/cylc-src/{workflow_name} folder and " + "re-running the command") + fre_logger.error(" - pass the --force-checkout option to archive the workflow (move to " + "~/.fre-workflows/archived/) and clone a new workflow.") + return +# raise ValueError("Resolve yaml differences or pass --force-checkout to archive the workflow" +# "(moved to ~/.fre-workflows/archived) and clone a new workflow.") + + if not Path(f"{src_dir}/{workflow_name}").is_dir(): + fre_logger.info("Workflow does not exist; will create now") + clone_output = subprocess.run( ["git", "clone","--recursive", + f"--branch={tag}", + repo, f"{src_dir}/{workflow_name}"], + capture_output = True, text = True, check = True) + fre_logger.debug(clone_output) + fre_logger.info("(%s):(%s) check out ==> SUCCESSFUL", repo, tag) + + ## Move combined yaml to cylc-src location + current_dir = Path.cwd() + shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}") + fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py new file mode 100644 index 000000000..e2b485031 --- /dev/null +++ b/fre/workflow/freworkflow.py @@ -0,0 +1,40 @@ +''' fre workflow click interface for fre workflow subcommands''' +import os +import click +import logging +fre_logger = logging.getLogger(__name__) + +#fre tools +from . import checkout_script +#from . import install_script +#from . import run_script + +@click.group(help=click.style(" - workflow subcommands", fg=(57,139,210))) +def workflow_cli(): + ''' entry point to fre workflow click commands ''' + +@workflow_cli.command() +@click.option("-y", "--yamlfile", type=str, + help="Model yaml file", + required=True) +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +@click.option("-a", "--application", + type=click.Choice(['run', 'pp']), + help="Type of workflow to check out/clone", + required=True) +@click.option("--target-dir", + type=str, + help=f"""Target directory for the workflow to be cloned into. + If not defined, a default location of ~/.fre-workflows + will be used""") +@click.option("--force-checkout", + is_flag=True, + default=False, + help="If the checkout already, exists, remove and clone the desired repo again.") +def checkout(target_dir, yamlfile, experiment, application, force_checkout): + """ + Checkout/clone the workflow repository. + """ + checkout_script.workflow_checkout(target_dir, yamlfile, experiment, application, force_checkout) diff --git a/fre/workflow/tests/AM5_example/am5.yaml b/fre/workflow/tests/AM5_example/am5.yaml new file mode 100644 index 000000000..f40d51a0a --- /dev/null +++ b/fre/workflow/tests/AM5_example/am5.yaml @@ -0,0 +1,71 @@ +# reusable variables +fre_properties: + - &AM5_VERSION "am5f7b12r1" + - &FRE_STEM !join [am5/, *AM5_VERSION] + + # amip + - &EXP_AMIP_START "19790101T0000Z" + - &EXP_AMIP_END "20200101T0000Z" + - &ANA_AMIP_START "19800101T0000Z" + - &ANA_AMIP_END "20200101T0000Z" + + - &PP_AMIP_CHUNK96 "P1Y" + - &PP_AMIP_CHUNK384 "P1Y" + - &PP_XYINTERP96 "180,288" + - &PP_XYINTERP384 "720,1152" + + # climo + - &EXP_CLIMO_START96 "0001" + - &EXP_CLIMO_END96 "0011" + - &ANA_CLIMO_START96 "0002" + - &ANA_CLIMO_END96 "0011" + + - &EXP_CLIMO_START384 "0001" + - &EXP_CLIMO_END384 "0006" + - &ANA_CLIMO_START384 "0002" + - &ANA_CLIMO_END384 "0006" + + # coupled + - &PP_CPLD_CHUNK_A "P5Y" + - &PP_CPLD_CHUNK_B "P20Y" + + # grids + - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + + # compile information + - &release "f1a1r1" + - &INTEL "intel-classic" + - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +# compile information +build: + compileYaml: "compile.yaml" + platformYaml: "yaml_include/platforms.yaml" + +experiments: + - name: "c96L65_am5f7b12r1_amip_TESTING" + settings: "yaml_include/settings.yaml" + pp: + - "yaml_include/pp.c96_amip.yaml" + - "yaml_include/pp-test.c96_amip.yaml" + - name: "c96L65_am5f7b12r1_amip_TESTING_WRONG" + settings: "yaml_include/settings_WRONG.yaml" + pp: + - "yaml_include/pp.c96_amip.yaml" + - name: "c96L65_am5f7b12r1_pdclim1850F" + pp: + - "yaml_include/pp.c96_clim.yaml" + +# amip: +# settings: +# - shared/settings.yaml +# - shared/directories.yaml +# run: +# version: 1.1 +# - run/inputs.yaml +# - run/runtime.yaml +# postprocess: +# version: 2.0 +# - pp/components +# - analysis/legacy-bw.yaml diff --git a/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml b/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml new file mode 100644 index 000000000..76f078523 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml @@ -0,0 +1,33 @@ +# local reusable variable overrides +fre_properties: + - &custom_interp "200,200" + +#c96_amip_postprocess: +postprocess: + components: + - type: "atmos_cmip-TEST" + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos-TEST" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_level_cmip-TEST" + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml new file mode 100644 index 000000000..d20d05757 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,94 @@ +# local reusable variable overrides +fre_properties: + - &custom_interp "180,360" + +#c96_amip_postprocess: +postprocess: + # main pp instructions + components: + - type: "atmos_cmip" + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + - type: "atmos_level_cmip" + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_level" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_month_aer" + sources: + - history_file: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_diurnal" + sources: + - history_file: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_scalar" + sources: + - history_file: "atmos_scalar" + postprocess_on: True + - type: "aerosol_cmip" + xyInterp: *PP_XYINTERP96 + sources: + - history_file: "aerosol_month_cmip" + sourceGrid: "cubedsphere" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + - type: "land" + sources: + - history_file: "land_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + postprocess_on: False + - type: "land_cmip" + sources: + - history_file: "land_month_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + postprocess_on: False + - type: "tracer_level" + sources: + - history_file: "atmos_tracer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml new file mode 100644 index 000000000..44e2ab797 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -0,0 +1,26 @@ +#c96_amip_directories: +directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + workflow: + repository: "https://github.com/NOAA-GFDL/fre-workflows.git" + version: "main" + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunks: [*PP_AMIP_CHUNK96] + pp_grid_spec: *GRID_SPEC96 + switches: + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + do_analysis_only: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml new file mode 100644 index 000000000..f620f8028 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -0,0 +1,26 @@ +#c96_amip_directories: +directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + workflow: + repository: "" + version: "main" + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunks: [*PP_AMIP_CHUNK96] + pp_grid_spec: *GRID_SPEC96 + switches: + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + do_analysis_only: False diff --git a/fre/workflow/tests/test_checkout_script.py b/fre/workflow/tests/test_checkout_script.py new file mode 100644 index 000000000..34d0a39b0 --- /dev/null +++ b/fre/workflow/tests/test_checkout_script.py @@ -0,0 +1,120 @@ +''' fre workflow checkout tests ''' +import re +from pathlib import Path +import pytest +from fre.workflow import checkout_script + +TEST_CONFIGS = "fre/workflow/tests/AM5_example/" +EXPERIMENT = "c96L65_am5f7b12r1_amip_TESTING" + +def test_cylc_src_creation_fail(tmp_path): + """ + Test for the expected failure if the cylc-src + directory cannot be created. + + This test simulates the instance where a file with the name + 'cylc-src' already exists, causing a permission error in HOME. + """ + cylc_src_file = Path(f"{tmp_path}/cylc-src") + with open(cylc_src_file, "w", encoding='utf-8') as f: + f.write("testing 123") + + # run checkout to create cylc-src + directory = Path(f"{tmp_path}/cylc-src") + expected_error = f"(checkoutScript) directory {directory} wasn't able to be created. exit!" + with pytest.raises(OSError, match = re.escape(expected_error)): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = tmp_path) + +def test_checkout_invalid_resolved_yaml(tmp_path): + """ + Test for the expected error if the repository is not + defined in the settings.yaml and the yamls cannot + be combined + """ + experiment = "c96L65_am5f7b12r1_amip_TESTING_WRONG" + repo = None + tag = "main" + with pytest.raises(ValueError, match = f"Combined yaml is not valid. Please fix the errors and try again."): + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = experiment, + application = "pp", + target_dir = tmp_path) + +def test_pp_workflow_checkout(tmp_path, caplog): + """ + Test for a successful post-processing workflow checkout. + """ + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = tmp_path) + + expected_repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + expected_tag = "main" + assert all([Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").exists(), + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}").is_dir(), + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/flow.cylc").exists(), + Path(f"{tmp_path}/cylc-src/{EXPERIMENT}/config.yaml").exists()]) + +def test_pp_workflow_checkout_exists_already(tmp_path, caplog): + """ + Test for the expected output message if the checkout already exists. + """ + # 1st checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = tmp_path) + + # 2nd checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = tmp_path) + + repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + expected_tag = "main" + expected_output = [ + (f"Checkout exists ('{tmp_path}/cylc-src/c96L65_am5f7b12r1_amip_TESTING'), " + f"and matches '{expected_tag}'") + ] + + for string in expected_output: + assert string in caplog.text + +def test_pp_workflow_checkout_force_checkout(tmp_path, caplog): + """ + Test successful re-cloning of the workflow repo + when force-checkout=True. + """ + # 1st checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = tmp_path) + + # 2nd checkout + checkout_script.workflow_checkout(yamlfile = f"{TEST_CONFIGS}/am5.yaml", + experiment = EXPERIMENT, + application = "pp", + target_dir = tmp_path, + force_checkout = True) + + src_dir = f"{tmp_path}/cylc-src" + workflow_name = EXPERIMENT + repo = "https://github.com/NOAA-GFDL/fre-workflows.git" + tag = "main" + expected_output = f" *** REMOVING {src_dir}/{workflow_name} *** " + + assert expected_output in caplog.text + +#def test_run_workflow_checkout(caplog): +# """ +# Test for a successful run workflow checkout. +# """ +# checkout_script.workflow_checkout(yamlfile, +# experiment = "c96L65_am5f7b12r1_amip_TESTING", +# application = "run")