diff --git a/fre/fre.py b/fre/fre.py index 25b717504..df924978c 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -22,7 +22,9 @@ # click and lazy group loading @click.group( cls = LazyGroup, - lazy_subcommands = {"pp": ".pp.frepp.pp_cli", + + lazy_subcommands = {"workflow": ".workflow.freworkflow.workflow_cli", + "pp": ".pp.frepp.pp_cli", "catalog": ".catalog.frecatalog.catalog_cli", "list": ".list_.frelist.list_cli", "check": ".check.frecheck.check_cli", diff --git a/fre/workflow/README b/fre/workflow/README new file mode 100644 index 000000000..e69de29bb diff --git a/fre/workflow/__init__.py b/fre/workflow/__init__.py new file mode 100644 index 000000000..dc62bb794 --- /dev/null +++ b/fre/workflow/__init__.py @@ -0,0 +1,27 @@ +from typing import Optional + +def make_workflow_name(experiment : Optional[str] = None) -> str: + """ + Function that takes in a triplet of tags for a model experiment, platform, and target, and + returns a directory name for the corresponding pp workflow. Because this is often given by + user to the shell being used by python, we split/reform the string to remove semi-colons or + spaces that may be used to execute an arbitrary command with elevated privileges. + + :param experiment: One of the postprocessing experiment names from the yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None + :type experiment: str + :param platform: The location + compiler that was used to run the model (e.g. gfdl.ncrc5-deploy), default None + :type platform: str + :param target: Options used for the model compiler (e.g. prod-openmp), default None + :type target: str + :return: string created in specific format from the input strings + :rtype: str + + .. note:: if any arguments are None, then "None" will appear in the workflow name + """ + name = f'{experiment}__{platform}__{target}' + return ''.join( + (''.join( + name.split(' ') + ) + ).split(';') + ) # user-input sanitation, prevents some malicious cmds from being executed with privileges diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py new file mode 100644 index 000000000..1ba5431a9 --- /dev/null +++ b/fre/workflow/freworkflow.py @@ -0,0 +1,39 @@ +''' fre workflow ''' +import os +import click +import logging +fre_logger = logging.getLogger(__name__) + +#fre tools +#from . import checkout_script +from . import install_script +#from . import run_script + +@click.group(help=click.style(" - workflow subcommands", fg=(57,139,210))) +def workflow_cli(): + ''' entry point to fre workflow click commands ''' + +@workflow_cli.command() +@click.option("-e", "--experiment", + type=str, + required=True, + help="Experiment name") +@click.option("--src-dir", + type=str, + envvar="TMPDIR", + default=os.path.expanduser("~/.fre"), + required=True, + help="Path to cylc-src directory") +@click.option("--target-dir", + type=str, + help="""Target directory to install the cylc + workflow into. Default location is + ~/cylc-run/""") +@click.option("--force-install", + type=bool, + help="If cylc-run/[workflow_name] exists") +def install(experiment, src_dir, target_dir): + """ + Install workflow configuration + """ + install_script.workflow_install(experiment, src_dir, target_dir) diff --git a/fre/workflow/install_script.py b/fre/workflow/install_script.py new file mode 100644 index 000000000..4de7d62a5 --- /dev/null +++ b/fre/workflow/install_script.py @@ -0,0 +1,73 @@ +''' fre workflow install ''' +from pathlib import Path +import subprocess +import logging +from fre.app.helpers import change_directory + +fre_logger =logging.getLogger(__name__) + +def workflow_install(experiment: str, src_dir: str, target_dir: str, force_install: bool): + """ + Install the Cylc workflow definition located in + cylc-src/$(experiment) to cylc-run/$(experiment) + + :param experiment: Experiment names from the yaml displayed by + fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip); + default None + :type experiment: str + :param src_dir: src_dir/workflow_name + :type src_dir: + :param target_dir: + :type target_dir: + """ + workflow_name = experiment + + # Check src_dir exists + if not Path(src_dir): + raise ValueError("""Cylc source directory ({src_dir}) could not be found! Try specifying + path by passing --src-dir [path].""") + + # if the cylc-run directory already exists, + # then check whether the cylc expanded definition (cylc config) + # is identical. If the same, good. If not, bad. + install_dir = Path(f"{target_dir}/cylc-run") + if Path(install_dir).is_dir(): + fre_logger.warning(" *** PREVIOUS INSTALL FOUND: %s ***", install_dir) + if force_install: + fre_logger.warning(" *** REMOVING %s *** ", install_dir) + install_output = subprocess.run(["cylc", "clean", f"{install_dir}/{workflow_name}"], + capture_output = True, + text = True, + check = True) + fre_logger.debug(install_output) + else: + # must convert from bytes to string for proper comparison + installed_def = subprocess.run(["cylc", "config", workflow_name], + capture_output=True, + check=True).stdout.decode('utf-8') + with change_directory(src_dir): + source_def = subprocess.run(['cylc', 'config', '.'], + capture_output=True, + check=True).stdout.decode('utf-8') + + if installed_def == source_def: + fre_logger.warning("""NOTE: Workflow '%s/%s}' already ", + installed, and the definition is unchanged""", install_dir, workflow_name) + else: + raise ValueError(f"""ERROR: Workflow '{install_dir}/{workflow_name}' already + installed, and the definition has changed! + + Please remove and re-install the workflow with one of these options: + - fre workflow install -e {experiment} --src-dir {src_dir} --target-dir {target_dir} --force-install" + - cylc clean {install_dir}/{workflow_name}, then re-run install command""") + + if not Path(install_dir).is_dir(): + fre_logger.warning("NOTE: About to install workflow into ~/cylc-run/%s", workflow_name) + if not target_dir: + # install workflow in default home location + cmd = f"cylc install --no-run-name {src_dir}" + subprocess.run(cmd, shell=True, check=True) + else: + # symlink the workflow and associated files in target_dir + cmd = f"cylc install --no-run-name {src_dir} --symlink-dirs='{target_dir}/{workflow_name}'" + subprocess.run(cmd, shell=True, check=True) diff --git a/fre/workflow/tests/AM5_example/am5.yaml b/fre/workflow/tests/AM5_example/am5.yaml new file mode 100644 index 000000000..26b19bcd7 --- /dev/null +++ b/fre/workflow/tests/AM5_example/am5.yaml @@ -0,0 +1,68 @@ +# reusable variables +fre_properties: + - &AM5_VERSION "am5f7b12r1" + - &FRE_STEM !join [am5/, *AM5_VERSION] + + # amip + - &EXP_AMIP_START "19790101T0000Z" + - &EXP_AMIP_END "20200101T0000Z" + - &ANA_AMIP_START "19800101T0000Z" + - &ANA_AMIP_END "20200101T0000Z" + + - &PP_AMIP_CHUNK96 "P1Y" + - &PP_AMIP_CHUNK384 "P1Y" + - &PP_XYINTERP96 "180,288" + - &PP_XYINTERP384 "720,1152" + + # climo + - &EXP_CLIMO_START96 "0001" + - &EXP_CLIMO_END96 "0011" + - &ANA_CLIMO_START96 "0002" + - &ANA_CLIMO_END96 "0011" + + - &EXP_CLIMO_START384 "0001" + - &EXP_CLIMO_END384 "0006" + - &ANA_CLIMO_START384 "0002" + - &ANA_CLIMO_END384 "0006" + + # coupled + - &PP_CPLD_CHUNK_A "P5Y" + - &PP_CPLD_CHUNK_B "P20Y" + + # grids + - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + + # compile information + - &release "f1a1r1" + - &INTEL "intel-classic" + - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +# compile information +build: + compileYaml: "compile.yaml" + platformYaml: "yaml_include/platforms.yaml" + +experiments: + - name: "c96L65_am5f7b12r1_amip_TESTING" + settings: "yaml_include/settings.yaml" + pp: + - "yaml_include/pp.c96_amip.yaml" + - "yaml_include/pp-test.c96_amip.yaml" + - name: "c96L65_am5f7b12r1_amip_TESTING_WRONG" + settings: "yaml_include/settings_WRONG.yaml" + pp: + - "yaml_include/pp.c96_amip.yaml" + +# amip: +# settings: +# - shared/settings.yaml +# - shared/directories.yaml +# run: +# version: 1.1 +# - run/inputs.yaml +# - run/runtime.yaml +# postprocess: +# version: 2.0 +# - pp/components +# - analysis/legacy-bw.yaml diff --git a/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml b/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml new file mode 100644 index 000000000..76f078523 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml @@ -0,0 +1,33 @@ +# local reusable variable overrides +fre_properties: + - &custom_interp "200,200" + +#c96_amip_postprocess: +postprocess: + components: + - type: "atmos_cmip-TEST" + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos-TEST" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_level_cmip-TEST" + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml new file mode 100644 index 000000000..d20d05757 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,94 @@ +# local reusable variable overrides +fre_properties: + - &custom_interp "180,360" + +#c96_amip_postprocess: +postprocess: + # main pp instructions + components: + - type: "atmos_cmip" + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + - type: "atmos_level_cmip" + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_level" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_month_aer" + sources: + - history_file: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_diurnal" + sources: + - history_file: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_scalar" + sources: + - history_file: "atmos_scalar" + postprocess_on: True + - type: "aerosol_cmip" + xyInterp: *PP_XYINTERP96 + sources: + - history_file: "aerosol_month_cmip" + sourceGrid: "cubedsphere" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + - type: "land" + sources: + - history_file: "land_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + postprocess_on: False + - type: "land_cmip" + sources: + - history_file: "land_month_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + postprocess_on: False + - type: "tracer_level" + sources: + - history_file: "atmos_tracer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml new file mode 100644 index 000000000..791c0c437 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -0,0 +1,32 @@ +#workflow repositories +workflow: + run_workflow: + repo: "https://github.com/NOAA-GFDL" + version: "tbd" + pp_workflow: + repo: "https://github.com/NOAA-GFDL/fre-workflows.git" + version: "main" + +#c96_amip_directories: +directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunks: [*PP_AMIP_CHUNK96] + pp_grid_spec: *GRID_SPEC96 + switches: + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + do_analysis_only: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml new file mode 100644 index 000000000..c973591e8 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -0,0 +1,32 @@ +#workflow repositories +workflow: + run_workflow: + repo: "https://github.com/NOAA-GFDL" + version: "tbd" + pp_workflow: + repo: + version: "main" + +#c96_amip_directories: +directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunks: [*PP_AMIP_CHUNK96] + pp_grid_spec: *GRID_SPEC96 + switches: + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + do_analysis_only: False diff --git a/fre/workflow/tests/test_install_script.py b/fre/workflow/tests/test_install_script.py new file mode 100644 index 000000000..07134303c --- /dev/null +++ b/fre/workflow/tests/test_install_script.py @@ -0,0 +1,72 @@ +''' fre workflow checkout tests ''' +import stat +import re +import os +from pathlib import Path +import pytest +from fre.workflow import install_script + +TEST_CONFIGS = "fre/workflow/tests/AM5_example/" +EXPERIMENT = "c96L65_am5f7b12r1_amip_TESTING" + +@pytest.fixture(autouse=True, name="fake_home") +def fake_home_fixture(tmp_path, monkeypatch): + """ + Set the tmp_path as HOME for the cylc-src directory + to be created in. + """ + ## Mock HOME for cylc-src and cylc-run + fake_home = Path(tmp_path) + monkeypatch.setenv("HOME", str(fake_home)) + + return fake_home + +@pytest.fixture(autouse=True, name="fake_exp_src_dir") +def fake_exp_src_dir_fixture(fake_home): + """ + """ + src_dir = f"{fake_home}/cylc-src/{EXPERIMENT}" + Path(src_dir).mkdir(parents=True) + + # create workflow definition + definition_content = "flow.cylc content" + Path(f"{src_dir}/flow.cylc").write_text(definition_content) + + yaml_content = "name: ah" + Path(f"{src_dir}/config.yaml").write_text(yaml_content) + + #####should I dump the yaml instead??? + + assert Path(f"{src_dir}/flow.cylc").exists() + assert Path(f"{src_dir}/flow.cylc").is_file() + assert Path(f"{src_dir}/config.yaml").exists() + assert Path(f"{src_dir}/config.yaml").is_file() + +##mock cylc-src, mock exp in cylc-src, mock flow.cylc +##test install +def test_install_script(fake_home, caplog, capfd): + """ + """ + try: + install_script.workflow_install(experiment = EXPERIMENT) + except: + assert False + +# print(caplog.text) +# print(type(capfd.readouterr().err)) +# ah + assert all([Path(f"{fake_home}/cylc-run/{EXPERIMENT}").exists(), + Path(f"{fake_home}/cylc-run/{EXPERIMENT}").is_dir(), + Path(f"{fake_home}/cylc-run/{EXPERIMENT}/flow.cylc").exists(), + Path(f"{fake_home}/cylc-run/{EXPERIMENT}/flow.cylc").is_file(), + f"INSTALLED {EXPERIMENT} from {fake_home}/cylc-src/{EXPERIMENT}" in capfd.readouterr().out, + #"INFO - Passing resolved YAML information to install." in capfd.readouterr().err, + f"NOTE: About to install workflow into ~/cylc-run/{EXPERIMENT}" in caplog.text]) + + with open(Path(f"{fake_home}/cylc-run/{EXPERIMENT}/flow.cylc"), "r") as f: + expected_content = "flow.cylc content" + wdf = f.read() + if expected_content in wdf: + assert True + else: + assert False