From 503ed3030c89a61c74a59696ecdf37303b4ac209 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 14:14:08 -0500 Subject: [PATCH 1/4] #698 Move install functionality to `fre workflow install` --- fre/workflow/README | 0 fre/workflow/__init__.py | 27 +++++ fre/workflow/freworkflow.py | 101 ++++++++++++++++++ fre/workflow/install_script.py | 49 +++++++++ fre/workflow/tests/AM5_example/am5.yaml | 68 ++++++++++++ .../yaml_include/pp-test.c96_amip.yaml | 33 ++++++ .../AM5_example/yaml_include/pp.c96_amip.yaml | 94 ++++++++++++++++ .../AM5_example/yaml_include/settings.yaml | 32 ++++++ .../yaml_include/settings_WRONG.yaml | 32 ++++++ fre/workflow/tests/test_install_script.py | 72 +++++++++++++ 10 files changed, 508 insertions(+) create mode 100644 fre/workflow/README create mode 100644 fre/workflow/__init__.py create mode 100644 fre/workflow/freworkflow.py create mode 100644 fre/workflow/install_script.py create mode 100644 fre/workflow/tests/AM5_example/am5.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/settings.yaml create mode 100644 fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml create mode 100644 fre/workflow/tests/test_install_script.py diff --git a/fre/workflow/README b/fre/workflow/README new file mode 100644 index 000000000..e69de29bb diff --git a/fre/workflow/__init__.py b/fre/workflow/__init__.py new file mode 100644 index 000000000..dc62bb794 --- /dev/null +++ b/fre/workflow/__init__.py @@ -0,0 +1,27 @@ +from typing import Optional + +def make_workflow_name(experiment : Optional[str] = None) -> str: + """ + Function that takes in a triplet of tags for a model experiment, platform, and target, and + returns a directory name for the corresponding pp workflow. Because this is often given by + user to the shell being used by python, we split/reform the string to remove semi-colons or + spaces that may be used to execute an arbitrary command with elevated privileges. + + :param experiment: One of the postprocessing experiment names from the yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None + :type experiment: str + :param platform: The location + compiler that was used to run the model (e.g. gfdl.ncrc5-deploy), default None + :type platform: str + :param target: Options used for the model compiler (e.g. prod-openmp), default None + :type target: str + :return: string created in specific format from the input strings + :rtype: str + + .. note:: if any arguments are None, then "None" will appear in the workflow name + """ + name = f'{experiment}__{platform}__{target}' + return ''.join( + (''.join( + name.split(' ') + ) + ).split(';') + ) # user-input sanitation, prevents some malicious cmds from being executed with privileges diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py new file mode 100644 index 000000000..0503fe5fc --- /dev/null +++ b/fre/workflow/freworkflow.py @@ -0,0 +1,101 @@ +''' fre workflow ''' + +import click +import logging +fre_logger = logging.getLogger(__name__) + +#fre tools +#from . import checkout_script +from . import install_script +#from . import run_script + +@click.group(help=click.style(" - workflow subcommands", fg=(57,139,210))) +def workflow_cli(): + ''' entry point to fre workflow click commands ''' + +#@workflow_cli.command() +#@click.option("-y", "--yamlfile", type=str, +# help="Model yaml file", +# required=True) +#@click.option("-e", "--experiment", type=str, +# help="Experiment name", +# required=True) +##@click.option("-p", "--platform", type=str, +## help="Platform name") +##@click.option("-t", "--target", type=str, +## help="Target name") +#@click.option("-b", "--branch", type =str, +# required=False, default = None, +# help="fre-workflows branch/tag to clone; default is $(fre --version)") +#@click.option("-a", "--application", +# type=click.Choice(['run', 'pp']), +# help="Use case for checked out workflow", +# required=True) +#def checkout(yamlfile, experiment, application, branch=None): +# """ +# Checkout/extract fre workflow +# """ +# checkout_script.workflow_checkout(yamlfile, experiment, application, branch) + +@workflow_cli.command() +@click.option("-e", "--experiment", type=str, + help="Experiment name", + required=True) +#@click.option("-p", "--platform", type=str, +# help="Platform name", +# required=True) +#@click.option("-t", "--target", type=str, +# help="Target name", +# required=True) +def install(experiment): + """ + Install workflow configuration + """ + install_script.workflow_install(experiment) + +#@workflow_cli.command() +#@click.option("-e", "--experiment", type=str, +# help="Experiment name", +# required=True) +##@click.option("-p", "--platform", type=str, +## help="Platform name", +## required=True) +##@click.option("-t", "--target", type=str, +## help="Target name", +## required=True) +#@click.option("--pause", is_flag=True, default=False, +# help="Pause the workflow immediately on start up", +# required=False) +#@click.option("--no_wait", is_flag=True, default=False, +# help="after submission, do not wait to ping the scheduler and confirm success", +# required=False) +#def run(experiment, pause, no_wait): +# """ +# Run workflow configuration +# """ +# run_script.workflow_run(experiment, pause, no_wait) +# +#@workflow_cli.command() +#@click.option("-e", "--experiment", type=str, +# help="Experiment name", +# required=True) +##@click.option("-p", "--platform", type=str, +## help="Platform name", +## required=True) +##@click.option("-T", "--target", type=str, +## help="Target name", +## required=True) +#@click.option("-c", "--config-file", type=str, +# help="Path to a configuration file in either XML or YAML", +# required=True) +#@click.option("-b", "--branch", +# required=False, default=None, +# help="fre-workflows branch/tag to clone; default is $(fre --version)") +#@click.option("-t", "--time", +# required=False, default=None, +# help="Time whose history files are ready") +#def all(experiment, platform, target, config_file, branch, time): +# """ +# Execute all fre workflow initialization steps in order +# """ +# wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time) diff --git a/fre/workflow/install_script.py b/fre/workflow/install_script.py new file mode 100644 index 000000000..98b41ee67 --- /dev/null +++ b/fre/workflow/install_script.py @@ -0,0 +1,49 @@ +''' fre workflow install ''' + +from pathlib import Path +import os +import subprocess +import logging +fre_logger =logging.getLogger(__name__) + +from . import make_workflow_name +from fre.app.helpers import change_directory + +def workflow_install(experiment): + """ + Install the Cylc workflow definition located in + + ~/cylc-src/$(experiment) + + to + + ~/cylc-run/$(experiment) + + :param experiment: Experiment names from the yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None + :type experiment: str + """ + + #name = experiment + '__' + platform + '__' + target +# workflow_name = make_workflow_name(experiment, platform, target) + workflow_name = experiment + + # if the cylc-run directory already exists, + # then check whether the cylc expanded definition (cylc config) + # is identical. If the same, good. If not, bad. + source_dir = Path(os.path.expanduser("~/cylc-src"), workflow_name) + install_dir = Path(os.path.expanduser("~/cylc-run"), workflow_name) + if os.path.isdir(install_dir): + with change_directory(source_dir): + # must convert from bytes to string for proper comparison + installed_def = subprocess.run(["cylc", "config", workflow_name],capture_output=True).stdout.decode('utf-8') + source_def = subprocess.run(['cylc', 'config', '.'], capture_output=True).stdout.decode('utf-8') + if installed_def == source_def: + fre_logger.warning(f"NOTE: Workflow '{install_dir}' already installed, and the definition is unchanged") + else: + fre_logger.error(f"ERROR: Please remove installed workflow with 'cylc clean {workflow_name}'" + " or move the workflow run directory '{install_dir}'") + raise Exception(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") + else: + fre_logger.warning(f"NOTE: About to install workflow into ~/cylc-run/{workflow_name}") + cmd = f"cylc install --no-run-name {workflow_name}" + subprocess.run(cmd, shell=True, check=True) diff --git a/fre/workflow/tests/AM5_example/am5.yaml b/fre/workflow/tests/AM5_example/am5.yaml new file mode 100644 index 000000000..26b19bcd7 --- /dev/null +++ b/fre/workflow/tests/AM5_example/am5.yaml @@ -0,0 +1,68 @@ +# reusable variables +fre_properties: + - &AM5_VERSION "am5f7b12r1" + - &FRE_STEM !join [am5/, *AM5_VERSION] + + # amip + - &EXP_AMIP_START "19790101T0000Z" + - &EXP_AMIP_END "20200101T0000Z" + - &ANA_AMIP_START "19800101T0000Z" + - &ANA_AMIP_END "20200101T0000Z" + + - &PP_AMIP_CHUNK96 "P1Y" + - &PP_AMIP_CHUNK384 "P1Y" + - &PP_XYINTERP96 "180,288" + - &PP_XYINTERP384 "720,1152" + + # climo + - &EXP_CLIMO_START96 "0001" + - &EXP_CLIMO_END96 "0011" + - &ANA_CLIMO_START96 "0002" + - &ANA_CLIMO_END96 "0011" + + - &EXP_CLIMO_START384 "0001" + - &EXP_CLIMO_END384 "0006" + - &ANA_CLIMO_START384 "0002" + - &ANA_CLIMO_END384 "0006" + + # coupled + - &PP_CPLD_CHUNK_A "P5Y" + - &PP_CPLD_CHUNK_B "P20Y" + + # grids + - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + + # compile information + - &release "f1a1r1" + - &INTEL "intel-classic" + - &FMSincludes "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include" + - &momIncludes "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include" + +# compile information +build: + compileYaml: "compile.yaml" + platformYaml: "yaml_include/platforms.yaml" + +experiments: + - name: "c96L65_am5f7b12r1_amip_TESTING" + settings: "yaml_include/settings.yaml" + pp: + - "yaml_include/pp.c96_amip.yaml" + - "yaml_include/pp-test.c96_amip.yaml" + - name: "c96L65_am5f7b12r1_amip_TESTING_WRONG" + settings: "yaml_include/settings_WRONG.yaml" + pp: + - "yaml_include/pp.c96_amip.yaml" + +# amip: +# settings: +# - shared/settings.yaml +# - shared/directories.yaml +# run: +# version: 1.1 +# - run/inputs.yaml +# - run/runtime.yaml +# postprocess: +# version: 2.0 +# - pp/components +# - analysis/legacy-bw.yaml diff --git a/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml b/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml new file mode 100644 index 000000000..76f078523 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml @@ -0,0 +1,33 @@ +# local reusable variable overrides +fre_properties: + - &custom_interp "200,200" + +#c96_amip_postprocess: +postprocess: + components: + - type: "atmos_cmip-TEST" + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos-TEST" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_level_cmip-TEST" + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml b/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml new file mode 100644 index 000000000..d20d05757 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,94 @@ +# local reusable variable overrides +fre_properties: + - &custom_interp "180,360" + +#c96_amip_postprocess: +postprocess: + # main pp instructions + components: + - type: "atmos_cmip" + sources: + - history_file: "atmos_month_cmip" + - history_file: "atmos_8xdaily_cmip" + - history_file: "atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: True + - type: "atmos_level_cmip" + sources: + - history_file: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_level" + sources: + - history_file: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_month_aer" + sources: + - history_file: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_diurnal" + sources: + - history_file: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + postprocess_on: False + - type: "atmos_scalar" + sources: + - history_file: "atmos_scalar" + postprocess_on: True + - type: "aerosol_cmip" + xyInterp: *PP_XYINTERP96 + sources: + - history_file: "aerosol_month_cmip" + sourceGrid: "cubedsphere" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False + - type: "land" + sources: + - history_file: "land_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + postprocess_on: False + - type: "land_cmip" + sources: + - history_file: "land_month_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + postprocess_on: False + - type: "tracer_level" + sources: + - history_file: "atmos_tracer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + postprocess_on: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml new file mode 100644 index 000000000..791c0c437 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/settings.yaml @@ -0,0 +1,32 @@ +#workflow repositories +workflow: + run_workflow: + repo: "https://github.com/NOAA-GFDL" + version: "tbd" + pp_workflow: + repo: "https://github.com/NOAA-GFDL/fre-workflows.git" + version: "main" + +#c96_amip_directories: +directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunks: [*PP_AMIP_CHUNK96] + pp_grid_spec: *GRID_SPEC96 + switches: + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + do_analysis_only: False diff --git a/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml new file mode 100644 index 000000000..c973591e8 --- /dev/null +++ b/fre/workflow/tests/AM5_example/yaml_include/settings_WRONG.yaml @@ -0,0 +1,32 @@ +#workflow repositories +workflow: + run_workflow: + repo: "https://github.com/NOAA-GFDL" + version: "tbd" + pp_workflow: + repo: + version: "main" + +#c96_amip_directories: +directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + +#c96_amip_postprocess: +postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunks: [*PP_AMIP_CHUNK96] + pp_grid_spec: *GRID_SPEC96 + switches: + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + do_analysis_only: False diff --git a/fre/workflow/tests/test_install_script.py b/fre/workflow/tests/test_install_script.py new file mode 100644 index 000000000..07134303c --- /dev/null +++ b/fre/workflow/tests/test_install_script.py @@ -0,0 +1,72 @@ +''' fre workflow checkout tests ''' +import stat +import re +import os +from pathlib import Path +import pytest +from fre.workflow import install_script + +TEST_CONFIGS = "fre/workflow/tests/AM5_example/" +EXPERIMENT = "c96L65_am5f7b12r1_amip_TESTING" + +@pytest.fixture(autouse=True, name="fake_home") +def fake_home_fixture(tmp_path, monkeypatch): + """ + Set the tmp_path as HOME for the cylc-src directory + to be created in. + """ + ## Mock HOME for cylc-src and cylc-run + fake_home = Path(tmp_path) + monkeypatch.setenv("HOME", str(fake_home)) + + return fake_home + +@pytest.fixture(autouse=True, name="fake_exp_src_dir") +def fake_exp_src_dir_fixture(fake_home): + """ + """ + src_dir = f"{fake_home}/cylc-src/{EXPERIMENT}" + Path(src_dir).mkdir(parents=True) + + # create workflow definition + definition_content = "flow.cylc content" + Path(f"{src_dir}/flow.cylc").write_text(definition_content) + + yaml_content = "name: ah" + Path(f"{src_dir}/config.yaml").write_text(yaml_content) + + #####should I dump the yaml instead??? + + assert Path(f"{src_dir}/flow.cylc").exists() + assert Path(f"{src_dir}/flow.cylc").is_file() + assert Path(f"{src_dir}/config.yaml").exists() + assert Path(f"{src_dir}/config.yaml").is_file() + +##mock cylc-src, mock exp in cylc-src, mock flow.cylc +##test install +def test_install_script(fake_home, caplog, capfd): + """ + """ + try: + install_script.workflow_install(experiment = EXPERIMENT) + except: + assert False + +# print(caplog.text) +# print(type(capfd.readouterr().err)) +# ah + assert all([Path(f"{fake_home}/cylc-run/{EXPERIMENT}").exists(), + Path(f"{fake_home}/cylc-run/{EXPERIMENT}").is_dir(), + Path(f"{fake_home}/cylc-run/{EXPERIMENT}/flow.cylc").exists(), + Path(f"{fake_home}/cylc-run/{EXPERIMENT}/flow.cylc").is_file(), + f"INSTALLED {EXPERIMENT} from {fake_home}/cylc-src/{EXPERIMENT}" in capfd.readouterr().out, + #"INFO - Passing resolved YAML information to install." in capfd.readouterr().err, + f"NOTE: About to install workflow into ~/cylc-run/{EXPERIMENT}" in caplog.text]) + + with open(Path(f"{fake_home}/cylc-run/{EXPERIMENT}/flow.cylc"), "r") as f: + expected_content = "flow.cylc content" + wdf = f.read() + if expected_content in wdf: + assert True + else: + assert False From 9bc00944a7877b7c8f440f1ed20c94295425a790 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 12 Feb 2026 14:16:15 -0500 Subject: [PATCH 2/4] #698 Add workflow tool --- fre/fre.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fre/fre.py b/fre/fre.py index 25b717504..df924978c 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -22,7 +22,9 @@ # click and lazy group loading @click.group( cls = LazyGroup, - lazy_subcommands = {"pp": ".pp.frepp.pp_cli", + + lazy_subcommands = {"workflow": ".workflow.freworkflow.workflow_cli", + "pp": ".pp.frepp.pp_cli", "catalog": ".catalog.frecatalog.catalog_cli", "list": ".list_.frelist.list_cli", "check": ".check.frecheck.check_cli", From 15c0802197359b3c35316e259cca4416329313ad Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Feb 2026 16:30:53 -0500 Subject: [PATCH 3/4] #698 Update install script --- fre/workflow/freworkflow.py | 104 +++++++-------------------------- fre/workflow/install_script.py | 83 ++++++++++++++++---------- 2 files changed, 74 insertions(+), 113 deletions(-) diff --git a/fre/workflow/freworkflow.py b/fre/workflow/freworkflow.py index 0503fe5fc..1ba5431a9 100644 --- a/fre/workflow/freworkflow.py +++ b/fre/workflow/freworkflow.py @@ -1,5 +1,5 @@ ''' fre workflow ''' - +import os import click import logging fre_logger = logging.getLogger(__name__) @@ -13,89 +13,27 @@ def workflow_cli(): ''' entry point to fre workflow click commands ''' -#@workflow_cli.command() -#@click.option("-y", "--yamlfile", type=str, -# help="Model yaml file", -# required=True) -#@click.option("-e", "--experiment", type=str, -# help="Experiment name", -# required=True) -##@click.option("-p", "--platform", type=str, -## help="Platform name") -##@click.option("-t", "--target", type=str, -## help="Target name") -#@click.option("-b", "--branch", type =str, -# required=False, default = None, -# help="fre-workflows branch/tag to clone; default is $(fre --version)") -#@click.option("-a", "--application", -# type=click.Choice(['run', 'pp']), -# help="Use case for checked out workflow", -# required=True) -#def checkout(yamlfile, experiment, application, branch=None): -# """ -# Checkout/extract fre workflow -# """ -# checkout_script.workflow_checkout(yamlfile, experiment, application, branch) - @workflow_cli.command() -@click.option("-e", "--experiment", type=str, - help="Experiment name", - required=True) -#@click.option("-p", "--platform", type=str, -# help="Platform name", -# required=True) -#@click.option("-t", "--target", type=str, -# help="Target name", -# required=True) -def install(experiment): +@click.option("-e", "--experiment", + type=str, + required=True, + help="Experiment name") +@click.option("--src-dir", + type=str, + envvar="TMPDIR", + default=os.path.expanduser("~/.fre"), + required=True, + help="Path to cylc-src directory") +@click.option("--target-dir", + type=str, + help="""Target directory to install the cylc + workflow into. Default location is + ~/cylc-run/""") +@click.option("--force-install", + type=bool, + help="If cylc-run/[workflow_name] exists") +def install(experiment, src_dir, target_dir): """ Install workflow configuration """ - install_script.workflow_install(experiment) - -#@workflow_cli.command() -#@click.option("-e", "--experiment", type=str, -# help="Experiment name", -# required=True) -##@click.option("-p", "--platform", type=str, -## help="Platform name", -## required=True) -##@click.option("-t", "--target", type=str, -## help="Target name", -## required=True) -#@click.option("--pause", is_flag=True, default=False, -# help="Pause the workflow immediately on start up", -# required=False) -#@click.option("--no_wait", is_flag=True, default=False, -# help="after submission, do not wait to ping the scheduler and confirm success", -# required=False) -#def run(experiment, pause, no_wait): -# """ -# Run workflow configuration -# """ -# run_script.workflow_run(experiment, pause, no_wait) -# -#@workflow_cli.command() -#@click.option("-e", "--experiment", type=str, -# help="Experiment name", -# required=True) -##@click.option("-p", "--platform", type=str, -## help="Platform name", -## required=True) -##@click.option("-T", "--target", type=str, -## help="Target name", -## required=True) -#@click.option("-c", "--config-file", type=str, -# help="Path to a configuration file in either XML or YAML", -# required=True) -#@click.option("-b", "--branch", -# required=False, default=None, -# help="fre-workflows branch/tag to clone; default is $(fre --version)") -#@click.option("-t", "--time", -# required=False, default=None, -# help="Time whose history files are ready") -#def all(experiment, platform, target, config_file, branch, time): -# """ -# Execute all fre workflow initialization steps in order -# """ -# wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time) + install_script.workflow_install(experiment, src_dir, target_dir) diff --git a/fre/workflow/install_script.py b/fre/workflow/install_script.py index 98b41ee67..7053f081a 100644 --- a/fre/workflow/install_script.py +++ b/fre/workflow/install_script.py @@ -1,49 +1,72 @@ ''' fre workflow install ''' - from pathlib import Path -import os import subprocess import logging -fre_logger =logging.getLogger(__name__) - -from . import make_workflow_name from fre.app.helpers import change_directory -def workflow_install(experiment): +fre_logger =logging.getLogger(__name__) + +def workflow_install(experiment: str, src_dir: str, target_dir: str, force_install: bool): """ Install the Cylc workflow definition located in + cylc-src/$(experiment) to cylc-run/$(experiment) - ~/cylc-src/$(experiment) - - to - - ~/cylc-run/$(experiment) - - :param experiment: Experiment names from the yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None + :param experiment: Experiment names from the yaml displayed by + fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip); + default None :type experiment: str + :param src_dir: src_dir/workflow_name + :type src_dir: + :param target_dir: + :type target_dir: """ - - #name = experiment + '__' + platform + '__' + target -# workflow_name = make_workflow_name(experiment, platform, target) workflow_name = experiment + # Check src_dir exists + if not Path(src_dir): + raise ValueError("""Cylc source directory ({src_dir}) could not be found! Try specifying + path by passing --src-dir [path].""") + # if the cylc-run directory already exists, # then check whether the cylc expanded definition (cylc config) # is identical. If the same, good. If not, bad. - source_dir = Path(os.path.expanduser("~/cylc-src"), workflow_name) - install_dir = Path(os.path.expanduser("~/cylc-run"), workflow_name) - if os.path.isdir(install_dir): - with change_directory(source_dir): + install_dir = Path(f"{target_dir}/cylc-run") + if Path(install_dir).is_dir(): + fre_logger.warning(" *** PREVIOUS INSTALL FOUND: %s ***", install_dir) + if force_install: + fre_logger.warning(" *** REMOVING %s *** ", install_dir) + install_output = subprocess.run(["cylc", "clean", f"{install_dir}/{workflow_name}"], + capture_output = True, + text = True, + check = True) + fre_logger.debug(install_output) + else: # must convert from bytes to string for proper comparison - installed_def = subprocess.run(["cylc", "config", workflow_name],capture_output=True).stdout.decode('utf-8') - source_def = subprocess.run(['cylc', 'config', '.'], capture_output=True).stdout.decode('utf-8') + installed_def = subprocess.run(["cylc", "config", workflow_name], + capture_output=True, + check=True).stdout.decode('utf-8') + with change_directory(src_dir): + source_def = subprocess.run(['cylc', 'config', '.'], + capture_output=True, + check=True).stdout.decode('utf-8') + if installed_def == source_def: - fre_logger.warning(f"NOTE: Workflow '{install_dir}' already installed, and the definition is unchanged") + fre_logger.warning("""NOTE: Workflow '%s/%s}' already ", + installed, and the definition is unchanged""", install_dir, workflow_name) else: - fre_logger.error(f"ERROR: Please remove installed workflow with 'cylc clean {workflow_name}'" - " or move the workflow run directory '{install_dir}'") - raise Exception(f"ERROR: Workflow '{install_dir}' already installed, and the definition has changed!") - else: - fre_logger.warning(f"NOTE: About to install workflow into ~/cylc-run/{workflow_name}") - cmd = f"cylc install --no-run-name {workflow_name}" - subprocess.run(cmd, shell=True, check=True) + fre_logger.error("ERROR: Please remove installed workflow with one of these options:") + fre_logger.error(" - fre workflow install -e %s --src-dir %s --target-dir %s --force-install", experiment, src_dir, target_dir) + fre_logger.error(" - cylc clean %s/%s, then re-run install command", install_dir, workflow_name) + raise ValueError(f"""ERROR: Workflow '{install_dir}/{workflow_name}' already + installed, and the definition has changed!""") + + if not Path(install_dir).is_dir(): + fre_logger.warning("NOTE: About to install workflow into ~/cylc-run/%s", workflow_name) + if not target_dir: + # install workflow in default home location + cmd = f"cylc install --no-run-name {src_dir}" + subprocess.run(cmd, shell=True, check=True) + else: + # symlink the workflow and associated files in target_dir + cmd = f"cylc install --no-run-name {src_dir} --symlink-dirs='{target_dir}/{workflow_name}'" + subprocess.run(cmd, shell=True, check=True) From 48cb544eae3d9737d376cc0bb5f047dedde12e14 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 26 Feb 2026 16:49:20 -0500 Subject: [PATCH 4/4] #698 Update error message --- fre/workflow/install_script.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fre/workflow/install_script.py b/fre/workflow/install_script.py index 7053f081a..4de7d62a5 100644 --- a/fre/workflow/install_script.py +++ b/fre/workflow/install_script.py @@ -54,11 +54,12 @@ def workflow_install(experiment: str, src_dir: str, target_dir: str, force_insta fre_logger.warning("""NOTE: Workflow '%s/%s}' already ", installed, and the definition is unchanged""", install_dir, workflow_name) else: - fre_logger.error("ERROR: Please remove installed workflow with one of these options:") - fre_logger.error(" - fre workflow install -e %s --src-dir %s --target-dir %s --force-install", experiment, src_dir, target_dir) - fre_logger.error(" - cylc clean %s/%s, then re-run install command", install_dir, workflow_name) raise ValueError(f"""ERROR: Workflow '{install_dir}/{workflow_name}' already - installed, and the definition has changed!""") + installed, and the definition has changed! + + Please remove and re-install the workflow with one of these options: + - fre workflow install -e {experiment} --src-dir {src_dir} --target-dir {target_dir} --force-install" + - cylc clean {install_dir}/{workflow_name}, then re-run install command""") if not Path(install_dir).is_dir(): fre_logger.warning("NOTE: About to install workflow into ~/cylc-run/%s", workflow_name)