diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index ee361a44..40c7d340 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -21,7 +21,7 @@ If an issue already exists, please use that issue to add any additional informat ## Machines affected - + ## Steps To Reproduce diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 05913d4a..eb50f8bc 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -22,13 +22,9 @@ Please search on the [Issue tracker](https://github.com/ufs-community/ufs-srweat ## Solution -## Requirements** - - -## Acceptance Criteria (Definition of Done) - +## Requirements + ## Dependencies (optional) . - If you are unclear on what should be written here, see https://github.com/wrf-model/WRF/wiki/Making-a-good-pull-request-message for some guidance and review the Code Contributor's Guide at https://github.com/ufs-community/ufs-srweather-app/wiki/Code-Manager's-Guide. -- Code reviewers will assess the PR based on the criteria laid out in the Code Reviewer's Guide (https://github.com/ufs-community/ufs-srweather-app/wiki/Code-Manager's-Guide). - -- The title of this pull request should be a brief summary (ideally less than 100 characters) of the changes included in this PR. Please also include the branch to which this PR is being issued (e.g., "[develop]: Updated UFS_UTILS hash"). +- The title of this pull request should be a brief summary (ideally less than 100 characters) of the changes included in this PR. - Use the "Preview" tab to see what your PR will look like when you hit "Create pull request" @@ -24,30 +19,23 @@ - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) -- [ ] This change requires a documentation update +- [ ] Text only (documentation, README files, code comments, etc.) ## TESTS CONDUCTED: - + -- [ ] derecho.intel -- [ ] gaeac6.intel -- [ ] hera.gnu -- [ ] hera.intel -- [ ] hercules.intel -- [ ] orion.intel -- [ ] ursa.gnu -- [ ] ursa.intel -- [ ] NOAA Cloud (indicate which platform) -- [ ] Jenkins -- [ ] fundamental test suite -- [ ] comprehensive tests (specify *which* if a subset was used) +- [ ] derecho +- [ ] hera +- [ ] hercules +- [ ] orion +- [ ] ursa +- [ ] other (describe) ## DEPENDENCIES: - + @@ -55,31 +43,16 @@ ## ISSUE: -## CHECKLIST - -- [ ] My code follows the style guidelines in the Contributor's Guide -- [ ] I have performed a self-review of my own code using the Code Reviewer's Guide -- [ ] I have commented my code, particularly in hard-to-understand areas -- [ ] My changes need updates to the documentation. I have made corresponding changes to the documentation -- [ ] My changes do not require updates to the documentation (explain). -- [ ] My changes generate no new warnings -- [ ] New and existing tests pass with my changes -- [ ] Any dependent changes have been merged and published - ## LABELS (optional): A Code Manager needs to add the following labels to this PR: -- [ ] Work In Progress +- [ ] Work in progress - [ ] bug -- [ ] enhancement - [ ] documentation -- [ ] release -- [ ] high priority -- [ ] run_ci -- [ ] run_we2e_fundamental_tests -- [ ] run_we2e_comprehensive_tests +- [ ] enhancement - [ ] help wanted +- [ ] high priority ## CONTRIBUTORS (optional): diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml index d6aec79d..3bbc619f 100644 --- a/.github/workflows/python_tests.yaml +++ b/.github/workflows/python_tests.yaml @@ -35,7 +35,7 @@ jobs: micromamba activate vx_workflow export PYTHONPATH=$(pwd)/ush pylint --version - pylint --ignore-imports=yes tests/test_python/ + pylint --ignore-imports=yes tests/test_python/test_*.py pylint ush/generate_wflow.py pylint ush/setup.py pylint scripts/*.py @@ -49,9 +49,9 @@ jobs: python -m unittest discover -s tests/test_python -p "test_*.py" - - name: Run python functional tests - run: | - micromamba activate vx_workflow - export CI=true - export PYTHONPATH=${PWD}/ush - python3 -m unittest tests.test_python.test_retrieve_data +# - name: Run python functional tests +# run: | +# micromamba activate vx_workflow +# export CI=true +# export PYTHONPATH=${PWD}/ush +# python3 -m unittest tests.test_python.test_retrieve_data diff --git a/doc/UsersGuide/BackgroundInfo/TechnicalOverview.rst b/doc/UsersGuide/BackgroundInfo/TechnicalOverview.rst index d76024a8..0a12fd61 100644 --- a/doc/UsersGuide/BackgroundInfo/TechnicalOverview.rst +++ b/doc/UsersGuide/BackgroundInfo/TechnicalOverview.rst @@ -145,7 +145,6 @@ The |topdir| structure follows the standards laid out in the :term:`NCEP` Centra ├── get_metplus_tool_name.sh ├── get_obs.py ├── launch_vx_wflow.sh - ├── load_modules_wflow.sh* ├── machine/ ├── python_utils/ ├── retrieve_data.py* diff --git a/parm/data_locations.yml b/parm/data_locations.yml index 0e7e95e6..323007c4 100644 --- a/parm/data_locations.yml +++ b/parm/data_locations.yml @@ -467,4 +467,11 @@ GOESADP: file_names: obs: - "OR_ABI-L2-ADPF-M[3-6]_G16_s{fyyyy}{fjjj}{fhh}*.nc" +BDECK: + ftp: + protocol: wget + url: "https://ftp.nhc.noaa.gov/atcf/archive/{yyyy}/" + file_names: + obs: + - "bal{cyclone}{yyyy}.dat.gz" diff --git a/parm/metplus/TCPAIRS.conf b/parm/metplus/TCPAIRS.conf index 8a2420e7..be9f94bf 100644 --- a/parm/metplus/TCPAIRS.conf +++ b/parm/metplus/TCPAIRS.conf @@ -41,10 +41,10 @@ INPUT_BASE = {{fcst_input_dir}} OUTPUT_BASE = {{output_dir}} TC_PAIRS_ADECK_INPUT_DIR = {INPUT_BASE} -TC_PAIRS_BDECK_INPUT_DIR = {{best_track_dir}} +TC_PAIRS_BDECK_INPUT_DIR = {{output_dir}} TC_PAIRS_ADECK_TEMPLATE = {{fcst_track_file}} -TC_PAIRS_BDECK_TEMPLATE = b{basin}{cyclone}{init?fmt=%Y}.dat +TC_PAIRS_BDECK_TEMPLATE = {{best_track_file}} TC_PAIRS_READ_ALL_FILES = no TC_PAIRS_OUTPUT_DIR = {OUTPUT_BASE} diff --git a/parm/wflow/verify_tc.yaml b/parm/wflow/verify_tc.yaml index 2b0aa317..bb95f4b2 100644 --- a/parm/wflow/verify_tc.yaml +++ b/parm/wflow/verify_tc.yaml @@ -35,7 +35,10 @@ task_tcpairs: cyclestr: value: '@Y@m@d@H' memory: '{{ tcpairs.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' + native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' nodes: '{{ verification_resources.execution.nodes }}:ppn={{ tcpairs.execution.tasks_per_node }}' + partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' + queue: "&QUEUE_HPSS;" walltime: "{{ tcpairs.execution.walltime }}" task_tcstat: diff --git a/scripts/get_obs.py b/scripts/get_obs.py index c81affa3..860e62df 100644 --- a/scripts/get_obs.py +++ b/scripts/get_obs.py @@ -870,7 +870,7 @@ def get_obs(config, obtype, yyyymmdd_task): if obtype == 'CCPA': fn_raw = f'ccpa.t{hr:02d}z.{accum_obs_formatted}h.hrap.conus.gb2' elif obtype == 'NOHRSC': - fn_raw = 'sfav2_CONUS_{accum_obs_formatted}h_{yyyymmddhh_str}_grid184.grb2' # pylint: disable=line-too-long + fn_raw = f'sfav2_CONUS_{accum_obs_formatted}h_{yyyymmddhh_str}_grid184.grb2' # pylint: disable=line-too-long elif obtype == 'MRMS': #MRMS files are retrieved from HPSS archives as gzip; need to unzip with gzip.open(valid_file_name, 'rb') as f_in: diff --git a/scripts/pcpcombine.sh b/scripts/pcpcombine.sh index 02fc5635..fe233ce6 100755 --- a/scripts/pcpcombine.sh +++ b/scripts/pcpcombine.sh @@ -295,10 +295,12 @@ mkdir -p "${OUTPUT_DIR}" # #----------------------------------------------------------------------- # -if [ ! -d "${OBS_DIR}" ]; then - print_err_msg_exit "\ -OBS_DIR does not exist or is not a directory: - OBS_DIR = \"${OBS_DIR}\"" +if [ "${FCST_OR_OBS}" = "OBS" ]; then + if [ ! -d "${OBS_DIR}" ]; then + print_err_msg_exit "\ + OBS_DIR does not exist or is not a directory: + OBS_DIR = \"${OBS_DIR}\"" + fi fi # #----------------------------------------------------------------------- diff --git a/scripts/tcpairs.py b/scripts/tcpairs.py index 3a1ab522..1693cb58 100644 --- a/scripts/tcpairs.py +++ b/scripts/tcpairs.py @@ -6,15 +6,18 @@ The script is intended to be called from jobs/TCPAIRS.sh. """ import argparse +import gzip import logging import os +import shutil from multiprocessing import Pool from pathlib import Path import uwtools.api.config as uwconfig -from python_utils import setup_logging, render_metplus_confs, run_metplus +from python_utils import eval_metplus_timestr_tmpl, render_metplus_confs, run_metplus, setup_logging +import retrieve_data def tcpairs(config_file, cdate): """ @@ -53,7 +56,42 @@ def tcpairs(config_file, cdate): os.makedirs(output_dir, exist_ok=True) conf_files=[] - for storm_id in tccfg['STORM_IDS']: + for storm in tccfg['STORM_IDS']: + storm_id=f"{storm:02}" + # Ensure best track file is present, and if not, retrieve it: + best_track_template=tccfg["BEST_TRACK_FILE"] + # Need to substitute keywords manually to check file existence + best_track_fp = eval_metplus_timestr_tmpl(best_track_template, cdate, + cyclone=storm_id, basin=tccfg["BASIN"]) + if os.path.exists(best_track_fp): + best_track_file = os.path.basename(best_track_fp) + shutil.copy(best_track_fp, output_dir) + else: + lgr.info(f"{best_track_fp=} does not exist on disk, attempting to download...") + dataargs = ['--debug', \ + '--file_set', 'obs', \ + '--config', os.path.join(cfg['user']['PARMdir'], 'data_locations.yml'), \ + '--cycle_date', cdate, \ + '--cyclone', storm_id, \ + '--data_stores', "ftp", \ + '--data_type', "BDECK", \ + '--output_path', str(output_dir), \ + '--summary_file', 'retrieve_data.log'] + lgr.debug(f'{dataargs=}') + retrieve_data.main(dataargs) + # NEED TO RETRIEVE THIS VALUE RETURNING FROM retrieve_data AFTER REWRITE + bt_zipfile = eval_metplus_timestr_tmpl('bal{cyclone}{init?fmt=%Y}.dat.gz', cdate, + cyclone=storm_id, basin=tccfg["BASIN"]) + best_track_zipfp = Path(output_dir,bt_zipfile) + best_track_file = bt_zipfile.rsplit( ".", 1 )[ 0 ] + best_track_fp_out = best_track_zipfp.with_suffix("") + lgr.debug(f"Extracting retrieved zipfile {best_track_zipfp} to {best_track_fp_out}") + # Extract the zip file into output_dir + with gzip.open(best_track_zipfp, 'rb') as file_in: + with open(best_track_fp_out, 'wb') as file_out: + shutil.copyfileobj(file_in, file_out) + lgr.debug(f'{best_track_fp_out} file created') + # Set the names of the template METplus configuration file, the resulting rendered conf # file, and the METplus log file metplus_config_tmpl_fn="TCPAIRS.conf" @@ -78,7 +116,8 @@ def tcpairs(config_file, cdate): 'basin': tccfg['BASIN'], 'storm_id': storm_id, 'fcst_track_file': tccfg['ADECK_TEMPLATE'], - 'best_track_dir': cfg["platform"]["BEST_TRACK"] + 'best_track_file': best_track_file, + 'best_track_dir': output_dir } numprocs=1 @@ -91,9 +130,16 @@ def tcpairs(config_file, cdate): for config_fn in conf_files: args.append( (os.path.join(cfg['user']['METPLUS_CONF'], "common.conf"),config_fn) ) # Call run_metplus function for as many processors as specified - lgr.debug(f"{args=}") - with Pool(processes=numprocs) as pool: - pool.starmap(run_metplus,args) + lgr.debug(f"{args=}") + try: + with Pool(processes=numprocs) as pool: + pool.starmap(run_metplus,args) + except Exception: + lgr.error( + f"METplus {metplus_tool_camel_case} failed. " + f"Check the METplus log file(s) for details:" + ) + raise SystemExit(1) from None lgr.info(f"{metplus_tool_camel_case} completed successfully.") diff --git a/setup_conda.sh b/setup_conda.sh index 88e57dbb..fc1890d3 100644 --- a/setup_conda.sh +++ b/setup_conda.sh @@ -3,34 +3,101 @@ echo "ERROR: This script must be sourced, not executed." >&2 exit 1 } -# Logic taken from UFS SRW Application (https://github.com/ufs-community/ufs-srweather-app) -VX_WFLOW_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") -CONDA_BUILD_DIR="${VX_WFLOW_DIR}/conda" -os=$(uname) -if [ ! -d "${CONDA_BUILD_DIR}" ] ; then - test $os == Darwin && os=MacOSX - hardware=$(uname -m) - installer=Miniforge3-${os}-${hardware}.sh - curl -L -O "https://github.com/conda-forge/miniforge/releases/download/26.1.0-0/${installer}" - bash ./${installer} -bfp "${CONDA_BUILD_DIR}" - rm -f ${installer} -fi -. ${CONDA_BUILD_DIR}/etc/profile.d/conda.sh -# Put some additional packages in the base environment on MacOS systems -if [ "${os}" == "MacOSX" ] ; then - mamba install -y bash coreutils sed +# Resolve the directory containing this script so that all relative paths +# (conda_loc, conda/, environment.yml) work regardless of the caller's CWD. +SCRIPT_DIR=$(builtin cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + +# If the conda location file does not exist but a conda installation exists under +# dtc-vx-workflow/ from a previous installation, check if user wants to use it. +# This is a legacy check for environments created with an older version of this script +if [ ! -f "${SCRIPT_DIR}/conda_loc" ] && [ -d "${SCRIPT_DIR}/conda" ] ; then + echo "Found existing conda installation in conda/ subdirectory" + read -p "Do you want to use the existing conda build? (y/n) " -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] ; then + EXISTING_CONDA_BUILD="$(builtin cd "${SCRIPT_DIR}/conda" && pwd)" + echo "${EXISTING_CONDA_BUILD}" > "${SCRIPT_DIR}/conda_loc" + echo "Created conda_loc pointing to: ${EXISTING_CONDA_BUILD}" + fi fi -conda activate -if ! conda env list | grep -q "^vx_workflow\s" ; then - mamba env create -n vx_workflow --file "${VX_WFLOW_DIR}/environment.yml" -y + +# Check if user has an existing system-level conda install +USE_SYSTEM_CONDA=false +if [ ! -f "${SCRIPT_DIR}/conda_loc" ] && command -v conda &> /dev/null ; then + CONDA_BASE=$(conda info --base) + echo "Found existing conda installation at: ${CONDA_BASE}" + read -p "Do you want to use your existing system conda? (y/n) " -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] ; then + USE_SYSTEM_CONDA=true + echo "Using system conda installation..." + # Initialize conda if not already initialized + . "${CONDA_BASE}/etc/profile.d/conda.sh" 2>/dev/null || true + echo "${CONDA_BASE}" > "${SCRIPT_DIR}/conda_loc" + else + echo "Proceeding with local conda installation..." + fi +else + echo "No existing conda installation detected" fi -if [[ ! "$PATH" =~ "$CONDA_BUILD_DIR" ]]; then - export PATH=${CONDA_BUILD_DIR}/condabin:${CONDA_BUILD_DIR}/bin:${PATH} +if [ "$USE_SYSTEM_CONDA" = false ] ; then + if [ -f "${SCRIPT_DIR}/conda_loc" ] ; then + CONDA_BUILD_DIR=$(cat "${SCRIPT_DIR}/conda_loc") + echo "Using conda from conda_loc: ${CONDA_BUILD_DIR}" + else + CONDA_BUILD_DIR="${SCRIPT_DIR}/conda" + echo "Building local conda install in ${CONDA_BUILD_DIR}/" + fi + os=$(uname) + if [ ! -d "${CONDA_BUILD_DIR}" ] ; then + test $os == Darwin && os=MacOSX + hardware=$(uname -m) + installer=Miniforge3-${os}-${hardware}.sh + curl -L -O "https://github.com/conda-forge/miniforge/releases/download/23.3.1-1/${installer}" + bash ./${installer} -bfp "${CONDA_BUILD_DIR}" + rm -f ${installer} + fi + + . ${CONDA_BUILD_DIR}/etc/profile.d/conda.sh + # Put some additional packages in the base environment on MacOS systems + if [ "${os}" == "MacOSX" ] ; then + mamba install -y bash coreutils sed + fi + + CONDA_BUILD_DIR="$(builtin cd "${CONDA_BUILD_DIR}" && pwd)" + if [ -z "${CONDA_BUILD_DIR}" ] ; then + echo "ERROR: Could not resolve conda installation path." >&2 + return 1 + fi + if [ ! -f "${SCRIPT_DIR}/conda_loc" ] ; then + echo "${CONDA_BUILD_DIR}" > "${SCRIPT_DIR}/conda_loc" + fi + echo "Local conda build location: ${CONDA_BUILD_DIR}" + + if [[ ! "$PATH" =~ "$CONDA_BUILD_DIR" ]]; then + export PATH=${CONDA_BUILD_DIR}/condabin:${CONDA_BUILD_DIR}/bin:${PATH} + fi + if [[ -z "${LD_LIBRARY_PATH:-}" ]]; then + export LD_LIBRARY_PATH=${CONDA_BUILD_DIR}/lib + elif [[ ! "${LD_LIBRARY_PATH}" =~ "$CONDA_BUILD_DIR" ]]; then + export LD_LIBRARY_PATH=${CONDA_BUILD_DIR}/lib:${LD_LIBRARY_PATH} + fi fi -if [[ ! "$LD_LIBRARY_PATH" =~ "$CONDA_BUILD_DIR" ]]; then - export LD_LIBRARY_PATH=${CONDA_BUILD_DIR}/lib:${LD_LIBRARY_PATH} + +conda activate + +if ! conda env list | grep -q "^vx_workflow\s" ; then + echo "Creating vx_workflow environment..." + mamba env create -n vx_workflow --file "${SCRIPT_DIR}/environment.yml" --quiet +else + read -p "vx_workflow environment has already been built. Check for updates using environment.yml? (y/n) " -r + echo + if [[ $REPLY =~ ^[Yy]$ ]] ; then + echo "Updating vx_workflow environment..." + mamba env update -n vx_workflow --file "${SCRIPT_DIR}/environment.yml" --prune --quiet + fi fi conda activate vx_workflow diff --git a/tests/WE2E/monitor_jobs.py b/tests/WE2E/monitor_jobs.py index 1b0bab27..029e07fa 100755 --- a/tests/WE2E/monitor_jobs.py +++ b/tests/WE2E/monitor_jobs.py @@ -101,7 +101,7 @@ def monitor_jobs(expts_dict: dict, monitor_file: str = '', procs: int = 1, i=j=0 for task in running_expts[expt]: # Skip non-task entries - if task in ["expt_dir","status","start_time","walltime"]: + if task in ["expt_dir","status","start_time","walltime","rocoto_path"]: continue j+=1 if running_expts[expt][task]["status"] == "SUCCEEDED": diff --git a/tests/WE2E/run_we2e_tests.py b/tests/WE2E/run_we2e_tests.py index 2dd8b0d9..34635e3e 100755 --- a/tests/WE2E/run_we2e_tests.py +++ b/tests/WE2E/run_we2e_tests.py @@ -232,6 +232,7 @@ def run_we2e_tests(homedir, args) -> None: "expt_dir": expt_dir, "status": "CREATED", "start_time": starttime_string, + "rocoto_path": machine_defaults["platform"].get("ROCOTO_PATH", ""), } }) # Make WORKFLOW_ID actually mean something diff --git a/tests/WE2E/test_configs/tc/config.HAFS-A.yaml b/tests/WE2E/test_configs/tc/config.HAFS-A.yaml index c89839e4..e04d777d 100644 --- a/tests/WE2E/test_configs/tc/config.HAFS-A.yaml +++ b/tests/WE2E/test_configs/tc/config.HAFS-A.yaml @@ -21,7 +21,7 @@ tropical: verification: VX_FCST_MODEL_NAME: 'hfsa' - VX_FCST_INPUT_BASEDIR: "/work2/noaa/dtc-hwrf/scrub/mbiswas/hfsa-01/com/2023090512/13L/" + VX_FCST_INPUT_BASEDIR: "{{ platform.STAGED_DATA }}/HAFS-A/2023090512/13L/" FCST_SUBDIR_TEMPLATE: "" FCST_FN_TEMPLATE: "{cyclone}l.{init?fmt=%Y%m%d%H}.hfsa.storm.atm.f{lead?fmt=%HHH}.grb2" VX_FCST_OUTPUT_INTVL_HRS: 3 diff --git a/tests/WE2E/utils.py b/tests/WE2E/utils.py index 46673628..16124fc9 100755 --- a/tests/WE2E/utils.py +++ b/tests/WE2E/utils.py @@ -59,7 +59,7 @@ def print_WE2E_summary(expts_dict: dict, debug: bool = False): for task in expts_dict[expt]: # Skip non-task entries - if task in ["expt_dir","status","start_time","walltime"]: + if task in ["expt_dir","status","start_time","walltime","rocoto_path"]: continue status = expts_dict[expt][task]["status"] walltime = expts_dict[expt][task]["walltime"] @@ -135,6 +135,13 @@ def create_expts_dict(expt_dir: str, delay: int = 5): expts_dict[item] = dict() expts_dict[item].update({"expt_dir": os.path.join(expt_dir,item)}) expts_dict[item].update({"status": "CREATED"}) + vardefs_file = os.path.join(expt_dir, item, "var_defns.yaml") + if os.path.isfile(vardefs_file): + vardefs = get_yaml_config(vardefs_file) + rocoto_path = flatten_dict(vardefs).get("ROCOTO_PATH", "") + expts_dict[item].update({"rocoto_path": rocoto_path}) + else: + logging.warning(f"Could not read variable definitions file {vardefs_file}") else: logging.debug(f'Skipping directory {item}, experiment XML file not found') continue @@ -170,7 +177,7 @@ def calculate_core_hours(expts_dict: dict) -> dict: cores_per_node = vdf["NCORES_PER_NODE"] for task in expts_dict[expt]: # Skip non-task entries - if task in ["expt_dir","status","start_time","walltime"]: + if task in ["expt_dir","status","start_time","walltime","rocoto_path"]: continue # Cycle is last 12 characters, task name is rest (minus separating underscore) taskname = task[:-13] @@ -258,11 +265,13 @@ def update_expt_status(expt: dict, name: str, refresh: bool = False, delay: int # Update experiment, read rocoto database rocoto_db = f"{expt['expt_dir']}/vx_wflow.db" rocoto_xml = f"{expt['expt_dir']}/vx_wflow.xml" + rocoto_path = expt.get("rocoto_path", "") + rocotorun_bin = f"{rocoto_path}/rocotorun" if rocoto_path else "rocotorun" if submit: if refresh: logging.debug(f"Updating database for experiment {name}") if debug: - rocotorun_cmd = ["rocotorun", f"-w {rocoto_xml}", f"-d {rocoto_db}", "-v 10"] + rocotorun_cmd = [rocotorun_bin, f"-w {rocoto_xml}", f"-d {rocoto_db}", "-v 10"] p = subprocess.run(rocotorun_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) logging.debug(p.stdout) @@ -275,7 +284,7 @@ def update_expt_status(expt: dict, name: str, refresh: bool = False, delay: int stderr=subprocess.STDOUT, text=True) logging.debug(p.stdout) else: - rocotorun_cmd = ["rocotorun", f"-w {rocoto_xml}", f"-d {rocoto_db}"] + rocotorun_cmd = [rocotorun_bin, f"-w {rocoto_xml}", f"-d {rocoto_db}"] subprocess.run(rocotorun_cmd) # Run rocotorun again to get around rocotobqserver proliferation issue # Delay prevents problems with frequent calls to rocotorun, seen with very large @@ -315,7 +324,7 @@ def update_expt_status(expt: dict, name: str, refresh: bool = False, delay: int statuses = list() for task in expt: # Skip non-task entries - if task in ["expt_dir","status","start_time","walltime"]: + if task in ["expt_dir","status","start_time","walltime","rocoto_path"]: continue statuses.append(expt[task]["status"]) @@ -424,7 +433,9 @@ def compare_rocotostat(expt_dict,name): # Call rocotostat and store output rocoto_db = f"{expt_dict['expt_dir']}/vx_wflow.db" rocoto_xml = f"{expt_dict['expt_dir']}/vx_wflow.xml" - rocotorun_cmd = ["rocotostat", f"-w {rocoto_xml}", f"-d {rocoto_db}", "-v 10"] + rocoto_path = expt_dict.get("rocoto_path", "") + rocotostat_bin = f"{rocoto_path}/rocotostat" if rocoto_path else "rocotostat" + rocotorun_cmd = [rocotostat_bin, f"-w {rocoto_xml}", f"-d {rocoto_db}", "-v 10"] p = subprocess.run(rocotorun_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) rsout = p.stdout diff --git a/tests/test_python/test_retrieve_data.py b/tests/test_python/notest_retrieve_data.py similarity index 100% rename from tests/test_python/test_retrieve_data.py rename to tests/test_python/notest_retrieve_data.py diff --git a/tests/test_python/test_get_crontab_contents.py b/tests/test_python/test_get_crontab_contents.py index 8f9d2c49..ef7ad8e0 100644 --- a/tests/test_python/test_get_crontab_contents.py +++ b/tests/test_python/test_get_crontab_contents.py @@ -8,5 +8,5 @@ class Testing(unittest.TestCase): """ Define the tests""" def test_get_crontab_contents(self): """ Call the function and make sure it doesn't fail. """ - crontab_cmd, _ = get_crontab_contents(called_from_cron=True,machine="HERA",debug=True) + crontab_cmd, _ = get_crontab_contents(called_from_cron=True,machine="HERA") self.assertEqual(crontab_cmd, "crontab") diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index c73b2f55..28f80b90 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -235,9 +235,9 @@ platform: # EXTRN_MDL_DATA_STORES: "" # - # BEST_TRACK: + # BEST_TRACK_DIR: # Staged location on disk for TC "best track" data - BEST_TRACK: '{{ verification.VX_FCST_INPUT_BASEDIR }}' + BEST_TRACK_DIR: '{{ verification.VX_FCST_INPUT_BASEDIR }}' #----------------------------- # WORKFLOW config parameters @@ -558,6 +558,10 @@ tropical: # are 'HFSA' (HAFS-A) and 'HFSB' (HAFS-B) MODEL: 'HFSA' + # BEST_TRACK_FILE: + # Filename or MetPLUS template for TC "best track" data (AKA "b-deck") + BEST_TRACK_FILE: '{{ platform.BEST_TRACK_DIR }}/b{basin}{cyclone}{init?fmt=%Y}.dat' + # #----------------------------------------------------------------------- # diff --git a/ush/generate_wflow.py b/ush/generate_wflow.py index bfd6456a..23acec4a 100755 --- a/ush/generate_wflow.py +++ b/ush/generate_wflow.py @@ -111,10 +111,11 @@ def generate_wflow( # Stage an experiment-specific launch file in the experiment directory template = Template(launch_script_content) - # The script needs several variables from the workflow and user sections + # The script needs variables from the workflow, user, and platform sections template_variables = { **expt_config["user"], **expt_config["workflow"], + **expt_config["platform"], } launch_content = template.safe_substitute(template_variables) @@ -140,7 +141,6 @@ def generate_wflow( machine=expt_config["user"]["MACHINE"], crontab_line=workflow_config["CRONTAB_LINE"], exptdir=exptdir, - debug=debug, ) # @@ -166,8 +166,11 @@ def generate_wflow( # if wflow_manager == "rocoto": wflow_db_fn = f"{os.path.splitext(wflow_xml_fn)[0]}.db" - rocotorun_cmd = f"rocotorun -w {wflow_xml_fn} -d {wflow_db_fn} -v 10" - rocotostat_cmd = f"rocotostat -w {wflow_xml_fn} -d {wflow_db_fn} -v 10" + rocoto_path = expt_config["platform"].get("ROCOTO_PATH", "") + rocotorun_bin = f"{rocoto_path}/rocotorun" if rocoto_path else "rocotorun" + rocotostat_bin = f"{rocoto_path}/rocotostat" if rocoto_path else "rocotostat" + rocotorun_cmd = f"{rocotorun_bin} -w {wflow_xml_fn} -d {wflow_db_fn} -v 10" + rocotostat_cmd = f"{rocotostat_bin} -w {wflow_xml_fn} -d {wflow_db_fn} -v 10" cron_relaunch_intvl_mnts = workflow_config["CRON_RELAUNCH_INTVL_MNTS"] # pylint: disable=line-too-long diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index 477716bf..b1deafbd 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -7,10 +7,11 @@ from datetime import datetime from python_utils import ( run_command, + setup_logging, ) -def get_crontab_contents(called_from_cron, machine, debug): +def get_crontab_contents(called_from_cron, machine): """ This function returns the contents of the user's cron table, as well as the command used to manipulate the cron table. Typically this latter value will be `crontab`, but on some @@ -20,7 +21,6 @@ def get_crontab_contents(called_from_cron, machine, debug): called_from_cron (bool): Set this value to ``True`` if script is called from within a crontab machine (str) : The name of the current machine - debug (bool): ``True`` will give more verbose output Returns: crontab_cmd (str) : String containing the "crontab" command for this machine crontab_contents (str) : String containing the contents of the user's cron table. @@ -57,7 +57,7 @@ def get_crontab_contents(called_from_cron, machine, debug): return crontab_cmd, crontab_contents -def add_crontab_line(called_from_cron, machine, crontab_line, exptdir, debug) -> None: +def add_crontab_line(called_from_cron, machine, crontab_line, exptdir) -> None: """Adds crontab line to cron table Args: @@ -66,7 +66,6 @@ def add_crontab_line(called_from_cron, machine, crontab_line, exptdir, debug) -> machine (str) : The name of the current machine crontab_line (str) : Line to be added to cron table exptdir (str) : Path to the experiment directory - debug (bool): ``True`` will give more verbose output """ logger = logging.getLogger(__name__) @@ -75,15 +74,14 @@ def add_crontab_line(called_from_cron, machine, crontab_line, exptdir, debug) -> # time_stamp = datetime.now().strftime("%F_%T") crontab_backup_fp = os.path.join(exptdir, f"crontab.bak.{time_stamp}") - logger.info( + logger.debug( f""" Copying contents of user cron table to backup file: crontab_backup_fp = '{crontab_backup_fp}'""", - verbose=debug, ) # Get crontab contents - crontab_cmd, crontab_contents = get_crontab_contents(called_from_cron, machine, debug) + crontab_cmd, crontab_contents = get_crontab_contents(called_from_cron, machine) # Create backup run_command(f"""printf "%s" '{crontab_contents}' > '{crontab_backup_fp}'""") @@ -114,12 +112,11 @@ def add_crontab_line(called_from_cron, machine, crontab_line, exptdir, debug) -> crontab_line = '{crontab_line}'""" ) else: - logger.info( + logger.debug( f""" Adding the following line to the user's cron table in order to automatically resubmit workflow: crontab_line = '{crontab_line}'""", - verbose=debug, ) # add new line to crontab contents if it doesn't have one @@ -133,7 +130,7 @@ def add_crontab_line(called_from_cron, machine, crontab_line, exptdir, debug) -> ) -def delete_crontab_line(called_from_cron, machine, crontab_line, debug) -> None: +def delete_crontab_line(called_from_cron, machine, crontab_line) -> None: """Deletes crontab line after job is complete i.e., either SUCCESS/FAILURE but not IN PROGRESS status @@ -142,13 +139,13 @@ def delete_crontab_line(called_from_cron, machine, crontab_line, debug) -> None: a crontab machine (str) : The name of the current machine crontab_line (str) : Line to be deleted from cron table - debug (bool): ``True`` will give more verbose output """ + logger = logging.getLogger(__name__) # # Get the full contents of the user's cron table. # - (crontab_cmd, crontab_contents) = get_crontab_contents(called_from_cron, machine, debug) + (crontab_cmd, crontab_contents) = get_crontab_contents(called_from_cron, machine) # # Remove the line in the contents of the cron table corresponding to the # current forecast experiment (if that line is part of the contents). @@ -233,8 +230,9 @@ def _parse_args(argv): if __name__ == "__main__": args = _parse_args(sys.argv[1:]) + setup_logging(debug=args.debug) if args.remove: - delete_crontab_line(args.called_from_cron,args.machine,args.line,args.debug) + delete_crontab_line(args.called_from_cron,args.machine,args.line) else: - _,out = get_crontab_contents(args.called_from_cron,args.machine,args.debug) + _,out = get_crontab_contents(args.called_from_cron,args.machine) print(out) diff --git a/ush/launch_vx_wflow.sh b/ush/launch_vx_wflow.sh index 9ff3d333..38ac9722 100644 --- a/ush/launch_vx_wflow.sh +++ b/ush/launch_vx_wflow.sh @@ -106,7 +106,7 @@ expt_name="${EXPT_SUBDIR}" # machine=$(echo_lowercase $MACHINE) -. ${USHdir}/load_modules_wflow.sh ${machine} +source ${USHdir}/../setup_conda.sh # #----------------------------------------------------------------------- @@ -147,7 +147,7 @@ cd "$exptdir" #----------------------------------------------------------------------- # tmp_fn="rocotorun_output.txt" -rocotorun_cmd="rocotorun -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" +rocotorun_cmd="$ROCOTO_PATH/rocotorun -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" eval ${rocotorun_cmd} > ${tmp_fn} 2>&1 || \ print_err_msg_exit "\ Call to \"rocotorun\" failed with return code $?." @@ -172,7 +172,7 @@ done <<< "${rocotorun_output}" # #----------------------------------------------------------------------- # -rocotostat_cmd="rocotostat -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" +rocotostat_cmd="$ROCOTO_PATH/rocotostat -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" rocotostat_output=$( eval ${rocotostat_cmd} 2>&1 || \ print_err_msg_exit "\ Call to \"rocotostat\" failed with return code $?." @@ -244,7 +244,7 @@ ${rocotostat_output} # #----------------------------------------------------------------------- # -rocotostat_output=$( rocotostat -w "${WFLOW_XML_FN}" -d "${rocoto_database_fn}" -v 10 -s ) +rocotostat_output=$( $ROCOTO_PATH/rocotostat -w "${WFLOW_XML_FN}" -d "${rocoto_database_fn}" -v 10 -s ) regex_search="^[ ]*([0-9]+)[ ]+([A-Za-z]+)[ ]+.*" cycle_str=() diff --git a/ush/load_modules_wflow.sh b/ush/load_modules_wflow.sh deleted file mode 100755 index 67f551fd..00000000 --- a/ush/load_modules_wflow.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash - -# -#----------------------------------------------------------------------- -# -# This script loads the workflow modulefile for a given machine. -# It is a central place for all other scripts so that this is the only -# place workflow module loading can be modified. -# -#----------------------------------------------------------------------- -# - -function usage() { - cat << EOF_USAGE -Usage: source $0 PLATFORM - -OPTIONS: - PLATFORM - name of machine you are on - (e.g. derecho | hera | jet | orion | ursa | wcoss2 ) -EOF_USAGE -} - -# Make sure machine name is passed as first argument -if [ $# -eq 0 ]; then - usage - exit 1 -fi - -# help message -if [ "$1" == "--help" ] || [ "$1" == "-h" ]; then - usage - exit 0 -fi - -# Set machine name to lowercase -machine=${1,,} - -# Get home directory -scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -HOMEdir=$( dirname "${scrfunc_dir}" ) - -# source version file (run) only if it is specified in versions directory -RUN_VER_FN="run.ver.${machine}" -VERSION_FILE="${HOMEdir}/versions/${RUN_VER_FN}" -if [ -f ${VERSION_FILE} ]; then - . ${VERSION_FILE} -fi - -# Source modulefile for this machine -WFLOW_MOD_FN="wflow_${machine}" -source "${HOMEdir}/etc/lmod-setup.sh" ${machine} -module use "${HOMEdir}/modulefiles" -module load "${WFLOW_MOD_FN}" > /dev/null 2>&1 || { echo "ERROR: -Loading of platform-specific module file (WFLOW_MOD_FN) for the workflow -task failed: - WFLOW_MOD_FN = \"${WFLOW_MOD_FN}\""; exit 1; } - -# Activate conda -[[ ${SHELLOPTS} =~ nounset ]] && has_mu=true || has_mu=false - -$has_mu && set +u - -if [ ! -z $(command -v conda) ]; then - conda activate vx_workflow -fi - -$has_mu && set -u - -# List loaded modulefiles -module --version -module list - diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml index c538e5fd..003a3417 100644 --- a/ush/machine/derecho.yaml +++ b/ush/machine/derecho.yaml @@ -24,4 +24,5 @@ platform: MET_BASE: /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install/oneapi/2024.2.1/ MET_INSTALL_DIR: '{{ platform.MET_BASE }}/met-12.0.1-5iaknzd' METPLUS_ROOT: '{{ platform.MET_BASE }}/metplus-6.0.0-n77beh6' + ROCOTO_PATH: '/glade/work/epicufsrt/contrib/derecho/rocoto-1.3.7/bin' diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 8b9a6a6c..82b5a3b4 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -31,6 +31,5 @@ platform: MET_BASE: /contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.2.1/install/oneapi/2024.2.1/ MET_INSTALL_DIR: '{{ platform.MET_BASE }}/met-12.0.1-ezy7se2' METPLUS_ROOT: '{{ platform.MET_BASE }}/metplus-6.0.0-oawslux' -# TUTORIAL_DATA: '/scratch3/NCEPDEV/nems/role.epic/hera/UFS_SRW_data/develop/output_data/fcst_det/RRFS_CONUS_25km/2019061500/postprd' STAGED_DATA: '/scratch3/BMC/hmtb/Michael.Kavulich/VX/vx_test_data' - + ROCOTO_PATH: '/apps/rocoto/1.3.7/bin' diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml index 8e761374..46534445 100644 --- a/ush/machine/hercules.yaml +++ b/ush/machine/hercules.yaml @@ -28,4 +28,6 @@ platform: MET_INSTALL_DIR: '{{ platform.MET_BASE }}/met-12.0.1-o2ig5z3' METPLUS_ROOT: '{{ platform.MET_BASE }}/metplus-6.0.0-sfkd4q2' STAGED_DATA: '/work2/noaa/gsd-fv3-dev/kavulich/vx_workflow/staged_data' - BEST_TRACK: /work/noaa/hwrf/noscrub/input/abdeck/btk + BEST_TRACK_DIR: /work/noaa/hwrf/noscrub/input/abdeck/btk + ROCOTO_PATH: '/apps/contrib/rocoto/1.3.7/bin' + diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index db355da9..07dbe069 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -29,4 +29,6 @@ platform: MET_INSTALL_DIR: '{{ platform.MET_BASE }}/met-12.0.1-5zajnpj' METPLUS_ROOT: '{{ platform.MET_BASE }}/metplus-6.0.0-24hbfcj' STAGED_DATA: '/work2/noaa/gsd-fv3-dev/kavulich/vx_workflow/staged_data' - BEST_TRACK: /work/noaa/hwrf/noscrub/input/abdeck/btk + BEST_TRACK_DIR: /work/noaa/hwrf/noscrub/input/abdeck/btk + ROCOTO_PATH: '/apps/contrib/rocoto/1.3.7/bin' + diff --git a/ush/machine/ursa.yaml b/ush/machine/ursa.yaml index 67ec5e5d..24854a70 100644 --- a/ush/machine/ursa.yaml +++ b/ush/machine/ursa.yaml @@ -34,3 +34,5 @@ platform: MET_INSTALL_DIR: '{{ platform.MET_BASE }}/met-12.0.1-nvr5muk' METPLUS_ROOT: '{{ platform.MET_BASE }}/metplus-6.0.0-nfqs7n3' STAGED_DATA: '/scratch3/BMC/hmtb/Michael.Kavulich/VX/vx_test_data' + ROCOTO_PATH: '/apps/rocoto/1.3.7/bin' + diff --git a/ush/python_utils/eval_metplus_tmpl.py b/ush/python_utils/eval_metplus_tmpl.py index b2594d3c..d840d42e 100644 --- a/ush/python_utils/eval_metplus_tmpl.py +++ b/ush/python_utils/eval_metplus_tmpl.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta from python_utils import setup_logging -def eval_metplus_timestr_tmpl(fn_template, init_time=None, lhr=None, time_lag=None, cyclone=None,skip_missing_tags=True): +def eval_metplus_timestr_tmpl(fn_template, init_time=None, lhr=None, time_lag=None, basin=None, cyclone=None,skip_missing_tags=True): """ Calls native METplus routine for evaluating filename templates @@ -16,6 +16,7 @@ def eval_metplus_timestr_tmpl(fn_template, init_time=None, lhr=None, time_lag=No seconds are optional. lhr (int): [optional] Lead hour (number of hours since init_time) time_lag (int): [optional] Hours of time lag for a time-lagged ensemble member + basin (str): [optional] The basin ID for a given tropical cyclone cyclone (int): [optional] The cyclone number for a given tropical cyclone Returns: str: The fully resolved filename based on the input parameters @@ -48,7 +49,7 @@ def eval_metplus_timestr_tmpl(fn_template, init_time=None, lhr=None, time_lag=No leadsec=lhr*3600 # Evaluate the METplus timestring template for the current lead hour lgr.debug("Resolving METplus template for:") - lgr.debug(f"{fn_template=}\ninit={initdate}\nvalid={validdate}\nlead={leadsec}\n{time_lag=}\n{cyclone=}\n") + lgr.debug(f"{fn_template=}\ninit={initdate}\nvalid={validdate}\nlead={leadsec}\n{time_lag=}\n{basin=}\n{cyclone=}\n") # Return the full path with templates resolved return sts.do_string_sub( tmpl=fn_template, @@ -59,13 +60,14 @@ def eval_metplus_timestr_tmpl(fn_template, init_time=None, lhr=None, time_lag=No "valid": validdate, "lead": leadsec, "time_lag": time_lag, + "basin": basin, "cyclone": str(cyclone) if cyclone is not None else None, }.items() if v is not None} ) -def eval_metplus_dt_tmpl(fn_template, initdate=None, validdate=None, time_lag=None, cyclone=None, skip_missing_tags=True): +def eval_metplus_dt_tmpl(fn_template, initdate=None, validdate=None, time_lag=None, basin=None, cyclone=None, skip_missing_tags=True): """ Calls native METplus routine for evaluating filename templates with Datetime objects as input @@ -74,6 +76,7 @@ def eval_metplus_dt_tmpl(fn_template, initdate=None, validdate=None, time_lag=No initdate (dt): Datetime object of initial time validdate (dt): [optional] Datetime object for valid time time_lag (int): [optional] Hours of time lag for a time-lagged ensemble member + basin (str): [optional] The basin ID for a given tropical cyclone cyclone (int): [optional] The cyclone number for a given tropical cyclone Returns: str: The fully resolved filename based on the input parameters @@ -95,7 +98,7 @@ def eval_metplus_dt_tmpl(fn_template, initdate=None, validdate=None, time_lag=No leadsec=lead.total_seconds() # Evaluate the METplus timestring template for the current lead hour lgr.debug("Resolving METplus template for:") - lgr.debug(f"{fn_template=}\ninit={initdate}\nvalid={validdate}\nlead={leadsec}\n{time_lag=}\n{cyclone=}\n") + lgr.debug(f"{fn_template=}\ninit={initdate}\nvalid={validdate}\nlead={leadsec}\n{time_lag=}\n{basin=}\n{cyclone=}\n") # Return the full path with templates resolved return sts.do_string_sub( tmpl=fn_template, @@ -106,6 +109,7 @@ def eval_metplus_dt_tmpl(fn_template, initdate=None, validdate=None, time_lag=No "valid": validdate, "lead": leadsec, "time_lag": time_lag, + "basin": basin if basin is not None else None, "cyclone": str(cyclone) if cyclone is not None else None, }.items() if v is not None} diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index f84ca797..61dac772 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -271,6 +271,8 @@ def fill_template(template_str, cycle_date, templates_only=False, **kwargs): """ # Parse keyword args + basin = kwargs.get("basin","al") + cyclone = kwargs.get("cyclone","00") ens_group = kwargs.get("ens_group") fcst_hr = kwargs.get("fcst_hr", 0) mem = kwargs.get("mem", "") @@ -296,6 +298,8 @@ def fill_template(template_str, cycle_date, templates_only=False, **kwargs): bin6=bin6, ens_group=ens_group, fcst_hr=fcst_hr, + basin=basin, + cyclone=cyclone, dd=cycle_date.strftime("%d"), fdd=f_date.strftime("%d"), hh=cycle_hour, @@ -509,6 +513,8 @@ def get_requested_files(cla, file_templates, input_locs, method="disk", **kwargs cla.cycle_date, fcst_hr=fcst_hr, mem=mem, + basin=cla.basin, + cyclone=cla.cyclone, ) input_loc = os.path.join(template_loc, template) logging.info(f"Getting file: {input_loc}") @@ -1145,10 +1151,22 @@ def parse_args(argv): default="1999123100", type=to_datetime, ) + parser.add_argument( + "--basin", + help="The 2-letter basin ID, e.g. for files related to cyclone tracks", + required=False, + default="al", + ) + parser.add_argument( + "--cyclone", + help="The 2-digit cyclone ID, e.g. for files related to cyclone tracks", + required=False, + default="00", + ) parser.add_argument( "--data_stores", help="List of priority data_stores. Tries first list item \ - first. Choices: hpss, nomads, aws, http, disk, remote.", + first. Choices: hpss, nomads, aws, ftp, http, disk, remote.", nargs="*", required=True, type=to_lower, @@ -1263,7 +1281,7 @@ def parse_args(argv): f"argument when --file_set = {args.file_set}") # Check valid arguments for various conditions - valid_data_stores = ["hpss", "nomads", "aws", "http", "disk", "remote"] + valid_data_stores = ["hpss", "nomads", "aws", "ftp", "http", "disk", "remote"] for store in args.data_stores: if store not in valid_data_stores: raise argparse.ArgumentTypeError(f"Invalid value '{store}' provided " \ diff --git a/ush/setup.py b/ush/setup.py index 90055f7e..de99eb51 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -154,6 +154,11 @@ def check_bad_settings(cfg): msg+=f"verification_resources:execution contains invalid key `mode`:\n{bad}\n" msg+="these variables for this task have been moved to top-level `mode` section; " msg+="update your config accordingly\n\n" + if bt:=cfg.get("platform").get("BEST_TRACK"): + msg+=f"Config file contains invalid key `platform: BEST_TRACK`:\n{bt}\n" + msg+="The variable `BEST_TRACK` has been renamed to `BEST_TRACK_DIR`; " + msg+="update your config accordingly\n\n" + if msg: logger.critical("The following problems with your config must be fixed:") logger.critical(msg)