From b4b276476db3117f196911f25d0315e97dc36ad8 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 24 Jun 2026 20:48:07 -0600 Subject: [PATCH 01/25] Convert remaining bash ex-scripts to Python Converts all four remaining shell scripts in scripts/ to Python, completing the bash-to-Python migration: - check_post_output.py: calls set_leadhrs() to verify post-processed forecast output files exist within the allowed missing-file threshold - gridstat_or_pointstat_ensmean.py: GridStat/PointStat on ensemble mean output from GenEnsProd - gridstat_or_pointstat_ensprob.py: GridStat/PointStat on ensemble probability output from GenEnsProd - genensprod_or_ensemblestat.py: GenEnsProd or EnsembleStat with per-member forecast template list; GenEnsProd skips obs file check - pcpcombine.py: PcpCombine for FCST or OBS with double set_leadhrs call pattern; includes PM2.5/PM10 USER_DEFINED combine logic for AIRNOW Co-Authored-By: Claude Sonnet 4.6 --- scripts/check_post_output.py | 96 ++++++++ scripts/genensprod_or_ensemblestat.py | 264 ++++++++++++++++++++ scripts/gridstat_or_pointstat_ensmean.py | 228 +++++++++++++++++ scripts/gridstat_or_pointstat_ensprob.py | 229 +++++++++++++++++ scripts/pcpcombine.py | 301 +++++++++++++++++++++++ 5 files changed, 1118 insertions(+) create mode 100644 scripts/check_post_output.py create mode 100644 scripts/genensprod_or_ensemblestat.py create mode 100644 scripts/gridstat_or_pointstat_ensmean.py create mode 100644 scripts/gridstat_or_pointstat_ensprob.py create mode 100644 scripts/pcpcombine.py diff --git a/scripts/check_post_output.py b/scripts/check_post_output.py new file mode 100644 index 00000000..54ad27fc --- /dev/null +++ b/scripts/check_post_output.py @@ -0,0 +1,96 @@ +# pylint: disable=logging-fstring-interpolation +""" +Converted from scripts/check_post_output.sh, this script checks that the expected +post-processed forecast output files exist on disk, up to the allowed maximum +number of missing files. + +The script is intended to be called from jobs/CHECK_POST_OUTPUT.sh. +""" + +import argparse +import logging +import os + +import uwtools.api.config as uwconfig + +from python_utils import setup_logging +from set_leadhrs import set_leadhrs + + +def check_post_output(config_file: str, cdate: str, ensmem_index: int) -> None: + """Check that post-processed forecast output files exist for a given cycle and member. + + Calls set_leadhrs() to verify that no more than NUM_MISSING_FCST_FILES_MAX files + are absent from disk. Raises an exception if the missing-file threshold is exceeded. + + Parameters + ---------- + config_file : str + Path to the experiment configuration YAML. + cdate : str + Eight-digit cycle date in ``YYYYMMDDHH`` format. + ensmem_index : int + Ensemble member index (0 for deterministic runs, 1-based for ensemble members). + """ + lgr = logging.getLogger(__name__) + + cfg = uwconfig.get_yaml_config(config=config_file) + vxcfg = cfg["verification"] + wfcfg = cfg["workflow"] + enscfg = cfg["ensemble"] + + # Get time lag in seconds for this member; deterministic uses index 0 + i = max(ensmem_index - 1, 0) + time_lag = int(enscfg["ENS_TIME_LAG_HRS"][i]) * 3600 + + # Build forecast filename template, prepending subdir template if set + subdir = vxcfg.get("FCST_SUBDIR_TEMPLATE", "") + fn_template = os.path.join(subdir, vxcfg["FCST_FN_TEMPLATE"]) if subdir \ + else vxcfg["FCST_FN_TEMPLATE"] + + lgr.info( + f"Checking post-processed output files for cycle {cdate}, " + f"member index {ensmem_index}" + ) + lgr.debug(f"{fn_template=}") + lgr.debug(f"{time_lag=}") + + set_leadhrs( + date_init=cdate, + lhr_min=0, + lhr_max=wfcfg["FCST_LEN_HRS"], + lhr_intvl=vxcfg["VX_FCST_OUTPUT_INTVL_HRS"], + base_dir=vxcfg["VX_FCST_INPUT_BASEDIR"], + time_lag=time_lag, + fn_template=fn_template, + num_missing_files_max=vxcfg["NUM_MISSING_FCST_FILES_MAX"], + ) + + lgr.info(f"Post-processed output file check completed successfully for cycle {cdate}.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Check that post-processed forecast output files exist on disk" + ) + parser.add_argument( + "--config", default="config.yaml", type=str, + help="Path to the experiment configuration file in YAML format", + ) + parser.add_argument( + "--cycle_date", required=True, type=str, + help="Eight-digit cycle date (YYYYMMDDHH)", + ) + parser.add_argument( + "--ensmem_index", required=True, type=int, + help="Ensemble member index (0 for deterministic, 1-based for ensemble members)", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", + help="Enable verbose debug output", + ) + args = parser.parse_args() + + setup_logging(debug=args.verbose) + + check_post_output(args.config, args.cycle_date, args.ensmem_index) diff --git a/scripts/genensprod_or_ensemblestat.py b/scripts/genensprod_or_ensemblestat.py new file mode 100644 index 00000000..bd4a7f51 --- /dev/null +++ b/scripts/genensprod_or_ensemblestat.py @@ -0,0 +1,264 @@ +# pylint: disable=logging-fstring-interpolation +""" +Converted from scripts/genensprod_or_ensemblestat.sh, this script calls either the METplus +"GenEnsProd" tool to generate ensemble products or the "EnsembleStat" tool to perform +ensemble-based verification. + +The script is intended to be called from jobs/GENENSPROD_OR_ENSEMBLESTAT.sh. +""" + +import argparse +import logging +import os + +from multiprocessing import Pool +from pathlib import Path +from string import Template + +import uwtools.api.config as uwconfig + +from python_utils import run_metplus, render_metplus_confs, setup_logging +from set_leadhrs import set_leadhrs +from set_vx_params import set_vx_params + +# Maps uppercase Rocoto METPLUSTOOLNAME to (CamelCase, snake_case) tool name variants +_TOOL_NAME_MAP = { + "GENENSPROD": ("GenEnsProd", "gen_ens_prod"), + "ENSEMBLESTAT": ("EnsembleStat", "ensemble_stat"), +} + + +def genensprod_or_ensemblestat( + config_file: str, + cdate: str, + obs_dir: str, + field_group: str, + obtype: str, + accum_hh: int, + fcst_level: str, + fcst_thresh: str, + metplus_tool: str, +) -> None: + """Execute a METplus GenEnsProd or EnsembleStat ensemble verification task. + + Parameters + ---------- + config_file : str + Path to the experiment YAML configuration file. + cdate : str + Eight-digit cycle date in ``YYYYMMDDHH`` format. + obs_dir : str + Directory containing observation files for the chosen obtype. + field_group : str + Group of observation fields to verify (e.g. APCP, REFC, SFC). + obtype : str + Observation type (e.g. NOHRSC, CCPA, NDAS). + accum_hh : int + Accumulation hours for the observation type. + fcst_level : str + METplus forecast level (e.g. L0, A03). + fcst_thresh : str + Forecast threshold set (usually "all" or "none"). + metplus_tool : str + METplus tool to run: ``"GENENSPROD"`` or ``"ENSEMBLESTAT"`` (case-insensitive). + """ + lgr = logging.getLogger(__name__) + + key = metplus_tool.upper() + if key not in _TOOL_NAME_MAP: + raise ValueError( + f"Invalid metplus_tool '{metplus_tool}'. " + f"Valid options: {list(_TOOL_NAME_MAP.keys())}" + ) + metplus_tool_camel_case, metplus_tool_name = _TOOL_NAME_MAP[key] + + cfg = uwconfig.get_yaml_config(config=config_file) + vxcfg = cfg["verification"] + enscfg = cfg["ensemble"] + + if not Path(obs_dir).is_dir(): + raise FileNotFoundError(f"{obs_dir=} does not exist or is not a directory") + + geom, _, _, met_out_name, met_filedir_name = set_vx_params(obtype, field_group, accum_hh) + + exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + + # Build obs input dir/template and forecast base dir + if geom == "grid": + if "APCP" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + fcst_in_dir = Path(exptdir) + elif "ASNOW" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + fcst_in_dir = Path(exptdir) + elif met_filedir_name == "REFC": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + fcst_in_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) + elif met_filedir_name == "RETOP": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + fcst_in_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) + else: + raise ValueError( + f"Invalid field group for {metplus_tool_camel_case}: {field_group}" + ) + elif geom == "point": + obs_in_dir = Path(exptdir, "metprd", "Pb2nc_obs") + obs_in_fn_template = vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + fcst_in_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) + else: + raise ValueError(f"Invalid parameters: {obtype=}, {field_group=}, {accum_hh=}") + + # Build per-member forecast filename templates (comma-separated list for METplus) + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + } + fcst_in_fn_templates = [] + for i in range(enscfg["NUM_ENS_MEMBERS"]): + ensmem = f"mem{str(i + 1).zfill(vxcfg['VX_NDIGITS_ENSMEM_NAMES'])}" + if field_group in ("APCP", "ASNOW"): + tmpl = str(Path( + cdate, ensmem, "metprd", "PcpCombine_fcst", + Template(vxcfg["FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT"]).safe_substitute(subvars), + )) + else: + subdir = Template(vxcfg.get("FCST_SUBDIR_TEMPLATE", "")).safe_substitute(subvars) + fn = Template(vxcfg["FCST_FN_TEMPLATE"]).safe_substitute(subvars) + tmpl = str(Path(subdir, fn)) if subdir else fn + fcst_in_fn_templates.append(tmpl) + fcst_in_fn_template = ", ".join(fcst_in_fn_templates) + + output_dir = Path(exptdir, cdate, "metprd", metplus_tool_camel_case) + staging_dir = Path(exptdir, cdate, "stage", met_filedir_name) + os.makedirs(output_dir, exist_ok=True) + + if obtype in ("CCPA", "NOHRSC"): + vx_intvl = vx_hr_start = accum_hh + else: + vx_intvl = vxcfg["VX_FCST_OUTPUT_INTVL_HRS"] + vx_hr_start = 0 + + # GenEnsProd only needs forecast files; skip obs existence check + vx_leadhr_list = set_leadhrs( + date_init=cdate, + lhr_min=vx_hr_start, + lhr_max=cfg["workflow"]["FCST_LEN_HRS"], + lhr_intvl=vx_intvl, + base_dir=obs_in_dir, + time_lag=0, + fn_template=str(obs_in_fn_template), + num_missing_files_max=vxcfg["NUM_MISSING_OBS_FILES_MAX"], + skip_check_files=(metplus_tool_camel_case == "GenEnsProd"), + ) + + if not vx_leadhr_list: + raise RuntimeError( + f"set_leadhrs returned an empty list for cycle {cdate}, " + f"{obtype=}, {field_group=}" + ) + + vx_mask_files = [] + if vxcfg["VX_MASK"]: + for mask in vxcfg["VX_MASK"]: + if os.path.isfile(maskfile := f"{cfg['user']['METPLUS_CONF']}/{mask}.poly"): + vx_mask_files.append(maskfile) + else: + vx_mask_files.append( + f"{os.environ['MET_INSTALL_DIR']}/share/met/poly/{mask}.poly" + ) + + metplus_config_tmpl_fn = f"{metplus_tool_camel_case}.conf" + metplus_config_fn = f"{metplus_tool_camel_case}_{met_filedir_name}_{cdate}.conf.0" + metplus_log_fn = f"metplus.log.{metplus_tool_camel_case}_{met_filedir_name}_{cdate}.0" + + vx_config_dict = uwconfig.get_yaml_config( + config=f"{cfg['user']['METPLUS_CONF']}/{vxcfg['VX_CONFIG_ENS_FN']}" + ) + + settings = { + "metplus_tool_name": metplus_tool_name, + "MetplusToolName": metplus_tool_camel_case, + "METPLUS_TOOL_NAME": metplus_tool_name.upper(), + "metplus_verbosity_level": vxcfg["METPLUS_VERBOSITY_LEVEL"], + "cdate": cdate, + "vx_leadhr_list": ", ".join(map(str, vx_leadhr_list)), + "metplus_config_fn": metplus_config_fn, + "metplus_log_fn": metplus_log_fn, + "obs_input_dir": obs_in_dir, + "obs_input_fn_template": obs_in_fn_template, + "fcst_input_dir": fcst_in_dir, + "fcst_input_fn_template": fcst_in_fn_template, + "output_dir": output_dir, + "output_fn_template": "", + "staging_dir": staging_dir, + "vx_fcst_model_name": vxcfg["VX_FCST_MODEL_NAME"], + "num_ens_members": enscfg["NUM_ENS_MEMBERS"], + "ensmem_name": "", + "time_lag": 0, + "fieldname_in_obs_input": str(obs_in_dir), + "fieldname_in_fcst_input": str(fcst_in_dir), + "fieldname_in_met_output": met_out_name, + "fieldname_in_met_filedir_names": met_filedir_name, + "obtype": obtype, + "accum_hh": f"{accum_hh:02}", + "accum_no_pad": accum_hh, + "metplus_templates_dir": cfg["user"]["METPLUS_CONF"], + "input_field_group": field_group, + "input_level_fcst": fcst_level, + "input_thresh_fcst": fcst_thresh, + "vx_mask": ", ".join(vx_mask_files), + "vx_config_dict": vx_config_dict, + } + + numprocs = 1 + conf_files = render_metplus_confs( + cfg, settings, metplus_config_tmpl_fn, vx_leadhr_list, numprocs + ) + lgr.debug(f"{conf_files=}") + + lgr.info(f"Running {metplus_tool_camel_case} with METplus") + common_conf = os.path.join(cfg["user"]["METPLUS_CONF"], "common.conf") + with Pool(processes=numprocs) as pool: + pool.starmap(run_metplus, [(common_conf, fn) for fn in conf_files]) + + lgr.info(f"{metplus_tool_camel_case} completed successfully.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run METplus GenEnsProd or EnsembleStat for ensemble verification" + ) + parser.add_argument("--config", default="config.yaml", type=str, + help="Path to the experiment configuration file in YAML format") + parser.add_argument("--cycle_date", required=True, type=str, + help="Eight-digit cycle date (YYYYMMDDHH)") + parser.add_argument("--obs_dir", required=True, type=str, + help="Observation directory for this obtype") + parser.add_argument("--field_group", required=True, type=str, + help="Group of fields for this verification task (e.g. APCP, REFC, SFC)") + parser.add_argument("--obtype", required=True, type=str, + help="Observation type (e.g. NOHRSC, CCPA, NDAS)") + parser.add_argument("--accum_hh", default=1, type=int, + help="Accumulation hours for this observation type") + parser.add_argument("--fcst_level", default="", type=str, + help="METplus forecast level (e.g. L0, A03)") + parser.add_argument("--fcst_thresh", default="", type=str, + help="Forecast thresholds to verify against (e.g. all, none)") + parser.add_argument("--metplus_tool", required=True, type=str, + help="METplus tool to run: GENENSPROD or ENSEMBLESTAT") + parser.add_argument("-v", "--verbose", action="store_true", + help="Enable verbose debug output") + args = parser.parse_args() + + setup_logging(debug=args.verbose) + logging.debug(f"{os.environ['METPLUS_ROOT']=}") + + genensprod_or_ensemblestat( + args.config, args.cycle_date, args.obs_dir, args.field_group, + args.obtype, args.accum_hh, args.fcst_level, args.fcst_thresh, + args.metplus_tool, + ) diff --git a/scripts/gridstat_or_pointstat_ensmean.py b/scripts/gridstat_or_pointstat_ensmean.py new file mode 100644 index 00000000..03f6006e --- /dev/null +++ b/scripts/gridstat_or_pointstat_ensmean.py @@ -0,0 +1,228 @@ +# pylint: disable=logging-fstring-interpolation +""" +Converted from scripts/gridstat_or_pointstat_ensmean.sh, this script calls the METplus +"GridStat" or "PointStat" tool to verify the ensemble mean output produced by GenEnsProd. + +The script is intended to be called from jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh. +""" + +import argparse +import logging +import os + +from multiprocessing import Pool +from pathlib import Path + +import uwtools.api.config as uwconfig + +from python_utils import run_metplus, render_metplus_confs, setup_logging +from set_leadhrs import set_leadhrs +from set_vx_params import set_vx_params + + +def gridstat_or_pointstat_ensmean( + config_file: str, + cdate: str, + obs_dir: str, + field_group: str, + obtype: str, + accum_hh: int, + fcst_level: str, + fcst_thresh: str, +) -> None: + """Execute a METplus GridStat or PointStat verification task on ensemble mean output. + + Parameters + ---------- + config_file : str + Path to the experiment YAML configuration file. + cdate : str + Eight-digit cycle date in ``YYYYMMDDHH`` format. + obs_dir : str + Directory containing observation files for the chosen obtype. + field_group : str + Group of observation fields to verify (e.g. APCP, REFC, SFC). + obtype : str + Observation type (e.g. NOHRSC, CCPA, NDAS). + accum_hh : int + Accumulation hours for the observation type. + fcst_level : str + METplus forecast level (e.g. L0, A03). + fcst_thresh : str + Forecast threshold set (usually "all" or "none"). + """ + lgr = logging.getLogger(__name__) + + cfg = uwconfig.get_yaml_config(config=config_file) + vxcfg = cfg["verification"] + + if not Path(obs_dir).is_dir(): + raise FileNotFoundError(f"{obs_dir=} does not exist or is not a directory") + + geom, _, _, met_out_name, met_filedir_name = set_vx_params(obtype, field_group, accum_hh) + + exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + + if geom == "grid": + metplus_tool_name = "grid_stat" + metplus_tool_camel_case = "GridStat" + if "APCP" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + elif "ASNOW" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + elif met_filedir_name == "REFC": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + elif met_filedir_name == "RETOP": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + else: + raise ValueError(f"Invalid field group for GridStat ensmean: {field_group}") + fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") + + elif geom == "point": + metplus_tool_name = "point_stat" + metplus_tool_camel_case = "PointStat" + obs_in_dir = Path(exptdir, "metprd", "Pb2nc_obs") + obs_in_fn_template = vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") + + else: + raise ValueError(f"Invalid parameters: {obtype=}, {field_group=}, {accum_hh=}") + + fcst_in_fn_template = ( + f"gen_ens_prod_{vxcfg['VX_FCST_MODEL_NAME']}_{met_filedir_name}_{obtype}" + "_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc" + ) + + output_dir = Path(exptdir, cdate, "metprd", f"{metplus_tool_camel_case}_ensmean") + staging_dir = Path(exptdir, cdate, "stage", f"{met_filedir_name}_ensmean") + os.makedirs(output_dir, exist_ok=True) + + if obtype in ("CCPA", "NOHRSC"): + vx_intvl = vx_hr_start = accum_hh + else: + vx_intvl = vxcfg["VX_FCST_OUTPUT_INTVL_HRS"] + vx_hr_start = 0 + + vx_leadhr_list = set_leadhrs( + date_init=cdate, + lhr_min=vx_hr_start, + lhr_max=cfg["workflow"]["FCST_LEN_HRS"], + lhr_intvl=vx_intvl, + base_dir=obs_in_dir, + time_lag=0, + fn_template=str(obs_in_fn_template), + num_missing_files_max=vxcfg["NUM_MISSING_OBS_FILES_MAX"], + ) + + if not vx_leadhr_list: + raise RuntimeError( + f"set_leadhrs returned an empty list for cycle {cdate}, " + f"{obtype=}, {field_group=}" + ) + + vx_mask_files = [] + if vxcfg["VX_MASK"]: + for mask in vxcfg["VX_MASK"]: + if os.path.isfile(maskfile := f"{cfg['user']['METPLUS_CONF']}/{mask}.poly"): + vx_mask_files.append(maskfile) + else: + vx_mask_files.append( + f"{os.environ['MET_INSTALL_DIR']}/share/met/poly/{mask}.poly" + ) + + metplus_config_tmpl_fn = f"{metplus_tool_camel_case}_ensmean.conf" + metplus_config_fn = ( + f"{metplus_tool_camel_case}_{met_filedir_name}_{cdate}_ensmean.conf.0" + ) + metplus_log_fn = ( + f"metplus.log.{metplus_tool_camel_case}_{met_filedir_name}_{cdate}_ensmean.0" + ) + + vx_config_dict = uwconfig.get_yaml_config( + config=f"{cfg['user']['METPLUS_CONF']}/{vxcfg['VX_CONFIG_ENS_FN']}" + ) + + settings = { + "metplus_tool_name": metplus_tool_name, + "MetplusToolName": metplus_tool_camel_case, + "METPLUS_TOOL_NAME": metplus_tool_name.upper(), + "metplus_verbosity_level": vxcfg["METPLUS_VERBOSITY_LEVEL"], + "cdate": cdate, + "vx_leadhr_list": ", ".join(map(str, vx_leadhr_list)), + "metplus_config_fn": metplus_config_fn, + "metplus_log_fn": metplus_log_fn, + "obs_input_dir": obs_in_dir, + "obs_input_fn_template": obs_in_fn_template, + "fcst_input_dir": fcst_in_dir, + "fcst_input_fn_template": fcst_in_fn_template, + "output_dir": output_dir, + "output_fn_template": "", + "staging_dir": staging_dir, + "vx_fcst_model_name": vxcfg["VX_FCST_MODEL_NAME"], + "num_ens_members": cfg["ensemble"]["NUM_ENS_MEMBERS"], + "ensmem_name": "", + "time_lag": 0, + "fieldname_in_obs_input": str(obs_in_dir), + "fieldname_in_fcst_input": str(fcst_in_dir), + "fieldname_in_met_output": met_out_name, + "fieldname_in_met_filedir_names": met_filedir_name, + "obtype": obtype, + "accum_hh": f"{accum_hh:02}", + "accum_no_pad": accum_hh, + "metplus_templates_dir": cfg["user"]["METPLUS_CONF"], + "input_field_group": field_group, + "input_level_fcst": fcst_level, + "input_thresh_fcst": fcst_thresh, + "vx_mask": ", ".join(vx_mask_files), + "vx_config_dict": vx_config_dict, + } + + numprocs = 1 + conf_files = render_metplus_confs( + cfg, settings, metplus_config_tmpl_fn, vx_leadhr_list, numprocs + ) + lgr.debug(f"{conf_files=}") + + lgr.info(f"Running {metplus_tool_camel_case}_ensmean with METplus") + common_conf = os.path.join(cfg["user"]["METPLUS_CONF"], "common.conf") + with Pool(processes=numprocs) as pool: + pool.starmap(run_metplus, [(common_conf, fn) for fn in conf_files]) + + lgr.info(f"{metplus_tool_camel_case}_ensmean completed successfully.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run METplus GridStat or PointStat on ensemble mean output" + ) + parser.add_argument("--config", default="config.yaml", type=str, + help="Path to the experiment configuration file in YAML format") + parser.add_argument("--cycle_date", required=True, type=str, + help="Eight-digit cycle date (YYYYMMDDHH)") + parser.add_argument("--obs_dir", required=True, type=str, + help="Observation directory for this obtype") + parser.add_argument("--field_group", required=True, type=str, + help="Group of fields for this verification task (e.g. APCP, REFC, SFC)") + parser.add_argument("--obtype", required=True, type=str, + help="Observation type (e.g. NOHRSC, CCPA, NDAS)") + parser.add_argument("--accum_hh", default=1, type=int, + help="Accumulation hours for this observation type") + parser.add_argument("--fcst_level", required=True, type=str, + help="METplus forecast level (e.g. L0, A03)") + parser.add_argument("--fcst_thresh", required=True, type=str, + help="Forecast thresholds to verify against (e.g. all, none)") + parser.add_argument("-v", "--verbose", action="store_true", + help="Enable verbose debug output") + args = parser.parse_args() + + setup_logging(debug=args.verbose) + logging.debug(f"{os.environ['METPLUS_ROOT']=}") + + gridstat_or_pointstat_ensmean( + args.config, args.cycle_date, args.obs_dir, args.field_group, + args.obtype, args.accum_hh, args.fcst_level, args.fcst_thresh, + ) diff --git a/scripts/gridstat_or_pointstat_ensprob.py b/scripts/gridstat_or_pointstat_ensprob.py new file mode 100644 index 00000000..d4df4013 --- /dev/null +++ b/scripts/gridstat_or_pointstat_ensprob.py @@ -0,0 +1,229 @@ +# pylint: disable=logging-fstring-interpolation +""" +Converted from scripts/gridstat_or_pointstat_ensprob.sh, this script calls the METplus +"GridStat" or "PointStat" tool to verify ensemble frequency/probability output produced +by GenEnsProd. + +The script is intended to be called from jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh. +""" + +import argparse +import logging +import os + +from multiprocessing import Pool +from pathlib import Path + +import uwtools.api.config as uwconfig + +from python_utils import run_metplus, render_metplus_confs, setup_logging +from set_leadhrs import set_leadhrs +from set_vx_params import set_vx_params + + +def gridstat_or_pointstat_ensprob( + config_file: str, + cdate: str, + obs_dir: str, + field_group: str, + obtype: str, + accum_hh: int, + fcst_level: str, + fcst_thresh: str, +) -> None: + """Execute a METplus GridStat or PointStat verification task on ensemble probability output. + + Parameters + ---------- + config_file : str + Path to the experiment YAML configuration file. + cdate : str + Eight-digit cycle date in ``YYYYMMDDHH`` format. + obs_dir : str + Directory containing observation files for the chosen obtype. + field_group : str + Group of observation fields to verify (e.g. APCP, REFC, SFC). + obtype : str + Observation type (e.g. NOHRSC, CCPA, NDAS). + accum_hh : int + Accumulation hours for the observation type. + fcst_level : str + METplus forecast level (e.g. L0, A03). + fcst_thresh : str + Forecast threshold set (usually "all" or "none"). + """ + lgr = logging.getLogger(__name__) + + cfg = uwconfig.get_yaml_config(config=config_file) + vxcfg = cfg["verification"] + + if not Path(obs_dir).is_dir(): + raise FileNotFoundError(f"{obs_dir=} does not exist or is not a directory") + + geom, _, _, met_out_name, met_filedir_name = set_vx_params(obtype, field_group, accum_hh) + + exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + + if geom == "grid": + metplus_tool_name = "grid_stat" + metplus_tool_camel_case = "GridStat" + if "APCP" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + elif "ASNOW" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + elif met_filedir_name == "REFC": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + elif met_filedir_name == "RETOP": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + else: + raise ValueError(f"Invalid field group for GridStat ensprob: {field_group}") + fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") + + elif geom == "point": + metplus_tool_name = "point_stat" + metplus_tool_camel_case = "PointStat" + obs_in_dir = Path(exptdir, "metprd", "Pb2nc_obs") + obs_in_fn_template = vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") + + else: + raise ValueError(f"Invalid parameters: {obtype=}, {field_group=}, {accum_hh=}") + + fcst_in_fn_template = ( + f"gen_ens_prod_{vxcfg['VX_FCST_MODEL_NAME']}_{met_filedir_name}_{obtype}" + "_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc" + ) + + output_dir = Path(exptdir, cdate, "metprd", f"{metplus_tool_camel_case}_ensprob") + staging_dir = Path(exptdir, cdate, "stage", f"{met_filedir_name}_ensprob") + os.makedirs(output_dir, exist_ok=True) + + if obtype in ("CCPA", "NOHRSC"): + vx_intvl = vx_hr_start = accum_hh + else: + vx_intvl = vxcfg["VX_FCST_OUTPUT_INTVL_HRS"] + vx_hr_start = 0 + + vx_leadhr_list = set_leadhrs( + date_init=cdate, + lhr_min=vx_hr_start, + lhr_max=cfg["workflow"]["FCST_LEN_HRS"], + lhr_intvl=vx_intvl, + base_dir=obs_in_dir, + time_lag=0, + fn_template=str(obs_in_fn_template), + num_missing_files_max=vxcfg["NUM_MISSING_OBS_FILES_MAX"], + ) + + if not vx_leadhr_list: + raise RuntimeError( + f"set_leadhrs returned an empty list for cycle {cdate}, " + f"{obtype=}, {field_group=}" + ) + + vx_mask_files = [] + if vxcfg["VX_MASK"]: + for mask in vxcfg["VX_MASK"]: + if os.path.isfile(maskfile := f"{cfg['user']['METPLUS_CONF']}/{mask}.poly"): + vx_mask_files.append(maskfile) + else: + vx_mask_files.append( + f"{os.environ['MET_INSTALL_DIR']}/share/met/poly/{mask}.poly" + ) + + metplus_config_tmpl_fn = f"{metplus_tool_camel_case}_ensprob.conf" + metplus_config_fn = ( + f"{metplus_tool_camel_case}_{met_filedir_name}_{cdate}_ensprob.conf.0" + ) + metplus_log_fn = ( + f"metplus.log.{metplus_tool_camel_case}_{met_filedir_name}_{cdate}_ensprob.0" + ) + + vx_config_dict = uwconfig.get_yaml_config( + config=f"{cfg['user']['METPLUS_CONF']}/{vxcfg['VX_CONFIG_ENS_FN']}" + ) + + settings = { + "metplus_tool_name": metplus_tool_name, + "MetplusToolName": metplus_tool_camel_case, + "METPLUS_TOOL_NAME": metplus_tool_name.upper(), + "metplus_verbosity_level": vxcfg["METPLUS_VERBOSITY_LEVEL"], + "cdate": cdate, + "vx_leadhr_list": ", ".join(map(str, vx_leadhr_list)), + "metplus_config_fn": metplus_config_fn, + "metplus_log_fn": metplus_log_fn, + "obs_input_dir": obs_in_dir, + "obs_input_fn_template": obs_in_fn_template, + "fcst_input_dir": fcst_in_dir, + "fcst_input_fn_template": fcst_in_fn_template, + "output_dir": output_dir, + "output_fn_template": "", + "staging_dir": staging_dir, + "vx_fcst_model_name": vxcfg["VX_FCST_MODEL_NAME"], + "num_ens_members": cfg["ensemble"]["NUM_ENS_MEMBERS"], + "ensmem_name": "", + "time_lag": 0, + "fieldname_in_obs_input": str(obs_in_dir), + "fieldname_in_fcst_input": str(fcst_in_dir), + "fieldname_in_met_output": met_out_name, + "fieldname_in_met_filedir_names": met_filedir_name, + "obtype": obtype, + "accum_hh": f"{accum_hh:02}", + "accum_no_pad": accum_hh, + "metplus_templates_dir": cfg["user"]["METPLUS_CONF"], + "input_field_group": field_group, + "input_level_fcst": fcst_level, + "input_thresh_fcst": fcst_thresh, + "vx_mask": ", ".join(vx_mask_files), + "vx_config_dict": vx_config_dict, + } + + numprocs = 1 + conf_files = render_metplus_confs( + cfg, settings, metplus_config_tmpl_fn, vx_leadhr_list, numprocs + ) + lgr.debug(f"{conf_files=}") + + lgr.info(f"Running {metplus_tool_camel_case}_ensprob with METplus") + common_conf = os.path.join(cfg["user"]["METPLUS_CONF"], "common.conf") + with Pool(processes=numprocs) as pool: + pool.starmap(run_metplus, [(common_conf, fn) for fn in conf_files]) + + lgr.info(f"{metplus_tool_camel_case}_ensprob completed successfully.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run METplus GridStat or PointStat on ensemble probability output" + ) + parser.add_argument("--config", default="config.yaml", type=str, + help="Path to the experiment configuration file in YAML format") + parser.add_argument("--cycle_date", required=True, type=str, + help="Eight-digit cycle date (YYYYMMDDHH)") + parser.add_argument("--obs_dir", required=True, type=str, + help="Observation directory for this obtype") + parser.add_argument("--field_group", required=True, type=str, + help="Group of fields for this verification task (e.g. APCP, REFC, SFC)") + parser.add_argument("--obtype", required=True, type=str, + help="Observation type (e.g. NOHRSC, CCPA, NDAS)") + parser.add_argument("--accum_hh", default=1, type=int, + help="Accumulation hours for this observation type") + parser.add_argument("--fcst_level", required=True, type=str, + help="METplus forecast level (e.g. L0, A03)") + parser.add_argument("--fcst_thresh", required=True, type=str, + help="Forecast thresholds to verify against (e.g. all, none)") + parser.add_argument("-v", "--verbose", action="store_true", + help="Enable verbose debug output") + args = parser.parse_args() + + setup_logging(debug=args.verbose) + logging.debug(f"{os.environ['METPLUS_ROOT']=}") + + gridstat_or_pointstat_ensprob( + args.config, args.cycle_date, args.obs_dir, args.field_group, + args.obtype, args.accum_hh, args.fcst_level, args.fcst_thresh, + ) diff --git a/scripts/pcpcombine.py b/scripts/pcpcombine.py new file mode 100644 index 00000000..a22a363b --- /dev/null +++ b/scripts/pcpcombine.py @@ -0,0 +1,301 @@ +# pylint: disable=logging-fstring-interpolation +""" +Converted from scripts/pcpcombine.sh, this script calls the METplus "PcpCombine" tool to +combine sub-hourly or hourly fields to generate multi-hour accumulations. Input can come +from either observations or a forecast. + +The script is intended to be called from jobs/PCPCOMBINE.sh. +""" + +import argparse +import logging +import os + +from multiprocessing import Pool +from pathlib import Path +from string import Template + +import uwtools.api.config as uwconfig + +from python_utils import run_metplus, render_metplus_confs, setup_logging +from set_leadhrs import set_leadhrs +from set_vx_params import set_vx_params + + +def pcpcombine( + config_file: str, + cdate: str, + obs_dir: str, + field_group: str, + obtype: str, + accum_hh: int, + fcst_level: str, + fcst_thresh: str, + fcst_or_obs: str, + ensmem_index: int, +) -> None: + """Execute a METplus PcpCombine task for obs or forecast accumulation fields. + + Parameters + ---------- + config_file : str + Path to the experiment YAML configuration file. + cdate : str + Eight-digit cycle date in ``YYYYMMDDHH`` format. + obs_dir : str + Directory containing observation files for the chosen obtype. + field_group : str + Field group to combine (e.g. APCP, ASNOW, PM25, PM10). + obtype : str + Observation type (e.g. CCPA, NOHRSC, AIRNOW). + accum_hh : int + Target accumulation period in hours. + fcst_level : str + METplus forecast level (e.g. A06). + fcst_thresh : str + Forecast threshold set (usually "all" or "none"). + fcst_or_obs : str + Whether this task processes forecast (``"FCST"``) or observation (``"OBS"``) data. + ensmem_index : int + Ensemble member index (0 for deterministic, 1-based for ensemble members). + """ + lgr = logging.getLogger(__name__) + + fcst_or_obs = fcst_or_obs.upper() + if fcst_or_obs not in ("FCST", "OBS"): + raise ValueError(f"fcst_or_obs must be 'FCST' or 'OBS', got '{fcst_or_obs}'") + + metplus_tool_camel_case = "PcpCombine" + + cfg = uwconfig.get_yaml_config(config=config_file) + vxcfg = cfg["verification"] + enscfg = cfg["ensemble"] + + _, obs_fieldname, fcst_fieldname, met_out_name, met_filedir_name = set_vx_params( + obtype, field_group, accum_hh + ) + + exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + do_ensemble = enscfg["DO_ENSEMBLE"] + ensmem = f"mem{str(ensmem_index).zfill(vxcfg['VX_NDIGITS_ENSMEM_NAMES'])}" + + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + } + + pcp_combine_method = "ADD" + pcp_combine_command = "" + + if fcst_or_obs == "FCST": + time_lag = 0 + if do_ensemble: + time_lag = int(enscfg["ENS_TIME_LAG_HRS"][ensmem_index - 1]) * 3600 + + fcst_subdir = Template(vxcfg.get("FCST_SUBDIR_TEMPLATE", "")).safe_substitute(subvars) + fcst_fn = Template(vxcfg["FCST_FN_TEMPLATE"]).safe_substitute(subvars) + input_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) + input_fn_template = str(Path(fcst_subdir, fcst_fn)) if fcst_subdir else fcst_fn + output_fn_template = Template( + vxcfg["FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).safe_substitute(subvars) + + output_base = Path(exptdir, cdate, ensmem) if do_ensemble else Path(exptdir, cdate) + output_dir = output_base / "metprd" / f"{metplus_tool_camel_case}_fcst" + staging_dir = output_base / "stage" / met_filedir_name + + # AIRNOW requires USER_DEFINED combining to build PM2.5/PM10 from model aerosol fields + if obtype == "AIRNOW": + pcp_combine_method = "USER_DEFINED" + smoke_type = vxcfg["FCST_SMOKE_TYPE"] + if field_group == "PM25": + if smoke_type == "HRRR": + pcp_combine_command = ( + "-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " -field 'name=\"MASSDEN\"; level=\"Z8\"; convert(x)=x*1e9;'" + ) + elif smoke_type == "RRFS": + pcp_combine_command = ( + "-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62010;" + " convert(x)=x*1e9;'" + " {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001;" + " GRIB2_aerosol_interval_type=0; convert(x)=x*1e9;'" + ) + else: + raise ValueError( + f"Unsupported FCST_SMOKE_TYPE '{smoke_type}' for PM25" + ) + elif field_group == "PM10": + if smoke_type == "RRFS": + pcp_combine_command = ( + "-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62010;" + " convert(x)=x*1e9;'" + " {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001;" + " GRIB2_aerosol_interval_type=0; convert(x)=x*1e9;'" + " {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001;" + " GRIB2_aerosol_interval_type=2; convert(x)=x*1e9;'" + ) + else: + raise ValueError( + f"PM10 only available for RRFS output, not available for {smoke_type}" + ) + + suffix = f"_{ensmem}" + + else: # OBS + time_lag = 0 + input_dir = Path(obs_dir) + input_fn_template = vxcfg[f"OBS_{obtype}_FN_TEMPLATES"][1] + output_fn_template = Template( + vxcfg[f"OBS_{obtype}_{field_group}_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).safe_substitute(subvars) + + output_base = Path(exptdir, cdate, "obs") if do_ensemble else Path(exptdir, cdate) + output_dir = output_base / "metprd" / f"{metplus_tool_camel_case}_obs" + staging_dir = output_base / "stage" / met_filedir_name + + suffix = f"_{obtype}" + + os.makedirs(output_dir, exist_ok=True) + + # Sub-interval: model output interval for FCST, obs availability interval for OBS + if fcst_or_obs == "FCST": + subintvl = vxcfg["VX_FCST_OUTPUT_INTVL_HRS"] + else: + subintvl = vxcfg[f"{obtype}_OBS_AVAIL_INTVL_HRS"] + input_accum_hh = f"{subintvl:02d}" + + # First pass: get accumulation end hours without file existence check + lhr_min = 0 if obtype == "AIRNOW" else accum_hh + vx_leadhr_list = set_leadhrs( + date_init=cdate, + lhr_min=lhr_min, + lhr_max=cfg["workflow"]["FCST_LEN_HRS"], + lhr_intvl=accum_hh, + base_dir=input_dir, + time_lag=time_lag, + fn_template=str(input_fn_template), + num_missing_files_max=0, + skip_check_files=True, + ) + + if not vx_leadhr_list: + raise RuntimeError( + f"set_leadhrs returned an empty list for cycle {cdate}, " + f"{obtype=}, {field_group=}" + ) + + # Second pass: verify sub-interval input files exist for each accumulation window + for hr_end in vx_leadhr_list: + hr_start = hr_end - accum_hh + subintvl + set_leadhrs( + date_init=cdate, + lhr_min=hr_start, + lhr_max=hr_end, + lhr_intvl=subintvl, + base_dir=input_dir, + time_lag=time_lag, + fn_template=str(input_fn_template), + num_missing_files_max=0, + ) + + fcst_or_obs_lower = fcst_or_obs.lower() + metplus_config_tmpl_fn = f"{metplus_tool_camel_case}.conf" + metplus_config_fn = ( + f"{metplus_tool_camel_case}_{fcst_or_obs_lower}_{met_filedir_name}{suffix}.conf.0" + ) + metplus_log_fn = ( + f"metplus.log.{metplus_tool_camel_case}_{fcst_or_obs_lower}" + f"_{met_filedir_name}{suffix}_{cdate}.0" + ) + + settings = { + "metplus_tool_name": "pcpcombine", + "MetplusToolName": metplus_tool_camel_case, + "METPLUS_TOOL_NAME": "PCPCOMBINE", + "metplus_verbosity_level": vxcfg["METPLUS_VERBOSITY_LEVEL"], + "cdate": cdate, + "vx_leadhr_list": ", ".join(map(str, vx_leadhr_list)), + "metplus_config_fn": metplus_config_fn, + "metplus_log_fn": metplus_log_fn, + "input_dir": input_dir, + "input_fn_template": input_fn_template, + "output_dir": output_dir, + "output_fn_template": output_fn_template, + "staging_dir": staging_dir, + "vx_fcst_model_name": vxcfg["VX_FCST_MODEL_NAME"], + "num_ens_members": enscfg["NUM_ENS_MEMBERS"], + "ensmem_name": ensmem, + "time_lag": time_lag, + "fieldname_in_obs_input": obs_fieldname, + "fieldname_in_fcst_input": fcst_fieldname, + "fieldname_in_met_output": met_out_name, + "fieldname_in_met_filedir_names": met_filedir_name, + "obtype": obtype, + "FCST_OR_OBS": fcst_or_obs, + "input_accum_hh": input_accum_hh, + "output_accum_hh": f"{accum_hh:02}", + "accum_no_pad": accum_hh, + "metplus_templates_dir": cfg["user"]["METPLUS_CONF"], + "input_field_group": field_group, + "input_level_fcst": fcst_level, + "input_thresh_fcst": fcst_thresh, + "pcp_combine_method": pcp_combine_method, + "pcp_combine_command": pcp_combine_command, + } + + numprocs = 1 + conf_files = render_metplus_confs( + cfg, settings, metplus_config_tmpl_fn, vx_leadhr_list, numprocs + ) + lgr.debug(f"{conf_files=}") + + lgr.info(f"Running {metplus_tool_camel_case} ({fcst_or_obs}) with METplus") + common_conf = os.path.join(cfg["user"]["METPLUS_CONF"], "common.conf") + with Pool(processes=numprocs) as pool: + pool.starmap(run_metplus, [(common_conf, fn) for fn in conf_files]) + + lgr.info(f"{metplus_tool_camel_case} completed successfully.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run METplus PcpCombine for multi-hour accumulation of obs or forecast fields" + ) + parser.add_argument("--config", default="config.yaml", type=str, + help="Path to the experiment configuration file in YAML format") + parser.add_argument("--cycle_date", required=True, type=str, + help="Eight-digit cycle date (YYYYMMDDHH)") + parser.add_argument("--obs_dir", required=True, type=str, + help="Observation directory for this obtype") + parser.add_argument("--field_group", required=True, type=str, + help="Field group to combine (e.g. APCP, ASNOW, PM25, PM10)") + parser.add_argument("--obtype", required=True, type=str, + help="Observation type (e.g. CCPA, NOHRSC, AIRNOW)") + parser.add_argument("--accum_hh", required=True, type=int, + help="Target accumulation period in hours") + parser.add_argument("--fcst_level", default="", type=str, + help="METplus forecast level (e.g. A06)") + parser.add_argument("--fcst_thresh", default="", type=str, + help="Forecast thresholds to verify against (e.g. all, none)") + parser.add_argument("--fcst_or_obs", required=True, type=str, + help="Whether processing forecast (FCST) or observation (OBS) data") + parser.add_argument("--ensmem_index", required=True, type=int, + help="Ensemble member index (0 for deterministic, 1-based for ensemble members)") + parser.add_argument("-v", "--verbose", action="store_true", + help="Enable verbose debug output") + args = parser.parse_args() + + setup_logging(debug=args.verbose) + logging.debug(f"{os.environ['METPLUS_ROOT']=}") + + pcpcombine( + args.config, args.cycle_date, args.obs_dir, args.field_group, + args.obtype, args.accum_hh, args.fcst_level, args.fcst_thresh, + args.fcst_or_obs, args.ensmem_index, + ) From 9e84507d321671cb14d86276ee7973c4a3f144bd Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 2 Jul 2026 15:20:56 -0600 Subject: [PATCH 02/25] Suppress broken pipe error from conda env list in setup_conda.sh grep -q exits immediately on first match, closing the pipe before conda env list finishes writing. Conda's Python runtime catches the resulting SIGPIPE and prints a noisy error report to stderr. Adding 2>/dev/null suppresses it without affecting the grep result or exit code. Co-Authored-By: Claude Sonnet 4.6 --- setup_conda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup_conda.sh b/setup_conda.sh index 5a166399..4d4849ff 100644 --- a/setup_conda.sh +++ b/setup_conda.sh @@ -98,7 +98,7 @@ else ENV_NAME=vx_workflow fi -if ! conda env list | grep -q "^${ENV_NAME}\s" ; then +if ! conda env list 2>/dev/null | grep -q "^${ENV_NAME}\s" ; then echo "Creating ${ENV_NAME} environment..." mamba env create -n ${ENV_NAME} --file "${ENV_YAML}" --quiet else From b9a11123a926a38b244f85f8ff802b94e12c655d Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 6 Jul 2026 18:30:07 +0000 Subject: [PATCH 03/25] - Update CHECK_POST_OUTPUT.sh to call new python script. For now this requires a string-subst dictionary in check_post_output.py; should be able to get rid of this logic in all scripts by the end. - Convert ENS_TIME_LAG_HRS to a true Python list. This removes need for strange "literal_eval" logic in gridstat_or_pointstat.py --- jobs/CHECK_POST_OUTPUT.sh | 10 +- scripts/check_post_output.py | 13 +- scripts/check_post_output.sh | 131 ------------------ scripts/gridstat_or_pointstat.py | 4 +- ...nsemble_verification_only_vx_time_lag.yaml | 2 +- ush/config_defaults.yaml | 2 +- 6 files changed, 23 insertions(+), 139 deletions(-) delete mode 100755 scripts/check_post_output.sh diff --git a/jobs/CHECK_POST_OUTPUT.sh b/jobs/CHECK_POST_OUTPUT.sh index 55a720da..0d22ab42 100755 --- a/jobs/CHECK_POST_OUTPUT.sh +++ b/jobs/CHECK_POST_OUTPUT.sh @@ -39,6 +39,8 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done +# Sets up PYTHONPATH and VERBOSE environment variables +. $USHdir/set_job_env.sh # #----------------------------------------------------------------------- # @@ -60,9 +62,13 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -$SCRIPTSdir/check_post_output.sh || \ +python $SCRIPTSdir/check_post_output.py ${VERBOSE_FLAG} \ + --config="${GLOBAL_VAR_DEFNS_FP}" \ + --cycle_date="${YYMMDD}${HH}" \ + --ensmem_index="${ENSMEM_INDX}" || \ print_err_msg_exit "\ -Call to script \"check_post_output.sh\" from \"${scrfunc_fn}\" failed." +Call to \"check_post_output.py\" from \"${scrfunc_fn}\" failed." + # #----------------------------------------------------------------------- # diff --git a/scripts/check_post_output.py b/scripts/check_post_output.py index 54ad27fc..4e17f2a0 100644 --- a/scripts/check_post_output.py +++ b/scripts/check_post_output.py @@ -15,6 +15,7 @@ from python_utils import setup_logging from set_leadhrs import set_leadhrs +from string import Template def check_post_output(config_file: str, cdate: str, ensmem_index: int) -> None: @@ -43,11 +44,21 @@ def check_post_output(config_file: str, cdate: str, ensmem_index: int) -> None: i = max(ensmem_index - 1, 0) time_lag = int(enscfg["ENS_TIME_LAG_HRS"][i]) * 3600 - # Build forecast filename template, prepending subdir template if set + # Make a dictionary of variables that may need to be substituted; these will be used to replace + # bash-like variables in some strings. This is needed to maintain some functionality while we + # still have a mix of bash and python exscripts. + subvars = { + "ensmem_name": f"mem{str(ensmem_index).zfill(vxcfg['VX_NDIGITS_ENSMEM_NAMES'])}", + "time_lag": time_lag, + } + +# Build forecast filename template, prepending subdir template if set subdir = vxcfg.get("FCST_SUBDIR_TEMPLATE", "") fn_template = os.path.join(subdir, vxcfg["FCST_FN_TEMPLATE"]) if subdir \ else vxcfg["FCST_FN_TEMPLATE"] + lgr.debug(f"{fn_template=}") + fn_template=Template(fn_template).substitute(subvars) lgr.info( f"Checking post-processed output files for cycle {cdate}, " f"member index {ensmem_index}" diff --git a/scripts/check_post_output.sh b/scripts/check_post_output.sh deleted file mode 100755 index c2f18c24..00000000 --- a/scripts/check_post_output.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash -# -#----------------------------------------------------------------------- -# -# The ex-script for checking the post output. -# -# Run-time environment variables: -# -# ACCUM_HH -# ENSMEM_INDX -# HH -# GLOBAL_VAR_DEFNS_FP -# METPLUS_ROOT (used by ush/set_leadhrs.py) -# YYMMDD -# -# Experiment variables -# -# user: -# USHdir -# -# workflow: -# FCST_LEN_HRS -# -# ensemble: -# DO_ENSEMBLE -# ENS_TIME_LAG_HRS -# -# verification: -# FCST_FN_TEMPLATE -# FCST_SUBDIR_TEMPLATE -# NUM_MISSING_FCST_FILES_MAX -# VX_FCST_INPUT_BASEDIR -# VX_NDIGITS_ENSMEM_NAMES -# -#----------------------------------------------------------------------- -# - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -sections=( - user - workflow - ensemble - verification -) -for sect in ${sections[*]} ; do - source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} -done -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the ex-script for the task that checks that no more than -NUM_MISSING_FCST_FILES_MAX of each forecast's (ensemble member's) post- -processed output files are missing. -========================================================================" -# -#----------------------------------------------------------------------- -# -# Get the time lag for the current ensemble member. -# -#----------------------------------------------------------------------- -# -i="0" -if [[ "${DO_ENSEMBLE}" == "True" ]]; then - i=$( bc -l <<< "${ENSMEM_INDX}-1" ) -fi -time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*3600" ) -# -#----------------------------------------------------------------------- -# -# Check to ensure that all the expected post-processed forecast output -# files are present on disk. This is done by the set_leadhrs function -# below. -# -#----------------------------------------------------------------------- -# -ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" $(( 10#${ENSMEM_INDX}))) -ensmem_name="mem${ensmem_indx}" -FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_SUBDIR_TEMPLATE:+${FCST_SUBDIR_TEMPLATE}/}${FCST_FN_TEMPLATE} ) - -FHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --date_init="${YYMMDD}${HH}" \ - --lhr_min="0" \ - --lhr_max="${FCST_LEN_HRS}" \ - --lhr_intvl="${VX_FCST_OUTPUT_INTVL_HRS}" \ - --base_dir="${VX_FCST_INPUT_BASEDIR}" \ - --fn_template="${FCST_INPUT_FN_TEMPLATE}" \ - --num_missing_files_max="${NUM_MISSING_FCST_FILES_MAX}" \ - --time_lag="${time_lag%.*}") || \ -print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Done checking for existence of post-processed files for ensemble member ${ENSMEM_INDX}. - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" diff --git a/scripts/gridstat_or_pointstat.py b/scripts/gridstat_or_pointstat.py index 14339511..e9763d63 100644 --- a/scripts/gridstat_or_pointstat.py +++ b/scripts/gridstat_or_pointstat.py @@ -8,7 +8,6 @@ """ import argparse -import ast import logging import math import os @@ -95,8 +94,7 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, lgr.debug(f"{vxcfg['VX_NDIGITS_ENSMEM_NAMES']=}") time_lag = 0 if do_ens: - time_lag_hrs = ast.literal_eval(cfg['ensemble']['ENS_TIME_LAG_HRS'])[ensmem_index-1] - time_lag = time_lag_hrs*3600 + time_lag = cfg['ensemble']['ENS_TIME_LAG_HRS'][ensmem_index-1]*3600 # Make a dictionary of variables that may need to be substituted; these will be used to replace # bash-like variables in some strings. This is needed to maintain some functionality while we diff --git a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml index 51500972..0a9a3e56 100644 --- a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml +++ b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml @@ -16,7 +16,7 @@ workflow: ensemble: DO_ENSEMBLE: true NUM_ENS_MEMBERS: 2 - ENS_TIME_LAG_HRS: '[ 0, 12 ]' + ENS_TIME_LAG_HRS: !list '[ 0, 12 ]' verification: diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 28f80b90..cec2ccc5 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -525,7 +525,7 @@ ensemble: DO_ENSEMBLE: false NUM_ENS_MEMBERS: 0 ENSMEM_NAMES: '{% for m in range(ensemble.NUM_ENS_MEMBERS) %}{{ "mem%03d, " % m }}{% endfor %}' - ENS_TIME_LAG_HRS: '[ {% for m in range([1,ensemble.NUM_ENS_MEMBERS]|max) %} 0, {% endfor %} ]' + ENS_TIME_LAG_HRS: !list '[ {% for m in range([1,ensemble.NUM_ENS_MEMBERS]|max) %} 0, {% endfor %} ]' tropical: #----------------------------- From 110875cc61ec25b7d52f2dea3313b5e145eda43e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 6 Jul 2026 19:51:04 +0000 Subject: [PATCH 04/25] Converting jobs/GENENSPROD_OR_ENSEMBLESTAT.sh to use new python script; almost working --- jobs/GENENSPROD_OR_ENSEMBLESTAT.sh | 15 +++++++++-- parm/metplus/EnsembleStat.conf | 2 +- parm/metplus/GenEnsProd.conf | 2 +- scripts/genensprod_or_ensemblestat.py | 37 +++++++++++++++++++-------- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh b/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh index 798cd77a..d874acdc 100755 --- a/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh +++ b/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh @@ -35,6 +35,8 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done +# Sets up PYTHONPATH and VERBOSE environment variables +. $USHdir/set_job_env.sh # #----------------------------------------------------------------------- # @@ -56,7 +58,16 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -$SCRIPTSdir/genensprod_or_ensemblestat.sh || \ +python $SCRIPTSdir/genensprod_or_ensemblestat.py ${VERBOSE_FLAG} \ + --config="${GLOBAL_VAR_DEFNS_FP}" \ + --cycle_date="${YYMMDD}${HH}" \ + --field_group="${FIELD_GROUP}" \ + --obs_dir="${OBS_DIR}" \ + --obtype="${OBTYPE}" \ + --accum_hh="${ACCUM_HH}" \ + --fcst_level="${FCST_LEVEL}" \ + --fcst_thresh="${FCST_THRESH}" \ + --metplus_tool="${METPLUSTOOLNAME}" || \ print_err_msg_exit "\ -Call to \"genensprod_or_ensemblestat.sh\" from \"${scrfunc_fn}\" failed." +Call to \"genensprod_or_ensemblestat.py\" from \"${scrfunc_fn}\" failed." diff --git a/parm/metplus/EnsembleStat.conf b/parm/metplus/EnsembleStat.conf index 242a69c0..e32cc1f0 100644 --- a/parm/metplus/EnsembleStat.conf +++ b/parm/metplus/EnsembleStat.conf @@ -221,7 +221,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Jinja requires certain variables to be defined globally within the template diff --git a/parm/metplus/GenEnsProd.conf b/parm/metplus/GenEnsProd.conf index 6c27a985..5179d4d6 100644 --- a/parm/metplus/GenEnsProd.conf +++ b/parm/metplus/GenEnsProd.conf @@ -109,7 +109,7 @@ STAGING_DIR = {{staging_dir}} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Jinja requires certain variables to be defined globally within the template diff --git a/scripts/genensprod_or_ensemblestat.py b/scripts/genensprod_or_ensemblestat.py index bd4a7f51..d8835e40 100644 --- a/scripts/genensprod_or_ensemblestat.py +++ b/scripts/genensprod_or_ensemblestat.py @@ -83,23 +83,36 @@ def genensprod_or_ensemblestat( exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + } + # Build obs input dir/template and forecast base dir if geom == "grid": if "APCP" in met_filedir_name: obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") - obs_in_fn_template = vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + obs_in_fn_template = Template( + vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) fcst_in_dir = Path(exptdir) elif "ASNOW" in met_filedir_name: obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") - obs_in_fn_template = vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + obs_in_fn_template = Template( + vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) fcst_in_dir = Path(exptdir) elif met_filedir_name == "REFC": obs_in_dir = Path(obs_dir) - obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + ).substitute(subvars) fcst_in_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) elif met_filedir_name == "RETOP": obs_in_dir = Path(obs_dir) - obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + ).substitute(subvars) fcst_in_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) else: raise ValueError( @@ -107,19 +120,23 @@ def genensprod_or_ensemblestat( ) elif geom == "point": obs_in_dir = Path(exptdir, "metprd", "Pb2nc_obs") - obs_in_fn_template = vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + obs_in_fn_template = Template( + vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + ).substitute(subvars) fcst_in_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) else: raise ValueError(f"Invalid parameters: {obtype=}, {field_group=}, {accum_hh=}") - # Build per-member forecast filename templates (comma-separated list for METplus) - subvars = { - "FIELD_GROUP": field_group, - "ACCUM_HH": f"{accum_hh:02}", - } fcst_in_fn_templates = [] for i in range(enscfg["NUM_ENS_MEMBERS"]): + # Build per-member forecast filename templates (comma-separated list for METplus) ensmem = f"mem{str(i + 1).zfill(vxcfg['VX_NDIGITS_ENSMEM_NAMES'])}" + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + "time_lag": int(enscfg["ENS_TIME_LAG_HRS"][i]) * 3600, + "ensmem_name": ensmem, + } if field_group in ("APCP", "ASNOW"): tmpl = str(Path( cdate, ensmem, "metprd", "PcpCombine_fcst", From 0d1082df0d4f027cbd65498beca3a587b4810d9c Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 6 Jul 2026 20:11:07 +0000 Subject: [PATCH 05/25] Found error: set_vx_params.py was not consistent with set_vx_params.sh for ACCUM_HH definitions --- scripts/genensprod_or_ensemblestat.sh | 458 -------------------------- ush/set_vx_params.py | 4 +- 2 files changed, 2 insertions(+), 460 deletions(-) delete mode 100755 scripts/genensprod_or_ensemblestat.sh diff --git a/scripts/genensprod_or_ensemblestat.sh b/scripts/genensprod_or_ensemblestat.sh deleted file mode 100755 index 37c35481..00000000 --- a/scripts/genensprod_or_ensemblestat.sh +++ /dev/null @@ -1,458 +0,0 @@ -#!/usr/bin/env bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -sections=( - user - platform - workflow - ensemble - verification -) -for sect in ${sections[*]} ; do - source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} -done -# -#----------------------------------------------------------------------- -# -# Source files defining auxiliary functions for verification. -# -#----------------------------------------------------------------------- -# -. $USHdir/get_metplus_tool_name.sh -. $USHdir/set_vx_params.sh -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of the MET/METplus tool in different formats that may be -# needed from the variable METPLUSTOOLNAME. -# -#----------------------------------------------------------------------- -# -get_metplus_tool_name \ - METPLUSTOOLNAME="${METPLUSTOOLNAME}" \ - outvarname_metplus_tool_name="metplus_tool_name" \ - outvarname_MetplusToolName="MetplusToolName" \ - outvarname_METPLUS_TOOL_NAME="METPLUS_TOOL_NAME" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# True. -# -#----------------------------------------------------------------------- -# -print_input_args "valid_args" -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the ex-script for the task that runs the METplus ${MetplusToolName} -tool either to generate ensemble products without performing verification -(if running the GenEnsProd tool) or to perform ensemble-based verification -(if running the EnsembleStat tool). -========================================================================" -# -#----------------------------------------------------------------------- -# -# Get the cycle date and time in YYYYMMDDHH format. -# -#----------------------------------------------------------------------- -# -CDATE="${YYMMDD}${HH}" -# -#----------------------------------------------------------------------- -# -# Set various verification parameters associated with the field to be -# verified. Not all of these are necessarily used later below but are -# set here for consistency with other verification ex-scripts. -# -#----------------------------------------------------------------------- -# -FIELDNAME_IN_OBS_INPUT="" -FIELDNAME_IN_FCST_INPUT="" -FIELDNAME_IN_MET_OUTPUT="" -FIELDNAME_IN_MET_FILEDIR_NAMES="" - -set_vx_params \ - obtype="${OBTYPE}" \ - field_group="${FIELD_GROUP}" \ - accum_hh="${ACCUM_HH}" \ - outvarname_grid_or_point="grid_or_point" \ - outvarname_fieldname_in_obs_input="FIELDNAME_IN_OBS_INPUT" \ - outvarname_fieldname_in_fcst_input="FIELDNAME_IN_FCST_INPUT" \ - outvarname_fieldname_in_MET_output="FIELDNAME_IN_MET_OUTPUT" \ - outvarname_fieldname_in_MET_filedir_names="FIELDNAME_IN_MET_FILEDIR_NAMES" -# -#----------------------------------------------------------------------- -# -# Set paths and file templates for input to and output from the MET/ -# METplus tool to be run as well as other file/directory parameters. -# -#----------------------------------------------------------------------- -# -vx_fcst_input_basedir=$( eval echo "${VX_FCST_INPUT_BASEDIR}" ) -vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) - -if [ "${grid_or_point}" = "grid" ]; then - - case "${FIELDNAME_IN_MET_FILEDIR_NAMES}" in - "APCP"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - FCST_INPUT_DIR="${vx_output_basedir}" - ;; - "ASNOW"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - FCST_INPUT_DIR="${vx_output_basedir}" - ;; - "REFC") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[1]}" - FCST_INPUT_DIR="${vx_fcst_input_basedir}" - ;; - "RETOP") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[3]}" - FCST_INPUT_DIR="${vx_fcst_input_basedir}" - ;; - esac - -elif [ "${grid_or_point}" = "point" ]; then - - OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT}" - FCST_INPUT_DIR="${vx_fcst_input_basedir}" - -fi -OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) -# -# Construct variable that contains a METplus template of the paths to -# the files that the PcpCombine tool has generated (in previous workflow -# tasks). This will be exported to the environment and read by the -# METplus configuration files. -# -FCST_INPUT_FN_TEMPLATE="" -for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do - - ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" "$((i+1))") - ensmem_name="mem${ensmem_indx}" - - cdate_ensmem_subdir_or_null="${CDATE}/${ensmem_name}" - - time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*3600" ) - - if [ "${FIELD_GROUP}" = "APCP" ] || [ "${FIELD_GROUP}" = "ASNOW" ]; then - template="${cdate_ensmem_subdir_or_null:+${cdate_ensmem_subdir_or_null}/}metprd/PcpCombine_fcst/${FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - else - template="${FCST_SUBDIR_TEMPLATE}/${FCST_FN_TEMPLATE}" - fi - - if [ -z "${FCST_INPUT_FN_TEMPLATE}" ]; then - FCST_INPUT_FN_TEMPLATE="$(eval echo ${template})" - else - FCST_INPUT_FN_TEMPLATE="${FCST_INPUT_FN_TEMPLATE}, $(eval echo ${template})" - fi - -done - -OUTPUT_BASE="${vx_output_basedir}/${CDATE}" -OUTPUT_DIR="${OUTPUT_BASE}/metprd/${MetplusToolName}" -STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}" -# -#----------------------------------------------------------------------- -# -# Generate the list of forecast hours for which to run the specified -# METplus tool. -# -# If running the GenEnsProd tool, we set this to the list of forecast -# output times without filtering for the existence of observation files -# corresponding to those times. This is because GenEnsProd operates -# only on forecasts; it does not need observations. -# -# On the other hand, if running the EnsembleStat tool, we set the list of -# forecast hours to a set of times that takes into consideration whether -# or not observations exist. We do this by starting with the full list -# of forecast times for which there is forecast output and then removing -# from that list any times for which there is no corresponding observations. -# -#----------------------------------------------------------------------- -# -case "$OBTYPE" in - "CCPA"|"NOHRSC") - vx_intvl="$((10#${ACCUM_HH}))" - vx_hr_start="${vx_intvl}" - ;; - *) - vx_intvl="$((${VX_FCST_OUTPUT_INTVL_HRS}))" - vx_hr_start="0" - ;; -esac -vx_hr_end="${FCST_LEN_HRS}" - -if [ "${MetplusToolName}" = "GenEnsProd" ]; then - VX_LEADHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --lhr_min="${vx_hr_start}" \ - --lhr_max="${vx_hr_end}" \ - --lhr_intvl="${vx_intvl}" \ - --skip_check_files ) || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" - -elif [ "${MetplusToolName}" = "EnsembleStat" ]; then - VX_LEADHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --date_init="${CDATE}" \ - --lhr_min="${vx_hr_start}" \ - --lhr_max="${vx_hr_end}" \ - --lhr_intvl="${vx_intvl}" \ - --base_dir="${OBS_INPUT_DIR}" \ - --fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - --num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ - --time_lag="${time_lag%.*}" ) || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" -fi -# -#----------------------------------------------------------------------- -# -# Make sure the MET/METplus output directory(ies) exists. -# -#----------------------------------------------------------------------- -# -mkdir -p "${OUTPUT_DIR}" -# -#----------------------------------------------------------------------- -# -# Check for existence of top-level OBS_DIR, if necessary. -# -#----------------------------------------------------------------------- -# -if [ "${MetplusToolName}" = "EnsembleStat" ]; then - if [ ! -d "${OBS_DIR}" ]; then - print_err_msg_exit "\ - OBS_DIR does not exist or is not a directory: - OBS_DIR = \"${OBS_DIR}\"" - fi -fi -# -#----------------------------------------------------------------------- -# -# Export variables needed in the common METplus configuration file (at -# ${METPLUS_CONF}/common.conf). -# -#----------------------------------------------------------------------- -# -export METPLUS_CONF -export LOGDIR -# -#----------------------------------------------------------------------- -# -# Do not run METplus if there isn't at least one lead hour for which to -# run it. -# -#----------------------------------------------------------------------- -# -if [ -z "${VX_LEADHR_LIST}" ]; then - print_err_msg_exit "\ -The list of lead hours for which to run METplus is empty: - VX_LEADHR_LIST = [${VX_LEADHR_LIST}]" -fi -# -#----------------------------------------------------------------------- -# -# Populate VX_MASK_FILE_LIST based on user selections -# -#----------------------------------------------------------------------- -# -# This weird logic is needed because in older versions of bash empty arrays are treated as unset -if [ ${VX_MASK[@]} ]; then - VX_MASK_FILE_LIST="" - for i in "${VX_MASK[@]}"; do - if [ -f "${METPLUS_CONF}/${i}.poly" ]; then - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, ${METPLUS_CONF}/${i}.poly" - else - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, {MET_INSTALL_DIR}/share/met/poly/${i}.poly" - fi - done -fi -# -#----------------------------------------------------------------------- -# -# Set the names of the template METplus configuration file, the METplus -# configuration file generated from this template, and the METplus log -# file. -# -#----------------------------------------------------------------------- -# -# First, set the base file names. -# -metplus_config_tmpl_bn="${MetplusToolName}" -metplus_config_bn="${MetplusToolName}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${CDATE}" -metplus_log_bn="${metplus_config_bn}" -# -# Add prefixes and suffixes (extensions) to the base file names. -# -metplus_config_tmpl_fn="${metplus_config_tmpl_bn}.conf" -metplus_config_fn="${metplus_config_bn}.conf" -metplus_log_fn="metplus.log.${metplus_log_bn}" -# -#----------------------------------------------------------------------- -# -# Load the yaml-like file containing the configuration for ensemble -# verification. -# -#----------------------------------------------------------------------- -# -vx_config_fp="${METPLUS_CONF}/${VX_CONFIG_ENS_FN}" -vx_config_dict=$(<"${vx_config_fp}") -# Indent each line of vx_config_dict so that it is aligned properly when -# included in the yaml-formatted variable "settings" below. -vx_config_dict=$( printf "%s\n" "${vx_config_dict}" | sed 's/^/ /' ) -# -#----------------------------------------------------------------------- -# -# Generate the METplus configuration file from its jinja template. -# -#----------------------------------------------------------------------- -# -# Set the full paths to the jinja template METplus configuration file -# (which already exists) and the METplus configuration file that will be -# generated from it. -# -metplus_config_tmpl_fp="${METPLUS_CONF}/${metplus_config_tmpl_fn}" -metplus_config_fp="${OUTPUT_DIR}/${metplus_config_fn}" -# -# Define variables that appear in the jinja template. -# -settings="\ -# -# MET/METplus information. -# -'metplus_tool_name': '${metplus_tool_name}' -'MetplusToolName': '${MetplusToolName}' -'METPLUS_TOOL_NAME': '${METPLUS_TOOL_NAME}' -'metplus_verbosity_level': '${METPLUS_VERBOSITY_LEVEL}' -# -# Date and forecast hour information. -# -'cdate': '$CDATE' -'vx_leadhr_list': '${VX_LEADHR_LIST}' -# -# Input and output directory/file information. -# -'metplus_config_fn': '${metplus_config_fn:-}' -'metplus_log_fn': '${metplus_log_fn:-}' -'obs_input_dir': '${OBS_INPUT_DIR:-}' -'obs_input_fn_template': '${OBS_INPUT_FN_TEMPLATE:-}' -'fcst_input_dir': '${FCST_INPUT_DIR:-}' -'fcst_input_fn_template': '${FCST_INPUT_FN_TEMPLATE:-}' -'output_dir': '${OUTPUT_DIR}' -'output_fn_template': '${OUTPUT_FN_TEMPLATE:-}' -'staging_dir': '${STAGING_DIR}' -'vx_fcst_model_name': '${VX_FCST_MODEL_NAME}' -# -# Ensemble and member-specific information. -# -'num_ens_members': '${NUM_ENS_MEMBERS}' -'ensmem_name': '${ensmem_name:-}' -'time_lag': '${time_lag:-}' -# -# Field information. -# -'fieldname_in_obs_input': '${FIELDNAME_IN_OBS_INPUT}' -'fieldname_in_fcst_input': '${FIELDNAME_IN_FCST_INPUT}' -'fieldname_in_met_output': '${FIELDNAME_IN_MET_OUTPUT}' -'fieldname_in_met_filedir_names': '${FIELDNAME_IN_MET_FILEDIR_NAMES}' -'obtype': '${OBTYPE}' -'accum_hh': '${ACCUM_HH:-}' -'accum_no_pad': '${ACCUM_NO_PAD:-}' -'metplus_templates_dir': '${METPLUS_CONF:-}' -'input_field_group': '${FIELD_GROUP:-}' -'input_level_fcst': '${FCST_LEVEL:-}' -'input_thresh_fcst': '${FCST_THRESH:-}' -# -# Verification mask settings -# -'vx_mask': '${VX_MASK_FILE_LIST:-}' -# -# -# Verification configuration dictionary. -# -'vx_config_dict': -${vx_config_dict:-} -" - -# Render the template to create a METplus configuration file -tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") -printf "%s" "$settings" > "$tmpfile" -uw template render \ - -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} \ - --verbose \ - --values-file "${tmpfile}" \ - --search-path "/" - -err=$? -rm $tmpfile -if [ $err -ne 0 ]; then - message_txt="Error rendering template for METplus config. - Contents of input are: -$settings" - print_err_msg_exit "${message_txt}" -fi -# -#----------------------------------------------------------------------- -# -# Call METplus. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Calling METplus to run MET's ${metplus_tool_name} tool for field(s): ${FIELDNAME_IN_MET_FILEDIR_NAMES}" -${METPLUS_ROOT}/ush/run_metplus.py \ - -c ${METPLUS_CONF}/common.conf \ - -c ${metplus_config_fp} || \ -print_err_msg_exit " -Call to METplus failed with return code: $? -METplus configuration file used is: - metplus_config_fp = \"${metplus_config_fp}\"" -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -METplus ${MetplusToolName} tool completed successfully. - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" diff --git a/ush/set_vx_params.py b/ush/set_vx_params.py index 62c597f1..60e22e56 100644 --- a/ush/set_vx_params.py +++ b/ush/set_vx_params.py @@ -20,14 +20,14 @@ def set_vx_params(obtype,field_group,accum_hh): fieldname_in_obs_in = field_group fieldname_in_fcst_in = field_group fieldname_in_met_out = field_group - fieldname_in_met_filedir_names = f"{field_group}{accum_hh:02}" + fieldname_in_met_filedir_names = f"{field_group}{accum_hh:02}h" case "NOHRSC": grid_or_point = "grid" if field_group == "ASNOW": fieldname_in_obs_in = field_group fieldname_in_fcst_in = field_group fieldname_in_met_out = field_group - fieldname_in_met_filedir_names = f"{field_group}{accum_hh:02}" + fieldname_in_met_filedir_names = f"{field_group}{accum_hh:02}h" case "MRMS": grid_or_point = "grid" From 363e15fbe6340631446f9411976131b19cdd1a1d Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 6 Jul 2026 22:01:18 +0000 Subject: [PATCH 06/25] Update PCPCOMBINE.sh to use pcpcombine.py. Some PCP combine tasks have an ensemble member index and some don't, so we need to add this as another optional variable to set_job_env.sh --- jobs/CHECK_POST_OUTPUT.sh | 2 +- jobs/GENENSPROD_OR_ENSEMBLESTAT.sh | 6 +- jobs/GRIDSTAT_OR_POINTSTAT.sh | 8 +- jobs/PCPCOMBINE.sh | 16 +- scripts/pcpcombine.py | 15 +- scripts/pcpcombine.sh | 469 ----------------------------- ush/set_job_env.sh | 6 + 7 files changed, 38 insertions(+), 484 deletions(-) delete mode 100755 scripts/pcpcombine.sh diff --git a/jobs/CHECK_POST_OUTPUT.sh b/jobs/CHECK_POST_OUTPUT.sh index 0d22ab42..18d441b7 100755 --- a/jobs/CHECK_POST_OUTPUT.sh +++ b/jobs/CHECK_POST_OUTPUT.sh @@ -65,7 +65,7 @@ In directory: \"${scrfunc_dir}\" python $SCRIPTSdir/check_post_output.py ${VERBOSE_FLAG} \ --config="${GLOBAL_VAR_DEFNS_FP}" \ --cycle_date="${YYMMDD}${HH}" \ - --ensmem_index="${ENSMEM_INDX}" || \ + ${ENSMEM_ARG} || \ print_err_msg_exit "\ Call to \"check_post_output.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh b/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh index d874acdc..cc54863d 100755 --- a/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh +++ b/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh @@ -35,7 +35,7 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done -# Sets up PYTHONPATH and VERBOSE environment variables +# Sets up PYTHONPATH, ACCUM_ARG, and VERBOSE environment variables . $USHdir/set_job_env.sh # #----------------------------------------------------------------------- @@ -64,10 +64,10 @@ python $SCRIPTSdir/genensprod_or_ensemblestat.py ${VERBOSE_FLAG} \ --field_group="${FIELD_GROUP}" \ --obs_dir="${OBS_DIR}" \ --obtype="${OBTYPE}" \ - --accum_hh="${ACCUM_HH}" \ --fcst_level="${FCST_LEVEL}" \ --fcst_thresh="${FCST_THRESH}" \ - --metplus_tool="${METPLUSTOOLNAME}" || \ + --metplus_tool="${METPLUSTOOLNAME}" \ + ${ACCUM_ARG} || \ print_err_msg_exit "\ Call to \"genensprod_or_ensemblestat.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/GRIDSTAT_OR_POINTSTAT.sh b/jobs/GRIDSTAT_OR_POINTSTAT.sh index d940de7d..ee7a27f4 100755 --- a/jobs/GRIDSTAT_OR_POINTSTAT.sh +++ b/jobs/GRIDSTAT_OR_POINTSTAT.sh @@ -35,7 +35,7 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done -# Sets up PYTHONPATH, VERBOSE and ACCUM_ARG environment variables +# Sets up PYTHONPATH, ACCUM_ARG, ENSMEM_ARG and VERBOSE environment variables . $USHdir/set_job_env.sh # #----------------------------------------------------------------------- @@ -58,15 +58,15 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -python $SCRIPTSdir/gridstat_or_pointstat.py ${VERBOSE_FLAG} ${ACCUM_ARG} \ +python $SCRIPTSdir/gridstat_or_pointstat.py ${VERBOSE_FLAG} \ --config="${GLOBAL_VAR_DEFNS_FP}" \ --cycle_date="${YYMMDD}${HH}" \ - --ensmem_index="${ENSMEM_INDX}" \ --field_group="${FIELD_GROUP}" \ --fcst_level="${FCST_LEVEL}" \ --fcst_thresh="${FCST_THRESH}" \ --obtype="${OBTYPE}" \ - --obs_dir="${OBS_DIR}" || \ + --obs_dir="${OBS_DIR}" \ + ${ACCUM_ARG} ${ENSMEM_ARG} || \ print_err_msg_exit "\ Call to \"gridstat_or_pointstat.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/PCPCOMBINE.sh b/jobs/PCPCOMBINE.sh index d4cbdd98..cfe18987 100755 --- a/jobs/PCPCOMBINE.sh +++ b/jobs/PCPCOMBINE.sh @@ -37,6 +37,9 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done +set -x +# Sets up PYTHONPATH, ACCUM_ARG, ENSMEM_ARG, and VERBOSE environment variables +. $USHdir/set_job_env.sh # #----------------------------------------------------------------------- # @@ -58,7 +61,16 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -$SCRIPTSdir/pcpcombine.sh || \ +python $SCRIPTSdir/pcpcombine.py ${VERBOSE_FLAG} \ + --config="${GLOBAL_VAR_DEFNS_FP}" \ + --cycle_date="${YYMMDD}${HH}" \ + --field_group="${FIELD_GROUP}" \ + --fcst_or_obs="${FCST_OR_OBS}" \ + --obs_dir="${OBS_DIR}" \ + --obtype="${OBTYPE}" \ + --fcst_level="${FCST_LEVEL}" \ + --fcst_thresh="${FCST_THRESH}" \ + ${ACCUM_ARG} ${ENSMEM_ARG} || \ print_err_msg_exit "\ -Call to \"pcpcombine.sh\" from \"${scrfunc_fn}\" failed." +Call to \"pcpcombine.py\" from \"${scrfunc_fn}\" failed." diff --git a/scripts/pcpcombine.py b/scripts/pcpcombine.py index a22a363b..996ea4cc 100644 --- a/scripts/pcpcombine.py +++ b/scripts/pcpcombine.py @@ -79,19 +79,24 @@ def pcpcombine( do_ensemble = enscfg["DO_ENSEMBLE"] ensmem = f"mem{str(ensmem_index).zfill(vxcfg['VX_NDIGITS_ENSMEM_NAMES'])}" + # Make a dictionary of variables that may need to be substituted; these will be used to replace + # bash-like variables in some strings. This is needed to maintain some functionality while we + # still have a mix of bash and python exscripts. subvars = { - "FIELD_GROUP": field_group, - "ACCUM_HH": f"{accum_hh:02}", + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + "ensmem_name": f"mem{str(ensmem_index).zfill(vxcfg['VX_NDIGITS_ENSMEM_NAMES'])}", } pcp_combine_method = "ADD" pcp_combine_command = "" + time_lag = 0 if fcst_or_obs == "FCST": - time_lag = 0 if do_ensemble: time_lag = int(enscfg["ENS_TIME_LAG_HRS"][ensmem_index - 1]) * 3600 + subvars["time_lag"]=time_lag fcst_subdir = Template(vxcfg.get("FCST_SUBDIR_TEMPLATE", "")).safe_substitute(subvars) fcst_fn = Template(vxcfg["FCST_FN_TEMPLATE"]).safe_substitute(subvars) input_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) @@ -148,7 +153,7 @@ def pcpcombine( suffix = f"_{ensmem}" else: # OBS - time_lag = 0 + subvars["time_lag"]=time_lag input_dir = Path(obs_dir) input_fn_template = vxcfg[f"OBS_{obtype}_FN_TEMPLATES"][1] output_fn_template = Template( @@ -285,7 +290,7 @@ def pcpcombine( help="Forecast thresholds to verify against (e.g. all, none)") parser.add_argument("--fcst_or_obs", required=True, type=str, help="Whether processing forecast (FCST) or observation (OBS) data") - parser.add_argument("--ensmem_index", required=True, type=int, + parser.add_argument("--ensmem_index", type=int, default=0, help="Ensemble member index (0 for deterministic, 1-based for ensemble members)") parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose debug output") diff --git a/scripts/pcpcombine.sh b/scripts/pcpcombine.sh deleted file mode 100755 index fe233ce6..00000000 --- a/scripts/pcpcombine.sh +++ /dev/null @@ -1,469 +0,0 @@ -#!/usr/bin/env bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -sections=( - user - platform - workflow - ensemble - verification -) -for sect in ${sections[*]} ; do - source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} -done -# -#----------------------------------------------------------------------- -# -# Source files defining auxiliary functions for verification. -# -#----------------------------------------------------------------------- -# -. $USHdir/get_metplus_tool_name.sh -. $USHdir/set_vx_params.sh -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of the MET/METplus tool in different formats that may be -# needed from the variable METPLUSTOOLNAME. -# -#----------------------------------------------------------------------- -# -get_metplus_tool_name \ - METPLUSTOOLNAME="${METPLUSTOOLNAME}" \ - outvarname_metplus_tool_name="metplus_tool_name" \ - outvarname_MetplusToolName="MetplusToolName" \ - outvarname_METPLUS_TOOL_NAME="METPLUS_TOOL_NAME" -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the ex-script for the task that runs the METplus ${MetplusToolName} -tool to combine hourly accumulated precipitation (APCP) data to generate -files containing multi-hour accumulated precipitation (e.g. 3-hour, 6- -hour, 24-hour). The input files can come from either observations or -a forecast. -========================================================================" -# -#----------------------------------------------------------------------- -# -# Get the cycle date and time in YYYYMMDDHH format. -# -#----------------------------------------------------------------------- -# -CDATE="${YYMMDD}${HH}" -# -#----------------------------------------------------------------------- -# -# Set various verification parameters associated with the field to be -# verified. Not all of these are necessarily used later below but are -# set here for consistency with other verification ex-scripts. -# -#----------------------------------------------------------------------- -# -FIELDNAME_IN_OBS_INPUT="" -FIELDNAME_IN_FCST_INPUT="" -FIELDNAME_IN_MET_OUTPUT="" -FIELDNAME_IN_MET_FILEDIR_NAMES="" - -set_vx_params \ - obtype="${OBTYPE}" \ - field_group="${FIELD_GROUP}" \ - accum_hh="${ACCUM_HH}" \ - outvarname_grid_or_point="grid_or_point" \ - outvarname_fieldname_in_obs_input="FIELDNAME_IN_OBS_INPUT" \ - outvarname_fieldname_in_fcst_input="FIELDNAME_IN_FCST_INPUT" \ - outvarname_fieldname_in_MET_output="FIELDNAME_IN_MET_OUTPUT" \ - outvarname_fieldname_in_MET_filedir_names="FIELDNAME_IN_MET_FILEDIR_NAMES" -# -#----------------------------------------------------------------------- -# -# If performing forecast ensemble verification, get the time lag (if any) -# of the current ensemble forecast member. The time lag is the duration -# (in units of seconds) by which the current forecast member was initialized -# before the current cycle date and time (with the latter specified by -# CDATE). For example, a time lag of 3600 means that the current member -# was initialized 1 hour before the current CDATE, while a time lag of 0 -# means the current member was initialized on CDATE. -# -# Note that if we're not running ensemble verification (i.e. if we're -# running verification for a single deterministic forecast), the time -# lag gets set to 0. -# -#----------------------------------------------------------------------- -# -time_lag="0" -if [ "${FCST_OR_OBS}" = "FCST" ]; then - i="0" - if [ "${DO_ENSEMBLE}" = "True" ]; then - i=$( bc -l <<< "${ENSMEM_INDX}-1" ) - fi - time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*3600" ) -fi -# -#----------------------------------------------------------------------- -# -# Set paths and file templates for input to and output from the MET/ -# METplus tool to be run as well as other file/directory parameters. -# -#----------------------------------------------------------------------- -# -vx_fcst_input_basedir=$( eval echo "${VX_FCST_INPUT_BASEDIR}" ) -vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) -if [ "${FCST_OR_OBS}" = "FCST" ]; then - ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" $(( 10#${ENSMEM_INDX}))) - ensmem_name="mem${ensmem_indx}" - # - # Since other aspects of a deterministic run use the "mem000" string (e.g. - # in rocoto workflow task names, in log file names), it seems reasonable - # that a deterministic run create a "mem000" subdirectory under the $CDATE - # directory. But since that is currently not the case in in the run_fcst - # task, we need the following if-statement. If and when such a modification - # is made for the run_fcst task, we would remove this if-statement and - # simply set - # slash_ensmem_subdir_or_null="/${ensmem_name}" - # or, better, just remove this variale and code "/${ensmem_name}" where - # slash_ensmem_subdir_or_null currently appears below. - # - if [ "${DO_ENSEMBLE}" = "True" ]; then - slash_ensmem_subdir_or_null="/${ensmem_name}" - else - slash_ensmem_subdir_or_null="" - fi -elif [ "${FCST_OR_OBS}" = "OBS" ]; then - if [ "${DO_ENSEMBLE}" = "True" ]; then - slash_obs_or_null="/obs" - else - slash_obs_or_null="" - fi -fi - -OBS_INPUT_DIR="" -OBS_INPUT_FN_TEMPLATE="" -FCST_INPUT_DIR="" -FCST_INPUT_FN_TEMPLATE="" -PCP_COMBINE_METHOD="ADD" -PCP_COMBINE_COMMAND="" -if [ "${FCST_OR_OBS}" = "FCST" ]; then - - FCST_INPUT_DIR="${vx_fcst_input_basedir}" - FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_SUBDIR_TEMPLATE:+${FCST_SUBDIR_TEMPLATE}/}${FCST_FN_TEMPLATE} ) - - OUTPUT_BASE="${vx_output_basedir}/${CDATE}${slash_ensmem_subdir_or_null}" - OUTPUT_DIR="${OUTPUT_BASE}/metprd/${MetplusToolName}_fcst" - OUTPUT_FN_TEMPLATE=$( eval echo ${FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT} ) - STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}" - if [ "${OBTYPE}" = "AIRNOW" ]; then - PCP_COMBINE_METHOD="USER_DEFINED" - - if [ "${FIELD_GROUP}" = "PM25" ]; then - if [ "${FCST_SMOKE_TYPE}" = "HRRR" ]; then - # for HRRR, all PM 2.5 is a single variable, so just pass through with corrected units - PCP_COMBINE_COMMAND="-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} -field 'name=\"MASSDEN\"; level=\"Z8\"; convert(x)=x*1e9;'" - elif [ "${FCST_SMOKE_TYPE}" = "RRFS" ]; then - # Need to combine two fields (different PM types) and convert units from forecast files to create PM25 equivalent to obs - PCP_COMBINE_COMMAND="-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62010; convert(x)=x*1e9;' {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001; GRIB2_aerosol_interval_type=0; convert(x)=x*1e9;'" - fi - elif [ "${FIELD_GROUP}" = "PM10" ]; then - if [ "${FCST_SMOKE_TYPE}" = "RRFS" ]; then - # for PM10, need to combine original field with PM2.5 described above - PCP_COMBINE_COMMAND="-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62010; convert(x)=x*1e9;' {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001; GRIB2_aerosol_interval_type=0; convert(x)=x*1e9;' {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001; GRIB2_aerosol_interval_type=2; convert(x)=x*1e9;'" - else - print_err_msg_exit "PM10 only available for RRFS output, not available for ${FCST_SMOKE_TYPE}" - fi - fi - fi -elif [ "${FCST_OR_OBS}" = "OBS" ]; then - - OBS_INPUT_DIR="${OBS_DIR}" - fn_template=$(eval echo \${OBS_${OBTYPE}_FN_TEMPLATES[1]}) - OBS_INPUT_FN_TEMPLATE=$( eval echo ${fn_template} ) - - OUTPUT_BASE="${vx_output_basedir}/${CDATE}${slash_obs_or_null}" - OUTPUT_DIR="${OUTPUT_BASE}/metprd/${MetplusToolName}_obs" - fn_template=$(eval echo \${OBS_${OBTYPE}_${FIELD_GROUP}_FN_TEMPLATE_PCPCOMBINE_OUTPUT}) - OUTPUT_FN_TEMPLATE=$( eval echo ${fn_template} ) - STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}" - -fi -# -#----------------------------------------------------------------------- -# -# Set the array of lead hours for which to run the MET/METplus tool. -# -#----------------------------------------------------------------------- -# -vx_intvl="$((10#${ACCUM_HH}))" -#Airnow obs use PCP_Combine simply to combine two fields, so run for every hour -if [ "${OBTYPE}" = "AIRNOW" ]; then - lhr_min=0 -else - lhr_min=${vx_intvl} -fi -VX_LEADHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --lhr_min="${lhr_min}" \ - --lhr_max="${FCST_LEN_HRS}" \ - --lhr_intvl="${vx_intvl}" \ - --skip_check_files ) || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" -# -#----------------------------------------------------------------------- -# -# Check for the presence of files (either from observations or forecasts) -# needed to create required accumulation given by ACCUM_HH. -# -#----------------------------------------------------------------------- -# -if [ "${FCST_OR_OBS}" = "FCST" ]; then - base_dir="${FCST_INPUT_DIR}" - fn_template="${FCST_INPUT_FN_TEMPLATE}" - subintvl="${VX_FCST_OUTPUT_INTVL_HRS}" -elif [ "${FCST_OR_OBS}" = "OBS" ]; then - base_dir="${OBS_INPUT_DIR}" - fn_template="${OBS_INPUT_FN_TEMPLATE}" - subintvl="${OBS_AVAIL_INTVL_HRS}" -fi -num_missing_files_max="0" -input_accum_hh=$(printf "%02d" ${subintvl}) -# -# Convert the list of hours at which the PcpCombine tool will be run to -# an array. This represents the hours at which each accumulation period -# ends. Then use it to check the presence of all files requied to build -# the required accumulations from the sub-accumulations. -# -subintvl_end_hrs=($( echo ${VX_LEADHR_LIST} | $SED "s/,//g" )) -for hr_end in ${subintvl_end_hrs[@]}; do - hr_start=$((hr_end - vx_intvl + subintvl)) - print_info_msg " -Checking for the presence of files that will contribute to the ${vx_intvl}-hour -accumulation ending at lead hour ${hr_end} (relative to ${CDATE})... -" - python3 $USHdir/set_leadhrs.py \ - --date_init="${CDATE}" \ - --lhr_min="${hr_start}" \ - --lhr_max="${hr_end}" \ - --lhr_intvl="${subintvl}" \ - --base_dir="${base_dir}" \ - --fn_template="${fn_template}" \ - --num_missing_files_max="${num_missing_files_max}" \ - --time_lag="${time_lag%.*}" || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" -done - -print_info_msg " -${MetplusToolName} will be run for the following lead hours (relative to ${CDATE}): - VX_LEADHR_LIST = ${VX_LEADHR_LIST} -" -# -#----------------------------------------------------------------------- -# -# Make sure the MET/METplus output directory(ies) exists. -# -#----------------------------------------------------------------------- -# -mkdir -p "${OUTPUT_DIR}" -# -#----------------------------------------------------------------------- -# -# Check for existence of top-level OBS_DIR. -# -#----------------------------------------------------------------------- -# -if [ "${FCST_OR_OBS}" = "OBS" ]; then - if [ ! -d "${OBS_DIR}" ]; then - print_err_msg_exit "\ - OBS_DIR does not exist or is not a directory: - OBS_DIR = \"${OBS_DIR}\"" - fi -fi -# -#----------------------------------------------------------------------- -# -# Export variables needed in the common METplus configuration file (at -# ${METPLUS_CONF}/common.conf). -# -#----------------------------------------------------------------------- -# -export METPLUS_CONF -export LOGDIR -# -#----------------------------------------------------------------------- -# -# Do not run METplus if there isn't at least one lead hour for which to -# run it. -# -#----------------------------------------------------------------------- -# -if [ -z "${VX_LEADHR_LIST}" ]; then - print_err_msg_exit "\ -The list of lead hours for which to run METplus is empty: - VX_LEADHR_LIST = [${VX_LEADHR_LIST}]" -fi -# -#----------------------------------------------------------------------- -# -# Set the names of the template METplus configuration file, the METplus -# configuration file generated from this template, and the METplus log -# file. -# -#----------------------------------------------------------------------- -# -# First, set the base file names. -# -metplus_config_tmpl_fn="${MetplusToolName}" -if [ "${FCST_OR_OBS}" = "FCST" ]; then - suffix="${ENSMEM_INDX:+_${ensmem_name}}" -elif [ "${FCST_OR_OBS}" = "OBS" ]; then - suffix="_${OBTYPE}" -fi -metplus_config_fn="${metplus_config_tmpl_fn}_$(echo_lowercase ${FCST_OR_OBS})_${FIELDNAME_IN_MET_FILEDIR_NAMES}${suffix}" -metplus_log_fn="${metplus_config_fn}_$CDATE" -# -# Add prefixes and suffixes (extensions) to the base file names. -# -metplus_config_tmpl_fn="${metplus_config_tmpl_fn}.conf" -metplus_config_fn="${metplus_config_fn}.conf" -metplus_log_fn="metplus.log.${metplus_log_fn}" -# -#----------------------------------------------------------------------- -# -# Generate the METplus configuration file from its jinja template. -# -#----------------------------------------------------------------------- -# -# Set the full paths to the jinja template METplus configuration file -# (which already exists) and the METplus configuration file that will be -# generated from it. -# -metplus_config_tmpl_fp="${METPLUS_CONF}/${metplus_config_tmpl_fn}" -metplus_config_fp="${OUTPUT_DIR}/${metplus_config_fn}" -# -# Define variables that appear in the jinja template. -# -settings="\ -# -# MET/METplus information. -# - 'metplus_tool_name': '${metplus_tool_name}' - 'MetplusToolName': '${MetplusToolName}' - 'METPLUS_TOOL_NAME': '${METPLUS_TOOL_NAME}' - 'metplus_verbosity_level': '${METPLUS_VERBOSITY_LEVEL}' -# -# Date and forecast hour information. -# - 'cdate': '$CDATE' - 'vx_leadhr_list': '${VX_LEADHR_LIST}' -# -# Input and output directory/file information. -# - 'metplus_config_fn': '${metplus_config_fn:-}' - 'metplus_log_fn': '${metplus_log_fn:-}' - 'input_dir': '${FCST_INPUT_DIR:-${OBS_INPUT_DIR}}' - 'input_fn_template': '${FCST_INPUT_FN_TEMPLATE:-${OBS_INPUT_FN_TEMPLATE}}' - 'output_dir': '${OUTPUT_DIR}' - 'output_fn_template': '${OUTPUT_FN_TEMPLATE:-}' - 'staging_dir': '${STAGING_DIR}' - 'vx_fcst_model_name': '${VX_FCST_MODEL_NAME}' -# -# Ensemble and member-specific information. -# - 'num_ens_members': '${NUM_ENS_MEMBERS}' - 'ensmem_name': '${ensmem_name:-}' - 'time_lag': '${time_lag:-}' -# -# Field information. -# - 'fieldname_in_obs_input': '${FIELDNAME_IN_OBS_INPUT}' - 'fieldname_in_fcst_input': '${FIELDNAME_IN_FCST_INPUT}' - 'fieldname_in_met_output': '${FIELDNAME_IN_MET_OUTPUT}' - 'fieldname_in_met_filedir_names': '${FIELDNAME_IN_MET_FILEDIR_NAMES}' - 'obtype': '${OBTYPE}' - 'FCST_OR_OBS': '${FCST_OR_OBS}' - 'input_accum_hh': '${input_accum_hh}' - 'output_accum_hh': '${ACCUM_HH:-}' - 'accum_no_pad': '${ACCUM_NO_PAD:-}' - 'metplus_templates_dir': '${METPLUS_CONF:-}' - 'input_field_group': '${FIELD_GROUP:-}' - 'input_level_fcst': '${FCST_LEVEL:-}' - 'input_thresh_fcst': '${FCST_THRESH:-}' -# -# Configuration information -# - 'pcp_combine_method': '${PCP_COMBINE_METHOD}' -# NOTE: this command must remain un-quoted for proper rendering of nested quotes in command - 'pcp_combine_command': ${PCP_COMBINE_COMMAND} -" -# Render the template to create a METplus configuration file -tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") -printf "%s" "$settings" > "$tmpfile" -uw template render \ - -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} \ - --verbose \ - --values-file "${tmpfile}" \ - --search-path "/" - -err=$? -rm $tmpfile -if [ $err -ne 0 ]; then - message_txt="Error rendering template for METplus config. - Contents of input are: -$settings" - print_err_msg_exit "${message_txt}" -fi -# -#----------------------------------------------------------------------- -# -# Call METplus. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Calling METplus to run MET's ${metplus_tool_name} tool for field(s): ${FIELDNAME_IN_MET_FILEDIR_NAMES}" -${METPLUS_ROOT}/ush/run_metplus.py \ - -c ${METPLUS_CONF}/common.conf \ - -c ${metplus_config_fp} || \ -print_err_msg_exit " -Call to METplus failed with return code: $? -METplus configuration file used is: - metplus_config_fp = \"${metplus_config_fp}\"" -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -METplus ${MetplusToolName} tool completed successfully. - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" diff --git a/ush/set_job_env.sh b/ush/set_job_env.sh index beda5a8f..412a2c8f 100644 --- a/ush/set_job_env.sh +++ b/ush/set_job_env.sh @@ -17,3 +17,9 @@ if [ ! -z "${ACCUM_HH}" ]; then ACCUM_ARG="--accum_hh=${ACCUM_HH}" fi +# For tasks that need an ensemble member index, set ENSMEM_INDX +ENSMEM_ARG="" +if [ ! -z "${ENSMEM_INDX}" ]; then + ENSMEM_ARG="--ensmem_index=${ENSMEM_INDX}" +fi + From 75d66cc9932bf97178104a3514f36dd487efc733 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Tue, 7 Jul 2026 02:20:12 +0000 Subject: [PATCH 07/25] All run scripts pythonized! --- jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh | 15 +- jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh | 15 +- jobs/PCPCOMBINE.sh | 1 - parm/metplus/GridStat_ensmean.conf | 2 +- parm/metplus/GridStat_ensprob.conf | 2 +- parm/metplus/PointStat_ensmean.conf | 2 +- parm/metplus/PointStat_ensprob.conf | 2 +- scripts/gridstat_or_pointstat_ensmean.py | 29 +- scripts/gridstat_or_pointstat_ensmean.sh | 402 ---------------------- scripts/gridstat_or_pointstat_ensprob.py | 29 +- scripts/gridstat_or_pointstat_ensprob.sh | 403 ----------------------- 11 files changed, 77 insertions(+), 825 deletions(-) delete mode 100755 scripts/gridstat_or_pointstat_ensmean.sh delete mode 100755 scripts/gridstat_or_pointstat_ensprob.sh diff --git a/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh b/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh index cf16fd03..3239671a 100755 --- a/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh +++ b/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh @@ -36,6 +36,8 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done +# Sets up PYTHONPATH and VERBOSE environment variables +. $USHdir/set_job_env.sh # #----------------------------------------------------------------------- # @@ -57,6 +59,15 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -$SCRIPTSdir/gridstat_or_pointstat_ensmean.sh || \ +python $SCRIPTSdir/gridstat_or_pointstat_ensmean.py ${VERBOSE_FLAG} \ + --config="${GLOBAL_VAR_DEFNS_FP}" \ + --cycle_date="${YYMMDD}${HH}" \ + --field_group="${FIELD_GROUP}" \ + --obs_dir="${OBS_DIR}" \ + --obtype="${OBTYPE}" \ + --fcst_level="${FCST_LEVEL}" \ + --fcst_thresh="${FCST_THRESH}" \ + ${ACCUM_ARG} || \ print_err_msg_exit "\ -Call to \"gridstat_or_pointstat_ensmean.sh\" from \"${scrfunc_fn}\" failed." +Call to \"gridstat_or_pointstat_ensmean.py\" from \"${scrfunc_fn}\" failed." + diff --git a/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh b/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh index c2caae2b..793d21d3 100755 --- a/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh +++ b/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh @@ -36,6 +36,8 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done +# Sets up PYTHONPATH and VERBOSE environment variables +. $USHdir/set_job_env.sh # #----------------------------------------------------------------------- # @@ -57,7 +59,14 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -$SCRIPTSdir/gridstat_or_pointstat_ensprob.sh || \ +python $SCRIPTSdir/gridstat_or_pointstat_ensprob.py ${VERBOSE_FLAG} \ + --config="${GLOBAL_VAR_DEFNS_FP}" \ + --cycle_date="${YYMMDD}${HH}" \ + --field_group="${FIELD_GROUP}" \ + --obs_dir="${OBS_DIR}" \ + --obtype="${OBTYPE}" \ + --fcst_level="${FCST_LEVEL}" \ + --fcst_thresh="${FCST_THRESH}" \ + ${ACCUM_ARG} || \ print_err_msg_exit "\ -Call to \"gridstat_or_pointstat_ensprob.sh\" from \"${scrfunc_fn}\" failed." - +Call to \"gridstat_or_pointstat_ensprob.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/PCPCOMBINE.sh b/jobs/PCPCOMBINE.sh index cfe18987..4b5ce647 100755 --- a/jobs/PCPCOMBINE.sh +++ b/jobs/PCPCOMBINE.sh @@ -37,7 +37,6 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done -set -x # Sets up PYTHONPATH, ACCUM_ARG, ENSMEM_ARG, and VERBOSE environment variables . $USHdir/set_job_env.sh # diff --git a/parm/metplus/GridStat_ensmean.conf b/parm/metplus/GridStat_ensmean.conf index 61b54d17..ddcf7c29 100644 --- a/parm/metplus/GridStat_ensmean.conf +++ b/parm/metplus/GridStat_ensmean.conf @@ -97,7 +97,7 @@ OBTYPE = {{obtype}} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Set the probabilistic threshold to be used for the forecast field. If diff --git a/parm/metplus/GridStat_ensprob.conf b/parm/metplus/GridStat_ensprob.conf index c47ed5de..5113c635 100644 --- a/parm/metplus/GridStat_ensprob.conf +++ b/parm/metplus/GridStat_ensprob.conf @@ -108,7 +108,7 @@ OBTYPE = {{obtype}} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Set the probabilistic threshold to be used for the forecast field. If diff --git a/parm/metplus/PointStat_ensmean.conf b/parm/metplus/PointStat_ensmean.conf index 6dc6bb4c..0de8e1b1 100644 --- a/parm/metplus/PointStat_ensmean.conf +++ b/parm/metplus/PointStat_ensmean.conf @@ -161,7 +161,7 @@ OBTYPE = {{obtype}} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Set the probabilistic threshold to be used for the forecast field. If diff --git a/parm/metplus/PointStat_ensprob.conf b/parm/metplus/PointStat_ensprob.conf index 223a5318..fa29f91e 100644 --- a/parm/metplus/PointStat_ensprob.conf +++ b/parm/metplus/PointStat_ensprob.conf @@ -163,7 +163,7 @@ OBTYPE = {{obtype}} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Set the probabilistic threshold to be used for the forecast field. If diff --git a/scripts/gridstat_or_pointstat_ensmean.py b/scripts/gridstat_or_pointstat_ensmean.py index 03f6006e..108d87ea 100644 --- a/scripts/gridstat_or_pointstat_ensmean.py +++ b/scripts/gridstat_or_pointstat_ensmean.py @@ -12,6 +12,7 @@ from multiprocessing import Pool from pathlib import Path +from string import Template import uwtools.api.config as uwconfig @@ -63,21 +64,37 @@ def gridstat_or_pointstat_ensmean( exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + # Make a dictionary of variables that may need to be substituted; these will be used to replace + # bash-like variables in some strings. This is needed to maintain some functionality while we + # still have a mix of bash and python exscripts. + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + } + if geom == "grid": metplus_tool_name = "grid_stat" metplus_tool_camel_case = "GridStat" if "APCP" in met_filedir_name: obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") - obs_in_fn_template = vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + obs_in_fn_template = Template( + vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) elif "ASNOW" in met_filedir_name: obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") - obs_in_fn_template = vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + obs_in_fn_template = Template( + vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) elif met_filedir_name == "REFC": obs_in_dir = Path(obs_dir) - obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + ).substitute(subvars) elif met_filedir_name == "RETOP": obs_in_dir = Path(obs_dir) - obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + ).substitute(subvars) else: raise ValueError(f"Invalid field group for GridStat ensmean: {field_group}") fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") @@ -86,7 +103,9 @@ def gridstat_or_pointstat_ensmean( metplus_tool_name = "point_stat" metplus_tool_camel_case = "PointStat" obs_in_dir = Path(exptdir, "metprd", "Pb2nc_obs") - obs_in_fn_template = vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + obs_in_fn_template = Template( + vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + ).substitute(subvars) fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") else: diff --git a/scripts/gridstat_or_pointstat_ensmean.sh b/scripts/gridstat_or_pointstat_ensmean.sh deleted file mode 100755 index c513cc8d..00000000 --- a/scripts/gridstat_or_pointstat_ensmean.sh +++ /dev/null @@ -1,402 +0,0 @@ -#!/usr/bin/env bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -sections=( - user - platform - workflow - ensemble - verification -) -for sect in ${sections[*]} ; do - source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} -done -# -#----------------------------------------------------------------------- -# -# Source files defining auxiliary functions for verification. -# -#----------------------------------------------------------------------- -# -. $USHdir/get_metplus_tool_name.sh -. $USHdir/set_vx_params.sh -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of the MET/METplus tool in different formats that may be -# needed from the variable METPLUSTOOLNAME. -# -#----------------------------------------------------------------------- -# -get_metplus_tool_name \ - METPLUSTOOLNAME="${METPLUSTOOLNAME}" \ - outvarname_metplus_tool_name="metplus_tool_name" \ - outvarname_MetplusToolName="MetplusToolName" \ - outvarname_METPLUS_TOOL_NAME="METPLUS_TOOL_NAME" -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the ex-script for the task that runs the METplus ${MetplusToolName} -tool to perform verification of the specified field group (FIELD_GROUP) -on the ensemble mean. -========================================================================" -# -#----------------------------------------------------------------------- -# -# Get the cycle date and time in YYYYMMDDHH format. -# -#----------------------------------------------------------------------- -# -CDATE="${YYMMDD}${HH}" -# -#----------------------------------------------------------------------- -# -# Set various verification parameters associated with the field to be -# verified. Not all of these are necessarily used later below but are -# set here for consistency with other verification ex-scripts. -# -#----------------------------------------------------------------------- -# -FIELDNAME_IN_OBS_INPUT="" -FIELDNAME_IN_FCST_INPUT="" -FIELDNAME_IN_MET_OUTPUT="" -FIELDNAME_IN_MET_FILEDIR_NAMES="" - -set_vx_params \ - obtype="${OBTYPE}" \ - field_group="${FIELD_GROUP}" \ - accum_hh="${ACCUM_HH}" \ - outvarname_grid_or_point="grid_or_point" \ - outvarname_fieldname_in_obs_input="FIELDNAME_IN_OBS_INPUT" \ - outvarname_fieldname_in_fcst_input="FIELDNAME_IN_FCST_INPUT" \ - outvarname_fieldname_in_MET_output="FIELDNAME_IN_MET_OUTPUT" \ - outvarname_fieldname_in_MET_filedir_names="FIELDNAME_IN_MET_FILEDIR_NAMES" -# -#----------------------------------------------------------------------- -# -# Set paths and file templates for input to and output from the MET/ -# METplus tool to be run as well as other file/directory parameters. -# -#----------------------------------------------------------------------- -# -vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) - -if [ "${grid_or_point}" = "grid" ]; then - - case "${FIELDNAME_IN_MET_FILEDIR_NAMES}" in - "APCP"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - ;; - "ASNOW"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - ;; - "REFC") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[1]}" - ;; - "RETOP") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[3]}" - ;; - esac - FCST_INPUT_DIR="${vx_output_basedir}/${CDATE}/metprd/GenEnsProd" - -elif [ "${grid_or_point}" = "point" ]; then - - OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT}" - FCST_INPUT_DIR="${vx_output_basedir}/${CDATE}/metprd/GenEnsProd" - -fi -OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) -FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - -OUTPUT_BASE="${vx_output_basedir}/${CDATE}" -OUTPUT_DIR="${OUTPUT_BASE}/metprd/${MetplusToolName}_ensmean" -STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}_ensmean" -# -#----------------------------------------------------------------------- -# -# Set the lead hours for which to run the MET/METplus tool. This is done -# by starting with the full list of lead hours for which we expect to -# find forecast output and then removing from that list any hours for -# which there is no corresponding observation data. -# -#----------------------------------------------------------------------- -# -case "$OBTYPE" in - "CCPA"|"NOHRSC") - vx_intvl="$((10#${ACCUM_HH}))" - vx_hr_start="${vx_intvl}" - ;; - *) - vx_intvl="$((${VX_FCST_OUTPUT_INTVL_HRS}))" - vx_hr_start="0" - ;; -esac -vx_hr_end="${FCST_LEN_HRS}" - -VX_LEADHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --date_init="${CDATE}" \ - --lhr_min="${vx_hr_start}" \ - --lhr_max="${vx_hr_end}" \ - --lhr_intvl="${vx_intvl}" \ - --base_dir="${OBS_INPUT_DIR}" \ - --fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - --num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" ) || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" - -# -#----------------------------------------------------------------------- -# -# Make sure the MET/METplus output directory(ies) exists. -# -#----------------------------------------------------------------------- -# -mkdir -p "${OUTPUT_DIR}" -# -#----------------------------------------------------------------------- -# -# Check for existence of top-level OBS_DIR. -# -#----------------------------------------------------------------------- -# -if [ ! -d "${OBS_DIR}" ]; then - print_err_msg_exit "\ -OBS_DIR does not exist or is not a directory: - OBS_DIR = \"${OBS_DIR}\"" -fi -# -#----------------------------------------------------------------------- -# -# Set variable containing accumulation period without leading zero -# padding. This may be needed in the METplus configuration files. -# -#----------------------------------------------------------------------- -# -ACCUM_NO_PAD=$( printf "%0d" "${ACCUM_HH}" ) -# -#----------------------------------------------------------------------- -# -# Export variables needed in the common METplus configuration file (at -# ${METPLUS_CONF}/common.conf). -# -#----------------------------------------------------------------------- -# -export METPLUS_CONF -export LOGDIR -# -#----------------------------------------------------------------------- -# -# Do not run METplus if there isn't at least one lead hour for which to -# run it. -# -#----------------------------------------------------------------------- -# -if [ -z "${VX_LEADHR_LIST}" ]; then - print_err_msg_exit "\ -The list of lead hours for which to run METplus is empty: - VX_LEADHR_LIST = [${VX_LEADHR_LIST}]" -fi -# -#----------------------------------------------------------------------- -# -# Populate VX_MASK_FILE_LIST based on user selections -# -#----------------------------------------------------------------------- -# -# This weird logic is needed because in older versions of bash empty arrays are treated as unset -if [ ${VX_MASK[@]} ]; then - VX_MASK_FILE_LIST="" - for i in "${VX_MASK[@]}"; do - if [ -f "${METPLUS_CONF}/${i}.poly" ]; then - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, ${METPLUS_CONF}/${i}.poly" - else - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, {MET_INSTALL_DIR}/share/met/poly/${i}.poly" - fi - done -fi -# -#----------------------------------------------------------------------- -# -# Set the names of the template METplus configuration file, the METplus -# configuration file generated from this template, and the METplus log -# file. -# -#----------------------------------------------------------------------- -# -# First, set the base file names. -# -metplus_config_tmpl_bn="${MetplusToolName}_ensmean" -metplus_config_bn="${MetplusToolName}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${CDATE}_ensmean" -metplus_log_bn="${metplus_config_bn}" -# -# Add prefixes and suffixes (extensions) to the base file names. -# -metplus_config_tmpl_fn="${metplus_config_tmpl_bn}.conf" -metplus_config_fn="${metplus_config_bn}.conf" -metplus_log_fn="metplus.log.${metplus_log_bn}" -# -#----------------------------------------------------------------------- -# -# Load the yaml-like file containing the configuration for ensemble -# verification. -# -#----------------------------------------------------------------------- -# -vx_config_fp="${METPLUS_CONF}/${VX_CONFIG_ENS_FN}" -vx_config_dict=$(<"${vx_config_fp}") -# Indent each line of vx_config_dict so that it is aligned properly when -# included in the yaml-formatted variable "settings" below. -vx_config_dict=$( printf "%s\n" "${vx_config_dict}" | sed 's/^/ /' ) -# -#----------------------------------------------------------------------- -# -# Generate the METplus configuration file from its jinja template. -# -#----------------------------------------------------------------------- -# -# Set the full paths to the jinja template METplus configuration file -# (which already exists) and the METplus configuration file that will be -# generated from it. -# -metplus_config_tmpl_fp="${METPLUS_CONF}/${metplus_config_tmpl_fn}" -metplus_config_fp="${OUTPUT_DIR}/${metplus_config_fn}" -# -# Define variables that appear in the jinja template. -# -settings="\ -# -# MET/METplus information. -# -'metplus_tool_name': '${metplus_tool_name}' -'MetplusToolName': '${MetplusToolName}' -'METPLUS_TOOL_NAME': '${METPLUS_TOOL_NAME}' -'metplus_verbosity_level': '${METPLUS_VERBOSITY_LEVEL}' -# -# Date and forecast hour information. -# -'cdate': '$CDATE' -'vx_leadhr_list': '${VX_LEADHR_LIST}' -# -# Input and output directory/file information. -# -'metplus_config_fn': '${metplus_config_fn:-}' -'metplus_log_fn': '${metplus_log_fn:-}' -'obs_input_dir': '${OBS_INPUT_DIR:-}' -'obs_input_fn_template': '${OBS_INPUT_FN_TEMPLATE:-}' -'fcst_input_dir': '${FCST_INPUT_DIR:-}' -'fcst_input_fn_template': '${FCST_INPUT_FN_TEMPLATE:-}' -'output_dir': '${OUTPUT_DIR}' -'output_fn_template': '${OUTPUT_FN_TEMPLATE:-}' -'staging_dir': '${STAGING_DIR}' -'vx_fcst_model_name': '${VX_FCST_MODEL_NAME}' -# -# Ensemble and member-specific information. -# -'num_ens_members': '${NUM_ENS_MEMBERS}' -'ensmem_name': '${ensmem_name:-}' -'time_lag': '${time_lag:-}' -# -# Field information. -# -'fieldname_in_obs_input': '${FIELDNAME_IN_OBS_INPUT}' -'fieldname_in_fcst_input': '${FIELDNAME_IN_FCST_INPUT}' -'fieldname_in_met_output': '${FIELDNAME_IN_MET_OUTPUT}' -'fieldname_in_met_filedir_names': '${FIELDNAME_IN_MET_FILEDIR_NAMES}' -'obtype': '${OBTYPE}' -'accum_hh': '${ACCUM_HH:-}' -'accum_no_pad': '${ACCUM_NO_PAD:-}' -'metplus_templates_dir': '${METPLUS_CONF:-}' -'input_field_group': '${FIELD_GROUP:-}' -'input_level_fcst': '${FCST_LEVEL:-}' -'input_thresh_fcst': '${FCST_THRESH:-}' -# -# Verification mask settings -# -'vx_mask': '${VX_MASK_FILE_LIST:-}' -# -# Verification configuration dictionary. -# -'vx_config_dict': -${vx_config_dict:-} -" - -# Render the template to create a METplus configuration file -tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") -printf "%s" "$settings" > "$tmpfile" -uw template render \ - -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} \ - --verbose \ - --values-file "${tmpfile}" \ - --search-path "/" - -err=$? -rm $tmpfile -if [ $err -ne 0 ]; then - message_txt="Error rendering template for METplus config. - Contents of input are: -$settings" - print_err_msg_exit "${message_txt}" -fi -# -#----------------------------------------------------------------------- -# -# Call METplus. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Calling METplus to run MET's ${metplus_tool_name} tool for field(s): ${FIELDNAME_IN_MET_FILEDIR_NAMES}" -${METPLUS_ROOT}/ush/run_metplus.py \ - -c ${METPLUS_CONF}/common.conf \ - -c ${metplus_config_fp} || \ -print_err_msg_exit " -Call to METplus failed with return code: $? -METplus configuration file used is: - metplus_config_fp = \"${metplus_config_fp}\"" -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -METplus ${MetplusToolName} tool completed successfully. - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" diff --git a/scripts/gridstat_or_pointstat_ensprob.py b/scripts/gridstat_or_pointstat_ensprob.py index d4df4013..a73f970a 100644 --- a/scripts/gridstat_or_pointstat_ensprob.py +++ b/scripts/gridstat_or_pointstat_ensprob.py @@ -13,6 +13,7 @@ from multiprocessing import Pool from pathlib import Path +from string import Template import uwtools.api.config as uwconfig @@ -64,21 +65,37 @@ def gridstat_or_pointstat_ensprob( exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + # Make a dictionary of variables that may need to be substituted; these will be used to replace + # bash-like variables in some strings. This is needed to maintain some functionality while we + # still have a mix of bash and python exscripts. + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + } + if geom == "grid": metplus_tool_name = "grid_stat" metplus_tool_camel_case = "GridStat" if "APCP" in met_filedir_name: obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") - obs_in_fn_template = vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + obs_in_fn_template = Template( + vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) elif "ASNOW" in met_filedir_name: obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") - obs_in_fn_template = vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + obs_in_fn_template = Template( + vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) elif met_filedir_name == "REFC": obs_in_dir = Path(obs_dir) - obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + ).substitute(subvars) elif met_filedir_name == "RETOP": obs_in_dir = Path(obs_dir) - obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + ).substitute(subvars) else: raise ValueError(f"Invalid field group for GridStat ensprob: {field_group}") fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") @@ -87,7 +104,9 @@ def gridstat_or_pointstat_ensprob( metplus_tool_name = "point_stat" metplus_tool_camel_case = "PointStat" obs_in_dir = Path(exptdir, "metprd", "Pb2nc_obs") - obs_in_fn_template = vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + obs_in_fn_template = Template( + vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + ).substitute(subvars) fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") else: diff --git a/scripts/gridstat_or_pointstat_ensprob.sh b/scripts/gridstat_or_pointstat_ensprob.sh deleted file mode 100755 index f59ebba1..00000000 --- a/scripts/gridstat_or_pointstat_ensprob.sh +++ /dev/null @@ -1,403 +0,0 @@ -#!/usr/bin/env bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -sections=( - user - platform - workflow - ensemble - verification -) -for sect in ${sections[*]} ; do - source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} -done -# -#----------------------------------------------------------------------- -# -# Source files defining auxiliary functions for verification. -# -#----------------------------------------------------------------------- -# -. $USHdir/get_metplus_tool_name.sh -. $USHdir/set_vx_params.sh -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of the MET/METplus tool in different formats that may be -# needed from the variable METPLUSTOOLNAME. -# -#----------------------------------------------------------------------- -# -get_metplus_tool_name \ - METPLUSTOOLNAME="${METPLUSTOOLNAME}" \ - outvarname_metplus_tool_name="metplus_tool_name" \ - outvarname_MetplusToolName="MetplusToolName" \ - outvarname_METPLUS_TOOL_NAME="METPLUS_TOOL_NAME" -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the ex-script for the task that runs the METplus ${MetplusToolName} -tool to perform verification of the specified field group (FIELD_GROUP) -on the ensemble -frequencies/probabilities. -========================================================================" -# -#----------------------------------------------------------------------- -# -# Get the cycle date and time in YYYYMMDDHH format. -# -#----------------------------------------------------------------------- -# -CDATE="${YYMMDD}${HH}" -# -#----------------------------------------------------------------------- -# -# Set various verification parameters associated with the field to be -# verified. Not all of these are necessarily used later below but are -# set here for consistency with other verification ex-scripts. -# -#----------------------------------------------------------------------- -# -FIELDNAME_IN_OBS_INPUT="" -FIELDNAME_IN_FCST_INPUT="" -FIELDNAME_IN_MET_OUTPUT="" -FIELDNAME_IN_MET_FILEDIR_NAMES="" - -set_vx_params \ - obtype="${OBTYPE}" \ - field_group="${FIELD_GROUP}" \ - accum_hh="${ACCUM_HH}" \ - outvarname_grid_or_point="grid_or_point" \ - outvarname_fieldname_in_obs_input="FIELDNAME_IN_OBS_INPUT" \ - outvarname_fieldname_in_fcst_input="FIELDNAME_IN_FCST_INPUT" \ - outvarname_fieldname_in_MET_output="FIELDNAME_IN_MET_OUTPUT" \ - outvarname_fieldname_in_MET_filedir_names="FIELDNAME_IN_MET_FILEDIR_NAMES" -# -#----------------------------------------------------------------------- -# -# Set paths and file templates for input to and output from the MET/ -# METplus tool to be run as well as other file/directory parameters. -# -#----------------------------------------------------------------------- -# -vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) - -if [ "${grid_or_point}" = "grid" ]; then - - case "${FIELDNAME_IN_MET_FILEDIR_NAMES}" in - "APCP"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - ;; - "ASNOW"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - ;; - "REFC") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[1]}" - ;; - "RETOP") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[3]}" - ;; - esac - -elif [ "${grid_or_point}" = "point" ]; then - - OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT}" - -fi -OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) -FCST_INPUT_DIR="${vx_output_basedir}/${CDATE}/metprd/GenEnsProd" -FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - -OUTPUT_BASE="${vx_output_basedir}/${CDATE}" -OUTPUT_DIR="${OUTPUT_BASE}/metprd/${MetplusToolName}_ensprob" -STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}_ensprob" -# -#----------------------------------------------------------------------- -# -# Set the lead hours for which to run the MET/METplus tool. This is done -# by starting with the full list of lead hours for which we expect to -# find forecast output and then removing from that list any hours for -# which there is no corresponding observation data. -# -#----------------------------------------------------------------------- -# -case "$OBTYPE" in - "CCPA"|"NOHRSC") - vx_intvl="$((10#${ACCUM_HH}))" - vx_hr_start="${vx_intvl}" - ;; - *) - vx_intvl="$((${VX_FCST_OUTPUT_INTVL_HRS}))" - vx_hr_start="0" - ;; -esac -vx_hr_end="${FCST_LEN_HRS}" - -VX_LEADHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --date_init="${CDATE}" \ - --lhr_min="${vx_hr_start}" \ - --lhr_max="${vx_hr_end}" \ - --lhr_intvl="${vx_intvl}" \ - --base_dir="${OBS_INPUT_DIR}" \ - --fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - --num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" ) || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" - -# -#----------------------------------------------------------------------- -# -# Make sure the MET/METplus output directory(ies) exists. -# -#----------------------------------------------------------------------- -# -mkdir -p "${OUTPUT_DIR}" -# -#----------------------------------------------------------------------- -# -# Check for existence of top-level OBS_DIR. -# -#----------------------------------------------------------------------- -# -if [ ! -d "${OBS_DIR}" ]; then - print_err_msg_exit "\ -OBS_DIR does not exist or is not a directory: - OBS_DIR = \"${OBS_DIR}\"" -fi -# -#----------------------------------------------------------------------- -# -# Set variable containing accumulation period without leading zero -# padding. This may be needed in the METplus configuration files. -# -#----------------------------------------------------------------------- -# -ACCUM_NO_PAD=$( printf "%0d" "${ACCUM_HH}" ) -# -#----------------------------------------------------------------------- -# -# Export variables needed in the common METplus configuration file (at -# ${METPLUS_CONF}/common.conf). -# -#----------------------------------------------------------------------- -# -export METPLUS_CONF -export LOGDIR -# -#----------------------------------------------------------------------- -# -# Do not run METplus if there isn't at least one lead hour for which to -# run it. -# -#----------------------------------------------------------------------- -# -if [ -z "${VX_LEADHR_LIST}" ]; then - print_err_msg_exit "\ -The list of lead hours for which to run METplus is empty: - VX_LEADHR_LIST = [${VX_LEADHR_LIST}]" -fi -# -#----------------------------------------------------------------------- -# -# Set the names of the template METplus configuration file, the METplus -# configuration file generated from this template, and the METplus log -# file. -# -#----------------------------------------------------------------------- -# -# First, set the base file names. -# -metplus_config_tmpl_bn="${MetplusToolName}_ensprob" -metplus_config_bn="${MetplusToolName}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${CDATE}_ensprob" -metplus_log_bn="${metplus_config_bn}" -# -# Add prefixes and suffixes (extensions) to the base file names. -# -metplus_config_tmpl_fn="${metplus_config_tmpl_bn}.conf" -metplus_config_fn="${metplus_config_bn}.conf" -metplus_log_fn="metplus.log.${metplus_log_bn}" -# -#----------------------------------------------------------------------- -# -# Populate VX_MASK_FILE_LIST based on user selections -# -#----------------------------------------------------------------------- -# -# This weird logic is needed because in older versions of bash empty arrays are treated as unset -if [ ${VX_MASK[@]} ]; then - VX_MASK_FILE_LIST="" - for i in "${VX_MASK[@]}"; do - if [ -f "${METPLUS_CONF}/${i}.poly" ]; then - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, ${METPLUS_CONF}/${i}.poly" - else - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, {MET_INSTALL_DIR}/share/met/poly/${i}.poly" - fi - done -fi -# -#----------------------------------------------------------------------- -# -# Load the yaml-like file containing the configuration for ensemble -# verification. -# -#----------------------------------------------------------------------- -# -vx_config_fp="${METPLUS_CONF}/${VX_CONFIG_ENS_FN}" -vx_config_dict=$(<"${vx_config_fp}") -# Indent each line of vx_config_dict so that it is aligned properly when -# included in the yaml-formatted variable "settings" below. -vx_config_dict=$( printf "%s\n" "${vx_config_dict}" | sed 's/^/ /' ) -# -#----------------------------------------------------------------------- -# -# Generate the METplus configuration file from its jinja template. -# -#----------------------------------------------------------------------- -# -# Set the full paths to the jinja template METplus configuration file -# (which already exists) and the METplus configuration file that will be -# generated from it. -# -metplus_config_tmpl_fp="${METPLUS_CONF}/${metplus_config_tmpl_fn}" -metplus_config_fp="${OUTPUT_DIR}/${metplus_config_fn}" -# -# Define variables that appear in the jinja template. -# -settings="\ -# -# MET/METplus information. -# -'metplus_tool_name': '${metplus_tool_name}' -'MetplusToolName': '${MetplusToolName}' -'METPLUS_TOOL_NAME': '${METPLUS_TOOL_NAME}' -'metplus_verbosity_level': '${METPLUS_VERBOSITY_LEVEL}' -# -# Date and forecast hour information. -# -'cdate': '$CDATE' -'vx_leadhr_list': '${VX_LEADHR_LIST}' -# -# Input and output directory/file information. -# -'metplus_config_fn': '${metplus_config_fn:-}' -'metplus_log_fn': '${metplus_log_fn:-}' -'obs_input_dir': '${OBS_INPUT_DIR:-}' -'obs_input_fn_template': '${OBS_INPUT_FN_TEMPLATE:-}' -'fcst_input_dir': '${FCST_INPUT_DIR:-}' -'fcst_input_fn_template': '${FCST_INPUT_FN_TEMPLATE:-}' -'output_dir': '${OUTPUT_DIR}' -'output_fn_template': '${OUTPUT_FN_TEMPLATE:-}' -'staging_dir': '${STAGING_DIR}' -'vx_fcst_model_name': '${VX_FCST_MODEL_NAME}' -# -# Ensemble and member-specific information. -# -'num_ens_members': '${NUM_ENS_MEMBERS}' -'ensmem_name': '${ensmem_name:-}' -'time_lag': '${time_lag:-}' -# -# Field information. -# -'fieldname_in_obs_input': '${FIELDNAME_IN_OBS_INPUT}' -'fieldname_in_fcst_input': '${FIELDNAME_IN_FCST_INPUT}' -'fieldname_in_met_output': '${FIELDNAME_IN_MET_OUTPUT}' -'fieldname_in_met_filedir_names': '${FIELDNAME_IN_MET_FILEDIR_NAMES}' -'obtype': '${OBTYPE}' -'accum_hh': '${ACCUM_HH:-}' -'accum_no_pad': '${ACCUM_NO_PAD:-}' -'metplus_templates_dir': '${METPLUS_CONF:-}' -'input_field_group': '${FIELD_GROUP:-}' -'input_level_fcst': '${FCST_LEVEL:-}' -'input_thresh_fcst': '${FCST_THRESH:-}' -# -# Verification mask settings -# -'vx_mask': '${VX_MASK_FILE_LIST:-}' -# -# -# Verification configuration dictionary. -# -'vx_config_dict': -${vx_config_dict:-} -" - -# Render the template to create a METplus configuration file -tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") -printf "%s" "$settings" > "$tmpfile" -uw template render \ - -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} \ - --verbose \ - --values-file "${tmpfile}" \ - --search-path "/" - -err=$? -rm $tmpfile -if [ $err -ne 0 ]; then - message_txt="Error rendering template for METplus config. - Contents of input are: -$settings" - print_err_msg_exit "${message_txt}" -fi -# -#----------------------------------------------------------------------- -# -# Call METplus. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Calling METplus to run MET's ${metplus_tool_name} tool for field(s): ${FIELDNAME_IN_MET_FILEDIR_NAMES}" -${METPLUS_ROOT}/ush/run_metplus.py \ - -c ${METPLUS_CONF}/common.conf \ - -c ${metplus_config_fp} || \ -print_err_msg_exit " -Call to METplus failed with return code: $? -METplus configuration file used is: - metplus_config_fp = \"${metplus_config_fp}\"" -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -METplus ${MetplusToolName} tool completed successfully. - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" From 10671761762f00b0917b7398630059c61d410d9d Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Tue, 7 Jul 2026 22:01:50 +0000 Subject: [PATCH 08/25] Update new genensprod_or_ensemblestat.py for parallel task submission --- scripts/genensprod_or_ensemblestat.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/genensprod_or_ensemblestat.py b/scripts/genensprod_or_ensemblestat.py index d8835e40..e3b55400 100644 --- a/scripts/genensprod_or_ensemblestat.py +++ b/scripts/genensprod_or_ensemblestat.py @@ -231,16 +231,19 @@ def genensprod_or_ensemblestat( "vx_config_dict": vx_config_dict, } - numprocs = 1 - conf_files = render_metplus_confs( - cfg, settings, metplus_config_tmpl_fn, vx_leadhr_list, numprocs - ) + numprocs=vxcfg['VX_TASKS'] + + conf_files = render_metplus_confs(cfg,settings,metplus_config_tmpl_fn,vx_leadhr_list,numprocs) lgr.debug(f"{conf_files=}") - lgr.info(f"Running {metplus_tool_camel_case} with METplus") - common_conf = os.path.join(cfg["user"]["METPLUS_CONF"], "common.conf") + lgr.info(f"Running {metplus_tool_camel_case} with METplus with {numprocs} tasks") + mpargs = [] + for config_fn in conf_files: + mpargs.append( (os.path.join(cfg['user']['METPLUS_CONF'], "common.conf"),config_fn) ) + # Call run_metplus function for as many processors as specified + lgr.debug(f"{mpargs=}") with Pool(processes=numprocs) as pool: - pool.starmap(run_metplus, [(common_conf, fn) for fn in conf_files]) + pool.starmap(run_metplus,mpargs) lgr.info(f"{metplus_tool_camel_case} completed successfully.") From 57283a6115c465a20300a0ec021f655036072d41 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Tue, 7 Jul 2026 22:04:36 +0000 Subject: [PATCH 09/25] Add gridstat: and pointstat: config sections, give them each an execution: section with tasks_per_node, walltime, and memory, as well as a TASKS: variable. Update some WE2E tests to use more parallelism --- parm/wflow/verify_det.yaml | 24 ++++++++--------- scripts/gridstat_or_pointstat.py | 7 ++--- ...lticyc_long-fcst-no-overlap_nssl-mpas.yaml | 5 ++++ ..._multicyc_long-fcst-overlap_nssl-mpas.yaml | 4 +++ .../config.MET_ensemble_verification.yaml | 5 ++++ ...fig.MET_ensemble_verification_only_vx.yaml | 5 ++++ ...nsemble_verification_only_vx_time_lag.yaml | 6 +++++ ...g.MET_ensemble_verification_winter_wx.yaml | 4 +++ ush/config_defaults.yaml | 27 ++++++++++++------- 9 files changed, 61 insertions(+), 26 deletions(-) diff --git a/parm/wflow/verify_det.yaml b/parm/wflow/verify_det.yaml index b3e5f5c3..180bca62 100644 --- a/parm/wflow/verify_det.yaml +++ b/parm/wflow/verify_det.yaml @@ -49,9 +49,9 @@ metatask_GridStat_APCP_all_accums_all_mems: ENSMEM_INDX: "#mem#" FCST_LEVEL: 'A#ACCUM_HH#' FCST_THRESH: 'all' - memory: '{{ verification_resources.execution.deterministic.gridstat.memory if user.MACHINE not in ["NOAACLOUD"] }}' - nodes: '{{ verification_resources.execution.nodes }}:ppn={{ verification_resources.execution.deterministic.gridstat.tasks_per_node }}' - walltime: "{{ verification_resources.execution.deterministic.gridstat.walltime }}" + memory: '{{ gridstat.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ gridstat.execution.tasks_per_node }}' + walltime: "{{ gridstat.execution.walltime }}" dependency: and: taskdep_pcpcombine_obs: @@ -83,9 +83,9 @@ metatask_GridStat_ASNOW_all_accums_all_mems: ENSMEM_INDX: "#mem#" FCST_LEVEL: 'A#ACCUM_HH#' FCST_THRESH: 'all' - memory: '{{ verification_resources.execution.deterministic.gridstat.memory if user.MACHINE not in ["NOAACLOUD"] }}' - nodes: '{{ verification_resources.execution.nodes }}:ppn={{ verification_resources.execution.deterministic.gridstat.tasks_per_node }}' - walltime: "{{ verification_resources.execution.deterministic.gridstat.walltime }}" + memory: '{{ gridstat.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ gridstat.execution.tasks_per_node }}' + walltime: "{{ gridstat.execution.walltime }}" dependency: and: taskdep_pcpcombine_obs: @@ -113,9 +113,9 @@ metatask_GridStat_REFC_RETOP_all_mems: ENSMEM_INDX: "#mem#" FCST_LEVEL: 'L0' FCST_THRESH: 'all' - memory: '{{ verification_resources.execution.deterministic.gridstat.memory if user.MACHINE not in ["NOAACLOUD"] }}' - nodes: '{{ verification_resources.execution.nodes }}:ppn={{ verification_resources.execution.deterministic.gridstat.tasks_per_node }}' - walltime: "{{ verification_resources.execution.deterministic.gridstat.walltime }}" + memory: '{{ gridstat.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ gridstat.execution.tasks_per_node }}' + walltime: "{{ gridstat.execution.walltime }}" dependency: and: # Check that the flag files that indicate that the get_obs_mrms tasks @@ -155,9 +155,9 @@ metatask_PointStat: ENSMEM_INDX: "#mem#" FCST_LEVEL: 'all' FCST_THRESH: 'all' - memory: '{{ verification_resources.execution.deterministic.gridstat.memory if user.MACHINE not in ["NOAACLOUD"] }}' - nodes: '{{ verification_resources.execution.nodes }}:ppn={{ verification_resources.execution.deterministic.pointstat.tasks_per_node }}' - walltime: "{{ verification_resources.execution.deterministic.pointstat.walltime }}" + memory: '{{ pointstat.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ pointstat.execution.tasks_per_node }}' + walltime: "{{ pointstat.execution.walltime }}" dependency: and: # Check that the flag files that indicate that the Pb2NC tasks are diff --git a/scripts/gridstat_or_pointstat.py b/scripts/gridstat_or_pointstat.py index e9763d63..e356b413 100644 --- a/scripts/gridstat_or_pointstat.py +++ b/scripts/gridstat_or_pointstat.py @@ -292,10 +292,7 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, 'vx_config_dict': vx_config_dict } - if field_group == "UPA": - numprocs=math.ceil(vxcfg['VX_TASKS']/2) - else: - numprocs=vxcfg['VX_TASKS'] + numprocs = cfg[metplus_tool_camel_case.lower()]["TASKS"] conf_files = render_metplus_confs(cfg,settings,metplus_config_tmpl_fn,vx_leadhr_list,numprocs) lgr.debug(f"{conf_files=}") @@ -313,7 +310,7 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, def run_metplus(common_config,config_fn): - """Calls the run_metplus script as a subprocess. If VX_TASKS > 1 and vx_leadhr_list > 1, + """Calls the run_metplus script as a subprocess. If TASKS > 1 and vx_leadhr_list > 1, calls in with starmap for the number of tasks specified.""" # Run METplus diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-no-overlap_nssl-mpas.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-no-overlap_nssl-mpas.yaml index 1657be5e..b7772a6f 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-no-overlap_nssl-mpas.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-no-overlap_nssl-mpas.yaml @@ -43,3 +43,8 @@ verification: VX_FCST_INPUT_BASEDIR: '{{ "%s/%s" % (platform.STAGED_DATA, verification.VX_FCST_MODEL_NAME) }}' FCST_FN_TEMPLATE: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%H?shift=${time_lag}}.grib2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}_${FIELD_GROUP}_a${ACCUM_HH}h.nc' +gridstat: + TASKS: 24 +pointstat: + TASKS: 24 + diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml index 62b6a57b..0ec59781 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml @@ -42,3 +42,7 @@ verification: VX_FCST_INPUT_BASEDIR: '{{ "%s/%s" % (platform.STAGED_DATA, verification.VX_FCST_MODEL_NAME) }}' FCST_FN_TEMPLATE: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%H?shift=${time_lag}}.grib2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}_${FIELD_GROUP}_a${ACCUM_HH}h.nc' +gridstat: + TASKS: 4 +pointstat: + TASKS: 4 diff --git a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification.yaml b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification.yaml index f9a38c45..4819bfa2 100644 --- a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification.yaml +++ b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification.yaml @@ -19,3 +19,8 @@ verification: VX_FCST_MODEL_NAME: srw.RRFS_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/MET_ensemble_verification' FCST_FN_TEMPLATE: 'srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' +gridstat: + TASKS: 4 +pointstat: + TASKS: 4 + diff --git a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx.yaml index 9fad2ac3..fa9532bb 100644 --- a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx.yaml @@ -23,3 +23,8 @@ verification: VX_MASK: ["CONUS"] VX_TASKS: 4 FCST_FN_TEMPLATE: 'rrfs.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' +gridstat: + TASKS: 4 +pointstat: + TASKS: 4 + diff --git a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml index 0a9a3e56..f99498ed 100644 --- a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml +++ b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml @@ -31,3 +31,9 @@ verification: VX_NDIGITS_ENSMEM_NAMES: 1 FCST_FN_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.t{init?fmt=%H?shift=-${time_lag}}z.bgdawpf{lead?fmt=%HHH?shift=${time_lag}}.tm00.grib2' VX_MASK: ["CONUS"] + VX_TASKS: 6 +gridstat: + TASKS: 6 +pointstat: + TASKS: 6 + diff --git a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_winter_wx.yaml b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_winter_wx.yaml index 2f2f1fcc..56d767f9 100644 --- a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_winter_wx.yaml +++ b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_winter_wx.yaml @@ -24,4 +24,8 @@ verification: VX_TASKS: 4 VX_MASK: ["CONUS"] FCST_FN_TEMPLATE: 'srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' +gridstat: + TASKS: 4 +pointstat: + TASKS: 4 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index cec2ccc5..af521202 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -574,15 +574,6 @@ verification_resources: execution: nodes: 1 tasks_per_node: 1 - deterministic: - gridstat: - tasks_per_node: '{{ verification.VX_TASKS }}' - walltime: 02:00:00 - memory: '{{ 2 * verification.VX_TASKS }}G' - pointstat: - tasks_per_node: '{{ verification.VX_TASKS }}' - walltime: 01:00:00 - memory: '{{ 2 * verification.VX_TASKS }}G' ensemble: genensprod: default: @@ -1007,6 +998,24 @@ verification: # sequential forecast hours, and so can be run simultaneously. VX_TASKS: 1 +gridstat: + # Runtime settings for deterministic GRIDSTAT jobs + execution: + tasks_per_node: '{{ gridstat.TASKS }}' + walltime: 01:00:00 + memory: '{{ 2 * gridstat.TASKS }}G' + # Number of GRIDSTAT instances to run in parallel + TASKS: 1 + +pointstat: + # Runtime settings for POINTSTAT jobs + execution: + tasks_per_node: '{{ pointstat.TASKS }}' + walltime: 01:00:00 + memory: '{{ 2 * pointstat.TASKS }}G' + # Number of POINTSTAT instances to run in parallel + TASKS: 1 + point2grid: execution: tasks_per_node: '{{ point2grid.TASKS }}' From 18aa8d97eb2365ae1ff25d823eb6ad7d849d536d Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Wed, 8 Jul 2026 03:08:25 +0000 Subject: [PATCH 10/25] For tools that use a staging directory, need a different one for each parallel task --- ush/python_utils/metplus_conf_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/python_utils/metplus_conf_utils.py b/ush/python_utils/metplus_conf_utils.py index fe86deda..d01d0366 100644 --- a/ush/python_utils/metplus_conf_utils.py +++ b/ush/python_utils/metplus_conf_utils.py @@ -114,6 +114,8 @@ def render_metplus_confs(cfg,settings,template_fn,vx_leadhr_list,tasks): logger.debug(f"metplus log file for task: {settings['metplus_log_fn']}") logger.debug(f"metplus final rendered conf for task: {outconf}") hours_per_task,remainder = divmod(num_fhrs,tasks) + if settings.get("staging_dir"): + settings['staging_dir'] = f"{settings['staging_dir']}.{i}" # For cases where things don't divide evenly, ensure we get best distribution if i >= remainder: vx_leadhr_list, task_fhrs = vx_leadhr_list[hours_per_task:],vx_leadhr_list[:hours_per_task] From b7477d0b8587716d3addcdfd21d489d028049569 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Wed, 8 Jul 2026 03:57:41 +0000 Subject: [PATCH 11/25] Add genensprod: and ensemblestat: config sections, give them each an execution: section with tasks_per_node, walltime, and memory, as well as a TASKS: variable, and apply memory and task settings to workflow definition file' --- parm/wflow/verify_ens.yaml | 32 ++++++++++++++++++++++++-------- ush/config_defaults.yaml | 33 ++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/parm/wflow/verify_ens.yaml b/parm/wflow/verify_ens.yaml index f55f46d7..e9bdc464 100644 --- a/parm/wflow/verify_ens.yaml +++ b/parm/wflow/verify_ens.yaml @@ -43,7 +43,9 @@ metatask_GenEnsProd_EnsembleStat_APCP_all_accums: FCST_LEVEL: 'A#ACCUM_HH#' FCST_THRESH: 'all' OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.genensprod.default.walltime }}" + memory: '{{ genensprod.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ genensprod.execution.tasks_per_node }}' + walltime: "{{ genensprod.execution.walltime }}" dependency: metataskdep_pcpcombine_fcst: attrs: @@ -54,7 +56,9 @@ metatask_GenEnsProd_EnsembleStat_APCP_all_accums: <<: *envars_GenEnsProd_CCPA METPLUSTOOLNAME: 'ENSEMBLESTAT' FCST_THRESH: 'none' - walltime: "{{ verification_resources.execution.ensemble.ensemblestat.walltime }}" + memory: '{{ ensemblestat.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ ensemblestat.execution.tasks_per_node }}' + walltime: "{{ ensemblestat.execution.walltime }}" dependency: and: taskdep_pcpcombine_obs_ccpa: &taskdep_pcpcombine_obs_ccpa @@ -80,7 +84,9 @@ metatask_GenEnsProd_EnsembleStat_ASNOW_all_accums: FCST_LEVEL: 'A#ACCUM_HH#' FCST_THRESH: 'all' OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.genensprod.default.walltime }}" + memory: '{{ genensprod.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ genensprod.execution.tasks_per_node }}' + walltime: "{{ genensprod.execution.walltime }}" dependency: and: metataskdep_pcpcombine_fcst: @@ -92,7 +98,9 @@ metatask_GenEnsProd_EnsembleStat_ASNOW_all_accums: <<: *envars_GenEnsProd_NOHRSC METPLUSTOOLNAME: 'ENSEMBLESTAT' FCST_THRESH: 'none' - walltime: "{{ verification_resources.execution.ensemble.ensemblestat.walltime }}" + memory: '{{ ensemblestat.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ ensemblestat.execution.tasks_per_node }}' + walltime: "{{ ensemblestat.execution.walltime }}" dependency: and: taskdep_pcpcombine_obs_nohrsc: &taskdep_pcpcombine_obs_nohrsc @@ -118,7 +126,9 @@ metatask_GenEnsProd_EnsembleStat_REFC_RETOP: FCST_LEVEL: 'L0' FCST_THRESH: 'all' OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.genensprod.default.walltime }}" + memory: '{{ genensprod.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ genensprod.execution.tasks_per_node }}' + walltime: "{{ genensprod.execution.walltime }}" dependency: and: metataskdep_check_post_output: &check_post_output @@ -131,7 +141,9 @@ metatask_GenEnsProd_EnsembleStat_REFC_RETOP: METPLUSTOOLNAME: 'ENSEMBLESTAT' FCST_LEVEL: 'L0' FCST_THRESH: 'none' - walltime: "{{ verification_resources.execution.ensemble.ensemblestat.walltime }}" + memory: '{{ ensemblestat.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ ensemblestat.execution.tasks_per_node }}' + walltime: "{{ ensemblestat.execution.walltime }}" dependency: and: sh_mrms_obs_available: &all_get_obs_mrms_complete @@ -161,7 +173,9 @@ metatask_GenEnsProd_EnsembleStat_SFC_UPA: FCST_LEVEL: 'all' FCST_THRESH: 'all' OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.genensprod.default.walltime }}" + memory: '{{ genensprod.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ genensprod.execution.tasks_per_node }}' + walltime: "{{ genensprod.execution.walltime }}" dependency: metataskdep_check_post_output: <<: *check_post_output @@ -170,7 +184,9 @@ metatask_GenEnsProd_EnsembleStat_SFC_UPA: envars: <<: *envars_GenEnsProd_NDAS METPLUSTOOLNAME: 'ENSEMBLESTAT' - walltime: "{{ verification_resources.execution.ensemble.ensemblestat.walltime }}" + memory: '{{ ensemblestat.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ ensemblestat.execution.tasks_per_node }}' + walltime: "{{ ensemblestat.execution.walltime }}" dependency: and: # Check that the flag files that indicate that the Pb2NC tasks diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index af521202..180c3485 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -575,13 +575,6 @@ verification_resources: nodes: 1 tasks_per_node: 1 ensemble: - genensprod: - default: - walltime: 04:45:00 - ndas: - walltime: 02:30:00 - ensemblestat: - walltime: 01:00:00 gridstat: walltime: 01:00:00 pointstat: @@ -1005,7 +998,7 @@ gridstat: walltime: 01:00:00 memory: '{{ 2 * gridstat.TASKS }}G' # Number of GRIDSTAT instances to run in parallel - TASKS: 1 + TASKS: '{{ verification.VX_TASKS }}' pointstat: # Runtime settings for POINTSTAT jobs @@ -1014,7 +1007,7 @@ pointstat: walltime: 01:00:00 memory: '{{ 2 * pointstat.TASKS }}G' # Number of POINTSTAT instances to run in parallel - TASKS: 1 + TASKS: '{{ verification.VX_TASKS }}' point2grid: execution: @@ -1031,6 +1024,24 @@ point2grid: # Number of POINT2GRID instances to run in parallel TASKS: '{{ verification.VX_TASKS }}' +genensprod: + # Runtime settings for GenEnsProd jobs + execution: + tasks_per_node: '{{ genensprod.TASKS }}' + walltime: 02:00:00 + memory: '{{ 2 * genensprod.TASKS }}G' + # Number of GenEnsProd instances to run in parallel + TASKS: '{{ verification.VX_TASKS }}' + +ensemblestat: + # Runtime settings for deterministic EnsembleStat jobs + execution: + tasks_per_node: '{{ ensemblestat.TASKS }}' + walltime: 01:00:00 + memory: '{{ 2 * ensemblestat.TASKS }}G' + # Number of EnsembleStat instances to run in parallel + TASKS: '{{ verification.VX_TASKS }}' + regriddataplane: # Runtime settings for REGRIDDATAPLANE jobs execution: @@ -1038,7 +1049,7 @@ regriddataplane: walltime: 00:30:00 memory: '{{ 10 * regriddataplane.TASKS }}G' # Number of REGRIDDATAPLANE instances to run in parallel - TASKS: 1 + TASKS: '{{ verification.VX_TASKS }}' # Regridding method. See https://metplus.readthedocs.io/projects/met/en/latest/Users_Guide/reformat_grid.html#optional-arguments-for-regrid-data-plane for valid options REGRID_METHOD: MAXGAUSS @@ -1057,7 +1068,7 @@ mode: # Memory to allocate to MODE job memory: '{{ 10 * mode.TASKS }}G' # Number of MODE instances to run in parallel - TASKS: 1 + TASKS: '{{ verification.VX_TASKS }}' # For more information on MODE variables see the MODE documentation: # https://metplus.readthedocs.io/projects/met/en/latest/Users_Guide/mode.html From 2d74ee9ca7f68d03a18d29c6a466fa299b8be160 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Wed, 8 Jul 2026 04:37:51 +0000 Subject: [PATCH 12/25] Only check for obs_dir for ensemblestat --- scripts/genensprod_or_ensemblestat.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/genensprod_or_ensemblestat.py b/scripts/genensprod_or_ensemblestat.py index e3b55400..29883012 100644 --- a/scripts/genensprod_or_ensemblestat.py +++ b/scripts/genensprod_or_ensemblestat.py @@ -76,8 +76,9 @@ def genensprod_or_ensemblestat( vxcfg = cfg["verification"] enscfg = cfg["ensemble"] - if not Path(obs_dir).is_dir(): - raise FileNotFoundError(f"{obs_dir=} does not exist or is not a directory") + if metplus_tool_name == "ensemble_stat": + if not Path(obs_dir).is_dir(): + raise FileNotFoundError(f"{obs_dir=} does not exist or is not a directory") geom, _, _, met_out_name, met_filedir_name = set_vx_params(obtype, field_group, accum_hh) From ddf8f8b4553818946fb1d190ef48f5b31fd566db Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Wed, 8 Jul 2026 04:54:35 +0000 Subject: [PATCH 13/25] Ensure that numprocs is always an integer --- scripts/genensprod_or_ensemblestat.py | 2 +- scripts/gridstat_or_pointstat.py | 2 +- scripts/mode.py | 2 +- scripts/pb2nc_obs.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/genensprod_or_ensemblestat.py b/scripts/genensprod_or_ensemblestat.py index 29883012..d3711b5c 100644 --- a/scripts/genensprod_or_ensemblestat.py +++ b/scripts/genensprod_or_ensemblestat.py @@ -232,7 +232,7 @@ def genensprod_or_ensemblestat( "vx_config_dict": vx_config_dict, } - numprocs=vxcfg['VX_TASKS'] + numprocs=int(vxcfg['VX_TASKS']) conf_files = render_metplus_confs(cfg,settings,metplus_config_tmpl_fn,vx_leadhr_list,numprocs) lgr.debug(f"{conf_files=}") diff --git a/scripts/gridstat_or_pointstat.py b/scripts/gridstat_or_pointstat.py index e356b413..a36554ee 100644 --- a/scripts/gridstat_or_pointstat.py +++ b/scripts/gridstat_or_pointstat.py @@ -292,7 +292,7 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, 'vx_config_dict': vx_config_dict } - numprocs = cfg[metplus_tool_camel_case.lower()]["TASKS"] + numprocs = int(cfg[metplus_tool_camel_case.lower()]["TASKS"]) conf_files = render_metplus_confs(cfg,settings,metplus_config_tmpl_fn,vx_leadhr_list,numprocs) lgr.debug(f"{conf_files=}") diff --git a/scripts/mode.py b/scripts/mode.py index 57ab3907..33e427dc 100644 --- a/scripts/mode.py +++ b/scripts/mode.py @@ -170,7 +170,7 @@ def mode(config_file,cdate,field_group,obtype): 'merge_flag': modecfg["MERGE_FLAG"], } - numprocs=modecfg['TASKS'] + numprocs=int(modecfg['TASKS']) conf_files = render_metplus_confs(cfg,settings,metplus_config_tmpl_fn,vx_leadhr_list, len(vx_leadhr_list)) lgr.debug(f"{conf_files=}") diff --git a/scripts/pb2nc_obs.py b/scripts/pb2nc_obs.py index 2cbb1fd0..237f6520 100644 --- a/scripts/pb2nc_obs.py +++ b/scripts/pb2nc_obs.py @@ -113,7 +113,7 @@ def pb2nc(config_file: str, cycle_date: str, obtype: str, verbose: bool = False) settings, metplus_config_tmpl_fn, vx_leadhr_list, - vxcfg["VX_TASKS"], + int(vxcfg["VX_TASKS"]), ) # Run METplus for each generated config file From 6314e525714840ca77340017f4e8b979edfbe656 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Fri, 10 Jul 2026 19:18:37 +0000 Subject: [PATCH 14/25] Remove unused/deprecated shell scripts --- ush/get_metplus_tool_name.sh | 124 ------------- ush/set_vx_params.sh | 327 ----------------------------------- 2 files changed, 451 deletions(-) delete mode 100644 ush/get_metplus_tool_name.sh delete mode 100644 ush/set_vx_params.sh diff --git a/ush/get_metplus_tool_name.sh b/ush/get_metplus_tool_name.sh deleted file mode 100644 index f4af028e..00000000 --- a/ush/get_metplus_tool_name.sh +++ /dev/null @@ -1,124 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that takes as input the name of a MET/METplus -# tool spelled in upper flat case, i.e. all-caps and without separators -# (e.g. METPLUSTOOLNAME) and returns that name converted to the following -# cases: -# -# 1) Snake case, i.e. in all lower-case with underscores as word separators, -# e.g. metplus_tool_name. -# 2) Pascal case, i.e. without separators and with the first letter of -# each word capitalized, e.g. MetplusToolName. -# 3) Screaming snake case, i.e. in all upper-case with underscores as -# word separators, e.g. METPLUS_TOOL_NAME. -# -#----------------------------------------------------------------------- -# -function get_metplus_tool_name() { -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "METPLUSTOOLNAME" \ - "outvarname_metplus_tool_name" \ - "outvarname_MetplusToolName" \ - "outvarname_METPLUS_TOOL_NAME" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# True. -# -#----------------------------------------------------------------------- -# - print_input_args "valid_args" -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local _metplus_tool_name_ \ - _MetplusToolName_ \ - _METPLUS_TOOL_NAME_ -# -#----------------------------------------------------------------------- -# -# Create array containing set of forecast hours for which we will check -# for the existence of corresponding observation or forecast file. -# -#----------------------------------------------------------------------- -# - valid_vals_METPLUSTOOLNAME=( \ - "ASCII2NC" "PB2NC" "PCPCOMBINE" "GRIDSTAT" "POINTSTAT" "GENENSPROD" "ENSEMBLESTAT" \ - ) - check_var_valid_value "METPLUSTOOLNAME" "valid_vals_METPLUSTOOLNAME" - - case "${METPLUSTOOLNAME}" in - "ASCII2NC") - _metplus_tool_name_="ascii2nc" - _MetplusToolName_="Ascii2nc" - ;; - "PB2NC") - _metplus_tool_name_="pb2nc" - _MetplusToolName_="Pb2nc" - ;; - "PCPCOMBINE") - _metplus_tool_name_="pcp_combine" - _MetplusToolName_="PcpCombine" - ;; - "GRIDSTAT") - _metplus_tool_name_="grid_stat" - _MetplusToolName_="GridStat" - ;; - "POINTSTAT") - _metplus_tool_name_="point_stat" - _MetplusToolName_="PointStat" - ;; - "GENENSPROD") - _metplus_tool_name_="gen_ens_prod" - _MetplusToolName_="GenEnsProd" - ;; - "ENSEMBLESTAT") - _metplus_tool_name_="ensemble_stat" - _MetplusToolName_="EnsembleStat" - ;; - *) - print_err_msg_exit "\ -Generic name specified for MET/METplus tool (METPLUSTOOLNAME) is -unupported: - METPLUSTOOLNAME = \"${METPLUSTOOLNAME}\"" - ;; - esac - - _METPLUS_TOOL_NAME_=$(echo_uppercase ${_metplus_tool_name_}) -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${outvarname_metplus_tool_name}" ]; then - printf -v ${outvarname_metplus_tool_name} "%s" "${_metplus_tool_name_}" - fi - - if [ ! -z "${outvarname_MetplusToolName}" ]; then - printf -v ${outvarname_MetplusToolName} "%s" "${_MetplusToolName_}" - fi - - if [ ! -z "${outvarname_METPLUS_TOOL_NAME}" ]; then - printf -v ${outvarname_METPLUS_TOOL_NAME} "%s" "${_METPLUS_TOOL_NAME_}" - fi -} diff --git a/ush/set_vx_params.sh b/ush/set_vx_params.sh deleted file mode 100644 index ca347546..00000000 --- a/ush/set_vx_params.sh +++ /dev/null @@ -1,327 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that sets various parameters needed when -# performing verification. The way these parameters are set depends on -# the field group being verified and, if the field group consists of a -# set of cumulative fields (e.g. accumulated precipitation or accumulated -# snowfall), the accumulation interval (both of which are inputs to this -# function). -# -# The verification workflow is designed for the MET/METplus software -# (MET = Model Evaluation Tools) developed at the DTC (Developmental -# Testbed Center). -# -#----------------------------------------------------------------------- -# -function set_vx_params() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "obtype" \ - "field_group" \ - "accum_hh" \ - "outvarname_grid_or_point" \ - "outvarname_fieldname_in_obs_input" \ - "outvarname_fieldname_in_fcst_input" \ - "outvarname_fieldname_in_MET_output" \ - "outvarname_fieldname_in_MET_filedir_names" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# True. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local _grid_or_point_ \ - fieldname_in_obs_input \ - fieldname_in_fcst_input \ - fieldname_in_MET_output \ - fieldname_in_MET_filedir_names -# -#----------------------------------------------------------------------- -# -# Make sure that accum_hh is a 2-digit integer. -# -#----------------------------------------------------------------------- -# - if [ "${obtype}" = "CCPA" ] || [ "${obtype}" = "NOHRSC" ]; then - if [[ ! "${accum_hh}" =~ ^[0-9]{2}$ ]]; then - print_err_msg_exit "\ -For the given observation type (obtype), the accumulation (accum_hh) must -be a 2-digit integer: - obtype = \"${obtype}\" - accum_hh = \"${accum_hh}\"" - fi - fi -# -#----------------------------------------------------------------------- -# -# Set the parameters. Definitions: -# -# grid_or_point: -# String that is set to either "grid" or "point" depending on whether -# obs type containing the field group is gridded or point-based. -# -# fieldname_in_obs_input: -# If the field group represents a single field, this is the string used -# to search for that field in the input observation files read in by MET. -# If not, this is set to a null string. -# -# fieldname_in_fcst_input: -# If the field group represents a single field, this is the string used -# to search for that field in the input forecast files read in by MET. -# If not, this is set to a null string. -# -# fieldname_in_MET_output: -# String that will be used in naming arrays defined in MET output files -# (e.g. NetCDF, stat, etc). -# -# fieldname_in_MET_filedir_names: -# String that will be used in naming directories and files (e.g. NetCDF -# files, stat files, log files, staging directories) generated by MET -# or METplus. -# -#----------------------------------------------------------------------- -# - _grid_or_point_="" - fieldname_in_obs_input="" - fieldname_in_fcst_input="" - fieldname_in_MET_output="" - fieldname_in_MET_filedir_names="" - - case "${obtype}" in - - "CCPA") - - _grid_or_point_="grid" - case "${field_group}" in - - "APCP") - fieldname_in_obs_input="${field_group}" - fieldname_in_fcst_input="${field_group}" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}${accum_hh}h" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - "NOHRSC") - - _grid_or_point_="grid" - case "${field_group}" in - - "ASNOW") - fieldname_in_obs_input="${field_group}" - fieldname_in_fcst_input="${field_group}" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}${accum_hh}h" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - "MRMS") - - _grid_or_point_="grid" - case "${field_group}" in - - "REFC") - fieldname_in_obs_input="MergedReflectivityQCComposite" - fieldname_in_fcst_input="${field_group}" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}" - ;; - - "RETOP") - fieldname_in_obs_input="EchoTop18" - fieldname_in_fcst_input="${field_group}" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - "NDAS") - - _grid_or_point_="point" - case "${field_group}" in - - "SFC") - fieldname_in_obs_input="" - fieldname_in_fcst_input="" - fieldname_in_MET_output="ADP${field_group}" - fieldname_in_MET_filedir_names="ADP${field_group}" - ;; - - "UPA") - fieldname_in_obs_input="" - fieldname_in_fcst_input="" - fieldname_in_MET_output="ADP${field_group}" - fieldname_in_MET_filedir_names="ADP${field_group}" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - "AERONET") - - _grid_or_point_="point" - case "${field_group}" in - - "AOD") - fieldname_in_obs_input="${field_group}" - fieldname_in_fcst_input="AOTK" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - "AIRNOW") - - _grid_or_point_="point" - case "${field_group}" in - - "PM25") - fieldname_in_obs_input="${field_group}" - fieldname_in_fcst_input="MASSDEN" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}" - ;; - "PM10") - fieldname_in_obs_input="${field_group}" - fieldname_in_fcst_input="MASSDEN" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field_group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype): - obtype = \"${obtype}\"" - ;; - - esac -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${outvarname_grid_or_point}" ]; then - printf -v ${outvarname_grid_or_point} "%s" "${_grid_or_point_}" - fi - - if [ ! -z "${outvarname_fieldname_in_obs_input}" ]; then - printf -v ${outvarname_fieldname_in_obs_input} "%s" "${fieldname_in_obs_input}" - fi - - if [ ! -z "${outvarname_fieldname_in_fcst_input}" ]; then - printf -v ${outvarname_fieldname_in_fcst_input} "%s" "${fieldname_in_fcst_input}" - fi - - if [ ! -z "${outvarname_fieldname_in_MET_output}" ]; then - printf -v ${outvarname_fieldname_in_MET_output} "%s" "${fieldname_in_MET_output}" - fi - - if [ ! -z "${outvarname_fieldname_in_MET_filedir_names}" ]; then - printf -v ${outvarname_fieldname_in_MET_filedir_names} "%s" "${fieldname_in_MET_filedir_names}" - fi - -} From 7746d974c243c8531265dfce7172ead8d047703d Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Fri, 10 Jul 2026 19:28:14 +0000 Subject: [PATCH 15/25] Deprecate and remove "echo_lowercase" and "echo_uppercase" bash functions --- jobs/GET_VERIF_OBS.sh | 3 +- ush/bash_utils/change_case.sh | 152 -------------------------------- ush/bash_utils/get_elem_inds.sh | 2 +- ush/launch_vx_wflow.sh | 3 +- ush/source_util_funcs.sh | 9 -- 5 files changed, 5 insertions(+), 164 deletions(-) delete mode 100644 ush/bash_utils/change_case.sh diff --git a/jobs/GET_VERIF_OBS.sh b/jobs/GET_VERIF_OBS.sh index 6cf198bc..c6956987 100755 --- a/jobs/GET_VERIF_OBS.sh +++ b/jobs/GET_VERIF_OBS.sh @@ -77,6 +77,7 @@ echo "CALLING: ${cmd[*]}" #----------------------------------------------------------------------- # mkdir -p ${WFLOW_FLAG_FILES_DIR} -file_bn="get_obs_$(echo_lowercase ${OBTYPE})" +# ${VARNAME,,} converts contents of VARNAME to lowercase +file_bn="get_obs_${OBTYPE,,}" touch "${WFLOW_FLAG_FILES_DIR}/${file_bn}_${YYMMDD}_complete.txt" diff --git a/ush/bash_utils/change_case.sh b/ush/bash_utils/change_case.sh deleted file mode 100644 index 27d8af6c..00000000 --- a/ush/bash_utils/change_case.sh +++ /dev/null @@ -1,152 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines functions used to change string to all uppercase or -# all lowercase -# -#----------------------------------------------------------------------- -# - - -# -#----------------------------------------------------------------------- -# -# Function to echo the given string as an uppercase string -# -#----------------------------------------------------------------------- -# -function echo_uppercase() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -# Get input string - - local input - - if [ "$#" -eq 1 ]; then - - input="$1" - -# -#----------------------------------------------------------------------- -# -# If no arguments or more than one, print out a usage message and exit. -# -#----------------------------------------------------------------------- -# - else - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} string - -where: - - string: - This is the string that should be converted to uppercase and echoed. -" - - fi - -# Echo the input string as upperercase - -echo $input| tr '[a-z]' '[A-Z]' - -} - - -# -#----------------------------------------------------------------------- -# -# Function to echo the given string as a lowercase string -# -#----------------------------------------------------------------------- -# -function echo_lowercase() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -# Get input string - - local input - - if [ "$#" -eq 1 ]; then - - input="$1" - -# -#----------------------------------------------------------------------- -# -# If no arguments or more than one, print out a usage message and exit. -# -#----------------------------------------------------------------------- -# - else - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} string - -where: - - string: - This is the string that should be converted to lowercase and echoed. -" - - fi - -# Echo the input string as lowercase - -echo $input| tr '[A-Z]' '[a-z]' - - -} - diff --git a/ush/bash_utils/get_elem_inds.sh b/ush/bash_utils/get_elem_inds.sh index 52c10e0f..a3940238 100644 --- a/ush/bash_utils/get_elem_inds.sh +++ b/ush/bash_utils/get_elem_inds.sh @@ -114,7 +114,7 @@ The arguments to this function are defined as follows: # #----------------------------------------------------------------------- # - inds_to_return=$(echo_lowercase $inds_to_return) + inds_to_return="${inds_to_return,,}" valid_vals_inds_to_return=( "first" "last" "all" ) check_var_valid_value "inds_to_return" "valid_vals_inds_to_return" # diff --git a/ush/launch_vx_wflow.sh b/ush/launch_vx_wflow.sh index 38ac9722..c1d2fecc 100644 --- a/ush/launch_vx_wflow.sh +++ b/ush/launch_vx_wflow.sh @@ -104,7 +104,8 @@ expt_name="${EXPT_SUBDIR}" # #----------------------------------------------------------------------- # -machine=$(echo_lowercase $MACHINE) +# Convert machine name to lowercase +machine="${MACHINE,,}" source ${USHdir}/../setup_conda.sh diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index 54bc20cb..5b67697c 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -52,15 +52,6 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # -# Source the file containing the functions that will echo given strings -# as uppercase or lowercase -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/change_case.sh -# -#----------------------------------------------------------------------- -# # Source the file containing the function that checks for preexisting # directories or files and handles them according to a specified method # (which can be one of "delete", "rename", and "quit"). From 562d342f697e5defd578fb44f5e0f08658094124 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Fri, 10 Jul 2026 19:50:29 +0000 Subject: [PATCH 16/25] Deprecate and remove print_info_msg() bash function per Issue #15, replace with printf --- jobs/ASCII2NC_OBS.sh | 4 +- jobs/CHECK_POST_OUTPUT.sh | 2 +- jobs/GENENSPROD_OR_ENSEMBLESTAT.sh | 2 +- jobs/GET_VERIF_OBS.sh | 4 +- jobs/GRIDSTAT_OR_POINTSTAT.sh | 2 +- jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh | 2 +- jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh | 2 +- jobs/MODE.sh | 2 +- jobs/PB2NC_OBS.sh | 4 +- jobs/PCPCOMBINE.sh | 2 +- jobs/POINT2GRID.sh | 2 +- jobs/REGRIDDATAPLANE.sh | 2 +- jobs/TCPAIRS.sh | 2 +- jobs/TCRMW.sh | 2 +- jobs/TCSTAT.sh | 2 +- ush/bash_utils/check_for_preexist_dir_file.sh | 5 +- ush/bash_utils/print_input_args.sh | 8 +- ush/bash_utils/print_msg.sh | 105 ------------------ 18 files changed, 26 insertions(+), 128 deletions(-) diff --git a/jobs/ASCII2NC_OBS.sh b/jobs/ASCII2NC_OBS.sh index 4cbb9f66..5f0e272b 100755 --- a/jobs/ASCII2NC_OBS.sh +++ b/jobs/ASCII2NC_OBS.sh @@ -51,7 +51,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -64,5 +64,5 @@ python $SCRIPTSdir/ascii2nc_obs.py ${VERBOSE_FLAG} \ --cycle_date="${YYMMDD}${HH}" \ --obtype="${OBTYPE}" || \ print_err_msg_exit "\ -Call to \"ascii2nc_obs.sh\" from \"${scrfunc_fn}\" failed." +Call to \"ascii2nc_obs.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/CHECK_POST_OUTPUT.sh b/jobs/CHECK_POST_OUTPUT.sh index 18d441b7..044772b6 100755 --- a/jobs/CHECK_POST_OUTPUT.sh +++ b/jobs/CHECK_POST_OUTPUT.sh @@ -54,7 +54,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh b/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh index cc54863d..60b0fc91 100755 --- a/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh +++ b/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/GET_VERIF_OBS.sh b/jobs/GET_VERIF_OBS.sh index c6956987..70b6a2e7 100755 --- a/jobs/GET_VERIF_OBS.sh +++ b/jobs/GET_VERIF_OBS.sh @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -66,7 +66,7 @@ cmd=( --obtype "${OBTYPE}" --obs_day "${YYMMDD}" ) -echo "CALLING: ${cmd[*]}" +printf "CALLING: ${cmd[*]}" "${cmd[@]}" || print_err_msg_exit "Error calling get_obs.py" # #----------------------------------------------------------------------- diff --git a/jobs/GRIDSTAT_OR_POINTSTAT.sh b/jobs/GRIDSTAT_OR_POINTSTAT.sh index ee7a27f4..d61e1ca8 100755 --- a/jobs/GRIDSTAT_OR_POINTSTAT.sh +++ b/jobs/GRIDSTAT_OR_POINTSTAT.sh @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh b/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh index 3239671a..319c809e 100755 --- a/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh +++ b/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh @@ -51,7 +51,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh b/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh index 793d21d3..562649c8 100755 --- a/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh +++ b/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh @@ -51,7 +51,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/MODE.sh b/jobs/MODE.sh index e2029c68..175d945e 100755 --- a/jobs/MODE.sh +++ b/jobs/MODE.sh @@ -54,7 +54,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/PB2NC_OBS.sh b/jobs/PB2NC_OBS.sh index b5907eac..fc56d0eb 100755 --- a/jobs/PB2NC_OBS.sh +++ b/jobs/PB2NC_OBS.sh @@ -51,7 +51,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -64,5 +64,5 @@ python $SCRIPTSdir/pb2nc_obs.py ${VERBOSE_FLAG} \ --cycle_date="${YYMMDD}${HH}" \ --obtype="${OBTYPE}" || \ print_err_msg_exit "\ -Call to \"pb2nc_obs.sh\" from \"${scrfunc_fn}\" failed." +Call to \"pb2nc_obs.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/PCPCOMBINE.sh b/jobs/PCPCOMBINE.sh index 4b5ce647..c64f1cb2 100755 --- a/jobs/PCPCOMBINE.sh +++ b/jobs/PCPCOMBINE.sh @@ -52,7 +52,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/POINT2GRID.sh b/jobs/POINT2GRID.sh index 434e8e5d..f6431f12 100755 --- a/jobs/POINT2GRID.sh +++ b/jobs/POINT2GRID.sh @@ -49,7 +49,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/REGRIDDATAPLANE.sh b/jobs/REGRIDDATAPLANE.sh index 61558a9c..2a70898a 100755 --- a/jobs/REGRIDDATAPLANE.sh +++ b/jobs/REGRIDDATAPLANE.sh @@ -54,7 +54,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/TCPAIRS.sh b/jobs/TCPAIRS.sh index 5d6887ee..22ccc9ad 100755 --- a/jobs/TCPAIRS.sh +++ b/jobs/TCPAIRS.sh @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/TCRMW.sh b/jobs/TCRMW.sh index 6a557396..3db282b9 100755 --- a/jobs/TCRMW.sh +++ b/jobs/TCRMW.sh @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/TCSTAT.sh b/jobs/TCSTAT.sh index 7feba1eb..92381b8d 100755 --- a/jobs/TCSTAT.sh +++ b/jobs/TCSTAT.sh @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/ush/bash_utils/check_for_preexist_dir_file.sh b/ush/bash_utils/check_for_preexist_dir_file.sh index eaf4b4b1..320e93e1 100644 --- a/ush/bash_utils/check_for_preexist_dir_file.sh +++ b/ush/bash_utils/check_for_preexist_dir_file.sh @@ -119,12 +119,13 @@ where the arguments are defined as follows: old_dir_or_file="${dir_or_file}_old${old_indx}" done - print_info_msg "$VERBOSE" " + if [ "$VERBOSE" -eq "True" ]; then + echo " Specified directory or file (dir_or_file) already exists: dir_or_file = \"${dir_or_file}\" Moving (renaming) preexisting directory or file to: old_dir_or_file = \"${old_dir_or_file}\"" - + fi mv "${dir_or_file}" "${old_dir_or_file}" ;; # diff --git a/ush/bash_utils/print_input_args.sh b/ush/bash_utils/print_input_args.sh index 9ea19f14..47321622 100644 --- a/ush/bash_utils/print_input_args.sh +++ b/ush/bash_utils/print_input_args.sh @@ -160,14 +160,16 @@ have been set as follows: #----------------------------------------------------------------------- # # If a global variable named DEBUG is not defined, print out the message. -# If it is defined, print out the message only if DEBUG is set to "TRUE". +# If it is defined, print out the message only if DEBUG is set to "True". # #----------------------------------------------------------------------- # if [ -z ${DEBUG+x} ]; then - print_info_msg "$msg" + printf "$msg" else - print_info_msg "$DEBUG" "$msg" + if [ "${DEBUG}" -eq "True" ]; then + printf "$msg" + fi fi } diff --git a/ush/bash_utils/print_msg.sh b/ush/bash_utils/print_msg.sh index 3c54deb7..c9d6afff 100644 --- a/ush/bash_utils/print_msg.sh +++ b/ush/bash_utils/print_msg.sh @@ -7,111 +7,6 @@ #----------------------------------------------------------------------- # -# -#----------------------------------------------------------------------- -# -# Function to print informational messages using printf. -# -#----------------------------------------------------------------------- -# -function print_info_msg() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local verbose \ - info_msg -# -#----------------------------------------------------------------------- -# -# If one argument is supplied, we assume it is the message to print out. -# between informational lines that are always printed. -# -#----------------------------------------------------------------------- -# - if [ "$#" -eq 1 ]; then - - verbose="True" - info_msg="$1" - - elif [ "$#" -eq 2 ]; then - - verbose="$1" - info_msg="$2" -# -#----------------------------------------------------------------------- -# -# If no arguments or more than two arguments are supplied, print out a -# usage message and exit. -# -#----------------------------------------------------------------------- -# - else - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} [verbose] info_msg - -where the arguments are defined as follows: - - verbose: - This is an optional argument. If set to \"TRUE\", info_msg will be - printed to stdout. Otherwise, info_msg will not be printed. - - info_msg: - This is the informational message to print to stdout. - -This function prints an informational message to stdout. If one argu- -ment is passed in, then that argument is assumed to be info_msg and is -printed. If two arguments are passed in, then the first is assumed to -be verbose and the second info_msg. In this case, info_msg gets printed -only if verbose is set to \"TRUE\". -" - - fi -# -#----------------------------------------------------------------------- -# -# If verbose is set to "TRUE", print out the message. -# -#----------------------------------------------------------------------- -# - if [ "$verbose" = "TRUE" ]; then - printf "%s\n" "${info_msg}" - fi -} - - - # #----------------------------------------------------------------------- From cf20e25918c86919ce28bf668a48d74d676ad541 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Fri, 10 Jul 2026 20:35:39 +0000 Subject: [PATCH 17/25] Deprecate and remove more unneeded bash functions: check_for_preexist_dir_file(), check_var_valid_value(), get_elem_inds(), is_element_of(), print_input_args(), and process_args() --- ush/bash_utils/check_for_preexist_dir_file.sh | 162 -------- ush/bash_utils/check_var_valid_value.sh | 128 ------ ush/bash_utils/get_elem_inds.sh | 162 -------- ush/bash_utils/is_element_of.sh | 135 ------- ush/bash_utils/print_input_args.sh | 176 -------- ush/bash_utils/process_args.sh | 380 ------------------ ush/launch_vx_wflow.sh | 34 +- ush/source_util_funcs.sh | 55 --- 8 files changed, 17 insertions(+), 1215 deletions(-) delete mode 100644 ush/bash_utils/check_for_preexist_dir_file.sh delete mode 100644 ush/bash_utils/check_var_valid_value.sh delete mode 100644 ush/bash_utils/get_elem_inds.sh delete mode 100644 ush/bash_utils/is_element_of.sh delete mode 100644 ush/bash_utils/print_input_args.sh delete mode 100644 ush/bash_utils/process_args.sh diff --git a/ush/bash_utils/check_for_preexist_dir_file.sh b/ush/bash_utils/check_for_preexist_dir_file.sh deleted file mode 100644 index 320e93e1..00000000 --- a/ush/bash_utils/check_for_preexist_dir_file.sh +++ /dev/null @@ -1,162 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that checks for a preexisting version of -# the specified directory or file and, if present, deals with it according -# to the specified method. -# -#----------------------------------------------------------------------- -# -function check_for_preexist_dir_file() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 2 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} dir_or_file method - -where the arguments are defined as follows: - - dir_or_file: - Name of directory or file to check for a preexisting version. - - method: - String specifying the action to take if a preexisting version of - dir_or_file is found. Valid values are \"delete\", \"reuse\", \"rename\", and \"quit\". -" - - fi -# -#----------------------------------------------------------------------- -# -# Set local variables to appropriate input arguments. -# -#----------------------------------------------------------------------- -# - local dir_or_file="$1" - local method="$2" -# -#----------------------------------------------------------------------- -# -# Set the valid values that method can take on and check to make sure -# the specified value is valid. -# -#----------------------------------------------------------------------- -# - local valid_vals_method=( "delete" "reuse" "rename" "quit" ) - check_var_valid_value "method" "valid_vals_method" -# -#----------------------------------------------------------------------- -# -# Check if dir_or_file already exists. If so, act depending on the value -# of method. -# -#----------------------------------------------------------------------- -# - if [ -e "${dir_or_file}" ]; then - - case "$method" in -# -#----------------------------------------------------------------------- -# -# If method is set to "delete", we remove the preexisting directory or -# file. -# -#----------------------------------------------------------------------- -# - "delete") - - rm -rf "${dir_or_file}" - ;; -# -#----------------------------------------------------------------------- -# -# If method is set to "rename", we move (rename) the preexisting directory -# or file. -# -#----------------------------------------------------------------------- -# - "rename") - - local i=1 - local old_indx=$( printf "%03d" "$i" ) - local old_dir_or_file="${dir_or_file}_old${old_indx}" - while [ -e "${old_dir_or_file}" ]; do - i=$[$i+1] - old_indx=$( printf "%03d" "$i" ) - old_dir_or_file="${dir_or_file}_old${old_indx}" - done - - if [ "$VERBOSE" -eq "True" ]; then - echo " -Specified directory or file (dir_or_file) already exists: - dir_or_file = \"${dir_or_file}\" -Moving (renaming) preexisting directory or file to: - old_dir_or_file = \"${old_dir_or_file}\"" - fi - mv "${dir_or_file}" "${old_dir_or_file}" - ;; -# -#----------------------------------------------------------------------- -# -# If method is set to "reuse", keep preexisting directory intact. -# -#----------------------------------------------------------------------- -# - "reuse") - ;; -# -#----------------------------------------------------------------------- -# -# If method is set to "quit", we simply exit with a nonzero status. Note -# that "exit" is different than "return" because it will cause the calling -# script (in which this file/function is sourced) to stop execution. -# -#----------------------------------------------------------------------- -# - "quit") - - print_err_msg_exit "\ -Specified directory or file (dir_or_file) already exists: - dir_or_file = \"${dir_or_file}\"" - ;; - - esac - - fi - -} - - diff --git a/ush/bash_utils/check_var_valid_value.sh b/ush/bash_utils/check_var_valid_value.sh deleted file mode 100644 index c8abaec0..00000000 --- a/ush/bash_utils/check_var_valid_value.sh +++ /dev/null @@ -1,128 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This function checks whether the specified variable contains a valid -# value (where the set of valid values is also specified). -# -#----------------------------------------------------------------------- -# -function check_var_valid_value() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} var_name valid_var_values_array_name [msg] - -where the arguments are defined as follows: - - var_name: - The name of the variable whose value we want to check for validity. - - valid_var_values_array_name: - The name of the array containing a list of valid values that var_name - can take on. - - msg - Optional argument specifying the first portion of the error message to - print out if var_name does not have a valid value. -" - - fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local var_name \ - valid_var_values_array_name \ - var_value \ - valid_var_values_at \ - valid_var_values \ - err_msg \ - valid_var_values_str -# -#----------------------------------------------------------------------- -# -# Set local variable values. -# -#----------------------------------------------------------------------- -# - var_name="$1" - valid_var_values_array_name="$2" - - var_value=${!var_name} - valid_var_values_at="$valid_var_values_array_name[@]" - valid_var_values=("${!valid_var_values_at:-}") - - if [ "$#" -eq 3 ]; then - err_msg="$3" - else - err_msg="\ -The value specified in ${var_name} is not supported: - ${var_name} = \"${var_value}\"" - fi -# -#----------------------------------------------------------------------- -# -# If var_value contains a dollar sign, we assume the corresponding variable -# (var_name) is a template variable, i.e. one whose value contains a -# reference to another variable, e.g. -# -# MY_VAR='\${ANOTHER_VAR}' -# -# In this case, we do nothing since it does not make sense to check -# whether var_value is a valid value (since its contents have not yet -# been expanded). If var_value doesn't contain a dollar sign, it must -# contain a literal string. In this case, we check whether it is equal -# to one of the elements of the array valid_var_values. If not, we -# print out an error message and exit the calling script. -# -#----------------------------------------------------------------------- -# - if [[ "${var_value}" != *'$'* ]]; then - is_element_of "valid_var_values" "${var_value}" || { \ - valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}"); - print_err_msg_exit "\ -${err_msg} -${var_name} must be set to one of the following: - ${valid_var_values_str}"; \ - } - fi - -} - diff --git a/ush/bash_utils/get_elem_inds.sh b/ush/bash_utils/get_elem_inds.sh deleted file mode 100644 index a3940238..00000000 --- a/ush/bash_utils/get_elem_inds.sh +++ /dev/null @@ -1,162 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# For a description of this function, see the usage message below. -# -#----------------------------------------------------------------------- -# -function get_elem_inds() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 2 ] && [ "$#" -ne 3 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} array_name str_to_match [inds_to_return] - -This function prints to stdout the indices of those elements of a given -array that match (i.e. are equal to) a given string. It can return the -index of the first matched element, the index of the last matched ele- -ment, or the indices of all matched elements. The return code -from this function will be zero if at least one match is found and non- -zero if no matches are found. - -The arguments to this function are defined as follows: - - array_name: - The name of the array in which to search for str_to_match. Note that - this is the name of the array, not the array itself. - - str_to_match: - The string to match in array_name. - - inds_to_return: - Optional argument that specifies the subset of the indices of the ar- - ray elements that match str_to_match to print to stdout. Must be set - to \"first\", \"last\", or \"all\" (but is case insensitive). If set to - \"first\", the index of only the first matched element is printed. If - set to \"last\", the index of only the last matched element is printed. - If set to \"all\", the indices of all matched elements are printed. De- - fault is \"all\". -" - - fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local array_name \ - str_to_match \ - inds_to_return \ - array_name_at \ - array \ - valid_vals_inds_to_return \ - match_inds \ - num_matches \ - num_elems \ - n -# -#----------------------------------------------------------------------- -# -# Set local variables to appropriate input arguments. -# -#----------------------------------------------------------------------- -# - array_name="$1" - str_to_match="$2" - - inds_to_return="all" - if [ "$#" -eq 3 ]; then - inds_to_return="$3" - fi - - array_name_at="$array_name[@]" - array=("${!array_name_at}") -# -#----------------------------------------------------------------------- -# -# Change all letters in inds_to_return to lower case. Then check whe- -# ther it has a valid value. -# -#----------------------------------------------------------------------- -# - inds_to_return="${inds_to_return,,}" - valid_vals_inds_to_return=( "first" "last" "all" ) - check_var_valid_value "inds_to_return" "valid_vals_inds_to_return" -# -#----------------------------------------------------------------------- -# -# Initialize the array match_inds to an empty array. This will contain -# the indices of any matched elements. Then loop through the elements -# of the given array and check whether each element is equal to str_to_- -# match. If so, save the index of that element as an element of match_- -# inds. If inds_to_return is set to "first", we break out of the loop -# after finding the first match in order to not waste computation. -# -#----------------------------------------------------------------------- -# - match_inds=() - num_matches=0 - - num_elems=${#array[@]} - for (( n=0; n<${num_elems}; n++ )); do - if [ "${array[$n]}" = "${str_to_match}" ]; then - match_inds[${num_matches}]=$n - num_matches=$((num_matches+1)) - if [ "${inds_to_return}" = "first" ]; then - break - fi - fi - done -# -#----------------------------------------------------------------------- -# -# Find the number of matches. If it is more than zero, print the indi- -# ces of the matched elements to stdout. -# -#----------------------------------------------------------------------- -# - num_matches=${#match_inds[@]} - if [ ${num_matches} -gt 0 ]; then - if [ "${inds_to_return}" = "last" ]; then - printf "%s\n" "${match_inds[-1]}" - else - printf "%s\n" "${match_inds[@]}" - fi - fi - -} diff --git a/ush/bash_utils/is_element_of.sh b/ush/bash_utils/is_element_of.sh deleted file mode 100644 index e2d1403e..00000000 --- a/ush/bash_utils/is_element_of.sh +++ /dev/null @@ -1,135 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# For a description of this function, see the usage message below. -# -#----------------------------------------------------------------------- -# -function is_element_of() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 2 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} array_name str_to_match - -This function checks whether the specified array contains the specified -string, i.e. whether at least one of the elements of the array is equal -to the string. The return code from this function will be zero if at -least one match is found and nonzero if no matches are found. - -The arguments to this function are defined as follows: - - array_name: - The name of the array in which to search for str_to_match. Note that - this is the name of the array, not the array itself. - - str_to_match: - The string to search for in array_name. - -Use this function in a script as follows: - - . ./is_element_of.sh - array_name=("1" "2" "3 4" "5") - - str_to_match="2" - is_element_of "${str_to_match}" array_name - echo $? # Should output 0. - - str_to_match="3 4" - is_element_of "${str_to_match}" array_name - echo $? # Should output 0. - - str_to_match="6" - is_element_of "${str_to_match}" array_name - echo $? # Should output 1. -" - - fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local array_name \ - str_to_match \ - array_name_at \ - array \ - found_match \ - num_elems \ - n -# -#----------------------------------------------------------------------- -# -# Set local variables to appropriate input arguments. -# -#----------------------------------------------------------------------- -# - array_name="$1" - str_to_match="$2" - - array_name_at="$array_name[@]" - array=("${!array_name_at:-}") -# -#----------------------------------------------------------------------- -# -# Initialize the return variable found_match to 1 (false). Then loop -# through the elements of the array and check whether each element is -# equal to str_to_match. Once a match is found, reset found_match to 0 -# (true) and break out of the loop. -# -#----------------------------------------------------------------------- -# - found_match=1 - num_elems=${#array[@]} - for (( n=0; n<${num_elems}; n++ )); do - if [ "${array[$n]}" = "${str_to_match}" ]; then - found_match=0 - break - fi - done -# -#----------------------------------------------------------------------- -# -# Return the variable found_match. -# -#----------------------------------------------------------------------- -# - return ${found_match} - -} - diff --git a/ush/bash_utils/print_input_args.sh b/ush/bash_utils/print_input_args.sh deleted file mode 100644 index 47321622..00000000 --- a/ush/bash_utils/print_input_args.sh +++ /dev/null @@ -1,176 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that prints to stdout the names and val- -# ues of a specified list of variables that are the valid arguments to -# the script or function that calls this function. It is mainly used -# for debugging to check that the argument values passed to the calling -# script/function have been set correctly. Note that if a global varia- -# ble named VERBOSE is not defined, the message will be printed out. If -# a global variable named VERBOSE is defined, then the message will be -# printed out only if VERBOSE is set to TRUE. -# -#----------------------------------------------------------------------- -# -function print_input_args() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Get information about the script or function that calls this function. -# Note that caller_name will be set as follows: -# -# 1) If the caller is a function, caller_name will be set to the name of -# that function. -# 2) If the caller is a sourced script, caller_name will be set to -# "script". Note that a sourced script cannot be the top level -# script since by defintion, it is sourced by another script or func- -# tion. -# 3) If the caller is the top-level script, caller_name will be set to -# "main". -# -# Thus, if caller_name is set to "script" or "main", the caller is a -# script, and if it is set to anything else, the caller is a function. -# -#----------------------------------------------------------------------- -# - local caller_fp=$( $READLINK -f "${BASH_SOURCE[1]}" ) - local caller_fn=$( basename "${caller_fp}" ) - local caller_dir=$( dirname "${caller_fp}" ) - local caller_name="${FUNCNAME[1]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 1 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} array_name_valid_caller_args - -where array_name_valid_caller_args is the name of the array containing -the names of valid arguments that can be passed to the calling script or -function. -" - - fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local array_name_valid_caller_args \ - valid_caller_args \ - script_or_function \ - msg \ - num_valid_args \ - i \ - line -# -#----------------------------------------------------------------------- -# -# Get the array containing the list of valid argument names that can be -# passed to the calling script/function. Note that if this is set to an -# empty array in the calling script/function [e.g. using the notation -# some_array=()], then it will (for whatever reason) not be defined in -# the scope of this function. For this reason, we need the if-statement -# below to check for this case. Then get the number of valid arguments. -# -#----------------------------------------------------------------------- -# - array_name_valid_caller_args="$1" - valid_arg_names_0th="${array_name_valid_caller_args}[0]" - if [ ${!valid_arg_names_0th:-"__unset__"} = "__unset__" ]; then - valid_caller_args=() - else - valid_caller_args_at="${array_name_valid_caller_args}[@]" - valid_caller_args=("${!valid_caller_args_at}") - fi - num_valid_caller_args="${#valid_caller_args[@]}" -# -#----------------------------------------------------------------------- -# -# Set the message to print to stdout. Note that if the number of valid -# arguments is zero, then we simply print out a message stating this fact. -# -#----------------------------------------------------------------------- -# - if [ "${caller_name}" = "main" ] || \ - [ "${caller_name}" = "script" ]; then - script_or_function="the script" - else - script_or_function="function \"${caller_name}\"" - fi - - if [ ${num_valid_caller_args} -eq 0 ]; then - - msg=" -No arguments have been passed to ${script_or_function} in file - - \"${caller_fp}\" -" - - else - - msg=" -The arguments to ${script_or_function} in file - - \"${caller_fp}\" - -have been set as follows: -" - - for (( i=0; i<${num_valid_caller_args}; i++ )); do - line=$( declare -p "${valid_caller_args[$i]}" ) - msg=$( printf "%s\n%s" "$msg" " $line" ) - done - - fi -# -#----------------------------------------------------------------------- -# -# If a global variable named DEBUG is not defined, print out the message. -# If it is defined, print out the message only if DEBUG is set to "True". -# -#----------------------------------------------------------------------- -# - if [ -z ${DEBUG+x} ]; then - printf "$msg" - else - if [ "${DEBUG}" -eq "True" ]; then - printf "$msg" - fi - fi - -} - diff --git a/ush/bash_utils/process_args.sh b/ush/bash_utils/process_args.sh deleted file mode 100644 index c6cb44ff..00000000 --- a/ush/bash_utils/process_args.sh +++ /dev/null @@ -1,380 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This function processes a list of variable name and value pairs passed -# to it as a set of arguments, starting with the second argument. We -# refer to these pairs as argument-value pairs (or "arg-val" pairs for -# short) because the variable names in these pairs represent the names -# of arguments to the script or function that calls this function (which -# we refer to here as the "caller"). The first argument to this func- -# tion being the name of an array that contains a list of valid argument -# names that the caller is allowed to accept. Each arg-val pair must -# have the form -# -# ARG_NAME=VAR_VALUE -# -# where ARG_NAME is the name of an argument and VAR_VALUE is the value -# to set that argument to. For each arg-val pair, this function creates -# a global variable named ARG_NAME and assigns to it the value VAR_VAL- -# UE. -# -# The purpose of this function is to provide a mechanism by which a pa- -# rent script, say parent.sh, can pass variable values to a child script -# or function, say child.sh, that makes it very clear which arguments of -# child.sh are being set and to what values. For example, parent.sh can -# call child.sh as follows: -# -# ... -# child.sh arg3="Hello" arg2="bye" arg4=("this" "is" "an" "array") -# ... -# -# Then child.sh can use this function (process_args) as follows to pro- -# cess the arg-val pairs passed to it: -# -# ... -# valid_args=( "arg1" "arg2" "arg3" "arg4" ) -# process_args valid_args "$@" -# ... -# -# Here, valid_args is an array that defines or "declares" the argument -# list for child.sh, i.e. it defines the variable names that child.sh is -# allowed to accept as arguments. Its name is passed to process_args as -# the first argument. The "$@" appearing in the call to process_args -# passes to process_args the list of arg-val pairs that parent.sh passes -# to child.sh as the second through N-th arguments. In the example -# above, "$@" represents: -# -# arg3="Hello" arg2="bye" arg4=("this" "is" "an" "array") -# -# After the call to process_args in child.sh, the variables arg1, arg2, -# arg3, and arg4 will be set as follows in child.sh: -# -# arg1="" -# arg2="bye" -# arg3="Hello" -# arg4=("this" "is" "an" "array") -# -# Note that: -# -# 1) The set of arg-val pairs may list only a subset of the list of arg- -# uments declared in valid_args; the unlisted arguments will be set -# to the null string. In the example above, arg1 is set to the null -# string because it is not specified in any of the arg-val pairs in -# the call to child.sh in parent.sh. -# -# 2) The arg-val pairs in the call to child.sh do not have to be in the -# same order as the list of "declared" arguments in child.sh. For -# instance, in the example above, the arg-val pair for arg3 is listed -# before the one for arg2. -# -# 3) An argument can be set to an array by starting and ending the value -# portion of its arg-val pair with opening and closing parentheses, -# repsectively, and listing the elements within (each one in a set of -# double-quotes and separated fromt the next by whitespace). In the -# example above, this is done for arg4. -# -# 4) If the value portion of an arg-val pair contains an argument that -# is not defined in the array valid_args in child.sh, the call to -# process_args in child.sh will result in an error message and exit -# from the caller. -# -#----------------------------------------------------------------------- -# -function process_args() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -lt 1 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} array_name_valid_arg_names \ - arg_val_pair1 \ - ... \ - arg_val_pairN - -where the arguments are defined as follows: - - array_name_valid_arg_names: - The name of the array containing a list of valid argument names. - - arg_val_pair1 ... arg_val_pairN: - A list of N argument-value pairs. These have the form - - arg1=\"val1\" ... argN=\"valN\" - - where each argument name (argI) needs to be in the list of valid argu- - ment names specified in array_name_valid_arg_names. Note that not all - the valid arguments listed in array_name_valid_arg_names need to be - set, and the argument-value pairs can be in any order, i.e. they don't - have to follow the order of arguments listed in valid_arg_names_ar- - ray_name. -" - - fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local array_name_valid_arg_names \ - valid_arg_names_at \ - valid_arg_names \ - num_valid_args \ - num_arg_val_pairs \ - i valid_arg_name arg_already_specified \ - arg_val_pair arg_name arg_value is_array \ - err_msg cmd_line -# -#----------------------------------------------------------------------- -# -# Get the array containing the list of valid argument names that can be -# passed to the calling script/function. Note that if this is set to an -# empty array in the calling script/function [e.g. using the notation -# some_array=()], then it will (for whatever reason) not be defined in -# the scope of this function. For this reason, we need the if-statement -# below to check for this case. -# -#----------------------------------------------------------------------- -# - array_name_valid_arg_names="$1" - valid_arg_names_0th="${array_name_valid_arg_names}[0]" - if [ ${!valid_arg_names_0th:-"__unset__"} = "__unset__" ]; then - valid_arg_names=() - else - valid_arg_names_at="${array_name_valid_arg_names}[@]" - valid_arg_names=("${!valid_arg_names_at}") - fi -# -#----------------------------------------------------------------------- -# -# Get the number of valid arguments. Also, set a string containing the -# list of all valid arguments with each one placed in double quotes. -# -#----------------------------------------------------------------------- -# - num_valid_args=${#valid_arg_names[@]} - if [ ${num_valid_args} -eq 0 ]; then - valid_arg_names_str="" - else - valid_arg_names_str=$( printf "\"%s\" " "${valid_arg_names[@]}" ) - fi - -# -# Instead of the if-statement above, the following could be used, but it -# is too difficult to understand... -# -# valid_arg_names_str=$( printf "\"%s\" " ${valid_arg_names[@]+"${valid_arg_names[@]}"} ) - -# -#----------------------------------------------------------------------- -# -# Get the number of argument-value pairs (or arg-val pairs, for short) -# being passed into this function. These consist of all arguments -# starting with the 2nd, so we subtract 1 from the total number of argu- -# ments. -# -#----------------------------------------------------------------------- -# - num_arg_val_pairs=$(( $# - 1 )) -# -#----------------------------------------------------------------------- -# -# Make sure that the number of arg-val pairs is less than or equal to -# the number of valid arguments. -# -#----------------------------------------------------------------------- -# - if [ "${num_arg_val_pairs}" -gt "${num_valid_args}" ]; then - print_err_msg_exit "\ -The number of argument-value pairs specified on the command line (num_- -arg_val_pairs) must be less than or equal to the number of valid argu- -ments (num_valid_args) specified in the array valid_arg_names: - num_arg_val_pairs = ${num_arg_val_pairs} - num_valid_args = ${num_valid_args} - valid_arg_names = ( ${valid_arg_names_str})" - fi -# -#----------------------------------------------------------------------- -# -# If the number of valid arguments is zero, i.e. the array valid_arg_names -# contains no elements, then there are no script/function arguments to -# set. In this case, reset the shell options to what they were before -# entering this function and simply return to the calling script/function. -# -#----------------------------------------------------------------------- -# - if [ ${num_valid_args} -eq 0 ]; then - return - fi -# -#----------------------------------------------------------------------- -# -# Make sure that none of the elements of the array containing the list -# of valid arguments contain spaces or are empty. -# -#----------------------------------------------------------------------- -# - for (( i=0; i<${num_valid_args}; i++ )); do - - valid_arg_name="${valid_arg_names[$i]}" - -# Remove spaces (if any exist) from the current valid argument name. - valid_arg_name_no_spaces=$( \ - printf "%s\n" "${valid_arg_name}" | $SED -r -e 's/[[:space:]]//g' ) - - if [ "${valid_arg_name_no_spaces}" != "${valid_arg_name}" ]; then - print_err_msg_exit "\ -The name of an argument in the list of valid arguments (valid_arg_names) -cannot contain any spaces, but the element with index i=${i} contains at -least one space: - valid_arg_names = ( ${valid_arg_names_str}) - valid_arg_names[$i] = \"${valid_arg_names[$i]}\"" - fi - - if [ -z ${valid_arg_name} ]; then - print_err_msg_exit "\ -The list of valid arguments (valid_arg_names) cannot contain empty elements, -but the element with index i=${i} is empty: - valid_arg_names = ( ${valid_arg_names_str}) - valid_arg_names[$i] = \"${valid_arg_names[$i]}\"" - fi - - done -# -#----------------------------------------------------------------------- -# -# Initialize all valid arguments to the null string. Note that the -# scope of this initialization is global, i.e. the calling script or -# function will be aware of these initializations. Also, initialize -# each element of the array arg_already_specified to "false". This ar- -# ray keeps track of whether each valid argument has already been set -# to a value by an arg-val specification. -# -#----------------------------------------------------------------------- -# - for (( i=0; i<${num_valid_args}; i++ )); do - valid_arg_name="${valid_arg_names[$i]}" - eval ${valid_arg_name}="" - arg_already_specified[$i]="false" - done -# -#----------------------------------------------------------------------- -# -# Loop over the list of arg-val pairs and set argument values. -# -#----------------------------------------------------------------------- -# -# Set the separator used in each arg=value pair. This is simply an equal -# sign. -# - sep="=" - for arg_val_pair in "${@:2}"; do -# -# Get the argument name and its value using bash variable substitution/ -# expansion. The %% operator deletes the longest trailing portion of -# arg_val_pair that matches the pattern that follows %%, while the # -# operator deletes the shortest leading portion of arg_val_pair that -# matches the pattern that follows #. -# - arg_name=${arg_val_pair%%"$sep"*} - arg_value=${arg_val_pair#*"$sep"} -# -# If the first character of the argument's value is an opening parenthe- -# sis and its last character is a closing parenthesis, then the argument -# is an array. Check for this and set the is_array flag accordingly. -# - is_array="false" - if [ "${arg_value:0:1}" = "(" ] && \ - [ "${arg_value: -1}" = ")" ]; then - is_array="true" - fi -# -#----------------------------------------------------------------------- -# -# Make sure that the argument name specified by the current argument- -# value pair is valid. -# -#----------------------------------------------------------------------- -# - err_msg="\ -The specified argument name (arg_name) in the current argument-value -pair (arg_val_pair) is not valid: - arg_val_pair = \"${arg_val_pair}\" - arg_name = \"${arg_name}\"" - check_var_valid_value "arg_name" "valid_arg_names" "${err_msg}" -# -#----------------------------------------------------------------------- -# -# Loop through the list of valid argument names and find the one that -# the current arg-val pair corresponds to. Then set that argument to -# the specified value. -# -#----------------------------------------------------------------------- -# - for (( i=0; i<${num_valid_args}; i++ )); do - - valid_arg_name="${valid_arg_names[$i]}" - if [ "${arg_name}" = "${valid_arg_name}" ]; then -# -# Check whether the current argument has already been set by a previous -# arg-val pair on the command line. If not, proceed to set the argument -# to the specified value. If so, print out an error message and exit -# the calling script. -# - if [ "${arg_already_specified[$i]}" = "false" ]; then - arg_already_specified[$i]="true" - if [ "${is_array}" = "true" ]; then - eval ${arg_name}=${arg_value} - else - eval ${arg_name}=\"${arg_value}\" - fi - else - cmd_line=$( printf "\'%s\' " "${@:1}" ) - print_err_msg_exit "\ -The current argument has already been assigned a value on the command -line: - arg_name = \"${arg_name}\" - cmd_line = ${cmd_line} -Please assign values to arguments only once on the command line." - fi - fi - - done - - done - -} - diff --git a/ush/launch_vx_wflow.sh b/ush/launch_vx_wflow.sh index c1d2fecc..54b02002 100644 --- a/ush/launch_vx_wflow.sh +++ b/ush/launch_vx_wflow.sh @@ -70,23 +70,23 @@ export USHdir=$USHdir # #----------------------------------------------------------------------- # -# Declare arguments. -# -#----------------------------------------------------------------------- -# -valid_args=( \ - "called_from_cron" \ - ) -process_args valid_args "$@" -print_input_args "valid_args" -# -#----------------------------------------------------------------------- -# -# Make sure called_from_cron is set to a valid value. -# -#----------------------------------------------------------------------- -# -called_from_cron=${called_from_cron:-"False"} +# Parse arguments. +# +#----------------------------------------------------------------------- +# +while [ $# -gt 0 ]; do + case "$1" in + --called_from_cron=*) + called_from_cron="${1#*=}" + ;; + *) + printf "******************************\n" + printf "* Error: Invalid argument $1.*\n" + printf "******************************\n" + exit 1 + esac + shift +done # #----------------------------------------------------------------------- # diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index 5b67697c..a21ae984 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -52,34 +52,6 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # -# Source the file containing the function that checks for preexisting -# directories or files and handles them according to a specified method -# (which can be one of "delete", "rename", and "quit"). -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/check_for_preexist_dir_file.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that searches an array for a -# specified string. -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/is_element_of.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that gets the indices of those -# elements of an array that match a given string. -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/get_elem_inds.sh -# -#----------------------------------------------------------------------- -# # Source the file containing the function that determines whether or not # a specified variable is an array. # @@ -89,33 +61,6 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # -# Source the file containing the function that checks the validity of a -# variable's value (given a set of valid values). -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/check_var_valid_value.sh -# -#----------------------------------------------------------------------- -# -# -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/print_input_args.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that processes a list of argu- -# ments to a script or function, where the list is comprised of a set of -# argument name-value pairs, e.g. arg1="value1", ... -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/process_args.sh -# -#----------------------------------------------------------------------- -# # Source the file containing the function that creates a symlink to a # file (including performing checks). # From 06549724e873266f1822fc1f6e5f41a4e00a12b3 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Fri, 10 Jul 2026 20:40:18 +0000 Subject: [PATCH 18/25] Remove the remaining unused bash functions: is_array() and create_symlink_to_file() --- ush/bash_utils/create_symlink_to_file.sh | 79 ----------------------- ush/bash_utils/is_array.sh | 82 ------------------------ ush/source_util_funcs.sh | 18 ------ 3 files changed, 179 deletions(-) delete mode 100644 ush/bash_utils/create_symlink_to_file.sh delete mode 100644 ush/bash_utils/is_array.sh diff --git a/ush/bash_utils/create_symlink_to_file.sh b/ush/bash_utils/create_symlink_to_file.sh deleted file mode 100644 index 514daa7a..00000000 --- a/ush/bash_utils/create_symlink_to_file.sh +++ /dev/null @@ -1,79 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that is used to create a symbolic link -# ("symlink") to the specified target file ("target"). It checks for -# the existence of the target file and fails (with an appropriate error -# message) if that target does not exist or is not a file. Also, the -# argument "relative" determines whether a relative or an absolute path -# to the symlink is used. Note that on some platforms, relative symlinks -# are not supported. In those cases, an absolute path is used regardless -# of the setting of "relative". -# -#----------------------------------------------------------------------- -# -function create_symlink_to_file() { -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# -if [[ $# -lt 2 ]]; then - print_err_msg_exit "Function create_symlink_to_file() requires at least two arguments" -fi - -target=$1 -symlink=$2 -relative=${3:-True} -if [ "$relative" != "True" ] && [ "$relative" != "False" ]; then - print_err_msg_exit "'relative' must be set to True or False" -fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local valid_vals_relative \ - relative_flag -# -#----------------------------------------------------------------------- -# -# Make sure that the target file exists and is a file. -# -#----------------------------------------------------------------------- -# - if [ ! -f "${target}" ]; then - print_err_msg_exit "\ -Cannot create symlink to specified target file because the latter does -not exist or is not a file: - target = \"$target\"" - fi -# -#----------------------------------------------------------------------- -# -# Set the flag that specifies whether or not a relative symlink should -# be created. -# -#----------------------------------------------------------------------- -# - relative_flag="" - if [ "${relative}" = "TRUE" ]; then - relative_flag="${RELATIVE_LINK_FLAG}" - fi -# -#----------------------------------------------------------------------- -# -# Create the symlink. -# -#----------------------------------------------------------------------- -# -ln -sf ${relative_flag} "$target" "$symlink" - -} - diff --git a/ush/bash_utils/is_array.sh b/ush/bash_utils/is_array.sh deleted file mode 100644 index cbc0a285..00000000 --- a/ush/bash_utils/is_array.sh +++ /dev/null @@ -1,82 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that is used to check whether a specified -# variable is a bash array. It is called as follows: -# -# is_array var_name -# -# Here, var_name is the name of the variable to check to determine whe- -# ther or not it is an array. If the variable is an array, this func- -# tion will return a 0, and if it is not, this function will return a 1. -# -#----------------------------------------------------------------------- -# -function is_array() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 1 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} var_name - -where var_name is the name of the variable to check to determine whether -or not it is an array. -" - - fi -# -#----------------------------------------------------------------------- -# -# Set local variables to appropriate input arguments. -# -#----------------------------------------------------------------------- -# - local var_name="$1" - local declare_output=$( declare -p "$var_name" 2> /dev/null ) - local regex="^declare -[aA] ${var_name}(=|$)" - printf "%s" "$declare_output" | grep --extended-regexp "$regex" >/dev/null - is_an_array="$?" -# -#----------------------------------------------------------------------- -# -# Return the variable "is_an_array". -# -#----------------------------------------------------------------------- -# - return ${is_an_array} - -} - diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index a21ae984..2585a8ab 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -52,24 +52,6 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # -# Source the file containing the function that determines whether or not -# a specified variable is an array. -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/is_array.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that creates a symlink to a -# file (including performing checks). -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/create_symlink_to_file.sh -# -#----------------------------------------------------------------------- -# # Source the file that sources YAML files as if they were bash # #----------------------------------------------------------------------- From 84e9e202c36493fee7300808d3d5c7f6c4b01083 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Fri, 10 Jul 2026 21:20:32 +0000 Subject: [PATCH 19/25] Add explicit !int tags for all TASKS: entry in config_defaults.yaml --- ush/config_defaults.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 180c3485..0ad282b8 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -998,7 +998,7 @@ gridstat: walltime: 01:00:00 memory: '{{ 2 * gridstat.TASKS }}G' # Number of GRIDSTAT instances to run in parallel - TASKS: '{{ verification.VX_TASKS }}' + TASKS: !int '{{ verification.VX_TASKS }}' pointstat: # Runtime settings for POINTSTAT jobs @@ -1007,7 +1007,7 @@ pointstat: walltime: 01:00:00 memory: '{{ 2 * pointstat.TASKS }}G' # Number of POINTSTAT instances to run in parallel - TASKS: '{{ verification.VX_TASKS }}' + TASKS: !int '{{ verification.VX_TASKS }}' point2grid: execution: @@ -1022,7 +1022,7 @@ point2grid: # 3: no retrieval quality flag GOES_QC_FLAGS: '0,1' # Number of POINT2GRID instances to run in parallel - TASKS: '{{ verification.VX_TASKS }}' + TASKS: !int '{{ verification.VX_TASKS }}' genensprod: # Runtime settings for GenEnsProd jobs @@ -1031,7 +1031,7 @@ genensprod: walltime: 02:00:00 memory: '{{ 2 * genensprod.TASKS }}G' # Number of GenEnsProd instances to run in parallel - TASKS: '{{ verification.VX_TASKS }}' + TASKS: !int '{{ verification.VX_TASKS }}' ensemblestat: # Runtime settings for deterministic EnsembleStat jobs @@ -1040,16 +1040,16 @@ ensemblestat: walltime: 01:00:00 memory: '{{ 2 * ensemblestat.TASKS }}G' # Number of EnsembleStat instances to run in parallel - TASKS: '{{ verification.VX_TASKS }}' + TASKS: !int '{{ verification.VX_TASKS }}' regriddataplane: # Runtime settings for REGRIDDATAPLANE jobs execution: tasks_per_node: '{{ regriddataplane.TASKS }}' walltime: 00:30:00 - memory: '{{ 10 * regriddataplane.TASKS }}G' + memory: '{{ 10 * (regriddataplane.TASKS | int) }}G' # Number of REGRIDDATAPLANE instances to run in parallel - TASKS: '{{ verification.VX_TASKS }}' + TASKS: !int '{{ verification.VX_TASKS }}' # Regridding method. See https://metplus.readthedocs.io/projects/met/en/latest/Users_Guide/reformat_grid.html#optional-arguments-for-regrid-data-plane for valid options REGRID_METHOD: MAXGAUSS @@ -1068,7 +1068,7 @@ mode: # Memory to allocate to MODE job memory: '{{ 10 * mode.TASKS }}G' # Number of MODE instances to run in parallel - TASKS: '{{ verification.VX_TASKS }}' + TASKS: !int '{{ verification.VX_TASKS }}' # For more information on MODE variables see the MODE documentation: # https://metplus.readthedocs.io/projects/met/en/latest/Users_Guide/mode.html From 8be5b4856864e3a0f1ccc4f5b683bb5416608b95 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 13 Jul 2026 18:00:59 +0000 Subject: [PATCH 20/25] Make sure one more TASKS entry is an int --- scripts/genensprod_or_ensemblestat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/genensprod_or_ensemblestat.py b/scripts/genensprod_or_ensemblestat.py index d3711b5c..36166ece 100644 --- a/scripts/genensprod_or_ensemblestat.py +++ b/scripts/genensprod_or_ensemblestat.py @@ -232,7 +232,7 @@ def genensprod_or_ensemblestat( "vx_config_dict": vx_config_dict, } - numprocs=int(vxcfg['VX_TASKS']) + numprocs = int(cfg[metplus_tool_camel_case.lower()]["TASKS"]) conf_files = render_metplus_confs(cfg,settings,metplus_config_tmpl_fn,vx_leadhr_list,numprocs) lgr.debug(f"{conf_files=}") From 99175bc3c4b71c763e7b4344dd6e48e59e88818f Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 13 Jul 2026 18:02:46 +0000 Subject: [PATCH 21/25] More processors! --- .../config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml | 4 ++-- .../config.MET_ensemble_verification_only_vx_time_lag.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml index 0ec59781..41917f71 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml @@ -43,6 +43,6 @@ verification: FCST_FN_TEMPLATE: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%H?shift=${time_lag}}.grib2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}_${FIELD_GROUP}_a${ACCUM_HH}h.nc' gridstat: - TASKS: 4 + TASKS: 12 pointstat: - TASKS: 4 + TASKS: 6 diff --git a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml index f99498ed..b3c632fc 100644 --- a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml +++ b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml @@ -33,7 +33,7 @@ verification: VX_MASK: ["CONUS"] VX_TASKS: 6 gridstat: - TASKS: 6 -pointstat: - TASKS: 6 + TASKS: 8 +genensprod: + TASKS: 8 From 87613b5095e10430249c94ece77b7c37adefea9e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Fri, 17 Jul 2026 17:31:17 +0000 Subject: [PATCH 22/25] Add checks for renamed/moved config sections in setup.py --- ush/setup.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/ush/setup.py b/ush/setup.py index de99eb51..052787e9 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -148,12 +148,31 @@ def check_bad_settings(cfg): msg+="update your config accordingly\n\n" if bad:=ex.get("regriddataplane"): msg+=f"verification_resources:execution contains invalid key `regriddataplane`:\n{bad}" - msg+="these variables for this task moved to top-level `regriddataplane` section; " + msg+="these variables for this task moved to top-level `regriddataplane` section; \n" msg+="update your config accordingly\n\n" if bad:=ex.get("mode"): msg+=f"verification_resources:execution contains invalid key `mode`:\n{bad}\n" - msg+="these variables for this task have been moved to top-level `mode` section; " + msg+="these variables for this task have been moved to top-level `mode` section; \n" msg+="update your config accordingly\n\n" + if bad:=ex.get("deterministic"): + msg+=f"verification_resources:execution contains invalid key `deterministic`:\n{bad}\n" + if bad.get("gridstat"): + msg+="gridstat settings have been moved to top-level `gridstat` section; \n" + if bad.get("pointstat"): + msg+="pointstat settings have been moved to top-level `pointstat` section; \n" + else: + msg+="unknown key {bad}, see config_defaults.yaml for valid settings" + msg+="update your config accordingly\n\n" + if bad:=ex.get("ensemble"): + if bad.get("genensprod"): + msg+=f"verification_resources:execution:ensemble contains invalid key:\n" + msg+="genensprod settings have been moved to top-level `genensprod` section; \n" + msg+="update your config accordingly\n\n" + if bad.get("ensemblestat"): + msg+=f"verification_resources:execution:ensemble contains invalid key:\n" + msg+="ensemblestat settings have been moved to top-level `ensemblestat` section; \n" + msg+="update your config accordingly\n\n" + # No else: here since verification_resources:execution:ensemble still has valid keys if bt:=cfg.get("platform").get("BEST_TRACK"): msg+=f"Config file contains invalid key `platform: BEST_TRACK`:\n{bt}\n" msg+="The variable `BEST_TRACK` has been renamed to `BEST_TRACK_DIR`; " From f5cffd2db63325ddb7e437fc7510e2c38ceb6cbc Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Fri, 17 Jul 2026 19:44:46 +0000 Subject: [PATCH 23/25] Missed moving gridstat/pointstat settings to new section for MODE_AOD test --- .../test_configs/mode/config.MODE_AOD.yaml | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/WE2E/test_configs/mode/config.MODE_AOD.yaml b/tests/WE2E/test_configs/mode/config.MODE_AOD.yaml index f5f89b5b..21a8bd80 100644 --- a/tests/WE2E/test_configs/mode/config.MODE_AOD.yaml +++ b/tests/WE2E/test_configs/mode/config.MODE_AOD.yaml @@ -62,17 +62,19 @@ mode: CONV_THRESH: '>=0.3' MERGE_THRESH: '>=5' MERGE_FLAG: ENGINE +gridstat: + execution: + tasks_per_node: '{{ verification.VX_TASKS }}' + walltime: 01:00:00 + memory: '{{ 5 * verification.VX_TASKS }}G' +pointstat: + execution: + tasks_per_node: '{{ verification.VX_TASKS }}' + walltime: 01:00:00 + memory: '{{ 5 * verification.VX_TASKS }}G' + verification_resources: execution: - deterministic: - gridstat: - tasks_per_node: '{{ verification.VX_TASKS }}' - walltime: 01:00:00 - memory: '{{ 5 * verification.VX_TASKS }}G' - pointstat: - tasks_per_node: '{{ verification.VX_TASKS }}' - walltime: 01:00:00 - memory: '{{ 5 * verification.VX_TASKS }}G' pcpcombine: fcst: walltime: 01:00:00 From 6ef422718f938dbddbb28176a244c03a90634d25 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Fri, 17 Jul 2026 23:04:44 +0000 Subject: [PATCH 24/25] Fix linting issues with ush/setup.py --- ush/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/setup.py b/ush/setup.py index 052787e9..2dbf8b43 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -165,11 +165,11 @@ def check_bad_settings(cfg): msg+="update your config accordingly\n\n" if bad:=ex.get("ensemble"): if bad.get("genensprod"): - msg+=f"verification_resources:execution:ensemble contains invalid key:\n" + msg+="verification_resources:execution:ensemble contains invalid key:\n" msg+="genensprod settings have been moved to top-level `genensprod` section; \n" msg+="update your config accordingly\n\n" if bad.get("ensemblestat"): - msg+=f"verification_resources:execution:ensemble contains invalid key:\n" + msg+="verification_resources:execution:ensemble contains invalid key:\n" msg+="ensemblestat settings have been moved to top-level `ensemblestat` section; \n" msg+="update your config accordingly\n\n" # No else: here since verification_resources:execution:ensemble still has valid keys From 2b426c9b6989b9c7d897a383cb9bf35509a24276 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Fri, 17 Jul 2026 23:11:02 +0000 Subject: [PATCH 25/25] Fix rest of the linting --- scripts/check_post_output.py | 2 +- scripts/genensprod_or_ensemblestat.py | 3 ++- scripts/gridstat_or_pointstat.py | 1 - scripts/gridstat_or_pointstat_ensmean.py | 1 + scripts/gridstat_or_pointstat_ensprob.py | 1 + scripts/pcpcombine.py | 1 + 6 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/check_post_output.py b/scripts/check_post_output.py index 4e17f2a0..dbd5a867 100644 --- a/scripts/check_post_output.py +++ b/scripts/check_post_output.py @@ -10,12 +10,12 @@ import argparse import logging import os +from string import Template import uwtools.api.config as uwconfig from python_utils import setup_logging from set_leadhrs import set_leadhrs -from string import Template def check_post_output(config_file: str, cdate: str, ensmem_index: int) -> None: diff --git a/scripts/genensprod_or_ensemblestat.py b/scripts/genensprod_or_ensemblestat.py index 36166ece..24b1abdb 100644 --- a/scripts/genensprod_or_ensemblestat.py +++ b/scripts/genensprod_or_ensemblestat.py @@ -62,6 +62,7 @@ def genensprod_or_ensemblestat( metplus_tool : str METplus tool to run: ``"GENENSPROD"`` or ``"ENSEMBLESTAT"`` (case-insensitive). """ + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-branches,too-many-statements lgr = logging.getLogger(__name__) key = metplus_tool.upper() @@ -93,7 +94,7 @@ def genensprod_or_ensemblestat( if geom == "grid": if "APCP" in met_filedir_name: obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") - obs_in_fn_template = Template( + obs_in_fn_template = Template( vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] ).substitute(subvars) fcst_in_dir = Path(exptdir) diff --git a/scripts/gridstat_or_pointstat.py b/scripts/gridstat_or_pointstat.py index a36554ee..a1bcc8f0 100644 --- a/scripts/gridstat_or_pointstat.py +++ b/scripts/gridstat_or_pointstat.py @@ -9,7 +9,6 @@ import argparse import logging -import math import os import subprocess diff --git a/scripts/gridstat_or_pointstat_ensmean.py b/scripts/gridstat_or_pointstat_ensmean.py index 108d87ea..0e069a06 100644 --- a/scripts/gridstat_or_pointstat_ensmean.py +++ b/scripts/gridstat_or_pointstat_ensmean.py @@ -52,6 +52,7 @@ def gridstat_or_pointstat_ensmean( fcst_thresh : str Forecast threshold set (usually "all" or "none"). """ + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-branches,too-many-statements lgr = logging.getLogger(__name__) cfg = uwconfig.get_yaml_config(config=config_file) diff --git a/scripts/gridstat_or_pointstat_ensprob.py b/scripts/gridstat_or_pointstat_ensprob.py index a73f970a..1fcec9f0 100644 --- a/scripts/gridstat_or_pointstat_ensprob.py +++ b/scripts/gridstat_or_pointstat_ensprob.py @@ -53,6 +53,7 @@ def gridstat_or_pointstat_ensprob( fcst_thresh : str Forecast threshold set (usually "all" or "none"). """ + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-branches,too-many-statements lgr = logging.getLogger(__name__) cfg = uwconfig.get_yaml_config(config=config_file) diff --git a/scripts/pcpcombine.py b/scripts/pcpcombine.py index 996ea4cc..f54e9f1f 100644 --- a/scripts/pcpcombine.py +++ b/scripts/pcpcombine.py @@ -59,6 +59,7 @@ def pcpcombine( ensmem_index : int Ensemble member index (0 for deterministic, 1-based for ensemble members). """ + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-branches,too-many-statements lgr = logging.getLogger(__name__) fcst_or_obs = fcst_or_obs.upper()