diff --git a/jobs/ASCII2NC_OBS.sh b/jobs/ASCII2NC_OBS.sh index 4cbb9f66..5f0e272b 100755 --- a/jobs/ASCII2NC_OBS.sh +++ b/jobs/ASCII2NC_OBS.sh @@ -51,7 +51,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -64,5 +64,5 @@ python $SCRIPTSdir/ascii2nc_obs.py ${VERBOSE_FLAG} \ --cycle_date="${YYMMDD}${HH}" \ --obtype="${OBTYPE}" || \ print_err_msg_exit "\ -Call to \"ascii2nc_obs.sh\" from \"${scrfunc_fn}\" failed." +Call to \"ascii2nc_obs.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/CHECK_POST_OUTPUT.sh b/jobs/CHECK_POST_OUTPUT.sh index 55a720da..044772b6 100755 --- a/jobs/CHECK_POST_OUTPUT.sh +++ b/jobs/CHECK_POST_OUTPUT.sh @@ -39,6 +39,8 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done +# Sets up PYTHONPATH and VERBOSE environment variables +. $USHdir/set_job_env.sh # #----------------------------------------------------------------------- # @@ -52,7 +54,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -60,9 +62,13 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -$SCRIPTSdir/check_post_output.sh || \ +python $SCRIPTSdir/check_post_output.py ${VERBOSE_FLAG} \ + --config="${GLOBAL_VAR_DEFNS_FP}" \ + --cycle_date="${YYMMDD}${HH}" \ + ${ENSMEM_ARG} || \ print_err_msg_exit "\ -Call to script \"check_post_output.sh\" from \"${scrfunc_fn}\" failed." +Call to \"check_post_output.py\" from \"${scrfunc_fn}\" failed." + # #----------------------------------------------------------------------- # diff --git a/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh b/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh index 798cd77a..60b0fc91 100755 --- a/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh +++ b/jobs/GENENSPROD_OR_ENSEMBLESTAT.sh @@ -35,6 +35,8 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done +# Sets up PYTHONPATH, ACCUM_ARG, and VERBOSE environment variables +. $USHdir/set_job_env.sh # #----------------------------------------------------------------------- # @@ -48,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -56,7 +58,16 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -$SCRIPTSdir/genensprod_or_ensemblestat.sh || \ +python $SCRIPTSdir/genensprod_or_ensemblestat.py ${VERBOSE_FLAG} \ + --config="${GLOBAL_VAR_DEFNS_FP}" \ + --cycle_date="${YYMMDD}${HH}" \ + --field_group="${FIELD_GROUP}" \ + --obs_dir="${OBS_DIR}" \ + --obtype="${OBTYPE}" \ + --fcst_level="${FCST_LEVEL}" \ + --fcst_thresh="${FCST_THRESH}" \ + --metplus_tool="${METPLUSTOOLNAME}" \ + ${ACCUM_ARG} || \ print_err_msg_exit "\ -Call to \"genensprod_or_ensemblestat.sh\" from \"${scrfunc_fn}\" failed." +Call to \"genensprod_or_ensemblestat.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/GET_VERIF_OBS.sh b/jobs/GET_VERIF_OBS.sh index 6cf198bc..70b6a2e7 100755 --- a/jobs/GET_VERIF_OBS.sh +++ b/jobs/GET_VERIF_OBS.sh @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -66,7 +66,7 @@ cmd=( --obtype "${OBTYPE}" --obs_day "${YYMMDD}" ) -echo "CALLING: ${cmd[*]}" +printf "CALLING: ${cmd[*]}" "${cmd[@]}" || print_err_msg_exit "Error calling get_obs.py" # #----------------------------------------------------------------------- @@ -77,6 +77,7 @@ echo "CALLING: ${cmd[*]}" #----------------------------------------------------------------------- # mkdir -p ${WFLOW_FLAG_FILES_DIR} -file_bn="get_obs_$(echo_lowercase ${OBTYPE})" +# ${VARNAME,,} converts contents of VARNAME to lowercase +file_bn="get_obs_${OBTYPE,,}" touch "${WFLOW_FLAG_FILES_DIR}/${file_bn}_${YYMMDD}_complete.txt" diff --git a/jobs/GRIDSTAT_OR_POINTSTAT.sh b/jobs/GRIDSTAT_OR_POINTSTAT.sh index d940de7d..d61e1ca8 100755 --- a/jobs/GRIDSTAT_OR_POINTSTAT.sh +++ b/jobs/GRIDSTAT_OR_POINTSTAT.sh @@ -35,7 +35,7 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done -# Sets up PYTHONPATH, VERBOSE and ACCUM_ARG environment variables +# Sets up PYTHONPATH, ACCUM_ARG, ENSMEM_ARG and VERBOSE environment variables . $USHdir/set_job_env.sh # #----------------------------------------------------------------------- @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -58,15 +58,15 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -python $SCRIPTSdir/gridstat_or_pointstat.py ${VERBOSE_FLAG} ${ACCUM_ARG} \ +python $SCRIPTSdir/gridstat_or_pointstat.py ${VERBOSE_FLAG} \ --config="${GLOBAL_VAR_DEFNS_FP}" \ --cycle_date="${YYMMDD}${HH}" \ - --ensmem_index="${ENSMEM_INDX}" \ --field_group="${FIELD_GROUP}" \ --fcst_level="${FCST_LEVEL}" \ --fcst_thresh="${FCST_THRESH}" \ --obtype="${OBTYPE}" \ - --obs_dir="${OBS_DIR}" || \ + --obs_dir="${OBS_DIR}" \ + ${ACCUM_ARG} ${ENSMEM_ARG} || \ print_err_msg_exit "\ Call to \"gridstat_or_pointstat.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh b/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh index cf16fd03..319c809e 100755 --- a/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh +++ b/jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh @@ -36,6 +36,8 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done +# Sets up PYTHONPATH and VERBOSE environment variables +. $USHdir/set_job_env.sh # #----------------------------------------------------------------------- # @@ -49,7 +51,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -57,6 +59,15 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -$SCRIPTSdir/gridstat_or_pointstat_ensmean.sh || \ +python $SCRIPTSdir/gridstat_or_pointstat_ensmean.py ${VERBOSE_FLAG} \ + --config="${GLOBAL_VAR_DEFNS_FP}" \ + --cycle_date="${YYMMDD}${HH}" \ + --field_group="${FIELD_GROUP}" \ + --obs_dir="${OBS_DIR}" \ + --obtype="${OBTYPE}" \ + --fcst_level="${FCST_LEVEL}" \ + --fcst_thresh="${FCST_THRESH}" \ + ${ACCUM_ARG} || \ print_err_msg_exit "\ -Call to \"gridstat_or_pointstat_ensmean.sh\" from \"${scrfunc_fn}\" failed." +Call to \"gridstat_or_pointstat_ensmean.py\" from \"${scrfunc_fn}\" failed." + diff --git a/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh b/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh index c2caae2b..562649c8 100755 --- a/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh +++ b/jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh @@ -36,6 +36,8 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done +# Sets up PYTHONPATH and VERBOSE environment variables +. $USHdir/set_job_env.sh # #----------------------------------------------------------------------- # @@ -49,7 +51,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -57,7 +59,14 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -$SCRIPTSdir/gridstat_or_pointstat_ensprob.sh || \ +python $SCRIPTSdir/gridstat_or_pointstat_ensprob.py ${VERBOSE_FLAG} \ + --config="${GLOBAL_VAR_DEFNS_FP}" \ + --cycle_date="${YYMMDD}${HH}" \ + --field_group="${FIELD_GROUP}" \ + --obs_dir="${OBS_DIR}" \ + --obtype="${OBTYPE}" \ + --fcst_level="${FCST_LEVEL}" \ + --fcst_thresh="${FCST_THRESH}" \ + ${ACCUM_ARG} || \ print_err_msg_exit "\ -Call to \"gridstat_or_pointstat_ensprob.sh\" from \"${scrfunc_fn}\" failed." - +Call to \"gridstat_or_pointstat_ensprob.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/MODE.sh b/jobs/MODE.sh index e2029c68..175d945e 100755 --- a/jobs/MODE.sh +++ b/jobs/MODE.sh @@ -54,7 +54,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/PB2NC_OBS.sh b/jobs/PB2NC_OBS.sh index b5907eac..fc56d0eb 100755 --- a/jobs/PB2NC_OBS.sh +++ b/jobs/PB2NC_OBS.sh @@ -51,7 +51,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -64,5 +64,5 @@ python $SCRIPTSdir/pb2nc_obs.py ${VERBOSE_FLAG} \ --cycle_date="${YYMMDD}${HH}" \ --obtype="${OBTYPE}" || \ print_err_msg_exit "\ -Call to \"pb2nc_obs.sh\" from \"${scrfunc_fn}\" failed." +Call to \"pb2nc_obs.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/PCPCOMBINE.sh b/jobs/PCPCOMBINE.sh index d4cbdd98..c64f1cb2 100755 --- a/jobs/PCPCOMBINE.sh +++ b/jobs/PCPCOMBINE.sh @@ -37,6 +37,8 @@ sections=( for sect in ${sections[*]} ; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done +# Sets up PYTHONPATH, ACCUM_ARG, ENSMEM_ARG, and VERBOSE environment variables +. $USHdir/set_job_env.sh # #----------------------------------------------------------------------- # @@ -50,7 +52,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" @@ -58,7 +60,16 @@ In directory: \"${scrfunc_dir}\" # # Call the run script # -$SCRIPTSdir/pcpcombine.sh || \ +python $SCRIPTSdir/pcpcombine.py ${VERBOSE_FLAG} \ + --config="${GLOBAL_VAR_DEFNS_FP}" \ + --cycle_date="${YYMMDD}${HH}" \ + --field_group="${FIELD_GROUP}" \ + --fcst_or_obs="${FCST_OR_OBS}" \ + --obs_dir="${OBS_DIR}" \ + --obtype="${OBTYPE}" \ + --fcst_level="${FCST_LEVEL}" \ + --fcst_thresh="${FCST_THRESH}" \ + ${ACCUM_ARG} ${ENSMEM_ARG} || \ print_err_msg_exit "\ -Call to \"pcpcombine.sh\" from \"${scrfunc_fn}\" failed." +Call to \"pcpcombine.py\" from \"${scrfunc_fn}\" failed." diff --git a/jobs/POINT2GRID.sh b/jobs/POINT2GRID.sh index 434e8e5d..f6431f12 100755 --- a/jobs/POINT2GRID.sh +++ b/jobs/POINT2GRID.sh @@ -49,7 +49,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/REGRIDDATAPLANE.sh b/jobs/REGRIDDATAPLANE.sh index 61558a9c..2a70898a 100755 --- a/jobs/REGRIDDATAPLANE.sh +++ b/jobs/REGRIDDATAPLANE.sh @@ -54,7 +54,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/TCPAIRS.sh b/jobs/TCPAIRS.sh index 5d6887ee..22ccc9ad 100755 --- a/jobs/TCPAIRS.sh +++ b/jobs/TCPAIRS.sh @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/TCRMW.sh b/jobs/TCRMW.sh index 6a557396..3db282b9 100755 --- a/jobs/TCRMW.sh +++ b/jobs/TCRMW.sh @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/jobs/TCSTAT.sh b/jobs/TCSTAT.sh index 7feba1eb..92381b8d 100755 --- a/jobs/TCSTAT.sh +++ b/jobs/TCSTAT.sh @@ -50,7 +50,7 @@ scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) scrfunc_fn=$( basename "${scrfunc_fp}" ) scrfunc_dir=$( dirname "${scrfunc_fp}" ) -print_info_msg " +printf " ======================================================================== Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" diff --git a/parm/metplus/EnsembleStat.conf b/parm/metplus/EnsembleStat.conf index 242a69c0..e32cc1f0 100644 --- a/parm/metplus/EnsembleStat.conf +++ b/parm/metplus/EnsembleStat.conf @@ -221,7 +221,7 @@ OBS_{{METPLUS_TOOL_NAME}}_WINDOW_END = {OBS_WINDOW_END} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Jinja requires certain variables to be defined globally within the template diff --git a/parm/metplus/GenEnsProd.conf b/parm/metplus/GenEnsProd.conf index 6c27a985..5179d4d6 100644 --- a/parm/metplus/GenEnsProd.conf +++ b/parm/metplus/GenEnsProd.conf @@ -109,7 +109,7 @@ STAGING_DIR = {{staging_dir}} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Jinja requires certain variables to be defined globally within the template diff --git a/parm/metplus/GridStat_ensmean.conf b/parm/metplus/GridStat_ensmean.conf index 61b54d17..ddcf7c29 100644 --- a/parm/metplus/GridStat_ensmean.conf +++ b/parm/metplus/GridStat_ensmean.conf @@ -97,7 +97,7 @@ OBTYPE = {{obtype}} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Set the probabilistic threshold to be used for the forecast field. If diff --git a/parm/metplus/GridStat_ensprob.conf b/parm/metplus/GridStat_ensprob.conf index c47ed5de..5113c635 100644 --- a/parm/metplus/GridStat_ensprob.conf +++ b/parm/metplus/GridStat_ensprob.conf @@ -108,7 +108,7 @@ OBTYPE = {{obtype}} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Set the probabilistic threshold to be used for the forecast field. If diff --git a/parm/metplus/PointStat_ensmean.conf b/parm/metplus/PointStat_ensmean.conf index 6dc6bb4c..0de8e1b1 100644 --- a/parm/metplus/PointStat_ensmean.conf +++ b/parm/metplus/PointStat_ensmean.conf @@ -161,7 +161,7 @@ OBTYPE = {{obtype}} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Set the probabilistic threshold to be used for the forecast field. If diff --git a/parm/metplus/PointStat_ensprob.conf b/parm/metplus/PointStat_ensprob.conf index 223a5318..fa29f91e 100644 --- a/parm/metplus/PointStat_ensprob.conf +++ b/parm/metplus/PointStat_ensprob.conf @@ -163,7 +163,7 @@ OBTYPE = {{obtype}} {#- Import the file containing jinja macros. #} -{%- import metplus_templates_dir ~ '/metplus_macros.jinja' as metplus_macros %} +{%- import 'metplus_macros.jinja' as metplus_macros %} {#- Set the probabilistic threshold to be used for the forecast field. If diff --git a/parm/wflow/get_obs.yaml b/parm/wflow/get_obs.yaml new file mode 100644 index 00000000..aac0c5fd --- /dev/null +++ b/parm/wflow/get_obs.yaml @@ -0,0 +1,127 @@ +default_task_get_obs: &default_task_get_obs + account: '&ACCOUNT;' + attrs: + cycledefs: cycledefs_obs_days_inst + maxtries: 1 + envars: &default_vars + GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;' + USHdir: '&USHdir;' + YYMMDD: + cyclestr: + value: '@Y@m@d' + HH: + cyclestr: + value: '@H' + MET_INSTALL_DIR: '&MET_INSTALL_DIR;' + METPLUS_ROOT: '&METPLUS_ROOT;' + LOGDIR: + cyclestr: + value: '&LOGDIR;' + nprocs: '{{ verification_resources.execution.nodes * verification_resources.execution.tasks_per_node }}' + join: + cyclestr: + value: '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' + memory: '{{ "2G" if user.MACHINE not in ["NOAACLOUD"] }}' + native: '{{ platform.SCHED_NATIVE_CMD }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ verification_resources.execution.tasks_per_node }}' + partition: '{{ "&PARTITION_DEFAULT;" if platform.get("PARTITION_DEFAULT") }}' + queue: '&QUEUE_DEFAULT;' + +task_get_obs_goes_aod: + <<: *default_task_get_obs + command: "&JOBSdir;/GET_VERIF_OBS.sh" + envars: + <<: *default_vars + OBTYPE: 'GOESAOD' + FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' + native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' + partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' + queue: "&QUEUE_HPSS;" + walltime: 02:00:00 + +task_get_obs_goes_adp: + <<: *default_task_get_obs + command: "&JOBSdir;/GET_VERIF_OBS.sh" + envars: + <<: *default_vars + OBTYPE: 'GOESADP' + FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' + native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' + partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' + queue: "&QUEUE_HPSS;" + walltime: 02:00:00 + +task_get_obs_aeronet: + <<: *default_task_get_obs + command: "&JOBSdir;/GET_VERIF_OBS.sh" + envars: + <<: *default_vars + OBTYPE: 'AERONET' + FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' + native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' + partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' + queue: "&QUEUE_HPSS;" + walltime: '{{ verification_resources.execution.get_obs.aeronet.walltime }}' + +task_get_obs_airnow: + <<: *default_task_get_obs + command: "&JOBSdir;/GET_VERIF_OBS.sh" + envars: + <<: *default_vars + OBTYPE: 'AIRNOW' + FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' + native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' + partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' + queue: "&QUEUE_HPSS;" + walltime: '{{ verification_resources.execution.get_obs.airnow.walltime }}' + + +task_get_obs_ccpa: + <<: *default_task_get_obs + attrs: + cycledefs: cycledefs_obs_days_cumul + maxtries: 1 + command: "&JOBSdir;/GET_VERIF_OBS.sh" + envars: + <<: *default_vars + OBTYPE: 'CCPA' + native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' + partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' + queue: "&QUEUE_HPSS;" + walltime: "{{ verification_resources.execution.get_obs.ccpa.walltime }}" + +task_get_obs_nohrsc: + <<: *default_task_get_obs + attrs: + cycledefs: cycledefs_obs_days_cumul + maxtries: 1 + command: "&JOBSdir;/GET_VERIF_OBS.sh" + envars: + <<: *default_vars + OBTYPE: 'NOHRSC' + native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' + partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' + queue: "&QUEUE_HPSS;" + walltime: "{{ verification_resources.execution.get_obs.nohrsc.walltime }}" + +task_get_obs_mrms: + <<: *default_task_get_obs + command: "&JOBSdir;/GET_VERIF_OBS.sh" + envars: + <<: *default_vars + OBTYPE: 'MRMS' + native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' + partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' + queue: "&QUEUE_HPSS;" + walltime: "{{ verification_resources.execution.get_obs.mrms.walltime }}" + +task_get_obs_ndas: + <<: *default_task_get_obs + command: "&JOBSdir;/GET_VERIF_OBS.sh" + envars: + <<: *default_vars + OBTYPE: 'NDAS' + queue: "&QUEUE_HPSS;" + native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' + partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' + walltime: "{{ verification_resources.execution.get_obs.ndas.walltime }}" diff --git a/parm/wflow/mode.yaml b/parm/wflow/mode.yaml index 1fa6ae30..3f1193cd 100644 --- a/parm/wflow/mode.yaml +++ b/parm/wflow/mode.yaml @@ -30,6 +30,7 @@ task_run_RegridDataPlane_GOESAOD: <<: *default_task_mode attrs: <<: *default_attrs + maxtries: !int "{{ regriddataplane.execution.tries }}" command: "&JOBSdir;/REGRIDDATAPLANE.sh" envars: <<: *default_vars @@ -48,12 +49,13 @@ task_run_MODE_GOESAOD: <<: *default_task_mode attrs: <<: *default_attrs + maxtries: !int "{{ mode.execution.tries }}" command: "&JOBSdir;/MODE.sh" envars: <<: *default_vars FIELD_GROUP: 'GOESAOD' OBTYPE: 'GOESAOD' - OMP_NUM_THREADS: 4 + OMP_NUM_THREADS: "{{ mode.execution.threads }}" memory: '{{ mode.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' nodes: '{{ verification_resources.execution.nodes }}:ppn={{ mode.execution.tasks_per_node }}' walltime: "{{ mode.execution.walltime }}" diff --git a/parm/wflow/verify_pre.yaml b/parm/wflow/preprocessing.yaml similarity index 66% rename from parm/wflow/verify_pre.yaml rename to parm/wflow/preprocessing.yaml index 78e37f52..f1f929a9 100644 --- a/parm/wflow/verify_pre.yaml +++ b/parm/wflow/preprocessing.yaml @@ -1,4 +1,4 @@ -default_task_verify_pre: &default_task_verify_pre +default_task_preprocessing: &default_task_preprocessing account: '&ACCOUNT;' attrs: cycledefs: cycledefs_obs_days_inst @@ -27,115 +27,18 @@ default_task_verify_pre: &default_task_verify_pre partition: '{{ "&PARTITION_DEFAULT;" if platform.get("PARTITION_DEFAULT") }}' queue: '&QUEUE_DEFAULT;' -task_get_obs_goes_aod: - <<: *default_task_verify_pre - command: "&JOBSdir;/GET_VERIF_OBS.sh" - envars: - <<: *default_vars - OBTYPE: 'GOESAOD' - FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' - native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' - partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' - queue: "&QUEUE_HPSS;" - walltime: 02:00:00 - -task_get_obs_goes_adp: - <<: *default_task_verify_pre - command: "&JOBSdir;/GET_VERIF_OBS.sh" - envars: - <<: *default_vars - OBTYPE: 'GOESADP' - FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' - native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' - partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' - queue: "&QUEUE_HPSS;" - walltime: 02:00:00 - -task_get_obs_aeronet: - <<: *default_task_verify_pre - command: "&JOBSdir;/GET_VERIF_OBS.sh" - envars: - <<: *default_vars - OBTYPE: 'AERONET' - FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' - native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' - partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' - queue: "&QUEUE_HPSS;" - walltime: '{{ verification_resources.execution.get_obs.aeronet.walltime }}' - -task_get_obs_airnow: - <<: *default_task_verify_pre - command: "&JOBSdir;/GET_VERIF_OBS.sh" - envars: - <<: *default_vars - OBTYPE: 'AIRNOW' - FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' - native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' - partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' - queue: "&QUEUE_HPSS;" - walltime: '{{ verification_resources.execution.get_obs.airnow.walltime }}' - - -task_get_obs_ccpa: - <<: *default_task_verify_pre - attrs: - cycledefs: cycledefs_obs_days_cumul - maxtries: 1 - command: "&JOBSdir;/GET_VERIF_OBS.sh" - envars: - <<: *default_vars - OBTYPE: 'CCPA' - native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' - partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' - queue: "&QUEUE_HPSS;" - walltime: "{{ verification_resources.execution.get_obs.ccpa.walltime }}" - -task_get_obs_nohrsc: - <<: *default_task_verify_pre - attrs: - cycledefs: cycledefs_obs_days_cumul - maxtries: 1 - command: "&JOBSdir;/GET_VERIF_OBS.sh" - envars: - <<: *default_vars - OBTYPE: 'NOHRSC' - native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' - partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' - queue: "&QUEUE_HPSS;" - walltime: "{{ verification_resources.execution.get_obs.nohrsc.walltime }}" - -task_get_obs_mrms: - <<: *default_task_verify_pre - command: "&JOBSdir;/GET_VERIF_OBS.sh" - envars: - <<: *default_vars - OBTYPE: 'MRMS' - native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' - partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' - queue: "&QUEUE_HPSS;" - walltime: "{{ verification_resources.execution.get_obs.mrms.walltime }}" - -task_get_obs_ndas: - <<: *default_task_verify_pre - command: "&JOBSdir;/GET_VERIF_OBS.sh" - envars: - <<: *default_vars - OBTYPE: 'NDAS' - queue: "&QUEUE_HPSS;" - native: '{{ platform.SCHED_NATIVE_CMD_HPSS if platform.get("SCHED_NATIVE_CMD_HPSS") else platform.SCHED_NATIVE_CMD }}' - partition: '{{ "&PARTITION_HPSS;" if platform.get("PARTITION_HPSS") }}' - walltime: "{{ verification_resources.execution.get_obs.ndas.walltime }}" - task_run_MET_Pb2nc_obs_NDAS: - <<: *default_task_verify_pre + <<: *default_task_preprocessing attrs: cycledefs: cycledefs_obs_days_inst - maxtries: 2 + maxtries: !int "{{ pb2nc.execution.tries }}" command: "&JOBSdir;/PB2NC_OBS.sh" envars: <<: *default_vars OBTYPE: NDAS - walltime: "{{ verification_resources.execution.pb2nc.walltime }}" + memory: '{{ pb2nc.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ pb2nc.execution.tasks_per_node }}' + walltime: "{{ pb2nc.execution.walltime }}" dependency: and: datadep: @@ -155,7 +58,7 @@ metatask_ASCII2nc_obs: METAOBS_DIR: '{% for var in verification.VX_FIELD_GROUPS %}{% if var =="AOD" %}&AERONET_OBS_DIR; {% elif var in ["PM25", "PM10"] %}&AIRNOW_OBS_DIR; {% endif %}{% endfor %}' TASKDEP: '{% for var in verification.VX_FIELD_GROUPS %}{% if var =="AOD" %}get_obs_aeronet {% elif var in ["PM25", "PM10"] %}get_obs_airnow {% endif %}{% endfor %}' task_run_MET_ASCII2nc_obs_#METAOBTYPE#: - <<: *default_task_verify_pre + <<: *default_task_preprocessing attrs: cycledefs: cycledefs_obs_days_inst maxtries: 2 @@ -180,10 +83,10 @@ metatask_ASCII2nc_obs: task: '#TASKDEP#' task_run_MET_Point2Grid_GOESAOD: - <<: *default_task_verify_pre + <<: *default_task_preprocessing attrs: cycledefs: forecast - maxtries: 2 + maxtries: !int "{{ point2grid.execution.tries }}" command: "&JOBSdir;/POINT2GRID.sh" memory: '{{ point2grid.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' walltime: "{{ point2grid.execution.walltime }}" @@ -219,7 +122,7 @@ metatask_PcpCombine_APCP_all_accums_obs_CCPA: var: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_PcpCombine_APCP#ACCUM_HH#h_obs_CCPA: - <<: *default_task_verify_pre + <<: *default_task_preprocessing attrs: cycledefs: forecast maxtries: 2 @@ -252,7 +155,7 @@ metatask_PcpCombine_ASNOW_all_accums_obs_NOHRSC: var: ACCUM_HH: '{% for ah in verification.VX_ASNOW_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_PcpCombine_ASNOW#ACCUM_HH#h_obs_NOHRSC: - <<: *default_task_verify_pre + <<: *default_task_preprocessing attrs: cycledefs: forecast maxtries: 2 @@ -283,9 +186,9 @@ metatask_PcpCombine_ASNOW_all_accums_obs_NOHRSC: metatask_check_post_output_all_mems: var: - mem: '{% if ensemble.DO_ENSEMBLE %}{% for m in range(1, ensemble.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% for m in range(0, ensemble.NUM_ENS_MEMBERS) %}{{ "%03d "%m }}{%- endfor -%}' task_check_post_output_mem#mem#: - <<: *default_task_verify_pre + <<: *default_task_preprocessing attrs: cycledefs: forecast maxtries: 1 @@ -306,9 +209,9 @@ metatask_PcpCombine_APCP_all_accums_all_mems: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' metatask_PcpCombine_APCP#ACCUM_HH#h_all_mems: var: - mem: '{% if ensemble.DO_ENSEMBLE %}{% for m in range(1, ensemble.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% for m in range(0, ensemble.NUM_ENS_MEMBERS) %}{{ "%03d "%m }}{%- endfor -%}' task_run_MET_PcpCombine_APCP#ACCUM_HH#h_fcst_mem#mem#: - <<: *default_task_verify_pre + <<: *default_task_preprocessing attrs: cycledefs: forecast maxtries: 2 @@ -336,9 +239,9 @@ metatask_PcpCombine_ASNOW_all_accums_all_mems: ACCUM_HH: '{% for ah in verification.VX_ASNOW_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' metatask_PcpCombine_ASNOW#ACCUM_HH#h_all_mems: var: - mem: '{% if ensemble.DO_ENSEMBLE %}{% for m in range(1, ensemble.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% for m in range(0, ensemble.NUM_ENS_MEMBERS) %}{{ "%03d "%m }}{%- endfor -%}' task_run_MET_PcpCombine_ASNOW#ACCUM_HH#h_fcst_mem#mem#: - <<: *default_task_verify_pre + <<: *default_task_preprocessing attrs: cycledefs: forecast maxtries: 2 @@ -366,9 +269,9 @@ metatask_PcpCombine_fcst_PM_all_mems: PM: '{% for var in verification.VX_FIELD_GROUPS %}{{ ("%s " % var) if var in ["PM25", "PM10"] }}{% endfor %}' metatask_PcpCombine_fcst_#PM#_all_mems: var: - mem: '{% if ensemble.DO_ENSEMBLE %}{% for m in range(1, ensemble.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% for m in range(0, ensemble.NUM_ENS_MEMBERS) %}{{ "%03d "%m }}{%- endfor -%}' task_run_MET_PcpCombine_fcst_#PM#_mem#mem#: - <<: *default_task_verify_pre + <<: *default_task_preprocessing attrs: cycledefs: forecast maxtries: 2 diff --git a/parm/wflow/verify_det.yaml b/parm/wflow/verify_det.yaml index b3e5f5c3..f86fd6dd 100644 --- a/parm/wflow/verify_det.yaml +++ b/parm/wflow/verify_det.yaml @@ -32,12 +32,12 @@ metatask_GridStat_APCP_all_accums_all_mems: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' metatask_GridStat_APCP#ACCUM_HH#h_all_mems: var: - mem: '{% if ensemble.DO_ENSEMBLE %}{% for m in range(1, ensemble.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% for m in range(0, ensemble.NUM_ENS_MEMBERS) %}{{ "%03d "%m }}{%- endfor -%}' task_run_MET_GridStat_vx_APCP#ACCUM_HH#h_mem#mem#: <<: *default_task_verify_det attrs: <<: *default_attrs - maxtries: 2 + maxtries: !int "{{ gridstat.execution.tries }}" command: "&JOBSdir;/GRIDSTAT_OR_POINTSTAT.sh" envars: <<: *default_vars @@ -49,9 +49,9 @@ metatask_GridStat_APCP_all_accums_all_mems: ENSMEM_INDX: "#mem#" FCST_LEVEL: 'A#ACCUM_HH#' FCST_THRESH: 'all' - memory: '{{ verification_resources.execution.deterministic.gridstat.memory if user.MACHINE not in ["NOAACLOUD"] }}' - nodes: '{{ verification_resources.execution.nodes }}:ppn={{ verification_resources.execution.deterministic.gridstat.tasks_per_node }}' - walltime: "{{ verification_resources.execution.deterministic.gridstat.walltime }}" + memory: '{{ gridstat.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ gridstat.execution.tasks_per_node }}' + walltime: "{{ gridstat.execution.walltime }}" dependency: and: taskdep_pcpcombine_obs: @@ -66,12 +66,12 @@ metatask_GridStat_ASNOW_all_accums_all_mems: ACCUM_HH: '{% for ah in verification.VX_ASNOW_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' metatask_GridStat_ASNOW#ACCUM_HH#h_all_mems: var: - mem: '{% if ensemble.DO_ENSEMBLE %}{% for m in range(1, ensemble.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% for m in range(0, ensemble.NUM_ENS_MEMBERS) %}{{ "%03d "%m }}{%- endfor -%}' task_run_MET_GridStat_vx_ASNOW#ACCUM_HH#h_mem#mem#: <<: *default_task_verify_det attrs: <<: *default_attrs - maxtries: 2 + maxtries: !int "{{ gridstat.execution.tries }}" command: "&JOBSdir;/GRIDSTAT_OR_POINTSTAT.sh" envars: <<: *default_vars @@ -83,9 +83,9 @@ metatask_GridStat_ASNOW_all_accums_all_mems: ENSMEM_INDX: "#mem#" FCST_LEVEL: 'A#ACCUM_HH#' FCST_THRESH: 'all' - memory: '{{ verification_resources.execution.deterministic.gridstat.memory if user.MACHINE not in ["NOAACLOUD"] }}' - nodes: '{{ verification_resources.execution.nodes }}:ppn={{ verification_resources.execution.deterministic.gridstat.tasks_per_node }}' - walltime: "{{ verification_resources.execution.deterministic.gridstat.walltime }}" + memory: '{{ gridstat.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ gridstat.execution.tasks_per_node }}' + walltime: "{{ gridstat.execution.walltime }}" dependency: and: taskdep_pcpcombine_obs: @@ -97,12 +97,15 @@ metatask_GridStat_ASNOW_all_accums_all_mems: metatask_GridStat_REFC_RETOP_all_mems: var: - mem: '{% if ensemble.DO_ENSEMBLE %}{% for m in range(1, ensemble.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% for m in range(0, ensemble.NUM_ENS_MEMBERS) %}{{ "%03d "%m }}{%- endfor -%}' metatask_GridStat_REFC_RETOP_mem#mem#: var: FIELD_GROUP: '{% for var in verification.VX_FIELD_GROUPS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_#FIELD_GROUP#_mem#mem#: <<: *default_task_verify_det + attrs: + <<: *default_attrs + maxtries: !int "{{ gridstat.execution.tries }}" command: "&JOBSdir;/GRIDSTAT_OR_POINTSTAT.sh" envars: <<: *default_vars @@ -113,9 +116,9 @@ metatask_GridStat_REFC_RETOP_all_mems: ENSMEM_INDX: "#mem#" FCST_LEVEL: 'L0' FCST_THRESH: 'all' - memory: '{{ verification_resources.execution.deterministic.gridstat.memory if user.MACHINE not in ["NOAACLOUD"] }}' - nodes: '{{ verification_resources.execution.nodes }}:ppn={{ verification_resources.execution.deterministic.gridstat.tasks_per_node }}' - walltime: "{{ verification_resources.execution.deterministic.gridstat.walltime }}" + memory: '{{ gridstat.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ gridstat.execution.tasks_per_node }}' + walltime: "{{ gridstat.execution.walltime }}" dependency: and: # Check that the flag files that indicate that the get_obs_mrms tasks @@ -142,9 +145,12 @@ metatask_PointStat: METAOB_AVAIL_HRS: '{% for var in verification.VX_FIELD_GROUPS %}{% if var =="SFC" or var =="UPA" %} {{ verification.NDAS_OBS_AVAIL_INTVL_HRS }} {% elif var =="AOD" %}{{ verification.AERONET_OBS_AVAIL_INTVL_HRS }} {% elif var =="PM25" or var =="PM10" %}{{ verification.AIRNOW_OBS_AVAIL_INTVL_HRS }} {% endif %}{% endfor %}' metatask_PointStat_#FIELD_GROUP#_all_mems: var: - mem: '{% if ensemble.DO_ENSEMBLE %}{% for m in range(1, ensemble.NUM_ENS_MEMBERS+1) %}{{ "%03d "%m }}{%- endfor -%} {% else %}{{ "000"|string }}{% endif %}' + mem: '{% for m in range(0, ensemble.NUM_ENS_MEMBERS) %}{{ "%03d "%m }}{%- endfor -%}' task_run_MET_PointStat_vx_#FIELD_GROUP#_mem#mem#: <<: *default_task_verify_det + attrs: + <<: *default_attrs + maxtries: !int "{{ pointstat.execution.tries }}" command: "&JOBSdir;/GRIDSTAT_OR_POINTSTAT.sh" envars: <<: *default_vars @@ -155,9 +161,9 @@ metatask_PointStat: ENSMEM_INDX: "#mem#" FCST_LEVEL: 'all' FCST_THRESH: 'all' - memory: '{{ verification_resources.execution.deterministic.gridstat.memory if user.MACHINE not in ["NOAACLOUD"] }}' - nodes: '{{ verification_resources.execution.nodes }}:ppn={{ verification_resources.execution.deterministic.pointstat.tasks_per_node }}' - walltime: "{{ verification_resources.execution.deterministic.pointstat.walltime }}" + memory: '{{ pointstat.execution.memory if user.MACHINE not in ["NOAACLOUD"] }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ pointstat.execution.tasks_per_node }}' + walltime: "{{ pointstat.execution.walltime }}" dependency: and: # Check that the flag files that indicate that the Pb2NC tasks are diff --git a/parm/wflow/verify_ens.yaml b/parm/wflow/verify_ens.yaml index f55f46d7..6bb8d43f 100644 --- a/parm/wflow/verify_ens.yaml +++ b/parm/wflow/verify_ens.yaml @@ -1,8 +1,8 @@ default_task_verify_ens: &default_task_verify_ens account: '&ACCOUNT;' - attrs: + attrs: &default_ens_attrs cycledefs: forecast - maxtries: 2 + maxtries: 1 envars: &default_vars GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;' USHdir: '&USHdir;' @@ -32,6 +32,9 @@ metatask_GenEnsProd_EnsembleStat_APCP_all_accums: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_GenEnsProd_vx_APCP#ACCUM_HH#h: &task_GenEnsProd_CCPA <<: *default_task_verify_ens + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ genensprod.execution.tries }}" command: "&JOBSdir;/GENENSPROD_OR_ENSEMBLESTAT.sh" envars: &envars_GenEnsProd_CCPA <<: *default_vars @@ -42,19 +45,27 @@ metatask_GenEnsProd_EnsembleStat_APCP_all_accums: OBTYPE: 'CCPA' FCST_LEVEL: 'A#ACCUM_HH#' FCST_THRESH: 'all' - OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.genensprod.default.walltime }}" + OMP_NUM_THREADS: '{{ genensprod.execution.threads }}' + memory: '{{ genensprod.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ genensprod.execution.tasks_per_node }}' + walltime: "{{ genensprod.execution.walltime }}" dependency: metataskdep_pcpcombine_fcst: attrs: metatask: PcpCombine_APCP#ACCUM_HH#h_all_mems task_run_MET_EnsembleStat_vx_APCP#ACCUM_HH#h: <<: *task_GenEnsProd_CCPA + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ ensemblestat.execution.tries }}" envars: <<: *envars_GenEnsProd_CCPA METPLUSTOOLNAME: 'ENSEMBLESTAT' FCST_THRESH: 'none' - walltime: "{{ verification_resources.execution.ensemble.ensemblestat.walltime }}" + OMP_NUM_THREADS: '{{ ensemblestat.execution.threads }}' + memory: '{{ ensemblestat.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ ensemblestat.execution.tasks_per_node }}' + walltime: "{{ ensemblestat.execution.walltime }}" dependency: and: taskdep_pcpcombine_obs_ccpa: &taskdep_pcpcombine_obs_ccpa @@ -69,6 +80,9 @@ metatask_GenEnsProd_EnsembleStat_ASNOW_all_accums: ACCUM_HH: '{% for ah in verification.VX_ASNOW_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_GenEnsProd_vx_ASNOW#ACCUM_HH#h: &task_GenEnsProd_NOHRSC <<: *default_task_verify_ens + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ genensprod.execution.tries }}" command: "&JOBSdir;/GENENSPROD_OR_ENSEMBLESTAT.sh" envars: &envars_GenEnsProd_NOHRSC <<: *default_vars @@ -79,8 +93,10 @@ metatask_GenEnsProd_EnsembleStat_ASNOW_all_accums: OBTYPE: 'NOHRSC' FCST_LEVEL: 'A#ACCUM_HH#' FCST_THRESH: 'all' - OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.genensprod.default.walltime }}" + OMP_NUM_THREADS: '{{ genensprod.execution.threads }}' + memory: '{{ genensprod.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ genensprod.execution.tasks_per_node }}' + walltime: "{{ genensprod.execution.walltime }}" dependency: and: metataskdep_pcpcombine_fcst: @@ -88,11 +104,17 @@ metatask_GenEnsProd_EnsembleStat_ASNOW_all_accums: metatask: PcpCombine_ASNOW#ACCUM_HH#h_all_mems task_run_MET_EnsembleStat_vx_ASNOW#ACCUM_HH#h: <<: *task_GenEnsProd_NOHRSC + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ ensemblestat.execution.tries }}" envars: <<: *envars_GenEnsProd_NOHRSC METPLUSTOOLNAME: 'ENSEMBLESTAT' FCST_THRESH: 'none' - walltime: "{{ verification_resources.execution.ensemble.ensemblestat.walltime }}" + OMP_NUM_THREADS: '{{ ensemblestat.execution.threads }}' + memory: '{{ ensemblestat.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ ensemblestat.execution.tasks_per_node }}' + walltime: "{{ ensemblestat.execution.walltime }}" dependency: and: taskdep_pcpcombine_obs_nohrsc: &taskdep_pcpcombine_obs_nohrsc @@ -107,6 +129,9 @@ metatask_GenEnsProd_EnsembleStat_REFC_RETOP: FIELD_GROUP: '{% for var in verification.VX_FIELD_GROUPS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GenEnsProd_vx_#FIELD_GROUP#: &task_GenEnsProd_MRMS <<: *default_task_verify_ens + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ genensprod.execution.tries }}" command: "&JOBSdir;/GENENSPROD_OR_ENSEMBLESTAT.sh" envars: &envars_GenEnsProd_MRMS <<: *default_vars @@ -117,8 +142,10 @@ metatask_GenEnsProd_EnsembleStat_REFC_RETOP: OBTYPE: 'MRMS' FCST_LEVEL: 'L0' FCST_THRESH: 'all' - OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.genensprod.default.walltime }}" + OMP_NUM_THREADS: '{{ genensprod.execution.threads }}' + memory: '{{ genensprod.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ genensprod.execution.tasks_per_node }}' + walltime: "{{ genensprod.execution.walltime }}" dependency: and: metataskdep_check_post_output: &check_post_output @@ -126,12 +153,18 @@ metatask_GenEnsProd_EnsembleStat_REFC_RETOP: metatask: check_post_output_all_mems task_run_MET_EnsembleStat_vx_#FIELD_GROUP#: <<: *task_GenEnsProd_MRMS + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ ensemblestat.execution.tries }}" envars: <<: *envars_GenEnsProd_MRMS METPLUSTOOLNAME: 'ENSEMBLESTAT' FCST_LEVEL: 'L0' FCST_THRESH: 'none' - walltime: "{{ verification_resources.execution.ensemble.ensemblestat.walltime }}" + OMP_NUM_THREADS: '{{ ensemblestat.execution.threads }}' + memory: '{{ ensemblestat.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ ensemblestat.execution.tasks_per_node }}' + walltime: "{{ ensemblestat.execution.walltime }}" dependency: and: sh_mrms_obs_available: &all_get_obs_mrms_complete @@ -150,6 +183,9 @@ metatask_GenEnsProd_EnsembleStat_SFC_UPA: FIELD_GROUP: '{% for var in verification.VX_FIELD_GROUPS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GenEnsProd_vx_#FIELD_GROUP#: &task_GenEnsProd_NDAS <<: *default_task_verify_ens + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ genensprod.execution.tries }}" command: "&JOBSdir;/GENENSPROD_OR_ENSEMBLESTAT.sh" envars: &envars_GenEnsProd_NDAS <<: *default_vars @@ -160,17 +196,25 @@ metatask_GenEnsProd_EnsembleStat_SFC_UPA: ACCUM_HH: '01' FCST_LEVEL: 'all' FCST_THRESH: 'all' - OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.genensprod.default.walltime }}" + OMP_NUM_THREADS: '{{ genensprod.execution.threads }}' + memory: '{{ genensprod.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ genensprod.execution.tasks_per_node }}' + walltime: "{{ genensprod.execution.walltime }}" dependency: metataskdep_check_post_output: <<: *check_post_output task_run_MET_EnsembleStat_vx_#FIELD_GROUP#: <<: *task_GenEnsProd_NDAS + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ ensemblestat.execution.tries }}" envars: <<: *envars_GenEnsProd_NDAS METPLUSTOOLNAME: 'ENSEMBLESTAT' - walltime: "{{ verification_resources.execution.ensemble.ensemblestat.walltime }}" + OMP_NUM_THREADS: '{{ ensemblestat.execution.threads }}' + memory: '{{ ensemblestat.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ ensemblestat.execution.tasks_per_node }}' + walltime: "{{ ensemblestat.execution.walltime }}" dependency: and: # Check that the flag files that indicate that the Pb2NC tasks @@ -195,6 +239,9 @@ metatask_GridStat_APCP_all_accums_ensmeanprob: ACCUM_HH: '{% for ah in verification.VX_APCP_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_APCP#ACCUM_HH#h_ens#statlc#: <<: *default_task_verify_ens + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ gridstat_ens.execution.tries }}" command: "&JOBSdir;/GRIDSTAT_OR_POINTSTAT_ENS#stat#.sh" envars: <<: *default_vars @@ -206,7 +253,9 @@ metatask_GridStat_APCP_all_accums_ensmeanprob: FCST_LEVEL: 'A#ACCUM_HH#' FCST_THRESH: 'all' OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.gridstat.walltime }}" + memory: '{{ gridstat_ens.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ gridstat_ens.execution.tasks_per_node }}' + walltime: "{{ gridstat_ens.execution.walltime }}" dependency: and: taskdep_pcpcombine_obs_ccpa: @@ -224,6 +273,9 @@ metatask_GridStat_ASNOW_all_accums_ensmeanprob: ACCUM_HH: '{% for ah in verification.VX_ASNOW_ACCUMS_HRS %}{% if workflow.FCST_LEN_HRS >= ah %}{{ "%02d " % ah }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_ASNOW#ACCUM_HH#h_ens#statlc#: <<: *default_task_verify_ens + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ gridstat_ens.execution.tries }}" command: "&JOBSdir;/GRIDSTAT_OR_POINTSTAT_ENS#stat#.sh" envars: <<: *default_vars @@ -235,7 +287,9 @@ metatask_GridStat_ASNOW_all_accums_ensmeanprob: FCST_LEVEL: 'A#ACCUM_HH#' FCST_THRESH: 'all' OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.gridstat.walltime }}" + memory: '{{ gridstat_ens.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ gridstat_ens.execution.tasks_per_node }}' + walltime: "{{ gridstat_ens.execution.walltime }}" dependency: and: taskdep_pcpcombine_obs_nohrsc: @@ -249,6 +303,9 @@ metatask_GridStat_REFC_RETOP_ensprob: FIELD_GROUP: '{% for var in verification.VX_FIELD_GROUPS %}{% if var in ["REFC", "RETOP"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_GridStat_vx_#FIELD_GROUP#_ensprob: <<: *default_task_verify_ens + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ gridstat_ens.execution.tries }}" command: "&JOBSdir;/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh" envars: <<: *default_vars @@ -260,7 +317,9 @@ metatask_GridStat_REFC_RETOP_ensprob: FCST_LEVEL: 'L0' FCST_THRESH: 'all' OMP_NUM_THREADS: 4 - walltime: "{{ verification_resources.execution.ensemble.gridstat.walltime }}" + memory: '{{ gridstat_ens.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ gridstat_ens.execution.tasks_per_node }}' + walltime: "{{ gridstat_ens.execution.walltime }}" dependency: and: sh_all_get_obs_mrms_complete: @@ -278,6 +337,9 @@ metatask_PointStat_SFC_UPA_ensmeanprob: FIELD_GROUP: '{% for var in verification.VX_FIELD_GROUPS %}{% if var in ["SFC", "UPA"] %}{{ "%s " % var }}{% endif %}{% endfor %}' task_run_MET_PointStat_vx_#FIELD_GROUP#_ens#statlc#: <<: *default_task_verify_ens + attrs: + <<: *default_ens_attrs + maxtries: !int "{{ pointstat_ens.execution.tries }}" command: "&JOBSdir;/GRIDSTAT_OR_POINTSTAT_ENS#stat#.sh" envars: <<: *default_vars @@ -288,7 +350,9 @@ metatask_PointStat_SFC_UPA_ensmeanprob: ACCUM_HH: '01' FCST_LEVEL: 'all' FCST_THRESH: 'all' - walltime: "{{ verification_resources.execution.ensemble.pointstat.walltime }}" + memory: '{{ pointstat_ens.execution.memory }}' + nodes: '{{ verification_resources.execution.nodes }}:ppn={{ pointstat_ens.execution.tasks_per_node }}' + walltime: "{{ pointstat_ens.execution.walltime }}" dependency: and: sh_all_pb2nc_obs_ndas_complete: diff --git a/parm/wflow/verify_tc.yaml b/parm/wflow/verify_tc.yaml index bb95f4b2..1d219db6 100644 --- a/parm/wflow/verify_tc.yaml +++ b/parm/wflow/verify_tc.yaml @@ -1,6 +1,6 @@ default_task_tc: &default_task_tc account: '&ACCOUNT;' - attrs: &default_attrs + attrs: &default_tc_attrs cycledefs: forecast maxtries: 1 envars: &default_vars @@ -27,6 +27,9 @@ default_task_tc: &default_task_tc task_tcpairs: <<: *default_task_tc + attrs: + <<: *default_tc_attrs + maxtries: !int "{{ tcpairs.execution.tries }}" command: "&JOBSdir;/TCPAIRS.sh" envars: <<: *default_vars @@ -43,6 +46,9 @@ task_tcpairs: task_tcstat: <<: *default_task_tc + attrs: + <<: *default_tc_attrs + maxtries: !int "{{ tcstat.execution.tries }}" command: "&JOBSdir;/TCSTAT.sh" envars: <<: *default_vars @@ -60,6 +66,9 @@ task_tcstat: task_tcrmw: <<: *default_task_tc + attrs: + <<: *default_tc_attrs + maxtries: !int "{{ tcrmw.execution.tries }}" command: "&JOBSdir;/TCRMW.sh" envars: <<: *default_vars diff --git a/scripts/check_post_output.py b/scripts/check_post_output.py new file mode 100644 index 00000000..78b4943f --- /dev/null +++ b/scripts/check_post_output.py @@ -0,0 +1,104 @@ +# pylint: disable=logging-fstring-interpolation +""" +Converted from scripts/check_post_output.sh, this script checks that the expected +post-processed forecast output files exist on disk, up to the allowed maximum +number of missing files. + +The script is intended to be called from jobs/CHECK_POST_OUTPUT.sh. +""" + +import argparse +import logging +from string import Template + +import uwtools.api.config as uwconfig + +from python_utils import setup_logging +from set_leadhrs import set_leadhrs + + +def check_post_output(config_file: str, cdate: str, ensmem_index: int) -> None: + """Check that post-processed forecast output files exist for a given cycle and member. + + Calls set_leadhrs() to verify that no more than NUM_MISSING_FCST_FILES_MAX files + are absent from disk. Raises an exception if the missing-file threshold is exceeded. + + Parameters + ---------- + config_file : str + Path to the experiment configuration YAML. + cdate : str + Eight-digit cycle date in ``YYYYMMDDHH`` format. + ensmem_index : int + Ensemble member index (0-based; use 0 for deterministic runs). + """ + lgr = logging.getLogger(__name__) + + cfg = uwconfig.get_yaml_config(config=config_file) + vxcfg = cfg["verification"] + wfcfg = cfg["workflow"] + enscfg = cfg["ensemble"] + + # Get time lag in seconds for this member; deterministic uses index 0 + i = max(ensmem_index - 1, 0) + time_lag = int(enscfg["ENS_TIME_LAG_HRS"][i]) * 3600 + + # Make a dictionary of variables that may need to be substituted; these will be used to replace + # bash-like variables in some strings. This is needed to maintain some functionality while we + # still have a mix of bash and python exscripts. + subvars = { + "ensmem_name": f"mem{str(ensmem_index).zfill(vxcfg['VX_NDIGITS_ENSMEM_NAMES'])}", + "time_lag": time_lag, + } + +# Build forecast filename template + fn_template = vxcfg["FCST_FN_TEMPLATE"][i] + + lgr.debug(f"{fn_template=}") + fn_template=Template(fn_template).substitute(subvars) + lgr.info( + f"Checking post-processed output files for cycle {cdate}, " + f"member index {ensmem_index}" + ) + lgr.debug(f"{fn_template=}") + lgr.debug(f"{time_lag=}") + + set_leadhrs( + date_init=cdate, + lhr_min=0, + lhr_max=wfcfg["FCST_LEN_HRS"], + lhr_intvl=vxcfg["VX_FCST_OUTPUT_INTVL_HRS"], + base_dir=vxcfg["VX_FCST_INPUT_BASEDIR"], + time_lag=time_lag, + fn_template=fn_template, + num_missing_files_max=vxcfg["NUM_MISSING_FCST_FILES_MAX"], + ) + + lgr.info(f"Post-processed output file check completed successfully for cycle {cdate}.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Check that post-processed forecast output files exist on disk" + ) + parser.add_argument( + "--config", default="config.yaml", type=str, + help="Path to the experiment configuration file in YAML format", + ) + parser.add_argument( + "--cycle_date", required=True, type=str, + help="Eight-digit cycle date (YYYYMMDDHH)", + ) + parser.add_argument( + "--ensmem_index", required=True, type=int, + help="Ensemble member index (use 0 for deterministic)", + ) + parser.add_argument( + "-v", "--verbose", action="store_true", + help="Enable verbose debug output", + ) + args = parser.parse_args() + + setup_logging(debug=args.verbose) + + check_post_output(args.config, args.cycle_date, args.ensmem_index) diff --git a/scripts/check_post_output.sh b/scripts/check_post_output.sh deleted file mode 100755 index c2f18c24..00000000 --- a/scripts/check_post_output.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash -# -#----------------------------------------------------------------------- -# -# The ex-script for checking the post output. -# -# Run-time environment variables: -# -# ACCUM_HH -# ENSMEM_INDX -# HH -# GLOBAL_VAR_DEFNS_FP -# METPLUS_ROOT (used by ush/set_leadhrs.py) -# YYMMDD -# -# Experiment variables -# -# user: -# USHdir -# -# workflow: -# FCST_LEN_HRS -# -# ensemble: -# DO_ENSEMBLE -# ENS_TIME_LAG_HRS -# -# verification: -# FCST_FN_TEMPLATE -# FCST_SUBDIR_TEMPLATE -# NUM_MISSING_FCST_FILES_MAX -# VX_FCST_INPUT_BASEDIR -# VX_NDIGITS_ENSMEM_NAMES -# -#----------------------------------------------------------------------- -# - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -sections=( - user - workflow - ensemble - verification -) -for sect in ${sections[*]} ; do - source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} -done -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the ex-script for the task that checks that no more than -NUM_MISSING_FCST_FILES_MAX of each forecast's (ensemble member's) post- -processed output files are missing. -========================================================================" -# -#----------------------------------------------------------------------- -# -# Get the time lag for the current ensemble member. -# -#----------------------------------------------------------------------- -# -i="0" -if [[ "${DO_ENSEMBLE}" == "True" ]]; then - i=$( bc -l <<< "${ENSMEM_INDX}-1" ) -fi -time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*3600" ) -# -#----------------------------------------------------------------------- -# -# Check to ensure that all the expected post-processed forecast output -# files are present on disk. This is done by the set_leadhrs function -# below. -# -#----------------------------------------------------------------------- -# -ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" $(( 10#${ENSMEM_INDX}))) -ensmem_name="mem${ensmem_indx}" -FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_SUBDIR_TEMPLATE:+${FCST_SUBDIR_TEMPLATE}/}${FCST_FN_TEMPLATE} ) - -FHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --date_init="${YYMMDD}${HH}" \ - --lhr_min="0" \ - --lhr_max="${FCST_LEN_HRS}" \ - --lhr_intvl="${VX_FCST_OUTPUT_INTVL_HRS}" \ - --base_dir="${VX_FCST_INPUT_BASEDIR}" \ - --fn_template="${FCST_INPUT_FN_TEMPLATE}" \ - --num_missing_files_max="${NUM_MISSING_FCST_FILES_MAX}" \ - --time_lag="${time_lag%.*}") || \ -print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Done checking for existence of post-processed files for ensemble member ${ENSMEM_INDX}. - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" diff --git a/scripts/genensprod_or_ensemblestat.py b/scripts/genensprod_or_ensemblestat.py new file mode 100644 index 00000000..ead82fd3 --- /dev/null +++ b/scripts/genensprod_or_ensemblestat.py @@ -0,0 +1,287 @@ +# pylint: disable=logging-fstring-interpolation +""" +Converted from scripts/genensprod_or_ensemblestat.sh, this script calls either the METplus +"GenEnsProd" tool to generate ensemble products or the "EnsembleStat" tool to perform +ensemble-based verification. + +The script is intended to be called from jobs/GENENSPROD_OR_ENSEMBLESTAT.sh. +""" + +import argparse +import logging +import os + +from multiprocessing import Pool +from pathlib import Path +from string import Template + +import uwtools.api.config as uwconfig + +from python_utils import run_metplus, render_metplus_confs, setup_logging +from set_leadhrs import set_leadhrs +from set_vx_params import set_vx_params + +# Maps uppercase Rocoto METPLUSTOOLNAME to (CamelCase, snake_case) tool name variants +_TOOL_NAME_MAP = { + "GENENSPROD": ("GenEnsProd", "gen_ens_prod"), + "ENSEMBLESTAT": ("EnsembleStat", "ensemble_stat"), +} + + +def genensprod_or_ensemblestat( + config_file: str, + cdate: str, + obs_dir: str, + field_group: str, + obtype: str, + accum_hh: int, + fcst_level: str, + fcst_thresh: str, + metplus_tool: str, +) -> None: + """Execute a METplus GenEnsProd or EnsembleStat ensemble verification task. + + Parameters + ---------- + config_file : str + Path to the experiment YAML configuration file. + cdate : str + Eight-digit cycle date in ``YYYYMMDDHH`` format. + obs_dir : str + Directory containing observation files for the chosen obtype. + field_group : str + Group of observation fields to verify (e.g. APCP, REFC, SFC). + obtype : str + Observation type (e.g. NOHRSC, CCPA, NDAS). + accum_hh : int + Accumulation hours for the observation type. + fcst_level : str + METplus forecast level (e.g. L0, A03). + fcst_thresh : str + Forecast threshold set (usually "all" or "none"). + metplus_tool : str + METplus tool to run: ``"GENENSPROD"`` or ``"ENSEMBLESTAT"`` (case-insensitive). + """ + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-branches,too-many-statements + lgr = logging.getLogger(__name__) + + key = metplus_tool.upper() + if key not in _TOOL_NAME_MAP: + raise ValueError( + f"Invalid metplus_tool '{metplus_tool}'. " + f"Valid options: {list(_TOOL_NAME_MAP.keys())}" + ) + metplus_tool_camel_case, metplus_tool_name = _TOOL_NAME_MAP[key] + + cfg = uwconfig.get_yaml_config(config=config_file) + vxcfg = cfg["verification"] + enscfg = cfg["ensemble"] + + if metplus_tool_name == "ensemble_stat": + if not Path(obs_dir).is_dir(): + raise FileNotFoundError(f"{obs_dir=} does not exist or is not a directory") + + geom, _, _, met_out_name, met_filedir_name = set_vx_params(obtype, field_group, accum_hh) + + exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + } + + # Build obs input dir/template and forecast base dir + if geom == "grid": + if "APCP" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = Template( + vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) + fcst_in_dir = Path(exptdir) + elif "ASNOW" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = Template( + vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) + fcst_in_dir = Path(exptdir) + elif met_filedir_name == "REFC": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + ).substitute(subvars) + fcst_in_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) + elif met_filedir_name == "RETOP": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + ).substitute(subvars) + fcst_in_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) + else: + raise ValueError( + f"Invalid field group for {metplus_tool_camel_case}: {field_group}" + ) + elif geom == "point": + obs_in_dir = Path(exptdir, "metprd", "Pb2nc_obs") + obs_in_fn_template = Template( + vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + ).substitute(subvars) + fcst_in_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) + else: + raise ValueError(f"Invalid parameters: {obtype=}, {field_group=}, {accum_hh=}") + + fcst_in_fn_templates = [] + for i in range(enscfg["NUM_ENS_MEMBERS"]): + # Build per-member forecast filename templates (comma-separated list for METplus) + ensmem = f"mem{str(i).zfill(vxcfg['VX_NDIGITS_ENSMEM_NAMES'])}" + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + "time_lag": int(enscfg["ENS_TIME_LAG_HRS"][i]) * 3600, + "ensmem_name": ensmem, + } + if field_group in ("APCP", "ASNOW"): + tmpl = str(Path( + cdate, ensmem, "metprd", "PcpCombine_fcst", + Template(vxcfg["FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT"]).safe_substitute(subvars), + )) + else: + tmpl = Template(vxcfg["FCST_FN_TEMPLATE"][i]).safe_substitute(subvars) + fcst_in_fn_templates.append(tmpl) + fcst_in_fn_template = ", ".join(fcst_in_fn_templates) + + output_dir = Path(exptdir, cdate, "metprd", metplus_tool_camel_case) + # The ".0" will be stripped from this dirname if using only one task, otherwise + # it will be replaced with the task number + staging_dir = Path(exptdir, cdate, "stage", f"{met_filedir_name}.0") + os.makedirs(output_dir, exist_ok=True) + + if obtype in ("CCPA", "NOHRSC"): + vx_intvl = vx_hr_start = accum_hh + else: + vx_intvl = vxcfg["VX_FCST_OUTPUT_INTVL_HRS"] + vx_hr_start = 0 + + # GenEnsProd only needs forecast files; skip obs existence check + vx_leadhr_list = set_leadhrs( + date_init=cdate, + lhr_min=vx_hr_start, + lhr_max=cfg["workflow"]["FCST_LEN_HRS"], + lhr_intvl=vx_intvl, + base_dir=obs_in_dir, + time_lag=0, + fn_template=str(obs_in_fn_template), + num_missing_files_max=vxcfg["NUM_MISSING_OBS_FILES_MAX"], + skip_check_files=(metplus_tool_camel_case == "GenEnsProd"), + ) + + if not vx_leadhr_list: + raise RuntimeError( + f"set_leadhrs returned an empty list for cycle {cdate}, " + f"{obtype=}, {field_group=}" + ) + + vx_mask_files = [] + if vxcfg["VX_MASK"]: + for mask in vxcfg["VX_MASK"]: + if os.path.isfile(maskfile := f"{cfg['user']['METPLUS_CONF']}/{mask}.poly"): + vx_mask_files.append(maskfile) + else: + vx_mask_files.append( + f"{os.environ['MET_INSTALL_DIR']}/share/met/poly/{mask}.poly" + ) + + metplus_config_tmpl_fn = f"{metplus_tool_camel_case}.conf" + metplus_config_fn = f"{metplus_tool_camel_case}_{met_filedir_name}_{cdate}.conf.0" + metplus_log_fn = f"metplus.log.{metplus_tool_camel_case}_{met_filedir_name}_{cdate}.0" + + vx_config_dict = uwconfig.get_yaml_config( + config=f"{cfg['user']['METPLUS_CONF']}/{vxcfg['VX_CONFIG_ENS_FN']}" + ) + + settings = { + "metplus_tool_name": metplus_tool_name, + "MetplusToolName": metplus_tool_camel_case, + "METPLUS_TOOL_NAME": metplus_tool_name.upper(), + "metplus_verbosity_level": vxcfg["METPLUS_VERBOSITY_LEVEL"], + "cdate": cdate, + "vx_leadhr_list": ", ".join(map(str, vx_leadhr_list)), + "metplus_config_fn": metplus_config_fn, + "metplus_log_fn": metplus_log_fn, + "obs_input_dir": obs_in_dir, + "obs_input_fn_template": obs_in_fn_template, + "fcst_input_dir": fcst_in_dir, + "fcst_input_fn_template": fcst_in_fn_template, + "output_dir": output_dir, + "output_fn_template": "", + "staging_dir": staging_dir, + "vx_fcst_model_name": vxcfg["VX_FCST_MODEL_NAME"], + "num_ens_members": enscfg["NUM_ENS_MEMBERS"], + "ensmem_name": "", + "time_lag": 0, + "fieldname_in_obs_input": str(obs_in_dir), + "fieldname_in_fcst_input": str(fcst_in_dir), + "fieldname_in_met_output": met_out_name, + "fieldname_in_met_filedir_names": met_filedir_name, + "obtype": obtype, + "accum_hh": f"{accum_hh:02}", + "accum_no_pad": accum_hh, + "metplus_templates_dir": cfg["user"]["METPLUS_CONF"], + "input_field_group": field_group, + "input_level_fcst": fcst_level, + "input_thresh_fcst": fcst_thresh, + "vx_mask": ", ".join(vx_mask_files), + "vx_config_dict": vx_config_dict, + } + + numprocs = int(cfg[metplus_tool_camel_case.lower()]["TASKS"]) + + conf_files = render_metplus_confs(cfg,settings,metplus_config_tmpl_fn,vx_leadhr_list,numprocs, + cfg["genensprod"].get("conf")) + lgr.debug(f"{conf_files=}") + + lgr.info(f"Running {metplus_tool_camel_case} with METplus with {numprocs} tasks") + mpargs = [] + for config_fn in conf_files: + mpargs.append( (os.path.join(cfg['user']['METPLUS_CONF'], "common.conf"),config_fn) ) + # Call run_metplus function for as many processors as specified + lgr.debug(f"{mpargs=}") + with Pool(processes=numprocs) as pool: + pool.starmap(run_metplus,mpargs) + + lgr.info(f"{metplus_tool_camel_case} completed successfully.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run METplus GenEnsProd or EnsembleStat for ensemble verification" + ) + parser.add_argument("--config", default="config.yaml", type=str, + help="Path to the experiment configuration file in YAML format") + parser.add_argument("--cycle_date", required=True, type=str, + help="Eight-digit cycle date (YYYYMMDDHH)") + parser.add_argument("--obs_dir", required=True, type=str, + help="Observation directory for this obtype") + parser.add_argument("--field_group", required=True, type=str, + help="Group of fields for this verification task (e.g. APCP, REFC, SFC)") + parser.add_argument("--obtype", required=True, type=str, + help="Observation type (e.g. NOHRSC, CCPA, NDAS)") + parser.add_argument("--accum_hh", default=1, type=int, + help="Accumulation hours for this observation type") + parser.add_argument("--fcst_level", default="", type=str, + help="METplus forecast level (e.g. L0, A03)") + parser.add_argument("--fcst_thresh", default="", type=str, + help="Forecast thresholds to verify against (e.g. all, none)") + parser.add_argument("--metplus_tool", required=True, type=str, + help="METplus tool to run: GENENSPROD or ENSEMBLESTAT") + parser.add_argument("-v", "--verbose", action="store_true", + help="Enable verbose debug output") + args = parser.parse_args() + + setup_logging(debug=args.verbose) + logging.debug(f"{os.environ['METPLUS_ROOT']=}") + + genensprod_or_ensemblestat( + args.config, args.cycle_date, args.obs_dir, args.field_group, + args.obtype, args.accum_hh, args.fcst_level, args.fcst_thresh, + args.metplus_tool, + ) diff --git a/scripts/genensprod_or_ensemblestat.sh b/scripts/genensprod_or_ensemblestat.sh deleted file mode 100755 index 37c35481..00000000 --- a/scripts/genensprod_or_ensemblestat.sh +++ /dev/null @@ -1,458 +0,0 @@ -#!/usr/bin/env bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -sections=( - user - platform - workflow - ensemble - verification -) -for sect in ${sections[*]} ; do - source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} -done -# -#----------------------------------------------------------------------- -# -# Source files defining auxiliary functions for verification. -# -#----------------------------------------------------------------------- -# -. $USHdir/get_metplus_tool_name.sh -. $USHdir/set_vx_params.sh -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of the MET/METplus tool in different formats that may be -# needed from the variable METPLUSTOOLNAME. -# -#----------------------------------------------------------------------- -# -get_metplus_tool_name \ - METPLUSTOOLNAME="${METPLUSTOOLNAME}" \ - outvarname_metplus_tool_name="metplus_tool_name" \ - outvarname_MetplusToolName="MetplusToolName" \ - outvarname_METPLUS_TOOL_NAME="METPLUS_TOOL_NAME" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# True. -# -#----------------------------------------------------------------------- -# -print_input_args "valid_args" -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the ex-script for the task that runs the METplus ${MetplusToolName} -tool either to generate ensemble products without performing verification -(if running the GenEnsProd tool) or to perform ensemble-based verification -(if running the EnsembleStat tool). -========================================================================" -# -#----------------------------------------------------------------------- -# -# Get the cycle date and time in YYYYMMDDHH format. -# -#----------------------------------------------------------------------- -# -CDATE="${YYMMDD}${HH}" -# -#----------------------------------------------------------------------- -# -# Set various verification parameters associated with the field to be -# verified. Not all of these are necessarily used later below but are -# set here for consistency with other verification ex-scripts. -# -#----------------------------------------------------------------------- -# -FIELDNAME_IN_OBS_INPUT="" -FIELDNAME_IN_FCST_INPUT="" -FIELDNAME_IN_MET_OUTPUT="" -FIELDNAME_IN_MET_FILEDIR_NAMES="" - -set_vx_params \ - obtype="${OBTYPE}" \ - field_group="${FIELD_GROUP}" \ - accum_hh="${ACCUM_HH}" \ - outvarname_grid_or_point="grid_or_point" \ - outvarname_fieldname_in_obs_input="FIELDNAME_IN_OBS_INPUT" \ - outvarname_fieldname_in_fcst_input="FIELDNAME_IN_FCST_INPUT" \ - outvarname_fieldname_in_MET_output="FIELDNAME_IN_MET_OUTPUT" \ - outvarname_fieldname_in_MET_filedir_names="FIELDNAME_IN_MET_FILEDIR_NAMES" -# -#----------------------------------------------------------------------- -# -# Set paths and file templates for input to and output from the MET/ -# METplus tool to be run as well as other file/directory parameters. -# -#----------------------------------------------------------------------- -# -vx_fcst_input_basedir=$( eval echo "${VX_FCST_INPUT_BASEDIR}" ) -vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) - -if [ "${grid_or_point}" = "grid" ]; then - - case "${FIELDNAME_IN_MET_FILEDIR_NAMES}" in - "APCP"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - FCST_INPUT_DIR="${vx_output_basedir}" - ;; - "ASNOW"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - FCST_INPUT_DIR="${vx_output_basedir}" - ;; - "REFC") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[1]}" - FCST_INPUT_DIR="${vx_fcst_input_basedir}" - ;; - "RETOP") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[3]}" - FCST_INPUT_DIR="${vx_fcst_input_basedir}" - ;; - esac - -elif [ "${grid_or_point}" = "point" ]; then - - OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT}" - FCST_INPUT_DIR="${vx_fcst_input_basedir}" - -fi -OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) -# -# Construct variable that contains a METplus template of the paths to -# the files that the PcpCombine tool has generated (in previous workflow -# tasks). This will be exported to the environment and read by the -# METplus configuration files. -# -FCST_INPUT_FN_TEMPLATE="" -for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do - - ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" "$((i+1))") - ensmem_name="mem${ensmem_indx}" - - cdate_ensmem_subdir_or_null="${CDATE}/${ensmem_name}" - - time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*3600" ) - - if [ "${FIELD_GROUP}" = "APCP" ] || [ "${FIELD_GROUP}" = "ASNOW" ]; then - template="${cdate_ensmem_subdir_or_null:+${cdate_ensmem_subdir_or_null}/}metprd/PcpCombine_fcst/${FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - else - template="${FCST_SUBDIR_TEMPLATE}/${FCST_FN_TEMPLATE}" - fi - - if [ -z "${FCST_INPUT_FN_TEMPLATE}" ]; then - FCST_INPUT_FN_TEMPLATE="$(eval echo ${template})" - else - FCST_INPUT_FN_TEMPLATE="${FCST_INPUT_FN_TEMPLATE}, $(eval echo ${template})" - fi - -done - -OUTPUT_BASE="${vx_output_basedir}/${CDATE}" -OUTPUT_DIR="${OUTPUT_BASE}/metprd/${MetplusToolName}" -STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}" -# -#----------------------------------------------------------------------- -# -# Generate the list of forecast hours for which to run the specified -# METplus tool. -# -# If running the GenEnsProd tool, we set this to the list of forecast -# output times without filtering for the existence of observation files -# corresponding to those times. This is because GenEnsProd operates -# only on forecasts; it does not need observations. -# -# On the other hand, if running the EnsembleStat tool, we set the list of -# forecast hours to a set of times that takes into consideration whether -# or not observations exist. We do this by starting with the full list -# of forecast times for which there is forecast output and then removing -# from that list any times for which there is no corresponding observations. -# -#----------------------------------------------------------------------- -# -case "$OBTYPE" in - "CCPA"|"NOHRSC") - vx_intvl="$((10#${ACCUM_HH}))" - vx_hr_start="${vx_intvl}" - ;; - *) - vx_intvl="$((${VX_FCST_OUTPUT_INTVL_HRS}))" - vx_hr_start="0" - ;; -esac -vx_hr_end="${FCST_LEN_HRS}" - -if [ "${MetplusToolName}" = "GenEnsProd" ]; then - VX_LEADHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --lhr_min="${vx_hr_start}" \ - --lhr_max="${vx_hr_end}" \ - --lhr_intvl="${vx_intvl}" \ - --skip_check_files ) || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" - -elif [ "${MetplusToolName}" = "EnsembleStat" ]; then - VX_LEADHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --date_init="${CDATE}" \ - --lhr_min="${vx_hr_start}" \ - --lhr_max="${vx_hr_end}" \ - --lhr_intvl="${vx_intvl}" \ - --base_dir="${OBS_INPUT_DIR}" \ - --fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - --num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" \ - --time_lag="${time_lag%.*}" ) || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" -fi -# -#----------------------------------------------------------------------- -# -# Make sure the MET/METplus output directory(ies) exists. -# -#----------------------------------------------------------------------- -# -mkdir -p "${OUTPUT_DIR}" -# -#----------------------------------------------------------------------- -# -# Check for existence of top-level OBS_DIR, if necessary. -# -#----------------------------------------------------------------------- -# -if [ "${MetplusToolName}" = "EnsembleStat" ]; then - if [ ! -d "${OBS_DIR}" ]; then - print_err_msg_exit "\ - OBS_DIR does not exist or is not a directory: - OBS_DIR = \"${OBS_DIR}\"" - fi -fi -# -#----------------------------------------------------------------------- -# -# Export variables needed in the common METplus configuration file (at -# ${METPLUS_CONF}/common.conf). -# -#----------------------------------------------------------------------- -# -export METPLUS_CONF -export LOGDIR -# -#----------------------------------------------------------------------- -# -# Do not run METplus if there isn't at least one lead hour for which to -# run it. -# -#----------------------------------------------------------------------- -# -if [ -z "${VX_LEADHR_LIST}" ]; then - print_err_msg_exit "\ -The list of lead hours for which to run METplus is empty: - VX_LEADHR_LIST = [${VX_LEADHR_LIST}]" -fi -# -#----------------------------------------------------------------------- -# -# Populate VX_MASK_FILE_LIST based on user selections -# -#----------------------------------------------------------------------- -# -# This weird logic is needed because in older versions of bash empty arrays are treated as unset -if [ ${VX_MASK[@]} ]; then - VX_MASK_FILE_LIST="" - for i in "${VX_MASK[@]}"; do - if [ -f "${METPLUS_CONF}/${i}.poly" ]; then - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, ${METPLUS_CONF}/${i}.poly" - else - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, {MET_INSTALL_DIR}/share/met/poly/${i}.poly" - fi - done -fi -# -#----------------------------------------------------------------------- -# -# Set the names of the template METplus configuration file, the METplus -# configuration file generated from this template, and the METplus log -# file. -# -#----------------------------------------------------------------------- -# -# First, set the base file names. -# -metplus_config_tmpl_bn="${MetplusToolName}" -metplus_config_bn="${MetplusToolName}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${CDATE}" -metplus_log_bn="${metplus_config_bn}" -# -# Add prefixes and suffixes (extensions) to the base file names. -# -metplus_config_tmpl_fn="${metplus_config_tmpl_bn}.conf" -metplus_config_fn="${metplus_config_bn}.conf" -metplus_log_fn="metplus.log.${metplus_log_bn}" -# -#----------------------------------------------------------------------- -# -# Load the yaml-like file containing the configuration for ensemble -# verification. -# -#----------------------------------------------------------------------- -# -vx_config_fp="${METPLUS_CONF}/${VX_CONFIG_ENS_FN}" -vx_config_dict=$(<"${vx_config_fp}") -# Indent each line of vx_config_dict so that it is aligned properly when -# included in the yaml-formatted variable "settings" below. -vx_config_dict=$( printf "%s\n" "${vx_config_dict}" | sed 's/^/ /' ) -# -#----------------------------------------------------------------------- -# -# Generate the METplus configuration file from its jinja template. -# -#----------------------------------------------------------------------- -# -# Set the full paths to the jinja template METplus configuration file -# (which already exists) and the METplus configuration file that will be -# generated from it. -# -metplus_config_tmpl_fp="${METPLUS_CONF}/${metplus_config_tmpl_fn}" -metplus_config_fp="${OUTPUT_DIR}/${metplus_config_fn}" -# -# Define variables that appear in the jinja template. -# -settings="\ -# -# MET/METplus information. -# -'metplus_tool_name': '${metplus_tool_name}' -'MetplusToolName': '${MetplusToolName}' -'METPLUS_TOOL_NAME': '${METPLUS_TOOL_NAME}' -'metplus_verbosity_level': '${METPLUS_VERBOSITY_LEVEL}' -# -# Date and forecast hour information. -# -'cdate': '$CDATE' -'vx_leadhr_list': '${VX_LEADHR_LIST}' -# -# Input and output directory/file information. -# -'metplus_config_fn': '${metplus_config_fn:-}' -'metplus_log_fn': '${metplus_log_fn:-}' -'obs_input_dir': '${OBS_INPUT_DIR:-}' -'obs_input_fn_template': '${OBS_INPUT_FN_TEMPLATE:-}' -'fcst_input_dir': '${FCST_INPUT_DIR:-}' -'fcst_input_fn_template': '${FCST_INPUT_FN_TEMPLATE:-}' -'output_dir': '${OUTPUT_DIR}' -'output_fn_template': '${OUTPUT_FN_TEMPLATE:-}' -'staging_dir': '${STAGING_DIR}' -'vx_fcst_model_name': '${VX_FCST_MODEL_NAME}' -# -# Ensemble and member-specific information. -# -'num_ens_members': '${NUM_ENS_MEMBERS}' -'ensmem_name': '${ensmem_name:-}' -'time_lag': '${time_lag:-}' -# -# Field information. -# -'fieldname_in_obs_input': '${FIELDNAME_IN_OBS_INPUT}' -'fieldname_in_fcst_input': '${FIELDNAME_IN_FCST_INPUT}' -'fieldname_in_met_output': '${FIELDNAME_IN_MET_OUTPUT}' -'fieldname_in_met_filedir_names': '${FIELDNAME_IN_MET_FILEDIR_NAMES}' -'obtype': '${OBTYPE}' -'accum_hh': '${ACCUM_HH:-}' -'accum_no_pad': '${ACCUM_NO_PAD:-}' -'metplus_templates_dir': '${METPLUS_CONF:-}' -'input_field_group': '${FIELD_GROUP:-}' -'input_level_fcst': '${FCST_LEVEL:-}' -'input_thresh_fcst': '${FCST_THRESH:-}' -# -# Verification mask settings -# -'vx_mask': '${VX_MASK_FILE_LIST:-}' -# -# -# Verification configuration dictionary. -# -'vx_config_dict': -${vx_config_dict:-} -" - -# Render the template to create a METplus configuration file -tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") -printf "%s" "$settings" > "$tmpfile" -uw template render \ - -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} \ - --verbose \ - --values-file "${tmpfile}" \ - --search-path "/" - -err=$? -rm $tmpfile -if [ $err -ne 0 ]; then - message_txt="Error rendering template for METplus config. - Contents of input are: -$settings" - print_err_msg_exit "${message_txt}" -fi -# -#----------------------------------------------------------------------- -# -# Call METplus. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Calling METplus to run MET's ${metplus_tool_name} tool for field(s): ${FIELDNAME_IN_MET_FILEDIR_NAMES}" -${METPLUS_ROOT}/ush/run_metplus.py \ - -c ${METPLUS_CONF}/common.conf \ - -c ${metplus_config_fp} || \ -print_err_msg_exit " -Call to METplus failed with return code: $? -METplus configuration file used is: - metplus_config_fp = \"${metplus_config_fp}\"" -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -METplus ${MetplusToolName} tool completed successfully. - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" diff --git a/scripts/get_obs.py b/scripts/get_obs.py index 860e62df..0413d483 100644 --- a/scripts/get_obs.py +++ b/scripts/get_obs.py @@ -873,10 +873,15 @@ def get_obs(config, obtype, yyyymmdd_task): fn_raw = f'sfav2_CONUS_{accum_obs_formatted}h_{yyyymmddhh_str}_grid184.grb2' # pylint: disable=line-too-long elif obtype == 'MRMS': #MRMS files are retrieved from HPSS archives as gzip; need to unzip - with gzip.open(valid_file_name, 'rb') as f_in: - with open(fn_raw:=valid_file_name.replace(".gz",""), 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - fn_raw = os.path.join('topofhour', fn_raw) + if valid_file_name: + with gzip.open(valid_file_name, 'rb') as f_in: + with open(fn_raw:=valid_file_name.replace(".gz",""), 'wb') as f_out: # pylint: disable=line-too-long + shutil.copyfileobj(f_in, f_out) + fn_raw = os.path.join('topofhour', fn_raw) + else: + logging.warning("No file found for MRMS for valid time\n") + logging.warning(f"{yyyymmddhh}, field group {fg}") + continue elif obtype == 'NDAS': time_ago = yyyymmddhh_arcv - yyyymmddhh hrs_ago = int(time_ago.seconds/3600) diff --git a/scripts/gridstat_or_pointstat.py b/scripts/gridstat_or_pointstat.py index 14339511..afffa684 100644 --- a/scripts/gridstat_or_pointstat.py +++ b/scripts/gridstat_or_pointstat.py @@ -8,9 +8,7 @@ """ import argparse -import ast import logging -import math import os import subprocess @@ -95,8 +93,7 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, lgr.debug(f"{vxcfg['VX_NDIGITS_ENSMEM_NAMES']=}") time_lag = 0 if do_ens: - time_lag_hrs = ast.literal_eval(cfg['ensemble']['ENS_TIME_LAG_HRS'])[ensmem_index-1] - time_lag = time_lag_hrs*3600 + time_lag = cfg['ensemble']['ENS_TIME_LAG_HRS'][ensmem_index]*3600 # Make a dictionary of variables that may need to be substituted; these will be used to replace # bash-like variables in some strings. This is needed to maintain some functionality while we @@ -126,10 +123,7 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] ).substitute(subvars) lgr.debug(f"{vxcfg['FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT']=}") - fcst_in_fn_template = Template( - vxcfg["FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] - ).substitute(subvars) - lgr.debug(f"{fcst_in_fn_template=}") + fcst_fn_tmpl = Template(vxcfg["FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT"]).substitute(subvars) elif "ASNOW" in met_filedir_name: if do_ens: obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") @@ -142,21 +136,19 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] ).substitute(subvars) ) - fcst_in_fn_template = Path( + fcst_fn_tmpl = Path( Template(vxcfg["FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT"]).substitute(subvars) ) elif met_filedir_name == "REFC": obs_in_dir = obs_dir fcst_in_dir = vxcfg["VX_FCST_INPUT_BASEDIR"] obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][1] - fcst_in_fn_template = Path(Template(vxcfg["FCST_SUBDIR_TEMPLATE"]).substitute(subvars), - Template(vxcfg["FCST_FN_TEMPLATE"]).substitute(subvars)) + fcst_fn_tmpl = Template(vxcfg["FCST_FN_TEMPLATE"][ensmem_index]).substitute(subvars) elif met_filedir_name == "RETOP": obs_in_dir = obs_dir fcst_in_dir = vxcfg["VX_FCST_INPUT_BASEDIR"] obs_in_fn_template = vxcfg["OBS_MRMS_FN_TEMPLATES"][3] - fcst_in_fn_template = Path(Template(vxcfg["FCST_SUBDIR_TEMPLATE"]).substitute(subvars), - Template(vxcfg["FCST_FN_TEMPLATE"]).substitute(subvars)) + fcst_fn_tmpl = Template(vxcfg["FCST_FN_TEMPLATE"][ensmem_index]).substitute(subvars) else: raise ValueError(f"Invalid OBTYPE for GridStat: {obtype}") @@ -167,16 +159,14 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, obs_in_dir = Path(exptdir, "metprd", "Pb2nc_obs") fcst_in_dir = vxcfg["VX_FCST_INPUT_BASEDIR"] obs_in_fn_template = vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] - fcst_in_fn_template = Path(Template(vxcfg["FCST_SUBDIR_TEMPLATE"]).substitute(subvars), - Template(vxcfg["FCST_FN_TEMPLATE"]).substitute(subvars)) + fcst_fn_tmpl = Template(vxcfg["FCST_FN_TEMPLATE"][ensmem_index]).substitute(subvars) elif obtype == "AERONET": #AERONET format has slightly different names for different tasks met_filedir_name = "AERONET_AOD" obs_in_dir = Path(exptdir, "metprd", "Ascii2nc_obs") fcst_in_dir = vxcfg["VX_FCST_INPUT_BASEDIR"] obs_in_fn_template = vxcfg["OBS_AERONET_FN_TEMPLATE_ASCII2NC_OUTPUT"] - fcst_in_fn_template = Path(Template(vxcfg["FCST_SUBDIR_TEMPLATE"]).substitute(subvars), - Template(vxcfg["FCST_FN_TEMPLATE"]).substitute(subvars)) + fcst_fn_tmpl = Template(vxcfg["FCST_FN_TEMPLATE"][ensmem_index]).substitute(subvars) elif obtype == "AIRNOW": # AIRNOW format has slightly different names for different tasks, and also differs # based on ob source @@ -193,13 +183,14 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, else: fcst_in_dir = Path(exptdir, cdate, "metprd", "PcpCombine_fcst") obs_in_fn_template = vxcfg["OBS_AIRNOW_FN_TEMPLATE_ASCII2NC_OUTPUT"] - fcst_in_fn_template = Path( + fcst_fn_tmpl = Path( Template(vxcfg["FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT"]).substitute(subvars) ) else: raise ValueError(f"Invalid OBTYPE for PointStat: {obtype}") else: raise ValueError(f"Invalid parameters:\n{obtype=}\n{field_group=}\n{accum_hh=}") + lgr.debug(f"{fcst_fn_tmpl=}") if do_ens: output_dir=Path(exptdir, cdate, ensmem, "metprd", metplus_tool_camel_case) @@ -269,7 +260,7 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, 'obs_input_dir': obs_in_dir, 'obs_input_fn_template': obs_in_fn_template, 'fcst_input_dir': fcst_in_dir, - 'fcst_input_fn_template': fcst_in_fn_template, + 'fcst_input_fn_template': fcst_fn_tmpl, 'output_dir': output_dir, 'staging_dir': staging_dir, 'vx_fcst_model_name': vxcfg['VX_FCST_MODEL_NAME'], @@ -294,10 +285,7 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, 'vx_config_dict': vx_config_dict } - if field_group == "UPA": - numprocs=math.ceil(vxcfg['VX_TASKS']/2) - else: - numprocs=vxcfg['VX_TASKS'] + numprocs = int(cfg[metplus_tool_camel_case.lower()]["TASKS"]) conf_files = render_metplus_confs(cfg,settings,metplus_config_tmpl_fn,vx_leadhr_list,numprocs) lgr.debug(f"{conf_files=}") @@ -315,7 +303,7 @@ def gridstat_or_pointstat(config_file,cdate,obs_dir,field_group,obtype,accum_hh, def run_metplus(common_config,config_fn): - """Calls the run_metplus script as a subprocess. If VX_TASKS > 1 and vx_leadhr_list > 1, + """Calls the run_metplus script as a subprocess. If TASKS > 1 and vx_leadhr_list > 1, calls in with starmap for the number of tasks specified.""" # Run METplus diff --git a/scripts/gridstat_or_pointstat_ensmean.py b/scripts/gridstat_or_pointstat_ensmean.py new file mode 100644 index 00000000..18b4ca93 --- /dev/null +++ b/scripts/gridstat_or_pointstat_ensmean.py @@ -0,0 +1,248 @@ +# pylint: disable=logging-fstring-interpolation +""" +Converted from scripts/gridstat_or_pointstat_ensmean.sh, this script calls the METplus +"GridStat" or "PointStat" tool to verify the ensemble mean output produced by GenEnsProd. + +The script is intended to be called from jobs/GRIDSTAT_OR_POINTSTAT_ENSMEAN.sh. +""" + +import argparse +import logging +import os + +from multiprocessing import Pool +from pathlib import Path +from string import Template + +import uwtools.api.config as uwconfig + +from python_utils import run_metplus, render_metplus_confs, setup_logging +from set_leadhrs import set_leadhrs +from set_vx_params import set_vx_params + + +def gridstat_or_pointstat_ensmean( + config_file: str, + cdate: str, + obs_dir: str, + field_group: str, + obtype: str, + accum_hh: int, + fcst_level: str, + fcst_thresh: str, +) -> None: + """Execute a METplus GridStat or PointStat verification task on ensemble mean output. + + Parameters + ---------- + config_file : str + Path to the experiment YAML configuration file. + cdate : str + Eight-digit cycle date in ``YYYYMMDDHH`` format. + obs_dir : str + Directory containing observation files for the chosen obtype. + field_group : str + Group of observation fields to verify (e.g. APCP, REFC, SFC). + obtype : str + Observation type (e.g. NOHRSC, CCPA, NDAS). + accum_hh : int + Accumulation hours for the observation type. + fcst_level : str + METplus forecast level (e.g. L0, A03). + fcst_thresh : str + Forecast threshold set (usually "all" or "none"). + """ + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-branches,too-many-statements + lgr = logging.getLogger(__name__) + + cfg = uwconfig.get_yaml_config(config=config_file) + vxcfg = cfg["verification"] + + if not Path(obs_dir).is_dir(): + raise FileNotFoundError(f"{obs_dir=} does not exist or is not a directory") + + geom, _, _, met_out_name, met_filedir_name = set_vx_params(obtype, field_group, accum_hh) + + exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + + # Make a dictionary of variables that may need to be substituted; these will be used to replace + # bash-like variables in some strings. This is needed to maintain some functionality while we + # still have a mix of bash and python exscripts. + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + } + + if geom == "grid": + metplus_tool_name = "grid_stat" + metplus_tool_camel_case = "GridStat" + if "APCP" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = Template( + vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) + elif "ASNOW" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = Template( + vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) + elif met_filedir_name == "REFC": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + ).substitute(subvars) + elif met_filedir_name == "RETOP": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + ).substitute(subvars) + else: + raise ValueError(f"Invalid field group for GridStat ensmean: {field_group}") + fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") + + elif geom == "point": + metplus_tool_name = "point_stat" + metplus_tool_camel_case = "PointStat" + obs_in_dir = Path(exptdir, "metprd", "Pb2nc_obs") + obs_in_fn_template = Template( + vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + ).substitute(subvars) + fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") + + else: + raise ValueError(f"Invalid parameters: {obtype=}, {field_group=}, {accum_hh=}") + + fcst_in_fn_template = ( + f"gen_ens_prod_{vxcfg['VX_FCST_MODEL_NAME']}_{met_filedir_name}_{obtype}" + "_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc" + ) + + output_dir = Path(exptdir, cdate, "metprd", f"{metplus_tool_camel_case}_ensmean") + staging_dir = Path(exptdir, cdate, "stage", f"{met_filedir_name}_ensmean") + os.makedirs(output_dir, exist_ok=True) + + if obtype in ("CCPA", "NOHRSC"): + vx_intvl = vx_hr_start = accum_hh + else: + vx_intvl = vxcfg["VX_FCST_OUTPUT_INTVL_HRS"] + vx_hr_start = 0 + + vx_leadhr_list = set_leadhrs( + date_init=cdate, + lhr_min=vx_hr_start, + lhr_max=cfg["workflow"]["FCST_LEN_HRS"], + lhr_intvl=vx_intvl, + base_dir=obs_in_dir, + time_lag=0, + fn_template=str(obs_in_fn_template), + num_missing_files_max=vxcfg["NUM_MISSING_OBS_FILES_MAX"], + ) + + if not vx_leadhr_list: + raise RuntimeError( + f"set_leadhrs returned an empty list for cycle {cdate}, " + f"{obtype=}, {field_group=}" + ) + + vx_mask_files = [] + if vxcfg["VX_MASK"]: + for mask in vxcfg["VX_MASK"]: + if os.path.isfile(maskfile := f"{cfg['user']['METPLUS_CONF']}/{mask}.poly"): + vx_mask_files.append(maskfile) + else: + vx_mask_files.append( + f"{os.environ['MET_INSTALL_DIR']}/share/met/poly/{mask}.poly" + ) + + metplus_config_tmpl_fn = f"{metplus_tool_camel_case}_ensmean.conf" + metplus_config_fn = ( + f"{metplus_tool_camel_case}_{met_filedir_name}_{cdate}_ensmean.conf.0" + ) + metplus_log_fn = ( + f"metplus.log.{metplus_tool_camel_case}_{met_filedir_name}_{cdate}_ensmean.0" + ) + + vx_config_dict = uwconfig.get_yaml_config( + config=f"{cfg['user']['METPLUS_CONF']}/{vxcfg['VX_CONFIG_ENS_FN']}" + ) + + settings = { + "metplus_tool_name": metplus_tool_name, + "MetplusToolName": metplus_tool_camel_case, + "METPLUS_TOOL_NAME": metplus_tool_name.upper(), + "metplus_verbosity_level": vxcfg["METPLUS_VERBOSITY_LEVEL"], + "cdate": cdate, + "vx_leadhr_list": ", ".join(map(str, vx_leadhr_list)), + "metplus_config_fn": metplus_config_fn, + "metplus_log_fn": metplus_log_fn, + "obs_input_dir": obs_in_dir, + "obs_input_fn_template": obs_in_fn_template, + "fcst_input_dir": fcst_in_dir, + "fcst_input_fn_template": fcst_in_fn_template, + "output_dir": output_dir, + "output_fn_template": "", + "staging_dir": staging_dir, + "vx_fcst_model_name": vxcfg["VX_FCST_MODEL_NAME"], + "num_ens_members": cfg["ensemble"]["NUM_ENS_MEMBERS"], + "ensmem_name": "", + "time_lag": 0, + "fieldname_in_obs_input": str(obs_in_dir), + "fieldname_in_fcst_input": str(fcst_in_dir), + "fieldname_in_met_output": met_out_name, + "fieldname_in_met_filedir_names": met_filedir_name, + "obtype": obtype, + "accum_hh": f"{accum_hh:02}", + "accum_no_pad": accum_hh, + "metplus_templates_dir": cfg["user"]["METPLUS_CONF"], + "input_field_group": field_group, + "input_level_fcst": fcst_level, + "input_thresh_fcst": fcst_thresh, + "vx_mask": ", ".join(vx_mask_files), + "vx_config_dict": vx_config_dict, + } + + numprocs = int(vxcfg["VX_TASKS"]) + conf_files = render_metplus_confs( + cfg, settings, metplus_config_tmpl_fn, vx_leadhr_list, numprocs + ) + lgr.debug(f"{conf_files=}") + + lgr.info(f"Running {metplus_tool_camel_case}_ensmean with METplus for {numprocs} tasks") + common_conf = os.path.join(cfg["user"]["METPLUS_CONF"], "common.conf") + with Pool(processes=numprocs) as pool: + pool.starmap(run_metplus, [(common_conf, fn) for fn in conf_files]) + + lgr.info(f"{metplus_tool_camel_case}_ensmean completed successfully.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run METplus GridStat or PointStat on ensemble mean output" + ) + parser.add_argument("--config", default="config.yaml", type=str, + help="Path to the experiment configuration file in YAML format") + parser.add_argument("--cycle_date", required=True, type=str, + help="Eight-digit cycle date (YYYYMMDDHH)") + parser.add_argument("--obs_dir", required=True, type=str, + help="Observation directory for this obtype") + parser.add_argument("--field_group", required=True, type=str, + help="Group of fields for this verification task (e.g. APCP, REFC, SFC)") + parser.add_argument("--obtype", required=True, type=str, + help="Observation type (e.g. NOHRSC, CCPA, NDAS)") + parser.add_argument("--accum_hh", default=1, type=int, + help="Accumulation hours for this observation type") + parser.add_argument("--fcst_level", required=True, type=str, + help="METplus forecast level (e.g. L0, A03)") + parser.add_argument("--fcst_thresh", required=True, type=str, + help="Forecast thresholds to verify against (e.g. all, none)") + parser.add_argument("-v", "--verbose", action="store_true", + help="Enable verbose debug output") + args = parser.parse_args() + + setup_logging(debug=args.verbose) + logging.debug(f"{os.environ['METPLUS_ROOT']=}") + + gridstat_or_pointstat_ensmean( + args.config, args.cycle_date, args.obs_dir, args.field_group, + args.obtype, args.accum_hh, args.fcst_level, args.fcst_thresh, + ) diff --git a/scripts/gridstat_or_pointstat_ensmean.sh b/scripts/gridstat_or_pointstat_ensmean.sh deleted file mode 100755 index c513cc8d..00000000 --- a/scripts/gridstat_or_pointstat_ensmean.sh +++ /dev/null @@ -1,402 +0,0 @@ -#!/usr/bin/env bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -sections=( - user - platform - workflow - ensemble - verification -) -for sect in ${sections[*]} ; do - source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} -done -# -#----------------------------------------------------------------------- -# -# Source files defining auxiliary functions for verification. -# -#----------------------------------------------------------------------- -# -. $USHdir/get_metplus_tool_name.sh -. $USHdir/set_vx_params.sh -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of the MET/METplus tool in different formats that may be -# needed from the variable METPLUSTOOLNAME. -# -#----------------------------------------------------------------------- -# -get_metplus_tool_name \ - METPLUSTOOLNAME="${METPLUSTOOLNAME}" \ - outvarname_metplus_tool_name="metplus_tool_name" \ - outvarname_MetplusToolName="MetplusToolName" \ - outvarname_METPLUS_TOOL_NAME="METPLUS_TOOL_NAME" -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the ex-script for the task that runs the METplus ${MetplusToolName} -tool to perform verification of the specified field group (FIELD_GROUP) -on the ensemble mean. -========================================================================" -# -#----------------------------------------------------------------------- -# -# Get the cycle date and time in YYYYMMDDHH format. -# -#----------------------------------------------------------------------- -# -CDATE="${YYMMDD}${HH}" -# -#----------------------------------------------------------------------- -# -# Set various verification parameters associated with the field to be -# verified. Not all of these are necessarily used later below but are -# set here for consistency with other verification ex-scripts. -# -#----------------------------------------------------------------------- -# -FIELDNAME_IN_OBS_INPUT="" -FIELDNAME_IN_FCST_INPUT="" -FIELDNAME_IN_MET_OUTPUT="" -FIELDNAME_IN_MET_FILEDIR_NAMES="" - -set_vx_params \ - obtype="${OBTYPE}" \ - field_group="${FIELD_GROUP}" \ - accum_hh="${ACCUM_HH}" \ - outvarname_grid_or_point="grid_or_point" \ - outvarname_fieldname_in_obs_input="FIELDNAME_IN_OBS_INPUT" \ - outvarname_fieldname_in_fcst_input="FIELDNAME_IN_FCST_INPUT" \ - outvarname_fieldname_in_MET_output="FIELDNAME_IN_MET_OUTPUT" \ - outvarname_fieldname_in_MET_filedir_names="FIELDNAME_IN_MET_FILEDIR_NAMES" -# -#----------------------------------------------------------------------- -# -# Set paths and file templates for input to and output from the MET/ -# METplus tool to be run as well as other file/directory parameters. -# -#----------------------------------------------------------------------- -# -vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) - -if [ "${grid_or_point}" = "grid" ]; then - - case "${FIELDNAME_IN_MET_FILEDIR_NAMES}" in - "APCP"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - ;; - "ASNOW"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - ;; - "REFC") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[1]}" - ;; - "RETOP") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[3]}" - ;; - esac - FCST_INPUT_DIR="${vx_output_basedir}/${CDATE}/metprd/GenEnsProd" - -elif [ "${grid_or_point}" = "point" ]; then - - OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT}" - FCST_INPUT_DIR="${vx_output_basedir}/${CDATE}/metprd/GenEnsProd" - -fi -OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) -FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - -OUTPUT_BASE="${vx_output_basedir}/${CDATE}" -OUTPUT_DIR="${OUTPUT_BASE}/metprd/${MetplusToolName}_ensmean" -STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}_ensmean" -# -#----------------------------------------------------------------------- -# -# Set the lead hours for which to run the MET/METplus tool. This is done -# by starting with the full list of lead hours for which we expect to -# find forecast output and then removing from that list any hours for -# which there is no corresponding observation data. -# -#----------------------------------------------------------------------- -# -case "$OBTYPE" in - "CCPA"|"NOHRSC") - vx_intvl="$((10#${ACCUM_HH}))" - vx_hr_start="${vx_intvl}" - ;; - *) - vx_intvl="$((${VX_FCST_OUTPUT_INTVL_HRS}))" - vx_hr_start="0" - ;; -esac -vx_hr_end="${FCST_LEN_HRS}" - -VX_LEADHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --date_init="${CDATE}" \ - --lhr_min="${vx_hr_start}" \ - --lhr_max="${vx_hr_end}" \ - --lhr_intvl="${vx_intvl}" \ - --base_dir="${OBS_INPUT_DIR}" \ - --fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - --num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" ) || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" - -# -#----------------------------------------------------------------------- -# -# Make sure the MET/METplus output directory(ies) exists. -# -#----------------------------------------------------------------------- -# -mkdir -p "${OUTPUT_DIR}" -# -#----------------------------------------------------------------------- -# -# Check for existence of top-level OBS_DIR. -# -#----------------------------------------------------------------------- -# -if [ ! -d "${OBS_DIR}" ]; then - print_err_msg_exit "\ -OBS_DIR does not exist or is not a directory: - OBS_DIR = \"${OBS_DIR}\"" -fi -# -#----------------------------------------------------------------------- -# -# Set variable containing accumulation period without leading zero -# padding. This may be needed in the METplus configuration files. -# -#----------------------------------------------------------------------- -# -ACCUM_NO_PAD=$( printf "%0d" "${ACCUM_HH}" ) -# -#----------------------------------------------------------------------- -# -# Export variables needed in the common METplus configuration file (at -# ${METPLUS_CONF}/common.conf). -# -#----------------------------------------------------------------------- -# -export METPLUS_CONF -export LOGDIR -# -#----------------------------------------------------------------------- -# -# Do not run METplus if there isn't at least one lead hour for which to -# run it. -# -#----------------------------------------------------------------------- -# -if [ -z "${VX_LEADHR_LIST}" ]; then - print_err_msg_exit "\ -The list of lead hours for which to run METplus is empty: - VX_LEADHR_LIST = [${VX_LEADHR_LIST}]" -fi -# -#----------------------------------------------------------------------- -# -# Populate VX_MASK_FILE_LIST based on user selections -# -#----------------------------------------------------------------------- -# -# This weird logic is needed because in older versions of bash empty arrays are treated as unset -if [ ${VX_MASK[@]} ]; then - VX_MASK_FILE_LIST="" - for i in "${VX_MASK[@]}"; do - if [ -f "${METPLUS_CONF}/${i}.poly" ]; then - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, ${METPLUS_CONF}/${i}.poly" - else - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, {MET_INSTALL_DIR}/share/met/poly/${i}.poly" - fi - done -fi -# -#----------------------------------------------------------------------- -# -# Set the names of the template METplus configuration file, the METplus -# configuration file generated from this template, and the METplus log -# file. -# -#----------------------------------------------------------------------- -# -# First, set the base file names. -# -metplus_config_tmpl_bn="${MetplusToolName}_ensmean" -metplus_config_bn="${MetplusToolName}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${CDATE}_ensmean" -metplus_log_bn="${metplus_config_bn}" -# -# Add prefixes and suffixes (extensions) to the base file names. -# -metplus_config_tmpl_fn="${metplus_config_tmpl_bn}.conf" -metplus_config_fn="${metplus_config_bn}.conf" -metplus_log_fn="metplus.log.${metplus_log_bn}" -# -#----------------------------------------------------------------------- -# -# Load the yaml-like file containing the configuration for ensemble -# verification. -# -#----------------------------------------------------------------------- -# -vx_config_fp="${METPLUS_CONF}/${VX_CONFIG_ENS_FN}" -vx_config_dict=$(<"${vx_config_fp}") -# Indent each line of vx_config_dict so that it is aligned properly when -# included in the yaml-formatted variable "settings" below. -vx_config_dict=$( printf "%s\n" "${vx_config_dict}" | sed 's/^/ /' ) -# -#----------------------------------------------------------------------- -# -# Generate the METplus configuration file from its jinja template. -# -#----------------------------------------------------------------------- -# -# Set the full paths to the jinja template METplus configuration file -# (which already exists) and the METplus configuration file that will be -# generated from it. -# -metplus_config_tmpl_fp="${METPLUS_CONF}/${metplus_config_tmpl_fn}" -metplus_config_fp="${OUTPUT_DIR}/${metplus_config_fn}" -# -# Define variables that appear in the jinja template. -# -settings="\ -# -# MET/METplus information. -# -'metplus_tool_name': '${metplus_tool_name}' -'MetplusToolName': '${MetplusToolName}' -'METPLUS_TOOL_NAME': '${METPLUS_TOOL_NAME}' -'metplus_verbosity_level': '${METPLUS_VERBOSITY_LEVEL}' -# -# Date and forecast hour information. -# -'cdate': '$CDATE' -'vx_leadhr_list': '${VX_LEADHR_LIST}' -# -# Input and output directory/file information. -# -'metplus_config_fn': '${metplus_config_fn:-}' -'metplus_log_fn': '${metplus_log_fn:-}' -'obs_input_dir': '${OBS_INPUT_DIR:-}' -'obs_input_fn_template': '${OBS_INPUT_FN_TEMPLATE:-}' -'fcst_input_dir': '${FCST_INPUT_DIR:-}' -'fcst_input_fn_template': '${FCST_INPUT_FN_TEMPLATE:-}' -'output_dir': '${OUTPUT_DIR}' -'output_fn_template': '${OUTPUT_FN_TEMPLATE:-}' -'staging_dir': '${STAGING_DIR}' -'vx_fcst_model_name': '${VX_FCST_MODEL_NAME}' -# -# Ensemble and member-specific information. -# -'num_ens_members': '${NUM_ENS_MEMBERS}' -'ensmem_name': '${ensmem_name:-}' -'time_lag': '${time_lag:-}' -# -# Field information. -# -'fieldname_in_obs_input': '${FIELDNAME_IN_OBS_INPUT}' -'fieldname_in_fcst_input': '${FIELDNAME_IN_FCST_INPUT}' -'fieldname_in_met_output': '${FIELDNAME_IN_MET_OUTPUT}' -'fieldname_in_met_filedir_names': '${FIELDNAME_IN_MET_FILEDIR_NAMES}' -'obtype': '${OBTYPE}' -'accum_hh': '${ACCUM_HH:-}' -'accum_no_pad': '${ACCUM_NO_PAD:-}' -'metplus_templates_dir': '${METPLUS_CONF:-}' -'input_field_group': '${FIELD_GROUP:-}' -'input_level_fcst': '${FCST_LEVEL:-}' -'input_thresh_fcst': '${FCST_THRESH:-}' -# -# Verification mask settings -# -'vx_mask': '${VX_MASK_FILE_LIST:-}' -# -# Verification configuration dictionary. -# -'vx_config_dict': -${vx_config_dict:-} -" - -# Render the template to create a METplus configuration file -tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") -printf "%s" "$settings" > "$tmpfile" -uw template render \ - -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} \ - --verbose \ - --values-file "${tmpfile}" \ - --search-path "/" - -err=$? -rm $tmpfile -if [ $err -ne 0 ]; then - message_txt="Error rendering template for METplus config. - Contents of input are: -$settings" - print_err_msg_exit "${message_txt}" -fi -# -#----------------------------------------------------------------------- -# -# Call METplus. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Calling METplus to run MET's ${metplus_tool_name} tool for field(s): ${FIELDNAME_IN_MET_FILEDIR_NAMES}" -${METPLUS_ROOT}/ush/run_metplus.py \ - -c ${METPLUS_CONF}/common.conf \ - -c ${metplus_config_fp} || \ -print_err_msg_exit " -Call to METplus failed with return code: $? -METplus configuration file used is: - metplus_config_fp = \"${metplus_config_fp}\"" -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -METplus ${MetplusToolName} tool completed successfully. - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" diff --git a/scripts/gridstat_or_pointstat_ensprob.py b/scripts/gridstat_or_pointstat_ensprob.py new file mode 100644 index 00000000..55d941a6 --- /dev/null +++ b/scripts/gridstat_or_pointstat_ensprob.py @@ -0,0 +1,249 @@ +# pylint: disable=logging-fstring-interpolation +""" +Converted from scripts/gridstat_or_pointstat_ensprob.sh, this script calls the METplus +"GridStat" or "PointStat" tool to verify ensemble frequency/probability output produced +by GenEnsProd. + +The script is intended to be called from jobs/GRIDSTAT_OR_POINTSTAT_ENSPROB.sh. +""" + +import argparse +import logging +import os + +from multiprocessing import Pool +from pathlib import Path +from string import Template + +import uwtools.api.config as uwconfig + +from python_utils import run_metplus, render_metplus_confs, setup_logging +from set_leadhrs import set_leadhrs +from set_vx_params import set_vx_params + + +def gridstat_or_pointstat_ensprob( + config_file: str, + cdate: str, + obs_dir: str, + field_group: str, + obtype: str, + accum_hh: int, + fcst_level: str, + fcst_thresh: str, +) -> None: + """Execute a METplus GridStat or PointStat verification task on ensemble probability output. + + Parameters + ---------- + config_file : str + Path to the experiment YAML configuration file. + cdate : str + Eight-digit cycle date in ``YYYYMMDDHH`` format. + obs_dir : str + Directory containing observation files for the chosen obtype. + field_group : str + Group of observation fields to verify (e.g. APCP, REFC, SFC). + obtype : str + Observation type (e.g. NOHRSC, CCPA, NDAS). + accum_hh : int + Accumulation hours for the observation type. + fcst_level : str + METplus forecast level (e.g. L0, A03). + fcst_thresh : str + Forecast threshold set (usually "all" or "none"). + """ + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-branches,too-many-statements + lgr = logging.getLogger(__name__) + + cfg = uwconfig.get_yaml_config(config=config_file) + vxcfg = cfg["verification"] + + if not Path(obs_dir).is_dir(): + raise FileNotFoundError(f"{obs_dir=} does not exist or is not a directory") + + geom, _, _, met_out_name, met_filedir_name = set_vx_params(obtype, field_group, accum_hh) + + exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + + # Make a dictionary of variables that may need to be substituted; these will be used to replace + # bash-like variables in some strings. This is needed to maintain some functionality while we + # still have a mix of bash and python exscripts. + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + } + + if geom == "grid": + metplus_tool_name = "grid_stat" + metplus_tool_camel_case = "GridStat" + if "APCP" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = Template( + vxcfg["OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) + elif "ASNOW" in met_filedir_name: + obs_in_dir = Path(exptdir, cdate, "obs", "metprd", "PcpCombine_obs") + obs_in_fn_template = Template( + vxcfg["OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).substitute(subvars) + elif met_filedir_name == "REFC": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][1] + ).substitute(subvars) + elif met_filedir_name == "RETOP": + obs_in_dir = Path(obs_dir) + obs_in_fn_template = Template( + vxcfg["OBS_MRMS_FN_TEMPLATES"][3] + ).substitute(subvars) + else: + raise ValueError(f"Invalid field group for GridStat ensprob: {field_group}") + fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") + + elif geom == "point": + metplus_tool_name = "point_stat" + metplus_tool_camel_case = "PointStat" + obs_in_dir = Path(exptdir, "metprd", "Pb2nc_obs") + obs_in_fn_template = Template( + vxcfg["OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT"] + ).substitute(subvars) + fcst_in_dir = Path(exptdir, cdate, "metprd", "GenEnsProd") + + else: + raise ValueError(f"Invalid parameters: {obtype=}, {field_group=}, {accum_hh=}") + + fcst_in_fn_template = ( + f"gen_ens_prod_{vxcfg['VX_FCST_MODEL_NAME']}_{met_filedir_name}_{obtype}" + "_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc" + ) + + output_dir = Path(exptdir, cdate, "metprd", f"{metplus_tool_camel_case}_ensprob") + staging_dir = Path(exptdir, cdate, "stage", f"{met_filedir_name}_ensprob") + os.makedirs(output_dir, exist_ok=True) + + if obtype in ("CCPA", "NOHRSC"): + vx_intvl = vx_hr_start = accum_hh + else: + vx_intvl = vxcfg["VX_FCST_OUTPUT_INTVL_HRS"] + vx_hr_start = 0 + + vx_leadhr_list = set_leadhrs( + date_init=cdate, + lhr_min=vx_hr_start, + lhr_max=cfg["workflow"]["FCST_LEN_HRS"], + lhr_intvl=vx_intvl, + base_dir=obs_in_dir, + time_lag=0, + fn_template=str(obs_in_fn_template), + num_missing_files_max=vxcfg["NUM_MISSING_OBS_FILES_MAX"], + ) + + if not vx_leadhr_list: + raise RuntimeError( + f"set_leadhrs returned an empty list for cycle {cdate}, " + f"{obtype=}, {field_group=}" + ) + + vx_mask_files = [] + if vxcfg["VX_MASK"]: + for mask in vxcfg["VX_MASK"]: + if os.path.isfile(maskfile := f"{cfg['user']['METPLUS_CONF']}/{mask}.poly"): + vx_mask_files.append(maskfile) + else: + vx_mask_files.append( + f"{os.environ['MET_INSTALL_DIR']}/share/met/poly/{mask}.poly" + ) + + metplus_config_tmpl_fn = f"{metplus_tool_camel_case}_ensprob.conf" + metplus_config_fn = ( + f"{metplus_tool_camel_case}_{met_filedir_name}_{cdate}_ensprob.conf.0" + ) + metplus_log_fn = ( + f"metplus.log.{metplus_tool_camel_case}_{met_filedir_name}_{cdate}_ensprob.0" + ) + + vx_config_dict = uwconfig.get_yaml_config( + config=f"{cfg['user']['METPLUS_CONF']}/{vxcfg['VX_CONFIG_ENS_FN']}" + ) + + settings = { + "metplus_tool_name": metplus_tool_name, + "MetplusToolName": metplus_tool_camel_case, + "METPLUS_TOOL_NAME": metplus_tool_name.upper(), + "metplus_verbosity_level": vxcfg["METPLUS_VERBOSITY_LEVEL"], + "cdate": cdate, + "vx_leadhr_list": ", ".join(map(str, vx_leadhr_list)), + "metplus_config_fn": metplus_config_fn, + "metplus_log_fn": metplus_log_fn, + "obs_input_dir": obs_in_dir, + "obs_input_fn_template": obs_in_fn_template, + "fcst_input_dir": fcst_in_dir, + "fcst_input_fn_template": fcst_in_fn_template, + "output_dir": output_dir, + "output_fn_template": "", + "staging_dir": staging_dir, + "vx_fcst_model_name": vxcfg["VX_FCST_MODEL_NAME"], + "num_ens_members": cfg["ensemble"]["NUM_ENS_MEMBERS"], + "ensmem_name": "", + "time_lag": 0, + "fieldname_in_obs_input": str(obs_in_dir), + "fieldname_in_fcst_input": str(fcst_in_dir), + "fieldname_in_met_output": met_out_name, + "fieldname_in_met_filedir_names": met_filedir_name, + "obtype": obtype, + "accum_hh": f"{accum_hh:02}", + "accum_no_pad": accum_hh, + "metplus_templates_dir": cfg["user"]["METPLUS_CONF"], + "input_field_group": field_group, + "input_level_fcst": fcst_level, + "input_thresh_fcst": fcst_thresh, + "vx_mask": ", ".join(vx_mask_files), + "vx_config_dict": vx_config_dict, + } + + numprocs = int(vxcfg["VX_TASKS"]) + conf_files = render_metplus_confs( + cfg, settings, metplus_config_tmpl_fn, vx_leadhr_list, numprocs + ) + lgr.debug(f"{conf_files=}") + + lgr.info(f"Running {metplus_tool_camel_case}_ensprob with METplus for {numprocs} tasks") + common_conf = os.path.join(cfg["user"]["METPLUS_CONF"], "common.conf") + with Pool(processes=numprocs) as pool: + pool.starmap(run_metplus, [(common_conf, fn) for fn in conf_files]) + + lgr.info(f"{metplus_tool_camel_case}_ensprob completed successfully.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run METplus GridStat or PointStat on ensemble probability output" + ) + parser.add_argument("--config", default="config.yaml", type=str, + help="Path to the experiment configuration file in YAML format") + parser.add_argument("--cycle_date", required=True, type=str, + help="Eight-digit cycle date (YYYYMMDDHH)") + parser.add_argument("--obs_dir", required=True, type=str, + help="Observation directory for this obtype") + parser.add_argument("--field_group", required=True, type=str, + help="Group of fields for this verification task (e.g. APCP, REFC, SFC)") + parser.add_argument("--obtype", required=True, type=str, + help="Observation type (e.g. NOHRSC, CCPA, NDAS)") + parser.add_argument("--accum_hh", default=1, type=int, + help="Accumulation hours for this observation type") + parser.add_argument("--fcst_level", required=True, type=str, + help="METplus forecast level (e.g. L0, A03)") + parser.add_argument("--fcst_thresh", required=True, type=str, + help="Forecast thresholds to verify against (e.g. all, none)") + parser.add_argument("-v", "--verbose", action="store_true", + help="Enable verbose debug output") + args = parser.parse_args() + + setup_logging(debug=args.verbose) + logging.debug(f"{os.environ['METPLUS_ROOT']=}") + + gridstat_or_pointstat_ensprob( + args.config, args.cycle_date, args.obs_dir, args.field_group, + args.obtype, args.accum_hh, args.fcst_level, args.fcst_thresh, + ) diff --git a/scripts/gridstat_or_pointstat_ensprob.sh b/scripts/gridstat_or_pointstat_ensprob.sh deleted file mode 100755 index f59ebba1..00000000 --- a/scripts/gridstat_or_pointstat_ensprob.sh +++ /dev/null @@ -1,403 +0,0 @@ -#!/usr/bin/env bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -sections=( - user - platform - workflow - ensemble - verification -) -for sect in ${sections[*]} ; do - source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} -done -# -#----------------------------------------------------------------------- -# -# Source files defining auxiliary functions for verification. -# -#----------------------------------------------------------------------- -# -. $USHdir/get_metplus_tool_name.sh -. $USHdir/set_vx_params.sh -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of the MET/METplus tool in different formats that may be -# needed from the variable METPLUSTOOLNAME. -# -#----------------------------------------------------------------------- -# -get_metplus_tool_name \ - METPLUSTOOLNAME="${METPLUSTOOLNAME}" \ - outvarname_metplus_tool_name="metplus_tool_name" \ - outvarname_MetplusToolName="MetplusToolName" \ - outvarname_METPLUS_TOOL_NAME="METPLUS_TOOL_NAME" -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the ex-script for the task that runs the METplus ${MetplusToolName} -tool to perform verification of the specified field group (FIELD_GROUP) -on the ensemble -frequencies/probabilities. -========================================================================" -# -#----------------------------------------------------------------------- -# -# Get the cycle date and time in YYYYMMDDHH format. -# -#----------------------------------------------------------------------- -# -CDATE="${YYMMDD}${HH}" -# -#----------------------------------------------------------------------- -# -# Set various verification parameters associated with the field to be -# verified. Not all of these are necessarily used later below but are -# set here for consistency with other verification ex-scripts. -# -#----------------------------------------------------------------------- -# -FIELDNAME_IN_OBS_INPUT="" -FIELDNAME_IN_FCST_INPUT="" -FIELDNAME_IN_MET_OUTPUT="" -FIELDNAME_IN_MET_FILEDIR_NAMES="" - -set_vx_params \ - obtype="${OBTYPE}" \ - field_group="${FIELD_GROUP}" \ - accum_hh="${ACCUM_HH}" \ - outvarname_grid_or_point="grid_or_point" \ - outvarname_fieldname_in_obs_input="FIELDNAME_IN_OBS_INPUT" \ - outvarname_fieldname_in_fcst_input="FIELDNAME_IN_FCST_INPUT" \ - outvarname_fieldname_in_MET_output="FIELDNAME_IN_MET_OUTPUT" \ - outvarname_fieldname_in_MET_filedir_names="FIELDNAME_IN_MET_FILEDIR_NAMES" -# -#----------------------------------------------------------------------- -# -# Set paths and file templates for input to and output from the MET/ -# METplus tool to be run as well as other file/directory parameters. -# -#----------------------------------------------------------------------- -# -vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) - -if [ "${grid_or_point}" = "grid" ]; then - - case "${FIELDNAME_IN_MET_FILEDIR_NAMES}" in - "APCP"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_CCPA_APCP_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - ;; - "ASNOW"*) - OBS_INPUT_DIR="${vx_output_basedir}/${CDATE}/obs/metprd/PcpCombine_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NOHRSC_ASNOW_FN_TEMPLATE_PCPCOMBINE_OUTPUT}" - ;; - "REFC") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[1]}" - ;; - "RETOP") - OBS_INPUT_DIR="${OBS_DIR}" - OBS_INPUT_FN_TEMPLATE="${OBS_MRMS_FN_TEMPLATES[3]}" - ;; - esac - -elif [ "${grid_or_point}" = "point" ]; then - - OBS_INPUT_DIR="${vx_output_basedir}/metprd/Pb2nc_obs" - OBS_INPUT_FN_TEMPLATE="${OBS_NDAS_SFCandUPA_FN_TEMPLATE_PB2NC_OUTPUT}" - -fi -OBS_INPUT_FN_TEMPLATE=$( eval echo ${OBS_INPUT_FN_TEMPLATE} ) -FCST_INPUT_DIR="${vx_output_basedir}/${CDATE}/metprd/GenEnsProd" -FCST_INPUT_FN_TEMPLATE=$( eval echo 'gen_ens_prod_${VX_FCST_MODEL_NAME}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${OBTYPE}_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V.nc' ) - -OUTPUT_BASE="${vx_output_basedir}/${CDATE}" -OUTPUT_DIR="${OUTPUT_BASE}/metprd/${MetplusToolName}_ensprob" -STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}_ensprob" -# -#----------------------------------------------------------------------- -# -# Set the lead hours for which to run the MET/METplus tool. This is done -# by starting with the full list of lead hours for which we expect to -# find forecast output and then removing from that list any hours for -# which there is no corresponding observation data. -# -#----------------------------------------------------------------------- -# -case "$OBTYPE" in - "CCPA"|"NOHRSC") - vx_intvl="$((10#${ACCUM_HH}))" - vx_hr_start="${vx_intvl}" - ;; - *) - vx_intvl="$((${VX_FCST_OUTPUT_INTVL_HRS}))" - vx_hr_start="0" - ;; -esac -vx_hr_end="${FCST_LEN_HRS}" - -VX_LEADHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --date_init="${CDATE}" \ - --lhr_min="${vx_hr_start}" \ - --lhr_max="${vx_hr_end}" \ - --lhr_intvl="${vx_intvl}" \ - --base_dir="${OBS_INPUT_DIR}" \ - --fn_template="${OBS_INPUT_FN_TEMPLATE}" \ - --num_missing_files_max="${NUM_MISSING_OBS_FILES_MAX}" ) || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" - -# -#----------------------------------------------------------------------- -# -# Make sure the MET/METplus output directory(ies) exists. -# -#----------------------------------------------------------------------- -# -mkdir -p "${OUTPUT_DIR}" -# -#----------------------------------------------------------------------- -# -# Check for existence of top-level OBS_DIR. -# -#----------------------------------------------------------------------- -# -if [ ! -d "${OBS_DIR}" ]; then - print_err_msg_exit "\ -OBS_DIR does not exist or is not a directory: - OBS_DIR = \"${OBS_DIR}\"" -fi -# -#----------------------------------------------------------------------- -# -# Set variable containing accumulation period without leading zero -# padding. This may be needed in the METplus configuration files. -# -#----------------------------------------------------------------------- -# -ACCUM_NO_PAD=$( printf "%0d" "${ACCUM_HH}" ) -# -#----------------------------------------------------------------------- -# -# Export variables needed in the common METplus configuration file (at -# ${METPLUS_CONF}/common.conf). -# -#----------------------------------------------------------------------- -# -export METPLUS_CONF -export LOGDIR -# -#----------------------------------------------------------------------- -# -# Do not run METplus if there isn't at least one lead hour for which to -# run it. -# -#----------------------------------------------------------------------- -# -if [ -z "${VX_LEADHR_LIST}" ]; then - print_err_msg_exit "\ -The list of lead hours for which to run METplus is empty: - VX_LEADHR_LIST = [${VX_LEADHR_LIST}]" -fi -# -#----------------------------------------------------------------------- -# -# Set the names of the template METplus configuration file, the METplus -# configuration file generated from this template, and the METplus log -# file. -# -#----------------------------------------------------------------------- -# -# First, set the base file names. -# -metplus_config_tmpl_bn="${MetplusToolName}_ensprob" -metplus_config_bn="${MetplusToolName}_${FIELDNAME_IN_MET_FILEDIR_NAMES}_${CDATE}_ensprob" -metplus_log_bn="${metplus_config_bn}" -# -# Add prefixes and suffixes (extensions) to the base file names. -# -metplus_config_tmpl_fn="${metplus_config_tmpl_bn}.conf" -metplus_config_fn="${metplus_config_bn}.conf" -metplus_log_fn="metplus.log.${metplus_log_bn}" -# -#----------------------------------------------------------------------- -# -# Populate VX_MASK_FILE_LIST based on user selections -# -#----------------------------------------------------------------------- -# -# This weird logic is needed because in older versions of bash empty arrays are treated as unset -if [ ${VX_MASK[@]} ]; then - VX_MASK_FILE_LIST="" - for i in "${VX_MASK[@]}"; do - if [ -f "${METPLUS_CONF}/${i}.poly" ]; then - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, ${METPLUS_CONF}/${i}.poly" - else - VX_MASK_FILE_LIST="${VX_MASK_FILE_LIST}, {MET_INSTALL_DIR}/share/met/poly/${i}.poly" - fi - done -fi -# -#----------------------------------------------------------------------- -# -# Load the yaml-like file containing the configuration for ensemble -# verification. -# -#----------------------------------------------------------------------- -# -vx_config_fp="${METPLUS_CONF}/${VX_CONFIG_ENS_FN}" -vx_config_dict=$(<"${vx_config_fp}") -# Indent each line of vx_config_dict so that it is aligned properly when -# included in the yaml-formatted variable "settings" below. -vx_config_dict=$( printf "%s\n" "${vx_config_dict}" | sed 's/^/ /' ) -# -#----------------------------------------------------------------------- -# -# Generate the METplus configuration file from its jinja template. -# -#----------------------------------------------------------------------- -# -# Set the full paths to the jinja template METplus configuration file -# (which already exists) and the METplus configuration file that will be -# generated from it. -# -metplus_config_tmpl_fp="${METPLUS_CONF}/${metplus_config_tmpl_fn}" -metplus_config_fp="${OUTPUT_DIR}/${metplus_config_fn}" -# -# Define variables that appear in the jinja template. -# -settings="\ -# -# MET/METplus information. -# -'metplus_tool_name': '${metplus_tool_name}' -'MetplusToolName': '${MetplusToolName}' -'METPLUS_TOOL_NAME': '${METPLUS_TOOL_NAME}' -'metplus_verbosity_level': '${METPLUS_VERBOSITY_LEVEL}' -# -# Date and forecast hour information. -# -'cdate': '$CDATE' -'vx_leadhr_list': '${VX_LEADHR_LIST}' -# -# Input and output directory/file information. -# -'metplus_config_fn': '${metplus_config_fn:-}' -'metplus_log_fn': '${metplus_log_fn:-}' -'obs_input_dir': '${OBS_INPUT_DIR:-}' -'obs_input_fn_template': '${OBS_INPUT_FN_TEMPLATE:-}' -'fcst_input_dir': '${FCST_INPUT_DIR:-}' -'fcst_input_fn_template': '${FCST_INPUT_FN_TEMPLATE:-}' -'output_dir': '${OUTPUT_DIR}' -'output_fn_template': '${OUTPUT_FN_TEMPLATE:-}' -'staging_dir': '${STAGING_DIR}' -'vx_fcst_model_name': '${VX_FCST_MODEL_NAME}' -# -# Ensemble and member-specific information. -# -'num_ens_members': '${NUM_ENS_MEMBERS}' -'ensmem_name': '${ensmem_name:-}' -'time_lag': '${time_lag:-}' -# -# Field information. -# -'fieldname_in_obs_input': '${FIELDNAME_IN_OBS_INPUT}' -'fieldname_in_fcst_input': '${FIELDNAME_IN_FCST_INPUT}' -'fieldname_in_met_output': '${FIELDNAME_IN_MET_OUTPUT}' -'fieldname_in_met_filedir_names': '${FIELDNAME_IN_MET_FILEDIR_NAMES}' -'obtype': '${OBTYPE}' -'accum_hh': '${ACCUM_HH:-}' -'accum_no_pad': '${ACCUM_NO_PAD:-}' -'metplus_templates_dir': '${METPLUS_CONF:-}' -'input_field_group': '${FIELD_GROUP:-}' -'input_level_fcst': '${FCST_LEVEL:-}' -'input_thresh_fcst': '${FCST_THRESH:-}' -# -# Verification mask settings -# -'vx_mask': '${VX_MASK_FILE_LIST:-}' -# -# -# Verification configuration dictionary. -# -'vx_config_dict': -${vx_config_dict:-} -" - -# Render the template to create a METplus configuration file -tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") -printf "%s" "$settings" > "$tmpfile" -uw template render \ - -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} \ - --verbose \ - --values-file "${tmpfile}" \ - --search-path "/" - -err=$? -rm $tmpfile -if [ $err -ne 0 ]; then - message_txt="Error rendering template for METplus config. - Contents of input are: -$settings" - print_err_msg_exit "${message_txt}" -fi -# -#----------------------------------------------------------------------- -# -# Call METplus. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Calling METplus to run MET's ${metplus_tool_name} tool for field(s): ${FIELDNAME_IN_MET_FILEDIR_NAMES}" -${METPLUS_ROOT}/ush/run_metplus.py \ - -c ${METPLUS_CONF}/common.conf \ - -c ${metplus_config_fp} || \ -print_err_msg_exit " -Call to METplus failed with return code: $? -METplus configuration file used is: - metplus_config_fp = \"${metplus_config_fp}\"" -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -METplus ${MetplusToolName} tool completed successfully. - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" diff --git a/scripts/mode.py b/scripts/mode.py index 57ab3907..4ebf0249 100644 --- a/scripts/mode.py +++ b/scripts/mode.py @@ -83,8 +83,7 @@ def mode(config_file,cdate,field_group,obtype): obs_in_dir = Path(exptdir, cdate, "metprd", "RegridDataPlane") obs_in_fn_template = f'regrid_{vxcfg["OBS_GOES_AOD_FN_TEMPLATE_POINT2GRID_OUTPUT"]}' fcst_in_dir = vxcfg["VX_FCST_INPUT_BASEDIR"] - fcst_in_fn_template = Path(Template(vxcfg["FCST_SUBDIR_TEMPLATE"]).substitute(subvars), - Template(vxcfg["FCST_FN_TEMPLATE"]).substitute(subvars)) + fcst_in_fn_template = Template(vxcfg["FCST_FN_TEMPLATE"][0]).substitute(subvars) # Get the list of all the times in the current day at which to retrieve obs. else: raise ValueError(f"Invalid OBTYPE for {metplus_tool_camel_case}: {obtype}") @@ -170,7 +169,7 @@ def mode(config_file,cdate,field_group,obtype): 'merge_flag': modecfg["MERGE_FLAG"], } - numprocs=modecfg['TASKS'] + numprocs=int(modecfg['TASKS']) conf_files = render_metplus_confs(cfg,settings,metplus_config_tmpl_fn,vx_leadhr_list, len(vx_leadhr_list)) lgr.debug(f"{conf_files=}") diff --git a/scripts/pb2nc_obs.py b/scripts/pb2nc_obs.py index 2cbb1fd0..f37b97a4 100644 --- a/scripts/pb2nc_obs.py +++ b/scripts/pb2nc_obs.py @@ -12,6 +12,7 @@ import subprocess from datetime import datetime from pathlib import Path +from multiprocessing import Pool import uwtools.api.config as uwconfig @@ -107,19 +108,25 @@ def pb2nc(config_file: str, cycle_date: str, obtype: str, verbose: bool = False) "obtype": obtype, } + numprocs = int(cfg[metplus_tool_camel_case.lower()]["TASKS"]) # Render the configuration file using the template conf_files = render_metplus_confs( cfg, settings, metplus_config_tmpl_fn, vx_leadhr_list, - vxcfg["VX_TASKS"], + numprocs, ) - - # Run METplus for each generated config file - common_conf = Path(cfg["user"]["METPLUS_CONF"], "common.conf") - for conf_file in conf_files: - run_metplus(str(common_conf), conf_file) + lgr.debug(f"{conf_files=}") + + lgr.info(f"Running {metplus_tool_camel_case} with METplus with {numprocs} tasks") + mpargs = [] + for config_fn in conf_files: + mpargs.append( (os.path.join(cfg['user']['METPLUS_CONF'], "common.conf"),config_fn) ) + # Call run_metplus function for as many processors as specified + lgr.debug(f"{mpargs=}") + with Pool(processes=numprocs) as pool: + pool.starmap(run_metplus,mpargs) # --------------------------------------------------------------------- # Create completion flag file diff --git a/scripts/pcpcombine.py b/scripts/pcpcombine.py new file mode 100644 index 00000000..1ded2ab7 --- /dev/null +++ b/scripts/pcpcombine.py @@ -0,0 +1,306 @@ +# pylint: disable=logging-fstring-interpolation +""" +Converted from scripts/pcpcombine.sh, this script calls the METplus "PcpCombine" tool to +combine sub-hourly or hourly fields to generate multi-hour accumulations. Input can come +from either observations or a forecast. + +The script is intended to be called from jobs/PCPCOMBINE.sh. +""" + +import argparse +import logging +import os + +from multiprocessing import Pool +from pathlib import Path +from string import Template + +import uwtools.api.config as uwconfig + +from python_utils import run_metplus, render_metplus_confs, setup_logging +from set_leadhrs import set_leadhrs +from set_vx_params import set_vx_params + + +def pcpcombine( + config_file: str, + cdate: str, + obs_dir: str, + field_group: str, + obtype: str, + accum_hh: int, + fcst_level: str, + fcst_thresh: str, + fcst_or_obs: str, + ensmem_index: int, +) -> None: + """Execute a METplus PcpCombine task for obs or forecast accumulation fields. + + Parameters + ---------- + config_file : str + Path to the experiment YAML configuration file. + cdate : str + Eight-digit cycle date in ``YYYYMMDDHH`` format. + obs_dir : str + Directory containing observation files for the chosen obtype. + field_group : str + Field group to combine (e.g. APCP, ASNOW, PM25, PM10). + obtype : str + Observation type (e.g. CCPA, NOHRSC, AIRNOW). + accum_hh : int + Target accumulation period in hours. + fcst_level : str + METplus forecast level (e.g. A06). + fcst_thresh : str + Forecast threshold set (usually "all" or "none"). + fcst_or_obs : str + Whether this task processes forecast (``"FCST"``) or observation (``"OBS"``) data. + ensmem_index : int + Ensemble member index (0 for deterministic). + """ + # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-branches,too-many-statements + lgr = logging.getLogger(__name__) + + fcst_or_obs = fcst_or_obs.upper() + if fcst_or_obs not in ("FCST", "OBS"): + raise ValueError(f"fcst_or_obs must be 'FCST' or 'OBS', got '{fcst_or_obs}'") + + metplus_tool_camel_case = "PcpCombine" + + cfg = uwconfig.get_yaml_config(config=config_file) + vxcfg = cfg["verification"] + enscfg = cfg["ensemble"] + + _, obs_fieldname, fcst_fieldname, met_out_name, met_filedir_name = set_vx_params( + obtype, field_group, accum_hh + ) + + exptdir = vxcfg["VX_OUTPUT_BASEDIR"] + do_ensemble = enscfg["DO_ENSEMBLE"] + ensmem = f"mem{str(ensmem_index).zfill(vxcfg['VX_NDIGITS_ENSMEM_NAMES'])}" + + # Make a dictionary of variables that may need to be substituted; these will be used to replace + # bash-like variables in some strings. This is needed to maintain some functionality while we + # still have a mix of bash and python exscripts. + subvars = { + "FIELD_GROUP": field_group, + "ACCUM_HH": f"{accum_hh:02}", + "ensmem_name": f"mem{str(ensmem_index).zfill(vxcfg['VX_NDIGITS_ENSMEM_NAMES'])}", + } + + pcp_combine_method = "ADD" + pcp_combine_command = "" + + time_lag = 0 + if fcst_or_obs == "FCST": + if do_ensemble: + time_lag = int(enscfg["ENS_TIME_LAG_HRS"][ensmem_index]) * 3600 + + subvars["time_lag"]=time_lag + input_fn_template = Template( + vxcfg["FCST_FN_TEMPLATE"][ensmem_index]).safe_substitute(subvars) + input_dir = Path(vxcfg["VX_FCST_INPUT_BASEDIR"]) + output_fn_template = Template( + vxcfg["FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).safe_substitute(subvars) + + output_base = Path(exptdir, cdate, ensmem) if do_ensemble else Path(exptdir, cdate) + output_dir = output_base / "metprd" / f"{metplus_tool_camel_case}_fcst" + staging_dir = output_base / "stage" / met_filedir_name + + # AIRNOW requires USER_DEFINED combining to build PM2.5/PM10 from model aerosol fields + if obtype == "AIRNOW": + pcp_combine_method = "USER_DEFINED" + smoke_type = vxcfg["FCST_SMOKE_TYPE"] + if field_group == "PM25": + if smoke_type == "HRRR": + pcp_combine_command = ( + "-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " -field 'name=\"MASSDEN\"; level=\"Z8\"; convert(x)=x*1e9;'" + ) + elif smoke_type == "RRFS": + pcp_combine_command = ( + "-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62010;" + " convert(x)=x*1e9;'" + " {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001;" + " GRIB2_aerosol_interval_type=0; convert(x)=x*1e9;'" + ) + else: + raise ValueError( + f"Unsupported FCST_SMOKE_TYPE '{smoke_type}' for PM25" + ) + elif field_group == "PM10": + if smoke_type == "RRFS": + pcp_combine_command = ( + "-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62010;" + " convert(x)=x*1e9;'" + " {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001;" + " GRIB2_aerosol_interval_type=0; convert(x)=x*1e9;'" + " {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE}" + " 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001;" + " GRIB2_aerosol_interval_type=2; convert(x)=x*1e9;'" + ) + else: + raise ValueError( + f"PM10 only available for RRFS output, not available for {smoke_type}" + ) + + suffix = f"_{ensmem}" + + else: # OBS + subvars["time_lag"]=time_lag + input_dir = Path(obs_dir) + input_fn_template = vxcfg[f"OBS_{obtype}_FN_TEMPLATES"][1] + output_fn_template = Template( + vxcfg[f"OBS_{obtype}_{field_group}_FN_TEMPLATE_PCPCOMBINE_OUTPUT"] + ).safe_substitute(subvars) + + output_base = Path(exptdir, cdate, "obs") if do_ensemble else Path(exptdir, cdate) + output_dir = output_base / "metprd" / f"{metplus_tool_camel_case}_obs" + staging_dir = output_base / "stage" / met_filedir_name + + suffix = f"_{obtype}" + + os.makedirs(output_dir, exist_ok=True) + + # Sub-interval: model output interval for FCST, obs availability interval for OBS + if fcst_or_obs == "FCST": + subintvl = vxcfg["VX_FCST_OUTPUT_INTVL_HRS"] + else: + subintvl = vxcfg[f"{obtype}_OBS_AVAIL_INTVL_HRS"] + input_accum_hh = f"{subintvl:02d}" + + # First pass: get accumulation end hours without file existence check + lhr_min = 0 if obtype == "AIRNOW" else accum_hh + vx_leadhr_list = set_leadhrs( + date_init=cdate, + lhr_min=lhr_min, + lhr_max=cfg["workflow"]["FCST_LEN_HRS"], + lhr_intvl=accum_hh, + base_dir=input_dir, + time_lag=time_lag, + fn_template=str(input_fn_template), + num_missing_files_max=0, + skip_check_files=True, + ) + + if not vx_leadhr_list: + raise RuntimeError( + f"set_leadhrs returned an empty list for cycle {cdate}, " + f"{obtype=}, {field_group=}" + ) + + # Second pass: verify sub-interval input files exist for each accumulation window + for hr_end in vx_leadhr_list: + hr_start = hr_end - accum_hh + subintvl + set_leadhrs( + date_init=cdate, + lhr_min=hr_start, + lhr_max=hr_end, + lhr_intvl=subintvl, + base_dir=input_dir, + time_lag=time_lag, + fn_template=str(input_fn_template), + num_missing_files_max=0, + ) + + fcst_or_obs_lower = fcst_or_obs.lower() + metplus_config_tmpl_fn = f"{metplus_tool_camel_case}.conf" + metplus_config_fn = ( + f"{metplus_tool_camel_case}_{fcst_or_obs_lower}_{met_filedir_name}{suffix}.conf.0" + ) + metplus_log_fn = ( + f"metplus.log.{metplus_tool_camel_case}_{fcst_or_obs_lower}" + f"_{met_filedir_name}{suffix}_{cdate}.0" + ) + + settings = { + "metplus_tool_name": "pcpcombine", + "MetplusToolName": metplus_tool_camel_case, + "METPLUS_TOOL_NAME": "PCPCOMBINE", + "metplus_verbosity_level": vxcfg["METPLUS_VERBOSITY_LEVEL"], + "cdate": cdate, + "vx_leadhr_list": ", ".join(map(str, vx_leadhr_list)), + "metplus_config_fn": metplus_config_fn, + "metplus_log_fn": metplus_log_fn, + "input_dir": input_dir, + "input_fn_template": input_fn_template, + "output_dir": output_dir, + "output_fn_template": output_fn_template, + "staging_dir": staging_dir, + "vx_fcst_model_name": vxcfg["VX_FCST_MODEL_NAME"], + "num_ens_members": enscfg["NUM_ENS_MEMBERS"], + "ensmem_name": ensmem, + "time_lag": time_lag, + "fieldname_in_obs_input": obs_fieldname, + "fieldname_in_fcst_input": fcst_fieldname, + "fieldname_in_met_output": met_out_name, + "fieldname_in_met_filedir_names": met_filedir_name, + "obtype": obtype, + "FCST_OR_OBS": fcst_or_obs, + "input_accum_hh": input_accum_hh, + "output_accum_hh": f"{accum_hh:02}", + "accum_no_pad": accum_hh, + "metplus_templates_dir": cfg["user"]["METPLUS_CONF"], + "input_field_group": field_group, + "input_level_fcst": fcst_level, + "input_thresh_fcst": fcst_thresh, + "pcp_combine_method": pcp_combine_method, + "pcp_combine_command": pcp_combine_command, + } + + numprocs = 1 + conf_files = render_metplus_confs( + cfg, settings, metplus_config_tmpl_fn, vx_leadhr_list, numprocs + ) + lgr.debug(f"{conf_files=}") + + lgr.info(f"Running {metplus_tool_camel_case} ({fcst_or_obs}) with METplus") + common_conf = os.path.join(cfg["user"]["METPLUS_CONF"], "common.conf") + with Pool(processes=numprocs) as pool: + pool.starmap(run_metplus, [(common_conf, fn) for fn in conf_files]) + + lgr.info(f"{metplus_tool_camel_case} completed successfully.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run METplus PcpCombine for multi-hour accumulation of obs or forecast fields" + ) + parser.add_argument("--config", default="config.yaml", type=str, + help="Path to the experiment configuration file in YAML format") + parser.add_argument("--cycle_date", required=True, type=str, + help="Eight-digit cycle date (YYYYMMDDHH)") + parser.add_argument("--obs_dir", required=True, type=str, + help="Observation directory for this obtype") + parser.add_argument("--field_group", required=True, type=str, + help="Field group to combine (e.g. APCP, ASNOW, PM25, PM10)") + parser.add_argument("--obtype", required=True, type=str, + help="Observation type (e.g. CCPA, NOHRSC, AIRNOW)") + parser.add_argument("--accum_hh", required=True, type=int, + help="Target accumulation period in hours") + parser.add_argument("--fcst_level", default="", type=str, + help="METplus forecast level (e.g. A06)") + parser.add_argument("--fcst_thresh", default="", type=str, + help="Forecast thresholds to verify against (e.g. all, none)") + parser.add_argument("--fcst_or_obs", required=True, type=str, + help="Whether processing forecast (FCST) or observation (OBS) data") + parser.add_argument("--ensmem_index", type=int, default=0, + help="Ensemble member index (0 for deterministic)") + parser.add_argument("-v", "--verbose", action="store_true", + help="Enable verbose debug output") + args = parser.parse_args() + + setup_logging(debug=args.verbose) + logging.debug(f"{os.environ['METPLUS_ROOT']=}") + + pcpcombine( + args.config, args.cycle_date, args.obs_dir, args.field_group, + args.obtype, args.accum_hh, args.fcst_level, args.fcst_thresh, + args.fcst_or_obs, args.ensmem_index, + ) diff --git a/scripts/pcpcombine.sh b/scripts/pcpcombine.sh deleted file mode 100755 index fe233ce6..00000000 --- a/scripts/pcpcombine.sh +++ /dev/null @@ -1,469 +0,0 @@ -#!/usr/bin/env bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -sections=( - user - platform - workflow - ensemble - verification -) -for sect in ${sections[*]} ; do - source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} -done -# -#----------------------------------------------------------------------- -# -# Source files defining auxiliary functions for verification. -# -#----------------------------------------------------------------------- -# -. $USHdir/get_metplus_tool_name.sh -. $USHdir/set_vx_params.sh -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of the MET/METplus tool in different formats that may be -# needed from the variable METPLUSTOOLNAME. -# -#----------------------------------------------------------------------- -# -get_metplus_tool_name \ - METPLUSTOOLNAME="${METPLUSTOOLNAME}" \ - outvarname_metplus_tool_name="metplus_tool_name" \ - outvarname_MetplusToolName="MetplusToolName" \ - outvarname_METPLUS_TOOL_NAME="METPLUS_TOOL_NAME" -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the ex-script for the task that runs the METplus ${MetplusToolName} -tool to combine hourly accumulated precipitation (APCP) data to generate -files containing multi-hour accumulated precipitation (e.g. 3-hour, 6- -hour, 24-hour). The input files can come from either observations or -a forecast. -========================================================================" -# -#----------------------------------------------------------------------- -# -# Get the cycle date and time in YYYYMMDDHH format. -# -#----------------------------------------------------------------------- -# -CDATE="${YYMMDD}${HH}" -# -#----------------------------------------------------------------------- -# -# Set various verification parameters associated with the field to be -# verified. Not all of these are necessarily used later below but are -# set here for consistency with other verification ex-scripts. -# -#----------------------------------------------------------------------- -# -FIELDNAME_IN_OBS_INPUT="" -FIELDNAME_IN_FCST_INPUT="" -FIELDNAME_IN_MET_OUTPUT="" -FIELDNAME_IN_MET_FILEDIR_NAMES="" - -set_vx_params \ - obtype="${OBTYPE}" \ - field_group="${FIELD_GROUP}" \ - accum_hh="${ACCUM_HH}" \ - outvarname_grid_or_point="grid_or_point" \ - outvarname_fieldname_in_obs_input="FIELDNAME_IN_OBS_INPUT" \ - outvarname_fieldname_in_fcst_input="FIELDNAME_IN_FCST_INPUT" \ - outvarname_fieldname_in_MET_output="FIELDNAME_IN_MET_OUTPUT" \ - outvarname_fieldname_in_MET_filedir_names="FIELDNAME_IN_MET_FILEDIR_NAMES" -# -#----------------------------------------------------------------------- -# -# If performing forecast ensemble verification, get the time lag (if any) -# of the current ensemble forecast member. The time lag is the duration -# (in units of seconds) by which the current forecast member was initialized -# before the current cycle date and time (with the latter specified by -# CDATE). For example, a time lag of 3600 means that the current member -# was initialized 1 hour before the current CDATE, while a time lag of 0 -# means the current member was initialized on CDATE. -# -# Note that if we're not running ensemble verification (i.e. if we're -# running verification for a single deterministic forecast), the time -# lag gets set to 0. -# -#----------------------------------------------------------------------- -# -time_lag="0" -if [ "${FCST_OR_OBS}" = "FCST" ]; then - i="0" - if [ "${DO_ENSEMBLE}" = "True" ]; then - i=$( bc -l <<< "${ENSMEM_INDX}-1" ) - fi - time_lag=$( bc -l <<< "${ENS_TIME_LAG_HRS[$i]}*3600" ) -fi -# -#----------------------------------------------------------------------- -# -# Set paths and file templates for input to and output from the MET/ -# METplus tool to be run as well as other file/directory parameters. -# -#----------------------------------------------------------------------- -# -vx_fcst_input_basedir=$( eval echo "${VX_FCST_INPUT_BASEDIR}" ) -vx_output_basedir=$( eval echo "${VX_OUTPUT_BASEDIR}" ) -if [ "${FCST_OR_OBS}" = "FCST" ]; then - ensmem_indx=$(printf "%0${VX_NDIGITS_ENSMEM_NAMES}d" $(( 10#${ENSMEM_INDX}))) - ensmem_name="mem${ensmem_indx}" - # - # Since other aspects of a deterministic run use the "mem000" string (e.g. - # in rocoto workflow task names, in log file names), it seems reasonable - # that a deterministic run create a "mem000" subdirectory under the $CDATE - # directory. But since that is currently not the case in in the run_fcst - # task, we need the following if-statement. If and when such a modification - # is made for the run_fcst task, we would remove this if-statement and - # simply set - # slash_ensmem_subdir_or_null="/${ensmem_name}" - # or, better, just remove this variale and code "/${ensmem_name}" where - # slash_ensmem_subdir_or_null currently appears below. - # - if [ "${DO_ENSEMBLE}" = "True" ]; then - slash_ensmem_subdir_or_null="/${ensmem_name}" - else - slash_ensmem_subdir_or_null="" - fi -elif [ "${FCST_OR_OBS}" = "OBS" ]; then - if [ "${DO_ENSEMBLE}" = "True" ]; then - slash_obs_or_null="/obs" - else - slash_obs_or_null="" - fi -fi - -OBS_INPUT_DIR="" -OBS_INPUT_FN_TEMPLATE="" -FCST_INPUT_DIR="" -FCST_INPUT_FN_TEMPLATE="" -PCP_COMBINE_METHOD="ADD" -PCP_COMBINE_COMMAND="" -if [ "${FCST_OR_OBS}" = "FCST" ]; then - - FCST_INPUT_DIR="${vx_fcst_input_basedir}" - FCST_INPUT_FN_TEMPLATE=$( eval echo ${FCST_SUBDIR_TEMPLATE:+${FCST_SUBDIR_TEMPLATE}/}${FCST_FN_TEMPLATE} ) - - OUTPUT_BASE="${vx_output_basedir}/${CDATE}${slash_ensmem_subdir_or_null}" - OUTPUT_DIR="${OUTPUT_BASE}/metprd/${MetplusToolName}_fcst" - OUTPUT_FN_TEMPLATE=$( eval echo ${FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT} ) - STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}" - if [ "${OBTYPE}" = "AIRNOW" ]; then - PCP_COMBINE_METHOD="USER_DEFINED" - - if [ "${FIELD_GROUP}" = "PM25" ]; then - if [ "${FCST_SMOKE_TYPE}" = "HRRR" ]; then - # for HRRR, all PM 2.5 is a single variable, so just pass through with corrected units - PCP_COMBINE_COMMAND="-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} -field 'name=\"MASSDEN\"; level=\"Z8\"; convert(x)=x*1e9;'" - elif [ "${FCST_SMOKE_TYPE}" = "RRFS" ]; then - # Need to combine two fields (different PM types) and convert units from forecast files to create PM25 equivalent to obs - PCP_COMBINE_COMMAND="-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62010; convert(x)=x*1e9;' {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001; GRIB2_aerosol_interval_type=0; convert(x)=x*1e9;'" - fi - elif [ "${FIELD_GROUP}" = "PM10" ]; then - if [ "${FCST_SMOKE_TYPE}" = "RRFS" ]; then - # for PM10, need to combine original field with PM2.5 described above - PCP_COMBINE_COMMAND="-add {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62010; convert(x)=x*1e9;' {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001; GRIB2_aerosol_interval_type=0; convert(x)=x*1e9;' {FCST_PCP_COMBINE_INPUT_DIR}/{FCST_PCP_COMBINE_INPUT_TEMPLATE} 'name=\"MASSDEN\"; level=\"Z8\"; GRIB2_aerosol_type=62001; GRIB2_aerosol_interval_type=2; convert(x)=x*1e9;'" - else - print_err_msg_exit "PM10 only available for RRFS output, not available for ${FCST_SMOKE_TYPE}" - fi - fi - fi -elif [ "${FCST_OR_OBS}" = "OBS" ]; then - - OBS_INPUT_DIR="${OBS_DIR}" - fn_template=$(eval echo \${OBS_${OBTYPE}_FN_TEMPLATES[1]}) - OBS_INPUT_FN_TEMPLATE=$( eval echo ${fn_template} ) - - OUTPUT_BASE="${vx_output_basedir}/${CDATE}${slash_obs_or_null}" - OUTPUT_DIR="${OUTPUT_BASE}/metprd/${MetplusToolName}_obs" - fn_template=$(eval echo \${OBS_${OBTYPE}_${FIELD_GROUP}_FN_TEMPLATE_PCPCOMBINE_OUTPUT}) - OUTPUT_FN_TEMPLATE=$( eval echo ${fn_template} ) - STAGING_DIR="${OUTPUT_BASE}/stage/${FIELDNAME_IN_MET_FILEDIR_NAMES}" - -fi -# -#----------------------------------------------------------------------- -# -# Set the array of lead hours for which to run the MET/METplus tool. -# -#----------------------------------------------------------------------- -# -vx_intvl="$((10#${ACCUM_HH}))" -#Airnow obs use PCP_Combine simply to combine two fields, so run for every hour -if [ "${OBTYPE}" = "AIRNOW" ]; then - lhr_min=0 -else - lhr_min=${vx_intvl} -fi -VX_LEADHR_LIST=$( python3 $USHdir/set_leadhrs.py \ - --lhr_min="${lhr_min}" \ - --lhr_max="${FCST_LEN_HRS}" \ - --lhr_intvl="${vx_intvl}" \ - --skip_check_files ) || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" -# -#----------------------------------------------------------------------- -# -# Check for the presence of files (either from observations or forecasts) -# needed to create required accumulation given by ACCUM_HH. -# -#----------------------------------------------------------------------- -# -if [ "${FCST_OR_OBS}" = "FCST" ]; then - base_dir="${FCST_INPUT_DIR}" - fn_template="${FCST_INPUT_FN_TEMPLATE}" - subintvl="${VX_FCST_OUTPUT_INTVL_HRS}" -elif [ "${FCST_OR_OBS}" = "OBS" ]; then - base_dir="${OBS_INPUT_DIR}" - fn_template="${OBS_INPUT_FN_TEMPLATE}" - subintvl="${OBS_AVAIL_INTVL_HRS}" -fi -num_missing_files_max="0" -input_accum_hh=$(printf "%02d" ${subintvl}) -# -# Convert the list of hours at which the PcpCombine tool will be run to -# an array. This represents the hours at which each accumulation period -# ends. Then use it to check the presence of all files requied to build -# the required accumulations from the sub-accumulations. -# -subintvl_end_hrs=($( echo ${VX_LEADHR_LIST} | $SED "s/,//g" )) -for hr_end in ${subintvl_end_hrs[@]}; do - hr_start=$((hr_end - vx_intvl + subintvl)) - print_info_msg " -Checking for the presence of files that will contribute to the ${vx_intvl}-hour -accumulation ending at lead hour ${hr_end} (relative to ${CDATE})... -" - python3 $USHdir/set_leadhrs.py \ - --date_init="${CDATE}" \ - --lhr_min="${hr_start}" \ - --lhr_max="${hr_end}" \ - --lhr_intvl="${subintvl}" \ - --base_dir="${base_dir}" \ - --fn_template="${fn_template}" \ - --num_missing_files_max="${num_missing_files_max}" \ - --time_lag="${time_lag%.*}" || \ - print_err_msg_exit "Call to set_leadhrs.py failed with return code: $?" -done - -print_info_msg " -${MetplusToolName} will be run for the following lead hours (relative to ${CDATE}): - VX_LEADHR_LIST = ${VX_LEADHR_LIST} -" -# -#----------------------------------------------------------------------- -# -# Make sure the MET/METplus output directory(ies) exists. -# -#----------------------------------------------------------------------- -# -mkdir -p "${OUTPUT_DIR}" -# -#----------------------------------------------------------------------- -# -# Check for existence of top-level OBS_DIR. -# -#----------------------------------------------------------------------- -# -if [ "${FCST_OR_OBS}" = "OBS" ]; then - if [ ! -d "${OBS_DIR}" ]; then - print_err_msg_exit "\ - OBS_DIR does not exist or is not a directory: - OBS_DIR = \"${OBS_DIR}\"" - fi -fi -# -#----------------------------------------------------------------------- -# -# Export variables needed in the common METplus configuration file (at -# ${METPLUS_CONF}/common.conf). -# -#----------------------------------------------------------------------- -# -export METPLUS_CONF -export LOGDIR -# -#----------------------------------------------------------------------- -# -# Do not run METplus if there isn't at least one lead hour for which to -# run it. -# -#----------------------------------------------------------------------- -# -if [ -z "${VX_LEADHR_LIST}" ]; then - print_err_msg_exit "\ -The list of lead hours for which to run METplus is empty: - VX_LEADHR_LIST = [${VX_LEADHR_LIST}]" -fi -# -#----------------------------------------------------------------------- -# -# Set the names of the template METplus configuration file, the METplus -# configuration file generated from this template, and the METplus log -# file. -# -#----------------------------------------------------------------------- -# -# First, set the base file names. -# -metplus_config_tmpl_fn="${MetplusToolName}" -if [ "${FCST_OR_OBS}" = "FCST" ]; then - suffix="${ENSMEM_INDX:+_${ensmem_name}}" -elif [ "${FCST_OR_OBS}" = "OBS" ]; then - suffix="_${OBTYPE}" -fi -metplus_config_fn="${metplus_config_tmpl_fn}_$(echo_lowercase ${FCST_OR_OBS})_${FIELDNAME_IN_MET_FILEDIR_NAMES}${suffix}" -metplus_log_fn="${metplus_config_fn}_$CDATE" -# -# Add prefixes and suffixes (extensions) to the base file names. -# -metplus_config_tmpl_fn="${metplus_config_tmpl_fn}.conf" -metplus_config_fn="${metplus_config_fn}.conf" -metplus_log_fn="metplus.log.${metplus_log_fn}" -# -#----------------------------------------------------------------------- -# -# Generate the METplus configuration file from its jinja template. -# -#----------------------------------------------------------------------- -# -# Set the full paths to the jinja template METplus configuration file -# (which already exists) and the METplus configuration file that will be -# generated from it. -# -metplus_config_tmpl_fp="${METPLUS_CONF}/${metplus_config_tmpl_fn}" -metplus_config_fp="${OUTPUT_DIR}/${metplus_config_fn}" -# -# Define variables that appear in the jinja template. -# -settings="\ -# -# MET/METplus information. -# - 'metplus_tool_name': '${metplus_tool_name}' - 'MetplusToolName': '${MetplusToolName}' - 'METPLUS_TOOL_NAME': '${METPLUS_TOOL_NAME}' - 'metplus_verbosity_level': '${METPLUS_VERBOSITY_LEVEL}' -# -# Date and forecast hour information. -# - 'cdate': '$CDATE' - 'vx_leadhr_list': '${VX_LEADHR_LIST}' -# -# Input and output directory/file information. -# - 'metplus_config_fn': '${metplus_config_fn:-}' - 'metplus_log_fn': '${metplus_log_fn:-}' - 'input_dir': '${FCST_INPUT_DIR:-${OBS_INPUT_DIR}}' - 'input_fn_template': '${FCST_INPUT_FN_TEMPLATE:-${OBS_INPUT_FN_TEMPLATE}}' - 'output_dir': '${OUTPUT_DIR}' - 'output_fn_template': '${OUTPUT_FN_TEMPLATE:-}' - 'staging_dir': '${STAGING_DIR}' - 'vx_fcst_model_name': '${VX_FCST_MODEL_NAME}' -# -# Ensemble and member-specific information. -# - 'num_ens_members': '${NUM_ENS_MEMBERS}' - 'ensmem_name': '${ensmem_name:-}' - 'time_lag': '${time_lag:-}' -# -# Field information. -# - 'fieldname_in_obs_input': '${FIELDNAME_IN_OBS_INPUT}' - 'fieldname_in_fcst_input': '${FIELDNAME_IN_FCST_INPUT}' - 'fieldname_in_met_output': '${FIELDNAME_IN_MET_OUTPUT}' - 'fieldname_in_met_filedir_names': '${FIELDNAME_IN_MET_FILEDIR_NAMES}' - 'obtype': '${OBTYPE}' - 'FCST_OR_OBS': '${FCST_OR_OBS}' - 'input_accum_hh': '${input_accum_hh}' - 'output_accum_hh': '${ACCUM_HH:-}' - 'accum_no_pad': '${ACCUM_NO_PAD:-}' - 'metplus_templates_dir': '${METPLUS_CONF:-}' - 'input_field_group': '${FIELD_GROUP:-}' - 'input_level_fcst': '${FCST_LEVEL:-}' - 'input_thresh_fcst': '${FCST_THRESH:-}' -# -# Configuration information -# - 'pcp_combine_method': '${PCP_COMBINE_METHOD}' -# NOTE: this command must remain un-quoted for proper rendering of nested quotes in command - 'pcp_combine_command': ${PCP_COMBINE_COMMAND} -" -# Render the template to create a METplus configuration file -tmpfile=$( $READLINK -f "$(mktemp ./met_plus_settings.XXXXXX.yaml)") -printf "%s" "$settings" > "$tmpfile" -uw template render \ - -i ${metplus_config_tmpl_fp} \ - -o ${metplus_config_fp} \ - --verbose \ - --values-file "${tmpfile}" \ - --search-path "/" - -err=$? -rm $tmpfile -if [ $err -ne 0 ]; then - message_txt="Error rendering template for METplus config. - Contents of input are: -$settings" - print_err_msg_exit "${message_txt}" -fi -# -#----------------------------------------------------------------------- -# -# Call METplus. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Calling METplus to run MET's ${metplus_tool_name} tool for field(s): ${FIELDNAME_IN_MET_FILEDIR_NAMES}" -${METPLUS_ROOT}/ush/run_metplus.py \ - -c ${METPLUS_CONF}/common.conf \ - -c ${metplus_config_fp} || \ -print_err_msg_exit " -Call to METplus failed with return code: $? -METplus configuration file used is: - metplus_config_fp = \"${metplus_config_fp}\"" -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -METplus ${MetplusToolName} tool completed successfully. - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" diff --git a/scripts/point2grid.py b/scripts/point2grid.py index 52ae3ca8..7ec3663f 100644 --- a/scripts/point2grid.py +++ b/scripts/point2grid.py @@ -95,8 +95,7 @@ def point2grid(config_file,cdate,obs_dir,field_group,obtype,fcst_level,fcst_thre #Point2Grid does not honor "time lag" shifts, so remove from template fcst_fn_template=Path( vxcfg["VX_FCST_INPUT_BASEDIR"], - Template(vxcfg["FCST_SUBDIR_TEMPLATE"]).substitute(subvars), - Template(vxcfg["FCST_FN_TEMPLATE"]).substitute(subvars) + Template(vxcfg["FCST_FN_TEMPLATE"][0]).substitute(subvars) ) output_dir=Path(exptdir, cdate, "metprd", metplus_tool_camel_case) # Make sure the MET/METplus output directory(ies) exists. diff --git a/scripts/regriddataplane.py b/scripts/regriddataplane.py index f6bcc23e..0e1e68c9 100644 --- a/scripts/regriddataplane.py +++ b/scripts/regriddataplane.py @@ -84,8 +84,7 @@ def regrid_data_plane(config_file,cdate,field_group,obtype): #RegridDataPlane does not honor "time lag" shifts, so remove from template fcst_fn_template=Path( vxcfg["VX_FCST_INPUT_BASEDIR"], - Template(vxcfg["FCST_SUBDIR_TEMPLATE"]).substitute(subvars), - Template(vxcfg["FCST_FN_TEMPLATE"]).substitute(subvars) + Template(vxcfg["FCST_FN_TEMPLATE"][0]).substitute(subvars) ) output_dir=Path(exptdir, cdate, "metprd", metplus_tool_camel_case) # Make sure the MET/METplus output directory(ies) exists. diff --git a/scripts/tcrmw.py b/scripts/tcrmw.py index 6709f875..e0608a4b 100644 --- a/scripts/tcrmw.py +++ b/scripts/tcrmw.py @@ -70,8 +70,7 @@ def tcrmw(config_file, cdate): subvars = { "time_lag": 0, } - fcst_fn_template=os.path.join(Template(vxcfg["FCST_SUBDIR_TEMPLATE"]).substitute(subvars), - Template(vxcfg["FCST_FN_TEMPLATE"]).substitute(subvars)) + fcst_fn_template = Template(vxcfg["FCST_FN_TEMPLATE"][0]).substitute(subvars) lgr.debug(f"{fcst_fn_template=}") # TCRMW does not accept "cyclone" keyword diff --git a/setup_conda.sh b/setup_conda.sh index 5a166399..4d4849ff 100644 --- a/setup_conda.sh +++ b/setup_conda.sh @@ -98,7 +98,7 @@ else ENV_NAME=vx_workflow fi -if ! conda env list | grep -q "^${ENV_NAME}\s" ; then +if ! conda env list 2>/dev/null | grep -q "^${ENV_NAME}\s" ; then echo "Creating ${ENV_NAME} environment..." mamba env create -n ${ENV_NAME} --file "${ENV_YAML}" --quiet else diff --git a/tests/WE2E/monitor_jobs.py b/tests/WE2E/monitor_jobs.py index 029e07fa..951f2ec5 100755 --- a/tests/WE2E/monitor_jobs.py +++ b/tests/WE2E/monitor_jobs.py @@ -11,10 +11,10 @@ from python_utils import check_python_version, load_config_file from utils import calculate_core_hours, write_monitor_file, update_expt_status,\ - update_expt_status_parallel, print_WE2E_summary + update_expt_status_parallel, print_WE2E_summary, DEFAULT_DELAY def monitor_jobs(expts_dict: dict, monitor_file: str = '', procs: int = 1, - mode: str = 'continuous', delay: int = 5, debug: bool = False) -> str: + mode: str = 'continuous', delay: int = DEFAULT_DELAY, debug: bool = False) -> str: """Monitors and runs jobs for the specified experiment using Rocoto Args: diff --git a/tests/WE2E/run_we2e_tests.py b/tests/WE2E/run_we2e_tests.py index 34635e3e..a84851c5 100755 --- a/tests/WE2E/run_we2e_tests.py +++ b/tests/WE2E/run_we2e_tests.py @@ -16,6 +16,7 @@ from textwrap import dedent from monitor_jobs import monitor_jobs, write_monitor_file +from utils import DEFAULT_DELAY from uwtools.api.config import get_yaml_config @@ -24,8 +25,6 @@ from generate_wflow import generate_wflow from python_utils import check_python_version - - def run_we2e_tests(homedir, args) -> None: """Runs the Workflow End-to-End (WE2E) tests selected by the user @@ -261,7 +260,7 @@ def run_we2e_tests(homedir, args) -> None: rerun_string=f"./monitor_jobs.py -y={monitor_file}" if args.procs>1: rerun_string+=f" -p={args.procs}" - if args.delay!=5: + if args.delay!=DEFAULT_DELAY: rerun_string+=f" --delay={args.delay}" logging.info(f"{rerun_string}\n") @@ -510,7 +509,7 @@ def setup_logging(logfile: str = "log.run_WE2E_tests", debug: bool = False) -> N help="Explicitly set VERBOSE=TRUE for all experiments", ) optional.add_argument( - '--delay', type=int, default=5, + '--delay', type=int, default=DEFAULT_DELAY, help='Pause this number of seconds between calls to rocotorun') user_args = ap.parse_args() diff --git a/tests/WE2E/test_configs/deterministic/config.MET_verification.yaml b/tests/WE2E/test_configs/deterministic/config.MET_verification.yaml index 210bc8fc..5ad33071 100644 --- a/tests/WE2E/test_configs/deterministic/config.MET_verification.yaml +++ b/tests/WE2E/test_configs/deterministic/config.MET_verification.yaml @@ -6,11 +6,8 @@ workflow: DATE_LAST_CYCL: '2019061518' FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: VX_FCST_MODEL_NAME: srw.RRFS_CONUS_25km VX_MASK: ["CONUS"] VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/MET_verification' - FCST_FN_TEMPLATE: 'srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H}/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' diff --git a/tests/WE2E/test_configs/deterministic/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/deterministic/config.MET_verification_only_vx.yaml index 65f40f3b..a9268fee 100644 --- a/tests/WE2E/test_configs/deterministic/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/deterministic/config.MET_verification_only_vx.yaml @@ -9,11 +9,12 @@ workflow: FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename taskgroups: - - parm/wflow/verify_pre.yaml + - parm/wflow/get_obs.yaml + - parm/wflow/preprocessing.yaml - parm/wflow/verify_det.yaml verification: VX_MASK: ["CONUS","EAST","EAST_OF_ROCKIES"] VX_FCST_MODEL_NAME: srw.RRFS_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/output_data/fcst_det/RRFS_CONUS_25km' - FCST_FN_TEMPLATE: 'rrfs.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H}/postprd/rrfs.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' diff --git a/tests/WE2E/test_configs/deterministic/config.MET_verification_smoke_only_vx.yaml b/tests/WE2E/test_configs/deterministic/config.MET_verification_smoke_only_vx.yaml index e0839bbb..6b68f884 100644 --- a/tests/WE2E/test_configs/deterministic/config.MET_verification_smoke_only_vx.yaml +++ b/tests/WE2E/test_configs/deterministic/config.MET_verification_smoke_only_vx.yaml @@ -8,16 +8,12 @@ workflow: DATE_LAST_CYCL: '2023071712' FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: VX_FCST_MODEL_NAME: RRFS_smoke_test VX_FIELD_GROUPS: [ "AOD", "PM25", "PM10" ] VX_FCST_MODEL_NAME: RRFS_CONUS VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/RRFS' - FCST_SUBDIR_TEMPLATE: '{init?fmt=%Y%m%d%H}' - FCST_FN_TEMPLATE: 'rrfs.t12z.prslev.f{lead?fmt=%HHH}.conus.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H}/rrfs.t12z.prslev.f{lead?fmt=%HHH}.conus.grib2' AERONET_OBS_DIR: "{{ workflow.EXPTDIR }}/obs_data/aeronet" AIRNOW_OBS_DIR: "{{ workflow.EXPTDIR }}/obs_data/airnow" OBS_DATA_STORE_AERONET: http diff --git a/tests/WE2E/test_configs/deterministic/config.MET_verification_winter_wx.yaml b/tests/WE2E/test_configs/deterministic/config.MET_verification_winter_wx.yaml index 55ba9063..33529653 100644 --- a/tests/WE2E/test_configs/deterministic/config.MET_verification_winter_wx.yaml +++ b/tests/WE2E/test_configs/deterministic/config.MET_verification_winter_wx.yaml @@ -10,9 +10,6 @@ workflow: DATE_LAST_CYCL: '2023021700' FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: VX_FIELD_GROUPS: - APCP @@ -26,3 +23,5 @@ verification: NOHRSC_OBS_DIR: '{{ workflow.EXPTDIR }}/NOHRSC_obs' VX_FCST_MODEL_NAME: custom_ESGgrid_Michigan_Ontario VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/MET_verification_winter_wx' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.{{ verification.VX_FCST_MODEL_NAME }}.grib2' + diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-fourcastnet.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-fourcastnet.yaml index 319d4273..5900d661 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-fourcastnet.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-fourcastnet.yaml @@ -1,7 +1,7 @@ metadata: description: |- This is a test for deterministic verification of the - FourCastNet (fcnv2) global AI model. Note that this test uses a custom + FourCastNet (fcnv2) global AI model. Note that this test uses a custom verification configuration file (as opposed to the default one in the SRW) because the AI model output often does not include many of the fields that exist in physics-based models. @@ -12,9 +12,6 @@ workflow: DATE_FIRST_CYCL: '2024073000' DATE_LAST_CYCL: '2024073000' FCST_LEN_HRS: 240 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: METPLUS_VERBOSITY_LEVEL: 5 @@ -36,8 +33,7 @@ verification: VX_FCST_MODEL_NAME: 'fcnv2' VX_CONFIG_DET_FN: 'vx_configs/vx_config_det.obs_gdas.model_aiml.yaml' VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}' - FCST_SUBDIR_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.{init?fmt=%Y%m%d}/{init?fmt=%H}' - FCST_FN_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.t00z.prslev.f{lead?fmt=%HHH}.grb2' + FCST_FN_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.{init?fmt=%Y%m%d}/{init?fmt=%H}/{{ verification.VX_FCST_MODEL_NAME }}.t00z.prslev.f{lead?fmt=%HHH}.grb2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: '{{ verification.VX_FCST_MODEL_NAME }}.t00z.prslev.f{lead?fmt=%HHH}_a${ACCUM_HH}h.nc' # VX_FCST_OUTPUT_INTVL_HRS: 6 diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-graphcast.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-graphcast.yaml index 8e87e783..6edf2ba6 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-graphcast.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-graphcast.yaml @@ -1,7 +1,7 @@ metadata: description: |- This is a test for deterministic verification of the - GraphCast (gc) global AI model. Note that this test uses a custom + GraphCast (gc) global AI model. Note that this test uses a custom verification configuration file because the AI model output does not include many of the fields that exist in physics-based models. @@ -16,9 +16,6 @@ workflow: DATE_FIRST_CYCL: '2024073000' DATE_LAST_CYCL: '2024073000' FCST_LEN_HRS: 240 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml task_get_extrn_lbcs: envvars: @@ -51,8 +48,7 @@ verification: VX_FCST_MODEL_NAME: 'gc' VX_CONFIG_DET_FN: 'vx_configs/vx_config_det.obs_gdas.model_aiml.yaml' VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}' - FCST_SUBDIR_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.{init?fmt=%Y%m%d}/{init?fmt=%H}' - FCST_FN_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.t00z.prslev.f{lead?fmt=%HHH}' + FCST_FN_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.{init?fmt=%Y%m%d}/{init?fmt=%H}/{{ verification.VX_FCST_MODEL_NAME }}.t00z.prslev.f{lead?fmt=%HHH}' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: '{{ verification.VX_FCST_MODEL_NAME }}.t00z.prslev.f{lead?fmt=%HHH}_a${ACCUM_HH}h.nc' # VX_FCST_OUTPUT_INTVL_HRS: 6 diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-panguweather.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-panguweather.yaml index 640efe35..0256bcdc 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-panguweather.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_aiml-panguweather.yaml @@ -1,7 +1,7 @@ metadata: description: |- SRW App configuration file to test deterministic verification of the - Pangu-Weather (pw) global AI model. Note that this test uses a custom + Pangu-Weather (pw) global AI model. Note that this test uses a custom verification configuration file because the AI model output does not include many of the fields that exist in physics-based models. @@ -10,9 +10,6 @@ workflow: DATE_FIRST_CYCL: '2024073000' DATE_LAST_CYCL: '2024073000' FCST_LEN_HRS: 240 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: METPLUS_VERBOSITY_LEVEL: 5 @@ -37,8 +34,7 @@ verification: VX_FCST_MODEL_NAME: 'pw' VX_CONFIG_DET_FN: 'vx_configs/vx_config_det.obs_gdas.model_aiml.yaml' VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}' - FCST_SUBDIR_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.{init?fmt=%Y%m%d}/{init?fmt=%H}' - FCST_FN_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.t00z.prslev.f{lead?fmt=%HHH}.grb2' + FCST_FN_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.{init?fmt=%Y%m%d}/{init?fmt=%H}/{{ verification.VX_FCST_MODEL_NAME }}.t00z.prslev.f{lead?fmt=%HHH}.grb2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: '{{ verification.VX_FCST_MODEL_NAME }}.t00z.prslev.f{lead?fmt=%HHH}_a${ACCUM_HH}h.nc' # VX_FCST_OUTPUT_INTVL_HRS: 6 diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_gfs.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_gfs.yaml index cba6d890..3cdee886 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_gfs.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_custom-vx-config_gfs.yaml @@ -14,9 +14,6 @@ workflow: DATE_FIRST_CYCL: '2024073000' DATE_LAST_CYCL: '2024073000' FCST_LEN_HRS: 240 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: METPLUS_VERBOSITY_LEVEL: 5 @@ -41,8 +38,7 @@ verification: VX_FCST_MODEL_NAME: 'gfs' VX_CONFIG_DET_FN: 'vx_configs/vx_config_det.obs_gdas.model_gfs.yaml' VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}' - FCST_SUBDIR_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.{init?fmt=%Y%m%d}/{init?fmt=%H}/atmos' - FCST_FN_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.t00z.pgrb2.0p25.f{lead?fmt=%HHH}' + FCST_FN_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.{init?fmt=%Y%m%d}/{init?fmt=%H}/atmos/{{ verification.VX_FCST_MODEL_NAME }}.t00z.pgrb2.0p25.f{lead?fmt=%HHH}' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: '{{ verification.VX_FCST_MODEL_NAME }}.t00z.pgrb2.0p25.f{lead?fmt=%HHH}_a${ACCUM_HH}h.nc' # VX_FCST_OUTPUT_INTVL_HRS: 6 diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_winter-wx_SRW-staged.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_winter-wx_SRW-staged.yaml index afbfa865..5c913633 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_winter-wx_SRW-staged.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_long-fcst_winter-wx_SRW-staged.yaml @@ -15,9 +15,6 @@ workflow: DATE_FIRST_CYCL: '2023021700' DATE_LAST_CYCL: '2023021700' FCST_LEN_HRS: 36 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: # Base directories in which to look for obs. If these do not exist and/or @@ -39,4 +36,4 @@ verification: VX_FCST_MODEL_NAME: 'Michigan_Ontario_snow_8km' VX_FIELD_GROUPS: [ "APCP", "ASNOW", "REFC", "RETOP", "SFC", "UPA" ] VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/output_data/fcst_det/custom_ESGgrid_Michigan_Ontario_8km/' - FCST_FN_TEMPLATE: 'srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.custom_esggrid_michigan_ontario.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.custom_esggrid_michigan_ontario.grib2' diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_fcst-overlap_ncep-hrrr.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_fcst-overlap_ncep-hrrr.yaml index 0c0726ce..3823998e 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_fcst-overlap_ncep-hrrr.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_fcst-overlap_ncep-hrrr.yaml @@ -15,9 +15,6 @@ workflow: DATE_FIRST_CYCL: '2024042902' DATE_LAST_CYCL: '2024043006' FCST_LEN_HRS: 9 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: METPLUS_VERBOSITY_LEVEL: 5 @@ -39,5 +36,5 @@ verification: # VX_FCST_MODEL_NAME: 'hrrr_ncep' VX_FCST_INPUT_BASEDIR: '{{ "%s/%s" % (platform.STAGED_DATA, verification.VX_FCST_MODEL_NAME) }}' - FCST_FN_TEMPLATE: 'hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/postprd/hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: 'hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}_${FIELD_GROUP}_a${ACCUM_HH}h.nc' diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_first-obs-00z_ncep-hrrr.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_first-obs-00z_ncep-hrrr.yaml index c7185688..5d46640e 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_first-obs-00z_ncep-hrrr.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_first-obs-00z_ncep-hrrr.yaml @@ -15,9 +15,6 @@ workflow: DATE_FIRST_CYCL: '2024042900' DATE_LAST_CYCL: '2024043000' FCST_LEN_HRS: 3 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: METPLUS_VERBOSITY_LEVEL: 5 @@ -39,5 +36,5 @@ verification: # VX_FCST_MODEL_NAME: 'hrrr_ncep' VX_FCST_INPUT_BASEDIR: '{{ "%s/%s" % (platform.STAGED_DATA, verification.VX_FCST_MODEL_NAME) }}' - FCST_FN_TEMPLATE: 'hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H}/postprd/hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: 'hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}_${FIELD_GROUP}_a${ACCUM_HH}h.nc' diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_last-obs-00z_ncep-hrrr.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_last-obs-00z_ncep-hrrr.yaml index 4737a9af..78165579 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_last-obs-00z_ncep-hrrr.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_last-obs-00z_ncep-hrrr.yaml @@ -15,9 +15,6 @@ workflow: DATE_FIRST_CYCL: '2024042921' DATE_LAST_CYCL: '2024043021' FCST_LEN_HRS: 3 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: METPLUS_VERBOSITY_LEVEL: 5 @@ -39,5 +36,5 @@ verification: # VX_FCST_MODEL_NAME: 'hrrr_ncep' VX_FCST_INPUT_BASEDIR: '{{ "%s/%s" % (platform.STAGED_DATA, verification.VX_FCST_MODEL_NAME) }}' - FCST_FN_TEMPLATE: 'hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H}/postprd/hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: 'hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}_${FIELD_GROUP}_a${ACCUM_HH}h.nc' diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-no-overlap_nssl-mpas.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-no-overlap_nssl-mpas.yaml index 1657be5e..1fa8dc29 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-no-overlap_nssl-mpas.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-no-overlap_nssl-mpas.yaml @@ -17,9 +17,6 @@ workflow: DATE_FIRST_CYCL: '2024042912' DATE_LAST_CYCL: '2024051112' FCST_LEN_HRS: 48 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: METPLUS_VERBOSITY_LEVEL: 5 @@ -41,5 +38,10 @@ verification: # VX_FCST_MODEL_NAME: 'NSSL-MPAS-HN' VX_FCST_INPUT_BASEDIR: '{{ "%s/%s" % (platform.STAGED_DATA, verification.VX_FCST_MODEL_NAME) }}' - FCST_FN_TEMPLATE: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%H?shift=${time_lag}}.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H}/postprd/mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%H?shift=${time_lag}}.grib2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}_${FIELD_GROUP}_a${ACCUM_HH}h.nc' +gridstat: + TASKS: 24 +pointstat: + TASKS: 24 + diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml index 62b6a57b..936a0cd0 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_long-fcst-overlap_nssl-mpas.yaml @@ -16,9 +16,6 @@ workflow: DATE_FIRST_CYCL: '2024042912' DATE_LAST_CYCL: '2024050212' FCST_LEN_HRS: 48 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: METPLUS_VERBOSITY_LEVEL: 5 @@ -40,5 +37,9 @@ verification: # VX_FCST_MODEL_NAME: 'NSSL-MPAS-HN' VX_FCST_INPUT_BASEDIR: '{{ "%s/%s" % (platform.STAGED_DATA, verification.VX_FCST_MODEL_NAME) }}' - FCST_FN_TEMPLATE: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%H?shift=${time_lag}}.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H}/postprd/mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%H?shift=${time_lag}}.grib2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}_${FIELD_GROUP}_a${ACCUM_HH}h.nc' +gridstat: + TASKS: 12 +pointstat: + TASKS: 6 diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_no-00z-obs_nssl-mpas.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_no-00z-obs_nssl-mpas.yaml index 0c695edc..e53c72fb 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_no-00z-obs_nssl-mpas.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_no-00z-obs_nssl-mpas.yaml @@ -17,9 +17,6 @@ workflow: DATE_FIRST_CYCL: '2024042912' DATE_LAST_CYCL: '2024043012' FCST_LEN_HRS: 3 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: METPLUS_VERBOSITY_LEVEL: 5 @@ -41,5 +38,5 @@ verification: # VX_FCST_MODEL_NAME: 'NSSL-MPAS-HN' VX_FCST_INPUT_BASEDIR: '{{ "%s/%s" % (platform.STAGED_DATA, verification.VX_FCST_MODEL_NAME) }}' - FCST_FN_TEMPLATE: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%H?shift=${time_lag}}.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H}/postprd/mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%H?shift=${time_lag}}.grib2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: 'mpashn4nssl_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}_${FIELD_GROUP}_a${ACCUM_HH}h.nc' diff --git a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_no-fcst-overlap_ncep-hrrr.yaml b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_no-fcst-overlap_ncep-hrrr.yaml index 241378f3..8fb31913 100644 --- a/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_no-fcst-overlap_ncep-hrrr.yaml +++ b/tests/WE2E/test_configs/deterministic/config.vx-det_multicyc_no-fcst-overlap_ncep-hrrr.yaml @@ -17,9 +17,6 @@ workflow: DATE_FIRST_CYCL: '2024042902' DATE_LAST_CYCL: '2024043022' FCST_LEN_HRS: 3 - taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml verification: METPLUS_VERBOSITY_LEVEL: 5 @@ -41,5 +38,5 @@ verification: # VX_FCST_MODEL_NAME: 'hrrr_ncep' VX_FCST_INPUT_BASEDIR: '{{"%s/%s" % (platform.STAGED_DATA, verification.VX_FCST_MODEL_NAME) }}' - FCST_FN_TEMPLATE: 'hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H}/postprd/hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' FCST_FN_TEMPLATE_PCPCOMBINE_OUTPUT: 'hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}_${FIELD_GROUP}_a${ACCUM_HH}h.nc' diff --git a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification.yaml b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification.yaml index f9a38c45..b0f30b83 100644 --- a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification.yaml +++ b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification.yaml @@ -8,7 +8,8 @@ workflow: FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename taskgroups: - - parm/wflow/verify_pre.yaml + - parm/wflow/get_obs.yaml + - parm/wflow/preprocessing.yaml - parm/wflow/verify_det.yaml - parm/wflow/verify_ens.yaml ensemble: @@ -18,4 +19,11 @@ verification: VX_MASK: ["AZCA"] VX_FCST_MODEL_NAME: srw.RRFS_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/MET_ensemble_verification' - FCST_FN_TEMPLATE: 'srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' + FCST_FN_TEMPLATE: + - '{init?fmt=%Y%m%d%H}/mem001/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' + - '{init?fmt=%Y%m%d%H}/mem002/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' +gridstat: + TASKS: 4 +pointstat: + TASKS: 4 + diff --git a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx.yaml index 9fad2ac3..f2c195c2 100644 --- a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx.yaml @@ -9,7 +9,8 @@ workflow: FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename taskgroups: - - parm/wflow/verify_pre.yaml + - parm/wflow/get_obs.yaml + - parm/wflow/preprocessing.yaml - parm/wflow/verify_det.yaml - parm/wflow/verify_ens.yaml @@ -22,4 +23,11 @@ verification: VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/output_data/fcst_ens/RRFS_CONUS_25km' VX_MASK: ["CONUS"] VX_TASKS: 4 - FCST_FN_TEMPLATE: 'rrfs.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' + FCST_FN_TEMPLATE: + - '{init?fmt=%Y%m%d%H}/mem001/postprd/rrfs.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' + - '{init?fmt=%Y%m%d%H}/mem002/postprd/rrfs.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conus_25km.grib2' +gridstat: + TASKS: 4 +pointstat: + TASKS: 4 + diff --git a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml index 51500972..54678f83 100644 --- a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml +++ b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_only_vx_time_lag.yaml @@ -9,14 +9,15 @@ workflow: FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename taskgroups: - - parm/wflow/verify_pre.yaml + - parm/wflow/get_obs.yaml + - parm/wflow/preprocessing.yaml - parm/wflow/verify_det.yaml - parm/wflow/verify_ens.yaml ensemble: DO_ENSEMBLE: true NUM_ENS_MEMBERS: 2 - ENS_TIME_LAG_HRS: '[ 0, 12 ]' + ENS_TIME_LAG_HRS: !list '[ 0, 12 ]' verification: @@ -29,5 +30,13 @@ verification: VX_FCST_MODEL_NAME: RRFSE_CONUS VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/output_data/fcst_ens/RRFS_CONUS_3km' VX_NDIGITS_ENSMEM_NAMES: 1 - FCST_FN_TEMPLATE: '{{ verification.VX_FCST_MODEL_NAME }}.t{init?fmt=%H?shift=-${time_lag}}z.bgdawpf{lead?fmt=%HHH?shift=${time_lag}}.tm00.grib2' + FCST_FN_TEMPLATE: + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem1/postprd/{{ verification.VX_FCST_MODEL_NAME }}.t{init?fmt=%H?shift=-${time_lag}}z.bgdawpf{lead?fmt=%HHH?shift=${time_lag}}.tm00.grib2' + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem2/postprd/{{ verification.VX_FCST_MODEL_NAME }}.t{init?fmt=%H?shift=-${time_lag}}z.bgdawpf{lead?fmt=%HHH?shift=${time_lag}}.tm00.grib2' VX_MASK: ["CONUS"] + VX_TASKS: 6 +gridstat: + TASKS: 8 +genensprod: + TASKS: 8 + diff --git a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_winter_wx.yaml b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_winter_wx.yaml index 2f2f1fcc..a22d1768 100644 --- a/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_winter_wx.yaml +++ b/tests/WE2E/test_configs/ensemble/config.MET_ensemble_verification_winter_wx.yaml @@ -9,7 +9,8 @@ workflow: FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename taskgroups: - - parm/wflow/verify_pre.yaml + - parm/wflow/get_obs.yaml + - parm/wflow/preprocessing.yaml - parm/wflow/verify_det.yaml - parm/wflow/verify_ens.yaml ensemble: @@ -23,5 +24,19 @@ verification: VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/MET_ensemble_verification_winter_wx' VX_TASKS: 4 VX_MASK: ["CONUS"] - FCST_FN_TEMPLATE: 'srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' + FCST_FN_TEMPLATE: + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem001/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem002/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem003/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem004/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem005/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem006/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem007/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem008/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem009/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' + - '{init?fmt=%Y%m%d%H?shift=-${time_lag}}/mem010/postprd/srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.rrfs_conuscompact_25km.grib2' +gridstat: + TASKS: 4 +pointstat: + TASKS: 4 diff --git a/tests/WE2E/test_configs/mode/config.MODE_AOD.yaml b/tests/WE2E/test_configs/mode/config.MODE_AOD.yaml index f5f89b5b..5f2e2177 100644 --- a/tests/WE2E/test_configs/mode/config.MODE_AOD.yaml +++ b/tests/WE2E/test_configs/mode/config.MODE_AOD.yaml @@ -11,9 +11,10 @@ workflow: FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename taskgroups: - - parm/wflow/verify_pre.yaml - - parm/wflow/verify_det.yaml - - parm/wflow/mode.yaml + - parm/wflow/get_obs.yaml + - parm/wflow/preprocessing.yaml + - parm/wflow/verify_det.yaml + - parm/wflow/mode.yaml verification: METPLUS_VERBOSITY_LEVEL: 4 VX_FIELD_GROUPS: @@ -29,8 +30,7 @@ verification: - GOESAOD VX_FCST_MODEL_NAME: RRFS_verification VX_FCST_INPUT_BASEDIR: '{{ platform.STAGED_DATA }}/RRFS' - FCST_SUBDIR_TEMPLATE: '{init?fmt=%Y%m%d%H}' - FCST_FN_TEMPLATE: 'rrfs.t12z.prslev.f{lead?fmt=%HHH}.conus.grib2' + FCST_FN_TEMPLATE: '{init?fmt=%Y%m%d%H}/rrfs.t12z.prslev.f{lead?fmt=%HHH}.conus.grib2' CCPA_OBS_DIR: "{{ workflow.EXPTDIR }}/obs_data/ccpa" NOHRSC_OBS_DIR: "{{ workflow.EXPTDIR }}/obs_data/nohrsc" MRMS_OBS_DIR: "{{ workflow.EXPTDIR }}/obs_data/mrms" @@ -62,17 +62,19 @@ mode: CONV_THRESH: '>=0.3' MERGE_THRESH: '>=5' MERGE_FLAG: ENGINE +gridstat: + execution: + tasks_per_node: '{{ verification.VX_TASKS }}' + walltime: 01:00:00 + memory: '{{ 5 * verification.VX_TASKS }}G' +pointstat: + execution: + tasks_per_node: '{{ verification.VX_TASKS }}' + walltime: 01:00:00 + memory: '{{ 5 * verification.VX_TASKS }}G' + verification_resources: execution: - deterministic: - gridstat: - tasks_per_node: '{{ verification.VX_TASKS }}' - walltime: 01:00:00 - memory: '{{ 5 * verification.VX_TASKS }}G' - pointstat: - tasks_per_node: '{{ verification.VX_TASKS }}' - walltime: 01:00:00 - memory: '{{ 5 * verification.VX_TASKS }}G' pcpcombine: fcst: walltime: 01:00:00 diff --git a/tests/WE2E/test_configs/tc/config.HAFS-A.yaml b/tests/WE2E/test_configs/tc/config.HAFS-A.yaml index e04d777d..bbea99cf 100644 --- a/tests/WE2E/test_configs/tc/config.HAFS-A.yaml +++ b/tests/WE2E/test_configs/tc/config.HAFS-A.yaml @@ -22,18 +22,17 @@ tropical: verification: VX_FCST_MODEL_NAME: 'hfsa' VX_FCST_INPUT_BASEDIR: "{{ platform.STAGED_DATA }}/HAFS-A/2023090512/13L/" - FCST_SUBDIR_TEMPLATE: "" FCST_FN_TEMPLATE: "{cyclone}l.{init?fmt=%Y%m%d%H}.hfsa.storm.atm.f{lead?fmt=%HHH}.grb2" VX_FCST_OUTPUT_INTVL_HRS: 3 # LOG_LEVEL # Python logging level for METplus wrappers. Details here: # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#log-level # - # METPLUS_VERBOSITY_LEVEL: + # METPLUS_VERBOSITY_LEVEL: # Logging verbosity level used by METplus verification tools. 0 to 9, # with 0 having the fewest log messages and 9 having the most. Levels 5 # and above can result in very large log files and slower tool execution. - # + # METPLUS_VERBOSITY_LEVEL: 2 LOG_MET_VERBOSITY: 2 LOG_LEVEL: INFO diff --git a/tests/WE2E/utils.py b/tests/WE2E/utils.py index 16124fc9..625b776b 100755 --- a/tests/WE2E/utils.py +++ b/tests/WE2E/utils.py @@ -27,6 +27,7 @@ REPORT_WIDTH = 100 EXPT_COLUMN_WIDTH = 65 TASK_COLUMN_WIDTH = 40 +DEFAULT_DELAY = 1 def print_WE2E_summary(expts_dict: dict, debug: bool = False): """Creates a summary of the specified experiment @@ -105,7 +106,7 @@ def print_WE2E_summary(expts_dict: dict, debug: bool = False): for line in expt_details: f.write(f"{line}\n") -def create_expts_dict(expt_dir: str, delay: int = 5): +def create_expts_dict(expt_dir: str, delay: int = DEFAULT_DELAY): """ Takes in a directory, searches that directory for subdirectories containing experiments, and creates a skeleton dictionary that can be filled out by ``update_expt_status()`` @@ -228,7 +229,7 @@ def write_monitor_file(monitor_file: str, expts_dict: dict): raise -def update_expt_status(expt: dict, name: str, refresh: bool = False, delay: int = 5, +def update_expt_status(expt: dict, name: str, refresh: bool = False, delay: int = DEFAULT_DELAY, debug: bool = False, submit: bool = True) -> dict: """ This function reads the dictionary for a given experiment, runs the ``rocotorun`` command to update the experiment (by running new jobs and updating the status of previously submitted ones), and reads the Rocoto database (``.db``) file to update the status of each job in the experiment dictionary. The function then uses a simple set of rules to combine the statuses of every task into a useful summary status for the whole experiment and returns the updated experiment dictionary. diff --git a/tests/test_python/test_python_utils.py b/tests/test_python/test_python_utils.py index 724a34b2..561c4051 100644 --- a/tests/test_python/test_python_utils.py +++ b/tests/test_python/test_python_utils.py @@ -22,12 +22,6 @@ class Testing(unittest.TestCase): """ Define the tests""" - def test_case_handlers(self): - """ Test that the case handling string manipulators work as - expected. """ - self.assertEqual(util.uppercase("upper"), "UPPER") - self.assertEqual(util.lowercase("LOWER"), "lower") - def test_pattern_finding(self): """ Test that find_pattern_in_file can work with a string or a file path""" diff --git a/ush/bash_utils/change_case.sh b/ush/bash_utils/change_case.sh deleted file mode 100644 index 27d8af6c..00000000 --- a/ush/bash_utils/change_case.sh +++ /dev/null @@ -1,152 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines functions used to change string to all uppercase or -# all lowercase -# -#----------------------------------------------------------------------- -# - - -# -#----------------------------------------------------------------------- -# -# Function to echo the given string as an uppercase string -# -#----------------------------------------------------------------------- -# -function echo_uppercase() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -# Get input string - - local input - - if [ "$#" -eq 1 ]; then - - input="$1" - -# -#----------------------------------------------------------------------- -# -# If no arguments or more than one, print out a usage message and exit. -# -#----------------------------------------------------------------------- -# - else - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} string - -where: - - string: - This is the string that should be converted to uppercase and echoed. -" - - fi - -# Echo the input string as upperercase - -echo $input| tr '[a-z]' '[A-Z]' - -} - - -# -#----------------------------------------------------------------------- -# -# Function to echo the given string as a lowercase string -# -#----------------------------------------------------------------------- -# -function echo_lowercase() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -# Get input string - - local input - - if [ "$#" -eq 1 ]; then - - input="$1" - -# -#----------------------------------------------------------------------- -# -# If no arguments or more than one, print out a usage message and exit. -# -#----------------------------------------------------------------------- -# - else - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} string - -where: - - string: - This is the string that should be converted to lowercase and echoed. -" - - fi - -# Echo the input string as lowercase - -echo $input| tr '[A-Z]' '[a-z]' - - -} - diff --git a/ush/bash_utils/check_for_preexist_dir_file.sh b/ush/bash_utils/check_for_preexist_dir_file.sh deleted file mode 100644 index eaf4b4b1..00000000 --- a/ush/bash_utils/check_for_preexist_dir_file.sh +++ /dev/null @@ -1,161 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that checks for a preexisting version of -# the specified directory or file and, if present, deals with it according -# to the specified method. -# -#----------------------------------------------------------------------- -# -function check_for_preexist_dir_file() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 2 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} dir_or_file method - -where the arguments are defined as follows: - - dir_or_file: - Name of directory or file to check for a preexisting version. - - method: - String specifying the action to take if a preexisting version of - dir_or_file is found. Valid values are \"delete\", \"reuse\", \"rename\", and \"quit\". -" - - fi -# -#----------------------------------------------------------------------- -# -# Set local variables to appropriate input arguments. -# -#----------------------------------------------------------------------- -# - local dir_or_file="$1" - local method="$2" -# -#----------------------------------------------------------------------- -# -# Set the valid values that method can take on and check to make sure -# the specified value is valid. -# -#----------------------------------------------------------------------- -# - local valid_vals_method=( "delete" "reuse" "rename" "quit" ) - check_var_valid_value "method" "valid_vals_method" -# -#----------------------------------------------------------------------- -# -# Check if dir_or_file already exists. If so, act depending on the value -# of method. -# -#----------------------------------------------------------------------- -# - if [ -e "${dir_or_file}" ]; then - - case "$method" in -# -#----------------------------------------------------------------------- -# -# If method is set to "delete", we remove the preexisting directory or -# file. -# -#----------------------------------------------------------------------- -# - "delete") - - rm -rf "${dir_or_file}" - ;; -# -#----------------------------------------------------------------------- -# -# If method is set to "rename", we move (rename) the preexisting directory -# or file. -# -#----------------------------------------------------------------------- -# - "rename") - - local i=1 - local old_indx=$( printf "%03d" "$i" ) - local old_dir_or_file="${dir_or_file}_old${old_indx}" - while [ -e "${old_dir_or_file}" ]; do - i=$[$i+1] - old_indx=$( printf "%03d" "$i" ) - old_dir_or_file="${dir_or_file}_old${old_indx}" - done - - print_info_msg "$VERBOSE" " -Specified directory or file (dir_or_file) already exists: - dir_or_file = \"${dir_or_file}\" -Moving (renaming) preexisting directory or file to: - old_dir_or_file = \"${old_dir_or_file}\"" - - mv "${dir_or_file}" "${old_dir_or_file}" - ;; -# -#----------------------------------------------------------------------- -# -# If method is set to "reuse", keep preexisting directory intact. -# -#----------------------------------------------------------------------- -# - "reuse") - ;; -# -#----------------------------------------------------------------------- -# -# If method is set to "quit", we simply exit with a nonzero status. Note -# that "exit" is different than "return" because it will cause the calling -# script (in which this file/function is sourced) to stop execution. -# -#----------------------------------------------------------------------- -# - "quit") - - print_err_msg_exit "\ -Specified directory or file (dir_or_file) already exists: - dir_or_file = \"${dir_or_file}\"" - ;; - - esac - - fi - -} - - diff --git a/ush/bash_utils/check_var_valid_value.sh b/ush/bash_utils/check_var_valid_value.sh deleted file mode 100644 index c8abaec0..00000000 --- a/ush/bash_utils/check_var_valid_value.sh +++ /dev/null @@ -1,128 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This function checks whether the specified variable contains a valid -# value (where the set of valid values is also specified). -# -#----------------------------------------------------------------------- -# -function check_var_valid_value() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} var_name valid_var_values_array_name [msg] - -where the arguments are defined as follows: - - var_name: - The name of the variable whose value we want to check for validity. - - valid_var_values_array_name: - The name of the array containing a list of valid values that var_name - can take on. - - msg - Optional argument specifying the first portion of the error message to - print out if var_name does not have a valid value. -" - - fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local var_name \ - valid_var_values_array_name \ - var_value \ - valid_var_values_at \ - valid_var_values \ - err_msg \ - valid_var_values_str -# -#----------------------------------------------------------------------- -# -# Set local variable values. -# -#----------------------------------------------------------------------- -# - var_name="$1" - valid_var_values_array_name="$2" - - var_value=${!var_name} - valid_var_values_at="$valid_var_values_array_name[@]" - valid_var_values=("${!valid_var_values_at:-}") - - if [ "$#" -eq 3 ]; then - err_msg="$3" - else - err_msg="\ -The value specified in ${var_name} is not supported: - ${var_name} = \"${var_value}\"" - fi -# -#----------------------------------------------------------------------- -# -# If var_value contains a dollar sign, we assume the corresponding variable -# (var_name) is a template variable, i.e. one whose value contains a -# reference to another variable, e.g. -# -# MY_VAR='\${ANOTHER_VAR}' -# -# In this case, we do nothing since it does not make sense to check -# whether var_value is a valid value (since its contents have not yet -# been expanded). If var_value doesn't contain a dollar sign, it must -# contain a literal string. In this case, we check whether it is equal -# to one of the elements of the array valid_var_values. If not, we -# print out an error message and exit the calling script. -# -#----------------------------------------------------------------------- -# - if [[ "${var_value}" != *'$'* ]]; then - is_element_of "valid_var_values" "${var_value}" || { \ - valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}"); - print_err_msg_exit "\ -${err_msg} -${var_name} must be set to one of the following: - ${valid_var_values_str}"; \ - } - fi - -} - diff --git a/ush/bash_utils/create_symlink_to_file.sh b/ush/bash_utils/create_symlink_to_file.sh deleted file mode 100644 index 514daa7a..00000000 --- a/ush/bash_utils/create_symlink_to_file.sh +++ /dev/null @@ -1,79 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that is used to create a symbolic link -# ("symlink") to the specified target file ("target"). It checks for -# the existence of the target file and fails (with an appropriate error -# message) if that target does not exist or is not a file. Also, the -# argument "relative" determines whether a relative or an absolute path -# to the symlink is used. Note that on some platforms, relative symlinks -# are not supported. In those cases, an absolute path is used regardless -# of the setting of "relative". -# -#----------------------------------------------------------------------- -# -function create_symlink_to_file() { -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# -if [[ $# -lt 2 ]]; then - print_err_msg_exit "Function create_symlink_to_file() requires at least two arguments" -fi - -target=$1 -symlink=$2 -relative=${3:-True} -if [ "$relative" != "True" ] && [ "$relative" != "False" ]; then - print_err_msg_exit "'relative' must be set to True or False" -fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local valid_vals_relative \ - relative_flag -# -#----------------------------------------------------------------------- -# -# Make sure that the target file exists and is a file. -# -#----------------------------------------------------------------------- -# - if [ ! -f "${target}" ]; then - print_err_msg_exit "\ -Cannot create symlink to specified target file because the latter does -not exist or is not a file: - target = \"$target\"" - fi -# -#----------------------------------------------------------------------- -# -# Set the flag that specifies whether or not a relative symlink should -# be created. -# -#----------------------------------------------------------------------- -# - relative_flag="" - if [ "${relative}" = "TRUE" ]; then - relative_flag="${RELATIVE_LINK_FLAG}" - fi -# -#----------------------------------------------------------------------- -# -# Create the symlink. -# -#----------------------------------------------------------------------- -# -ln -sf ${relative_flag} "$target" "$symlink" - -} - diff --git a/ush/bash_utils/get_elem_inds.sh b/ush/bash_utils/get_elem_inds.sh deleted file mode 100644 index 52c10e0f..00000000 --- a/ush/bash_utils/get_elem_inds.sh +++ /dev/null @@ -1,162 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# For a description of this function, see the usage message below. -# -#----------------------------------------------------------------------- -# -function get_elem_inds() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 2 ] && [ "$#" -ne 3 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} array_name str_to_match [inds_to_return] - -This function prints to stdout the indices of those elements of a given -array that match (i.e. are equal to) a given string. It can return the -index of the first matched element, the index of the last matched ele- -ment, or the indices of all matched elements. The return code -from this function will be zero if at least one match is found and non- -zero if no matches are found. - -The arguments to this function are defined as follows: - - array_name: - The name of the array in which to search for str_to_match. Note that - this is the name of the array, not the array itself. - - str_to_match: - The string to match in array_name. - - inds_to_return: - Optional argument that specifies the subset of the indices of the ar- - ray elements that match str_to_match to print to stdout. Must be set - to \"first\", \"last\", or \"all\" (but is case insensitive). If set to - \"first\", the index of only the first matched element is printed. If - set to \"last\", the index of only the last matched element is printed. - If set to \"all\", the indices of all matched elements are printed. De- - fault is \"all\". -" - - fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local array_name \ - str_to_match \ - inds_to_return \ - array_name_at \ - array \ - valid_vals_inds_to_return \ - match_inds \ - num_matches \ - num_elems \ - n -# -#----------------------------------------------------------------------- -# -# Set local variables to appropriate input arguments. -# -#----------------------------------------------------------------------- -# - array_name="$1" - str_to_match="$2" - - inds_to_return="all" - if [ "$#" -eq 3 ]; then - inds_to_return="$3" - fi - - array_name_at="$array_name[@]" - array=("${!array_name_at}") -# -#----------------------------------------------------------------------- -# -# Change all letters in inds_to_return to lower case. Then check whe- -# ther it has a valid value. -# -#----------------------------------------------------------------------- -# - inds_to_return=$(echo_lowercase $inds_to_return) - valid_vals_inds_to_return=( "first" "last" "all" ) - check_var_valid_value "inds_to_return" "valid_vals_inds_to_return" -# -#----------------------------------------------------------------------- -# -# Initialize the array match_inds to an empty array. This will contain -# the indices of any matched elements. Then loop through the elements -# of the given array and check whether each element is equal to str_to_- -# match. If so, save the index of that element as an element of match_- -# inds. If inds_to_return is set to "first", we break out of the loop -# after finding the first match in order to not waste computation. -# -#----------------------------------------------------------------------- -# - match_inds=() - num_matches=0 - - num_elems=${#array[@]} - for (( n=0; n<${num_elems}; n++ )); do - if [ "${array[$n]}" = "${str_to_match}" ]; then - match_inds[${num_matches}]=$n - num_matches=$((num_matches+1)) - if [ "${inds_to_return}" = "first" ]; then - break - fi - fi - done -# -#----------------------------------------------------------------------- -# -# Find the number of matches. If it is more than zero, print the indi- -# ces of the matched elements to stdout. -# -#----------------------------------------------------------------------- -# - num_matches=${#match_inds[@]} - if [ ${num_matches} -gt 0 ]; then - if [ "${inds_to_return}" = "last" ]; then - printf "%s\n" "${match_inds[-1]}" - else - printf "%s\n" "${match_inds[@]}" - fi - fi - -} diff --git a/ush/bash_utils/is_array.sh b/ush/bash_utils/is_array.sh deleted file mode 100644 index cbc0a285..00000000 --- a/ush/bash_utils/is_array.sh +++ /dev/null @@ -1,82 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that is used to check whether a specified -# variable is a bash array. It is called as follows: -# -# is_array var_name -# -# Here, var_name is the name of the variable to check to determine whe- -# ther or not it is an array. If the variable is an array, this func- -# tion will return a 0, and if it is not, this function will return a 1. -# -#----------------------------------------------------------------------- -# -function is_array() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 1 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} var_name - -where var_name is the name of the variable to check to determine whether -or not it is an array. -" - - fi -# -#----------------------------------------------------------------------- -# -# Set local variables to appropriate input arguments. -# -#----------------------------------------------------------------------- -# - local var_name="$1" - local declare_output=$( declare -p "$var_name" 2> /dev/null ) - local regex="^declare -[aA] ${var_name}(=|$)" - printf "%s" "$declare_output" | grep --extended-regexp "$regex" >/dev/null - is_an_array="$?" -# -#----------------------------------------------------------------------- -# -# Return the variable "is_an_array". -# -#----------------------------------------------------------------------- -# - return ${is_an_array} - -} - diff --git a/ush/bash_utils/is_element_of.sh b/ush/bash_utils/is_element_of.sh deleted file mode 100644 index e2d1403e..00000000 --- a/ush/bash_utils/is_element_of.sh +++ /dev/null @@ -1,135 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# For a description of this function, see the usage message below. -# -#----------------------------------------------------------------------- -# -function is_element_of() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 2 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} array_name str_to_match - -This function checks whether the specified array contains the specified -string, i.e. whether at least one of the elements of the array is equal -to the string. The return code from this function will be zero if at -least one match is found and nonzero if no matches are found. - -The arguments to this function are defined as follows: - - array_name: - The name of the array in which to search for str_to_match. Note that - this is the name of the array, not the array itself. - - str_to_match: - The string to search for in array_name. - -Use this function in a script as follows: - - . ./is_element_of.sh - array_name=("1" "2" "3 4" "5") - - str_to_match="2" - is_element_of "${str_to_match}" array_name - echo $? # Should output 0. - - str_to_match="3 4" - is_element_of "${str_to_match}" array_name - echo $? # Should output 0. - - str_to_match="6" - is_element_of "${str_to_match}" array_name - echo $? # Should output 1. -" - - fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local array_name \ - str_to_match \ - array_name_at \ - array \ - found_match \ - num_elems \ - n -# -#----------------------------------------------------------------------- -# -# Set local variables to appropriate input arguments. -# -#----------------------------------------------------------------------- -# - array_name="$1" - str_to_match="$2" - - array_name_at="$array_name[@]" - array=("${!array_name_at:-}") -# -#----------------------------------------------------------------------- -# -# Initialize the return variable found_match to 1 (false). Then loop -# through the elements of the array and check whether each element is -# equal to str_to_match. Once a match is found, reset found_match to 0 -# (true) and break out of the loop. -# -#----------------------------------------------------------------------- -# - found_match=1 - num_elems=${#array[@]} - for (( n=0; n<${num_elems}; n++ )); do - if [ "${array[$n]}" = "${str_to_match}" ]; then - found_match=0 - break - fi - done -# -#----------------------------------------------------------------------- -# -# Return the variable found_match. -# -#----------------------------------------------------------------------- -# - return ${found_match} - -} - diff --git a/ush/bash_utils/print_input_args.sh b/ush/bash_utils/print_input_args.sh deleted file mode 100644 index 9ea19f14..00000000 --- a/ush/bash_utils/print_input_args.sh +++ /dev/null @@ -1,174 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that prints to stdout the names and val- -# ues of a specified list of variables that are the valid arguments to -# the script or function that calls this function. It is mainly used -# for debugging to check that the argument values passed to the calling -# script/function have been set correctly. Note that if a global varia- -# ble named VERBOSE is not defined, the message will be printed out. If -# a global variable named VERBOSE is defined, then the message will be -# printed out only if VERBOSE is set to TRUE. -# -#----------------------------------------------------------------------- -# -function print_input_args() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Get information about the script or function that calls this function. -# Note that caller_name will be set as follows: -# -# 1) If the caller is a function, caller_name will be set to the name of -# that function. -# 2) If the caller is a sourced script, caller_name will be set to -# "script". Note that a sourced script cannot be the top level -# script since by defintion, it is sourced by another script or func- -# tion. -# 3) If the caller is the top-level script, caller_name will be set to -# "main". -# -# Thus, if caller_name is set to "script" or "main", the caller is a -# script, and if it is set to anything else, the caller is a function. -# -#----------------------------------------------------------------------- -# - local caller_fp=$( $READLINK -f "${BASH_SOURCE[1]}" ) - local caller_fn=$( basename "${caller_fp}" ) - local caller_dir=$( dirname "${caller_fp}" ) - local caller_name="${FUNCNAME[1]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -ne 1 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} array_name_valid_caller_args - -where array_name_valid_caller_args is the name of the array containing -the names of valid arguments that can be passed to the calling script or -function. -" - - fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local array_name_valid_caller_args \ - valid_caller_args \ - script_or_function \ - msg \ - num_valid_args \ - i \ - line -# -#----------------------------------------------------------------------- -# -# Get the array containing the list of valid argument names that can be -# passed to the calling script/function. Note that if this is set to an -# empty array in the calling script/function [e.g. using the notation -# some_array=()], then it will (for whatever reason) not be defined in -# the scope of this function. For this reason, we need the if-statement -# below to check for this case. Then get the number of valid arguments. -# -#----------------------------------------------------------------------- -# - array_name_valid_caller_args="$1" - valid_arg_names_0th="${array_name_valid_caller_args}[0]" - if [ ${!valid_arg_names_0th:-"__unset__"} = "__unset__" ]; then - valid_caller_args=() - else - valid_caller_args_at="${array_name_valid_caller_args}[@]" - valid_caller_args=("${!valid_caller_args_at}") - fi - num_valid_caller_args="${#valid_caller_args[@]}" -# -#----------------------------------------------------------------------- -# -# Set the message to print to stdout. Note that if the number of valid -# arguments is zero, then we simply print out a message stating this fact. -# -#----------------------------------------------------------------------- -# - if [ "${caller_name}" = "main" ] || \ - [ "${caller_name}" = "script" ]; then - script_or_function="the script" - else - script_or_function="function \"${caller_name}\"" - fi - - if [ ${num_valid_caller_args} -eq 0 ]; then - - msg=" -No arguments have been passed to ${script_or_function} in file - - \"${caller_fp}\" -" - - else - - msg=" -The arguments to ${script_or_function} in file - - \"${caller_fp}\" - -have been set as follows: -" - - for (( i=0; i<${num_valid_caller_args}; i++ )); do - line=$( declare -p "${valid_caller_args[$i]}" ) - msg=$( printf "%s\n%s" "$msg" " $line" ) - done - - fi -# -#----------------------------------------------------------------------- -# -# If a global variable named DEBUG is not defined, print out the message. -# If it is defined, print out the message only if DEBUG is set to "TRUE". -# -#----------------------------------------------------------------------- -# - if [ -z ${DEBUG+x} ]; then - print_info_msg "$msg" - else - print_info_msg "$DEBUG" "$msg" - fi - -} - diff --git a/ush/bash_utils/print_msg.sh b/ush/bash_utils/print_msg.sh index 3c54deb7..c9d6afff 100644 --- a/ush/bash_utils/print_msg.sh +++ b/ush/bash_utils/print_msg.sh @@ -7,111 +7,6 @@ #----------------------------------------------------------------------- # -# -#----------------------------------------------------------------------- -# -# Function to print informational messages using printf. -# -#----------------------------------------------------------------------- -# -function print_info_msg() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local verbose \ - info_msg -# -#----------------------------------------------------------------------- -# -# If one argument is supplied, we assume it is the message to print out. -# between informational lines that are always printed. -# -#----------------------------------------------------------------------- -# - if [ "$#" -eq 1 ]; then - - verbose="True" - info_msg="$1" - - elif [ "$#" -eq 2 ]; then - - verbose="$1" - info_msg="$2" -# -#----------------------------------------------------------------------- -# -# If no arguments or more than two arguments are supplied, print out a -# usage message and exit. -# -#----------------------------------------------------------------------- -# - else - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} [verbose] info_msg - -where the arguments are defined as follows: - - verbose: - This is an optional argument. If set to \"TRUE\", info_msg will be - printed to stdout. Otherwise, info_msg will not be printed. - - info_msg: - This is the informational message to print to stdout. - -This function prints an informational message to stdout. If one argu- -ment is passed in, then that argument is assumed to be info_msg and is -printed. If two arguments are passed in, then the first is assumed to -be verbose and the second info_msg. In this case, info_msg gets printed -only if verbose is set to \"TRUE\". -" - - fi -# -#----------------------------------------------------------------------- -# -# If verbose is set to "TRUE", print out the message. -# -#----------------------------------------------------------------------- -# - if [ "$verbose" = "TRUE" ]; then - printf "%s\n" "${info_msg}" - fi -} - - - # #----------------------------------------------------------------------- diff --git a/ush/bash_utils/process_args.sh b/ush/bash_utils/process_args.sh deleted file mode 100644 index c6cb44ff..00000000 --- a/ush/bash_utils/process_args.sh +++ /dev/null @@ -1,380 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This function processes a list of variable name and value pairs passed -# to it as a set of arguments, starting with the second argument. We -# refer to these pairs as argument-value pairs (or "arg-val" pairs for -# short) because the variable names in these pairs represent the names -# of arguments to the script or function that calls this function (which -# we refer to here as the "caller"). The first argument to this func- -# tion being the name of an array that contains a list of valid argument -# names that the caller is allowed to accept. Each arg-val pair must -# have the form -# -# ARG_NAME=VAR_VALUE -# -# where ARG_NAME is the name of an argument and VAR_VALUE is the value -# to set that argument to. For each arg-val pair, this function creates -# a global variable named ARG_NAME and assigns to it the value VAR_VAL- -# UE. -# -# The purpose of this function is to provide a mechanism by which a pa- -# rent script, say parent.sh, can pass variable values to a child script -# or function, say child.sh, that makes it very clear which arguments of -# child.sh are being set and to what values. For example, parent.sh can -# call child.sh as follows: -# -# ... -# child.sh arg3="Hello" arg2="bye" arg4=("this" "is" "an" "array") -# ... -# -# Then child.sh can use this function (process_args) as follows to pro- -# cess the arg-val pairs passed to it: -# -# ... -# valid_args=( "arg1" "arg2" "arg3" "arg4" ) -# process_args valid_args "$@" -# ... -# -# Here, valid_args is an array that defines or "declares" the argument -# list for child.sh, i.e. it defines the variable names that child.sh is -# allowed to accept as arguments. Its name is passed to process_args as -# the first argument. The "$@" appearing in the call to process_args -# passes to process_args the list of arg-val pairs that parent.sh passes -# to child.sh as the second through N-th arguments. In the example -# above, "$@" represents: -# -# arg3="Hello" arg2="bye" arg4=("this" "is" "an" "array") -# -# After the call to process_args in child.sh, the variables arg1, arg2, -# arg3, and arg4 will be set as follows in child.sh: -# -# arg1="" -# arg2="bye" -# arg3="Hello" -# arg4=("this" "is" "an" "array") -# -# Note that: -# -# 1) The set of arg-val pairs may list only a subset of the list of arg- -# uments declared in valid_args; the unlisted arguments will be set -# to the null string. In the example above, arg1 is set to the null -# string because it is not specified in any of the arg-val pairs in -# the call to child.sh in parent.sh. -# -# 2) The arg-val pairs in the call to child.sh do not have to be in the -# same order as the list of "declared" arguments in child.sh. For -# instance, in the example above, the arg-val pair for arg3 is listed -# before the one for arg2. -# -# 3) An argument can be set to an array by starting and ending the value -# portion of its arg-val pair with opening and closing parentheses, -# repsectively, and listing the elements within (each one in a set of -# double-quotes and separated fromt the next by whitespace). In the -# example above, this is done for arg4. -# -# 4) If the value portion of an arg-val pair contains an argument that -# is not defined in the array valid_args in child.sh, the call to -# process_args in child.sh will result in an error message and exit -# from the caller. -# -#----------------------------------------------------------------------- -# -function process_args() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# - if [ "$#" -lt 1 ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} array_name_valid_arg_names \ - arg_val_pair1 \ - ... \ - arg_val_pairN - -where the arguments are defined as follows: - - array_name_valid_arg_names: - The name of the array containing a list of valid argument names. - - arg_val_pair1 ... arg_val_pairN: - A list of N argument-value pairs. These have the form - - arg1=\"val1\" ... argN=\"valN\" - - where each argument name (argI) needs to be in the list of valid argu- - ment names specified in array_name_valid_arg_names. Note that not all - the valid arguments listed in array_name_valid_arg_names need to be - set, and the argument-value pairs can be in any order, i.e. they don't - have to follow the order of arguments listed in valid_arg_names_ar- - ray_name. -" - - fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local array_name_valid_arg_names \ - valid_arg_names_at \ - valid_arg_names \ - num_valid_args \ - num_arg_val_pairs \ - i valid_arg_name arg_already_specified \ - arg_val_pair arg_name arg_value is_array \ - err_msg cmd_line -# -#----------------------------------------------------------------------- -# -# Get the array containing the list of valid argument names that can be -# passed to the calling script/function. Note that if this is set to an -# empty array in the calling script/function [e.g. using the notation -# some_array=()], then it will (for whatever reason) not be defined in -# the scope of this function. For this reason, we need the if-statement -# below to check for this case. -# -#----------------------------------------------------------------------- -# - array_name_valid_arg_names="$1" - valid_arg_names_0th="${array_name_valid_arg_names}[0]" - if [ ${!valid_arg_names_0th:-"__unset__"} = "__unset__" ]; then - valid_arg_names=() - else - valid_arg_names_at="${array_name_valid_arg_names}[@]" - valid_arg_names=("${!valid_arg_names_at}") - fi -# -#----------------------------------------------------------------------- -# -# Get the number of valid arguments. Also, set a string containing the -# list of all valid arguments with each one placed in double quotes. -# -#----------------------------------------------------------------------- -# - num_valid_args=${#valid_arg_names[@]} - if [ ${num_valid_args} -eq 0 ]; then - valid_arg_names_str="" - else - valid_arg_names_str=$( printf "\"%s\" " "${valid_arg_names[@]}" ) - fi - -# -# Instead of the if-statement above, the following could be used, but it -# is too difficult to understand... -# -# valid_arg_names_str=$( printf "\"%s\" " ${valid_arg_names[@]+"${valid_arg_names[@]}"} ) - -# -#----------------------------------------------------------------------- -# -# Get the number of argument-value pairs (or arg-val pairs, for short) -# being passed into this function. These consist of all arguments -# starting with the 2nd, so we subtract 1 from the total number of argu- -# ments. -# -#----------------------------------------------------------------------- -# - num_arg_val_pairs=$(( $# - 1 )) -# -#----------------------------------------------------------------------- -# -# Make sure that the number of arg-val pairs is less than or equal to -# the number of valid arguments. -# -#----------------------------------------------------------------------- -# - if [ "${num_arg_val_pairs}" -gt "${num_valid_args}" ]; then - print_err_msg_exit "\ -The number of argument-value pairs specified on the command line (num_- -arg_val_pairs) must be less than or equal to the number of valid argu- -ments (num_valid_args) specified in the array valid_arg_names: - num_arg_val_pairs = ${num_arg_val_pairs} - num_valid_args = ${num_valid_args} - valid_arg_names = ( ${valid_arg_names_str})" - fi -# -#----------------------------------------------------------------------- -# -# If the number of valid arguments is zero, i.e. the array valid_arg_names -# contains no elements, then there are no script/function arguments to -# set. In this case, reset the shell options to what they were before -# entering this function and simply return to the calling script/function. -# -#----------------------------------------------------------------------- -# - if [ ${num_valid_args} -eq 0 ]; then - return - fi -# -#----------------------------------------------------------------------- -# -# Make sure that none of the elements of the array containing the list -# of valid arguments contain spaces or are empty. -# -#----------------------------------------------------------------------- -# - for (( i=0; i<${num_valid_args}; i++ )); do - - valid_arg_name="${valid_arg_names[$i]}" - -# Remove spaces (if any exist) from the current valid argument name. - valid_arg_name_no_spaces=$( \ - printf "%s\n" "${valid_arg_name}" | $SED -r -e 's/[[:space:]]//g' ) - - if [ "${valid_arg_name_no_spaces}" != "${valid_arg_name}" ]; then - print_err_msg_exit "\ -The name of an argument in the list of valid arguments (valid_arg_names) -cannot contain any spaces, but the element with index i=${i} contains at -least one space: - valid_arg_names = ( ${valid_arg_names_str}) - valid_arg_names[$i] = \"${valid_arg_names[$i]}\"" - fi - - if [ -z ${valid_arg_name} ]; then - print_err_msg_exit "\ -The list of valid arguments (valid_arg_names) cannot contain empty elements, -but the element with index i=${i} is empty: - valid_arg_names = ( ${valid_arg_names_str}) - valid_arg_names[$i] = \"${valid_arg_names[$i]}\"" - fi - - done -# -#----------------------------------------------------------------------- -# -# Initialize all valid arguments to the null string. Note that the -# scope of this initialization is global, i.e. the calling script or -# function will be aware of these initializations. Also, initialize -# each element of the array arg_already_specified to "false". This ar- -# ray keeps track of whether each valid argument has already been set -# to a value by an arg-val specification. -# -#----------------------------------------------------------------------- -# - for (( i=0; i<${num_valid_args}; i++ )); do - valid_arg_name="${valid_arg_names[$i]}" - eval ${valid_arg_name}="" - arg_already_specified[$i]="false" - done -# -#----------------------------------------------------------------------- -# -# Loop over the list of arg-val pairs and set argument values. -# -#----------------------------------------------------------------------- -# -# Set the separator used in each arg=value pair. This is simply an equal -# sign. -# - sep="=" - for arg_val_pair in "${@:2}"; do -# -# Get the argument name and its value using bash variable substitution/ -# expansion. The %% operator deletes the longest trailing portion of -# arg_val_pair that matches the pattern that follows %%, while the # -# operator deletes the shortest leading portion of arg_val_pair that -# matches the pattern that follows #. -# - arg_name=${arg_val_pair%%"$sep"*} - arg_value=${arg_val_pair#*"$sep"} -# -# If the first character of the argument's value is an opening parenthe- -# sis and its last character is a closing parenthesis, then the argument -# is an array. Check for this and set the is_array flag accordingly. -# - is_array="false" - if [ "${arg_value:0:1}" = "(" ] && \ - [ "${arg_value: -1}" = ")" ]; then - is_array="true" - fi -# -#----------------------------------------------------------------------- -# -# Make sure that the argument name specified by the current argument- -# value pair is valid. -# -#----------------------------------------------------------------------- -# - err_msg="\ -The specified argument name (arg_name) in the current argument-value -pair (arg_val_pair) is not valid: - arg_val_pair = \"${arg_val_pair}\" - arg_name = \"${arg_name}\"" - check_var_valid_value "arg_name" "valid_arg_names" "${err_msg}" -# -#----------------------------------------------------------------------- -# -# Loop through the list of valid argument names and find the one that -# the current arg-val pair corresponds to. Then set that argument to -# the specified value. -# -#----------------------------------------------------------------------- -# - for (( i=0; i<${num_valid_args}; i++ )); do - - valid_arg_name="${valid_arg_names[$i]}" - if [ "${arg_name}" = "${valid_arg_name}" ]; then -# -# Check whether the current argument has already been set by a previous -# arg-val pair on the command line. If not, proceed to set the argument -# to the specified value. If so, print out an error message and exit -# the calling script. -# - if [ "${arg_already_specified[$i]}" = "false" ]; then - arg_already_specified[$i]="true" - if [ "${is_array}" = "true" ]; then - eval ${arg_name}=${arg_value} - else - eval ${arg_name}=\"${arg_value}\" - fi - else - cmd_line=$( printf "\'%s\' " "${@:1}" ) - print_err_msg_exit "\ -The current argument has already been assigned a value on the command -line: - arg_name = \"${arg_name}\" - cmd_line = ${cmd_line} -Please assign values to arguments only once on the command line." - fi - fi - - done - - done - -} - diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 28f80b90..48715982 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -143,33 +143,15 @@ platform: REMOVE_MEMORY: False # #----------------------------------------------------------------------- - # - # Set run commands for platforms without a workflow manager. These values - # will be ignored unless WORKFLOW_MANAGER: "none". Definitions: - # - # RUN_CMD_PRDGEN: - # The run command for the product generation job. - # - # RUN_CMD_SERIAL: - # The run command for some serial jobs - # - #----------------------------------------------------------------------- - # - RUN_CMD_SERIAL: "" - RUN_CMD_PRDGEN: "" - # - #----------------------------------------------------------------------- - # + # SCHED_NATIVE_CMD: # Allows an extra parameter to be passed to SCHEDULER (SLURM/PBSPRO) via # XML Native command + # + # SCHED_NATIVE_CMD_HPSS: + # Same command, but for jobs submitted to "PARTITION_HPSS" # SCHED_NATIVE_CMD: "" - # - #----------------------------------------------------------------------- - # Pre task commands such as "ulimit" needed by tasks - #----------------------------------------------------------------------- - # - PRE_TASK_CMDS: "" + SCHED_NATIVE_CMD_HPSS: "" # #----------------------------------------------------------------------- # MET and METplus directories. MET_INSTALL_DIR is the directory containing @@ -251,7 +233,8 @@ workflow: #----------------------------------------------------------------------- # taskgroups: - - parm/wflow/verify_pre.yaml + - parm/wflow/get_obs.yaml + - parm/wflow/preprocessing.yaml - parm/wflow/verify_det.yaml # #----------------------------------------------------------------------- @@ -486,7 +469,7 @@ workflow: # #----------------------------------------------------------------------- # - VERBOSE: true + VERBOSE: false DEBUG: false @@ -511,10 +494,6 @@ ensemble: # For example, if this is set to 8, the member directories will be named # mem1, mem2, ..., mem8. Not used if DO_ENSEMBLE is set to false. # - # ENSMEM_NAMES: - # A list of names for the ensemble member names following the format - # mem001, mem002, etc. - # # ENS_TIME_LAG_HRS: # Time lag (in hours) to use for each ensemble member. For a deterministic # forecast, this is a one-element array. Default values of array elements @@ -523,9 +502,8 @@ ensemble: #----------------------------------------------------------------------- # DO_ENSEMBLE: false - NUM_ENS_MEMBERS: 0 - ENSMEM_NAMES: '{% for m in range(ensemble.NUM_ENS_MEMBERS) %}{{ "mem%03d, " % m }}{% endfor %}' - ENS_TIME_LAG_HRS: '[ {% for m in range([1,ensemble.NUM_ENS_MEMBERS]|max) %} 0, {% endfor %} ]' + NUM_ENS_MEMBERS: 1 + ENS_TIME_LAG_HRS: !list '[ {% for m in range([1,ensemble.NUM_ENS_MEMBERS]|max) %} 0, {% endfor %} ]' tropical: #----------------------------- @@ -574,27 +552,6 @@ verification_resources: execution: nodes: 1 tasks_per_node: 1 - deterministic: - gridstat: - tasks_per_node: '{{ verification.VX_TASKS }}' - walltime: 02:00:00 - memory: '{{ 2 * verification.VX_TASKS }}G' - pointstat: - tasks_per_node: '{{ verification.VX_TASKS }}' - walltime: 01:00:00 - memory: '{{ 2 * verification.VX_TASKS }}G' - ensemble: - genensprod: - default: - walltime: 04:45:00 - ndas: - walltime: 02:30:00 - ensemblestat: - walltime: 01:00:00 - gridstat: - walltime: 01:00:00 - pointstat: - walltime: 01:00:00 get_obs: aeronet: walltime: 02:00:00 @@ -608,8 +565,6 @@ verification_resources: walltime: 02:00:00 ndas: walltime: 02:00:00 - pb2nc: - walltime: 00:30:00 ascii2nc: walltime: 00:30:00 pcpcombine: @@ -961,15 +916,15 @@ verification: # VX_FCST_INPUT_BASEDIR: '{{ workflow.EXPTDIR }}' # - # FCST_SUBDIR_TEMPLATE: - # METplus template for the name of the subdirectory containing forecast - # files to use as inputs to the verification tasks. - # - FCST_SUBDIR_TEMPLATE: '{init?fmt=%Y%m%d%H?shift=-${time_lag}}{{ "/${ensmem_name}" if ensemble.DO_ENSEMBLE }}/postprd' - # # FCST_FN_TEMPLATE: # METplus template for the names of the forecast files to use as inputs - # to the verification tasks. + # to the verification tasks. This can be a string for deterministic runs + # or ensembles with a uniform naming template; otherwise, this should be + # a list of strings with length equal to the number of ensemble members + # + # If your directory structure includes METplus wildcards that must be + # substituted at runtime (e.g. {init?fmt=%H?shift=-${time_lag}}, those + # subdirectories should be included here rather than under VX_FCST_INPUT_BASEDIR # FCST_FN_TEMPLATE: 'srw.t{init?fmt=%H?shift=-${time_lag}}z.prslev.f{lead?fmt=%HHH?shift=${time_lag}}.{{ verification.VX_FCST_MODEL_NAME }}.grib2' # @@ -1007,11 +962,66 @@ verification: # sequential forecast hours, and so can be run simultaneously. VX_TASKS: 1 +pb2nc: + # Runtime settings for PB2NC jobs + execution: + # Number of PB2NC tasks; should typically be equal to the number of instances to run in parallel. Currently no support for multi-node jobs + tasks_per_node: '{{ gridstat.TASKS }}' + # Wallclock time to allocate to submitted PB2NC jobs + walltime: 00:30:00 + # Memory to allocate to PB2NC jobs + memory: '{{ 2 * gridstat.TASKS }}G' + # Number of times to submit a given task; in the case where tries > 1, the task will be retried on failure + tries: 1 + # Number of PB2NC instances to run in parallel + TASKS: !int '{{ verification.VX_TASKS }}' + +gridstat: + # Runtime settings for deterministic GRIDSTAT jobs + execution: + tasks_per_node: '{{ gridstat.TASKS }}' + walltime: 01:00:00 + memory: '{{ 2 * gridstat.TASKS }}G' + tries: 1 + # Number of GRIDSTAT instances to run in parallel + TASKS: !int '{{ verification.VX_TASKS }}' + +pointstat: + # Runtime settings for deterministic POINTSTAT jobs + execution: + tasks_per_node: '{{ pointstat.TASKS }}' + walltime: 01:00:00 + memory: '{{ 2 * pointstat.TASKS }}G' + tries: 1 + # Number of POINTSTAT instances to run in parallel + TASKS: !int '{{ verification.VX_TASKS }}' + +gridstat_ens: + # Runtime settings for ensemble GRIDSTAT jobs (ensmean/ensprob) + execution: + tasks_per_node: '{{ gridstat_ens.TASKS }}' + walltime: 01:00:00 + memory: '{{ 2 * gridstat_ens.TASKS }}G' + tries: 1 + # Number of ensemble GRIDSTAT instances to run in parallel + TASKS: !int '{{ verification.VX_TASKS }}' + +pointstat_ens: + # Runtime settings for ensemble POINTSTAT jobs (ensmean/ensprob) + execution: + tasks_per_node: '{{ pointstat_ens.TASKS }}' + walltime: 01:00:00 + memory: '{{ 2 * pointstat_ens.TASKS }}G' + tries: 1 + # Number of ensemble POINTSTAT instances to run in parallel + TASKS: !int '{{ verification.VX_TASKS }}' + point2grid: execution: tasks_per_node: '{{ point2grid.TASKS }}' walltime: 01:00:00 memory: '{{ 2 * point2grid.TASKS }}G' + tries: 1 REGRID_METHOD: BILIN # Comma-separated string indicating which DQF (quality control) flags to keep for GOES observations. Described in GOES_ABI_products_PUG-L2+-vol5.pdf # 0: high quality @@ -1020,16 +1030,44 @@ point2grid: # 3: no retrieval quality flag GOES_QC_FLAGS: '0,1' # Number of POINT2GRID instances to run in parallel - TASKS: '{{ verification.VX_TASKS }}' + TASKS: !int '{{ verification.VX_TASKS }}' + +genensprod: + # Runtime settings for GenEnsProd jobs + execution: + tasks_per_node: '{{ genensprod.TASKS }}' + walltime: 02:00:00 + memory: '{{ 2 * genensprod.TASKS }}G' + threads: 4 + tries: 1 + # Number of GenEnsProd instances to run in parallel + TASKS: !int '{{ verification.VX_TASKS }}' + + # Config file settings for the GenEnsProd task. These settings will be added directly to the configuration file + conf: + #Verbosity of MET logging output. + LOG_GEN_ENS_PROD_VERBOSITY: '{{ verification.METPLUS_VERBOSITY_LEVEL}}' + +ensemblestat: + # Runtime settings for deterministic EnsembleStat jobs + execution: + tasks_per_node: '{{ ensemblestat.TASKS }}' + walltime: 01:00:00 + memory: '{{ 2 * ensemblestat.TASKS }}G' + threads: 4 + tries: 1 + # Number of EnsembleStat instances to run in parallel + TASKS: !int '{{ verification.VX_TASKS }}' regriddataplane: # Runtime settings for REGRIDDATAPLANE jobs execution: tasks_per_node: '{{ regriddataplane.TASKS }}' walltime: 00:30:00 - memory: '{{ 10 * regriddataplane.TASKS }}G' + memory: '{{ 10 * (regriddataplane.TASKS | int) }}G' + tries: 1 # Number of REGRIDDATAPLANE instances to run in parallel - TASKS: 1 + TASKS: !int '{{ verification.VX_TASKS }}' # Regridding method. See https://metplus.readthedocs.io/projects/met/en/latest/Users_Guide/reformat_grid.html#optional-arguments-for-regrid-data-plane for valid options REGRID_METHOD: MAXGAUSS @@ -1041,14 +1079,18 @@ regriddataplane: mode: # Runtime settings for MODE jobs execution: - # Number of MODE instances to run in parallel. Currently no support for multi-node jobs + # Number of MODE tasks; should typically be equal to the number of instances to run in parallel. Currently no support for multi-node jobs tasks_per_node: '{{ mode.TASKS }}' # Wallclock time to allocate to submitted MODE job walltime: 01:00:00 # Memory to allocate to MODE job memory: '{{ 10 * mode.TASKS }}G' + # Number of threads to assign to each task + threads: 4 + # Number of times to submit a given task; in the case where tries > 1, the task will be retried on failure + tries: 1 # Number of MODE instances to run in parallel - TASKS: 1 + TASKS: !int '{{ verification.VX_TASKS }}' # For more information on MODE variables see the MODE documentation: # https://metplus.readthedocs.io/projects/met/en/latest/Users_Guide/mode.html @@ -1096,6 +1138,8 @@ tcpairs: walltime: 00:10:00 # Memory to allocate to TCPAIRS job memory: '2G' + # Number of times to submit a given task; in the case where tries > 1, the task will be retried on failure + tries: 1 # # TECH_ID_VX: # The identifier for the best track data to use for TCPAIRS comparison, as read in from the @@ -1118,6 +1162,8 @@ tcstat: walltime: 00:10:00 # Memory to allocate to TCSTAT job memory: '2G' + # Number of times to submit a given task; in the case where tries > 1, the task will be retried on failure + tries: 1 # SUMMARY_FILE: # Filename for TCstat summary output @@ -1136,6 +1182,8 @@ tcrmw: walltime: 00:30:00 # Memory to allocate to TCRMW job memory: '2G' + # Number of times to submit a given task; in the case where tries > 1, the task will be retried on failure + tries: 1 # OUTPUT_TEMPLATE: # Template for output file(s) from TCRMW. {cyclone} and {date} are substituted in the diff --git a/ush/configs/SFE_2026_ensembles/config.firsttest.yaml b/ush/configs/SFE_2026_ensembles/config.firsttest.yaml new file mode 100644 index 00000000..8e8160ca --- /dev/null +++ b/ush/configs/SFE_2026_ensembles/config.firsttest.yaml @@ -0,0 +1,90 @@ +metadata: + description: |- + First attempt at verifying REFS data, now with EAS calcs and + a few more task-specific parallelism settings +workflow: + DATE_FIRST_CYCL: '2026042712' + DATE_LAST_CYCL: '2026042712' + FCST_LEN_HRS: 42 + PREEXISTING_DIR_METHOD: rename + taskgroups: + - parm/wflow/verify_pre.yaml + - parm/wflow/verify_det.yaml + - parm/wflow/verify_ens.yaml + EXPT_SUBDIR: REFS_test_verification_EAS_moreprocs2 + USE_CRON_TO_RELAUNCH: false + EXPT_BASEDIR: rm_subdir + VERBOSE: true +ensemble: + DO_ENSEMBLE: true + NUM_ENS_MEMBERS: 14 + ENS_TIME_LAG_HRS: !list '[ 0, 0, 0, 0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6 ]' +verification: + VX_FCST_MODEL_NAME: REFS + VX_FIELD_GROUPS: + - APCP + - REFC + - RETOP + - SFC + - UPA + VX_FCST_INPUT_BASEDIR: '/scratch3/BMC/gsd-fv3-dev/Michael.Kavulich/RRFS_MPAS_testing/data/' + VX_MASK: + - CONUS + FCST_FN_TEMPLATE: + - '{init?fmt=%Y%m%d}/rrfs_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs01_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs02_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs03_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs04_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs05_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs01_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs02_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs03_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs04_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/rrfs05_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + - '{init?fmt=%Y%m%d}/hrrr_ncep_{init?fmt=%Y%m%d%H?shift=-${time_lag}}f{lead?fmt=%HHH?shift=${time_lag}}.grib2' + CCPA_OBS_DIR: '{{ verification.VX_FCST_INPUT_BASEDIR }}/obs_data/ccpa' + MRMS_OBS_DIR: '{{ verification.VX_FCST_INPUT_BASEDIR }}/obs_data/mrms' + NDAS_OBS_DIR: '{{ verification.VX_FCST_INPUT_BASEDIR }}/obs_data/ndas' + + VX_TASKS: 8 +gridstat: + TASKS: 16 +pointstat: + TASKS: 8 +ensemblestat: + execution: + walltime: 02:00:00 + memory: '120G' + threads: 32 + TASKS: 30 +genensprod: + execution: + walltime: 08:00:00 + TASKS: 30 + conf: + # Config file settings for the ensemble agreement scale (EAS) probability method. See the MET Users Guide for more details: + # https://metplus.readthedocs.io/projects/met/en/develop/Users_Guide/gen-ens-prod.html + # Set one of these to true to calculate EAS statistics + GEN_ENS_PROD_ENSEMBLE_FLAG_EAS: False + GEN_ENS_PROD_ENSEMBLE_FLAG_EAS_WIDTH: False + # A list of candidate widths (in number of gridpoints) in increasing order + GEN_ENS_PROD_EAS_PROB_WIDTH: '[ 3, 5, 7, 9, 11, 13, 15 ]' + # Valid shapes are CIRCLE or SQUARE + GEN_ENS_PROD_EAS_PROB_SHAPE: 'CIRCLE' + # A number between 0 and 1 specifying the required ratio of valid data in the neighborhood for an output value to be computed + GEN_ENS_PROD_EAS_PROB_VLD_THRESH: 0.0 + # Similarity criteria parameter α: related to the amount of bias tolerated. 0.5 is recommended. + GEN_ENS_PROD_EAS_PROB_ALPHA: 0.5 + # Definitions for gaussian smoother applied to raw EAS probability values + GEN_ENS_PROD_EAS_PROB_GAUSSIAN_DX: 81.27 + GEN_ENS_PROD_EAS_PROB_GAUSSIAN_RADIUS: 120 +gridstat_ens: + execution: + walltime: 02:00:00 + TASKS: 30 +user: + MACHINE: URSA + ACCOUNT: fv3lam diff --git a/ush/get_metplus_tool_name.sh b/ush/get_metplus_tool_name.sh deleted file mode 100644 index f4af028e..00000000 --- a/ush/get_metplus_tool_name.sh +++ /dev/null @@ -1,124 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that takes as input the name of a MET/METplus -# tool spelled in upper flat case, i.e. all-caps and without separators -# (e.g. METPLUSTOOLNAME) and returns that name converted to the following -# cases: -# -# 1) Snake case, i.e. in all lower-case with underscores as word separators, -# e.g. metplus_tool_name. -# 2) Pascal case, i.e. without separators and with the first letter of -# each word capitalized, e.g. MetplusToolName. -# 3) Screaming snake case, i.e. in all upper-case with underscores as -# word separators, e.g. METPLUS_TOOL_NAME. -# -#----------------------------------------------------------------------- -# -function get_metplus_tool_name() { -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "METPLUSTOOLNAME" \ - "outvarname_metplus_tool_name" \ - "outvarname_MetplusToolName" \ - "outvarname_METPLUS_TOOL_NAME" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# True. -# -#----------------------------------------------------------------------- -# - print_input_args "valid_args" -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local _metplus_tool_name_ \ - _MetplusToolName_ \ - _METPLUS_TOOL_NAME_ -# -#----------------------------------------------------------------------- -# -# Create array containing set of forecast hours for which we will check -# for the existence of corresponding observation or forecast file. -# -#----------------------------------------------------------------------- -# - valid_vals_METPLUSTOOLNAME=( \ - "ASCII2NC" "PB2NC" "PCPCOMBINE" "GRIDSTAT" "POINTSTAT" "GENENSPROD" "ENSEMBLESTAT" \ - ) - check_var_valid_value "METPLUSTOOLNAME" "valid_vals_METPLUSTOOLNAME" - - case "${METPLUSTOOLNAME}" in - "ASCII2NC") - _metplus_tool_name_="ascii2nc" - _MetplusToolName_="Ascii2nc" - ;; - "PB2NC") - _metplus_tool_name_="pb2nc" - _MetplusToolName_="Pb2nc" - ;; - "PCPCOMBINE") - _metplus_tool_name_="pcp_combine" - _MetplusToolName_="PcpCombine" - ;; - "GRIDSTAT") - _metplus_tool_name_="grid_stat" - _MetplusToolName_="GridStat" - ;; - "POINTSTAT") - _metplus_tool_name_="point_stat" - _MetplusToolName_="PointStat" - ;; - "GENENSPROD") - _metplus_tool_name_="gen_ens_prod" - _MetplusToolName_="GenEnsProd" - ;; - "ENSEMBLESTAT") - _metplus_tool_name_="ensemble_stat" - _MetplusToolName_="EnsembleStat" - ;; - *) - print_err_msg_exit "\ -Generic name specified for MET/METplus tool (METPLUSTOOLNAME) is -unupported: - METPLUSTOOLNAME = \"${METPLUSTOOLNAME}\"" - ;; - esac - - _METPLUS_TOOL_NAME_=$(echo_uppercase ${_metplus_tool_name_}) -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${outvarname_metplus_tool_name}" ]; then - printf -v ${outvarname_metplus_tool_name} "%s" "${_metplus_tool_name_}" - fi - - if [ ! -z "${outvarname_MetplusToolName}" ]; then - printf -v ${outvarname_MetplusToolName} "%s" "${_MetplusToolName_}" - fi - - if [ ! -z "${outvarname_METPLUS_TOOL_NAME}" ]; then - printf -v ${outvarname_METPLUS_TOOL_NAME} "%s" "${_METPLUS_TOOL_NAME_}" - fi -} diff --git a/ush/launch_vx_wflow.sh b/ush/launch_vx_wflow.sh index 38ac9722..54b02002 100644 --- a/ush/launch_vx_wflow.sh +++ b/ush/launch_vx_wflow.sh @@ -70,23 +70,23 @@ export USHdir=$USHdir # #----------------------------------------------------------------------- # -# Declare arguments. -# -#----------------------------------------------------------------------- -# -valid_args=( \ - "called_from_cron" \ - ) -process_args valid_args "$@" -print_input_args "valid_args" -# -#----------------------------------------------------------------------- -# -# Make sure called_from_cron is set to a valid value. -# -#----------------------------------------------------------------------- -# -called_from_cron=${called_from_cron:-"False"} +# Parse arguments. +# +#----------------------------------------------------------------------- +# +while [ $# -gt 0 ]; do + case "$1" in + --called_from_cron=*) + called_from_cron="${1#*=}" + ;; + *) + printf "******************************\n" + printf "* Error: Invalid argument $1.*\n" + printf "******************************\n" + exit 1 + esac + shift +done # #----------------------------------------------------------------------- # @@ -104,7 +104,8 @@ expt_name="${EXPT_SUBDIR}" # #----------------------------------------------------------------------- # -machine=$(echo_lowercase $MACHINE) +# Convert machine name to lowercase +machine="${MACHINE,,}" source ${USHdir}/../setup_conda.sh diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml index 003a3417..8e3f5f91 100644 --- a/ush/machine/derecho.yaml +++ b/ush/machine/derecho.yaml @@ -12,11 +12,6 @@ platform: QUEUE_DEFAULT: main QUEUE_FCST: main QUEUE_HPSS: main - RUN_CMD_FCST: mpiexec -n ${nprocs} - RUN_CMD_POST: mpiexec -n ${nprocs} - RUN_CMD_PRDGEN: mpiexec -n ${nprocs} - RUN_CMD_SERIAL: time - PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 82b5a3b4..79d7055d 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -16,14 +16,8 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch - RUN_CMD_FCST: srun --export=ALL - RUN_CMD_POST: srun --export=ALL - RUN_CMD_PRDGEN: srun --export=ALL - RUN_CMD_SERIAL: time - RUN_CMD_UTILS: srun --export=ALL SCHED_NATIVE_CMD: "--export=NONE" SCHED_NATIVE_CMD_HPSS: "-n 1 --export=NONE" - PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch3/NCEPDEV/nems/role.epic/hera/UFS_SRW_data/develop/input_model_data TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /scratch3/NCEPDEV/nems/role.epic/hera/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /scratch3/NCEPDEV/nems/role.epic/hera/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml index 46534445..25ac5f6f 100644 --- a/ush/machine/hercules.yaml +++ b/ush/machine/hercules.yaml @@ -15,13 +15,7 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch - RUN_CMD_FCST: srun --export=ALL - RUN_CMD_POST: srun --export=ALL - RUN_CMD_PRDGEN: srun --export=ALL - RUN_CMD_SERIAL: time - RUN_CMD_UTILS: srun --export=ALL -n $nprocs SCHED_NATIVE_CMD: --export=NONE - PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data EXTRN_MDL_DATA_STORES: aws nomads MET_BASE: /apps/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.1.0/install/oneapi/2024.2.1/ diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index 9f56a6c9..6a5a6ca3 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -15,14 +15,8 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch - RUN_CMD_FCST: srun --export=ALL - RUN_CMD_POST: srun --export=ALL - RUN_CMD_PRDGEN: srun --export=ALL - RUN_CMD_SERIAL: time - RUN_CMD_UTILS: srun --export=ALL SCHED_NATIVE_CMD: --export=NONE SCHED_NATIVE_CMD_HPSS: -n 1 --export=NONE - PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/input_model_data TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir diff --git a/ush/machine/linux.yaml b/ush/machine/linux.yaml index 2f742d95..bd404eea 100644 --- a/ush/machine/linux.yaml +++ b/ush/machine/linux.yaml @@ -10,12 +10,6 @@ platform: TEST_AERONET_OBS_DIR: /Users/username/DATA/UFS/obs_data/aeronet/proc TEST_AIRNOW_OBS_DIR: /Users/username/DATA/UFS/obs_data/airnow/proc TEST_GDAS_OBS_DIR: /Users/username/DATA/UFS/obs_data/gdas/proc - RUN_CMD_FCST: mpirun -n ${PE_MEMBER01} - RUN_CMD_POST: mpirun - RUN_CMD_PRDGEN: mpirun - RUN_CMD_SERIAL: time - RUN_CMD_UTILS: mpirun - PRE_TASK_CMDS: '{ ulimit -a; ulimit -s unlimited; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /home/username/DATA/UFS/input_model_data TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /home/username/DATA/UFS/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /home/username/DATA/UFS/dummy_FV3GFS_sys_dir diff --git a/ush/machine/macos.yaml b/ush/machine/macos.yaml index b5f990ca..9c7e9e2e 100644 --- a/ush/machine/macos.yaml +++ b/ush/machine/macos.yaml @@ -7,12 +7,6 @@ platform: TEST_MRMS_OBS_DIR: /Users/username/DATA/UFS/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /Users/username/DATA/UFS/obs_data/ndas/proc TEST_NOHRSC_OBS_DIR: /Users/username/DATA/UFS/obs_data/nohrsc/proc - RUN_CMD_FCST: mpirun -n ${PE_MEMBER01} - RUN_CMD_POST: mpirun - RUN_CMD_PRDGEN: mpirun - RUN_CMD_SERIAL: time - RUN_CMD_UTILS: mpirun - PRE_TASK_CMDS: '{ ulimit -a; ulimit -s unlimited; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /Users/username/DATA/UFS/input_model_data TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /Users/username/DATA/UFS/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /Users/username/DATA/UFS/dummy_FV3GFS_sys_dir diff --git a/ush/machine/noaacloud.yaml b/ush/machine/noaacloud.yaml index a6eeb533..36581ead 100644 --- a/ush/machine/noaacloud.yaml +++ b/ush/machine/noaacloud.yaml @@ -10,13 +10,7 @@ platform: TEST_AERONET_OBS_DIR: '{{ platform.WE2E_TEST_DATA }}/obs_data/aeronet' TEST_AIRNOW_OBS_DIR: '{{ platform.WE2E_TEST_DATA }}/obs_data/airnow' REMOVE_MEMORY: True - RUN_CMD_FCST: mpiexec -np $nprocs - RUN_CMD_POST: mpiexec -np $nprocs - RUN_CMD_PRDGEN: mpiexec -np $nprocs - RUN_CMD_SERIAL: time - RUN_CMD_UTILS: mpiexec -np $nprocs SCHED_NATIVE_CMD: --export=NONE - PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /contrib/EPIC/UFS_SRW_data/develop/input_model_data TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /contrib/EPIC/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /contrib/EPIC/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index 07dbe069..3686185c 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -15,14 +15,7 @@ platform: QUEUE_FCST: batch PARTITION_HPSS: service QUEUE_HPSS: batch - RUN_CMD_FCST: srun --export=ALL - RUN_CMD_POST: srun --export=ALL - RUN_CMD_PRDGEN: srun --export=ALL - RUN_CMD_SERIAL: time - RUN_CMD_UTILS: srun --export=ALL - RUN_CMD_NEXUS: srun --export=ALL SCHED_NATIVE_CMD: --export=NONE - PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data EXTRN_MDL_DATA_STORES: aws nomads MET_BASE: /apps/contrib/spack-stack/spack-stack-1.9.2/envs/ue-oneapi-2024.1.0/install/oneapi/2024.2.1 diff --git a/ush/machine/ursa.yaml b/ush/machine/ursa.yaml index 24854a70..767dcf46 100644 --- a/ush/machine/ursa.yaml +++ b/ush/machine/ursa.yaml @@ -17,15 +17,8 @@ platform: PARTITION_HPSS: u1-service EXCLUSIVE: exclusive QUEUE_HPSS: batch - RUN_CMD_FCST: srun --export=ALL - RUN_CMD_POST: srun --export=ALL - RUN_CMD_PRDGEN: srun --export=ALL - RUN_CMD_SERIAL: time - RUN_CMD_UTILS: srun --export=ALL - RUN_CMD_NEXUS: srun -n ${nprocs} --export=ALL SCHED_NATIVE_CMD: "--export=NONE" SCHED_NATIVE_CMD_HPSS: "-n 1 --export=NONE" - PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch3/NCEPDEV/nems/role.epic/ursa/UFS_SRW_data/develop/input_model_data TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /scratch3/NCEPDEV/nems/role.epic/ursa/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /scratch3/NCEPDEV/nems/role.epic/ursa/UFS_SRW_data/develop/dummy_FV3GFS_sys_dir diff --git a/ush/python_utils/__init__.py b/ush/python_utils/__init__.py index a7edc9af..59a66c19 100644 --- a/ush/python_utils/__init__.py +++ b/ush/python_utils/__init__.py @@ -1,4 +1,4 @@ -from .misc import uppercase, lowercase, find_pattern_in_str, find_pattern_in_file, dict_find +from .misc import find_pattern_in_str, find_pattern_in_file, dict_find from .check_for_preexist_dir_file import check_for_preexist_dir_file from .check_python_version import check_python_version from .check_var_valid_value import check_var_valid_value diff --git a/ush/python_utils/metplus_conf_utils.py b/ush/python_utils/metplus_conf_utils.py index fe86deda..8dddd8f2 100644 --- a/ush/python_utils/metplus_conf_utils.py +++ b/ush/python_utils/metplus_conf_utils.py @@ -84,13 +84,20 @@ def make_var_lists(vx_config_dict,field_group): return fcst_var_list,obs_var_list -def render_metplus_confs(cfg,settings,template_fn,vx_leadhr_list,tasks): +def render_metplus_confs(cfg,settings,template_fn,vx_leadhr_list,tasks,extra=None): """Renders metplus conf files from the appropriate template and user settings. If VX_TASKS > 1 and vx_leadhr_list > 1, renders a conf file for each parallel task. - Returns the filename(s) of metplus conf files that were rendered""" + Returns the filename(s) of metplus conf files that were rendered + + vx_leadhr_list (list) + tasks (int): The number of parallel tasks to prepare conf files for + extra (dict): A dictionary of additional settings to append to the conf file""" logger = logging.getLogger(__name__) + # initialize extra as an empty dictionary if not provided + if extra is None: + extra={} num_fhrs = len(vx_leadhr_list) outconfs = [] logger.debug(f"Loading METplus conf template file: {template_fn}") @@ -114,22 +121,33 @@ def render_metplus_confs(cfg,settings,template_fn,vx_leadhr_list,tasks): logger.debug(f"metplus log file for task: {settings['metplus_log_fn']}") logger.debug(f"metplus final rendered conf for task: {outconf}") hours_per_task,remainder = divmod(num_fhrs,tasks) + if settings.get("staging_dir"): + settings['staging_dir'] = f"{str(settings['staging_dir']).rsplit('.',1)[0]}.{i}" # For cases where things don't divide evenly, ensure we get best distribution + logger.debug(f"{vx_leadhr_list=}") if i >= remainder: vx_leadhr_list, task_fhrs = vx_leadhr_list[hours_per_task:],vx_leadhr_list[:hours_per_task] else: vx_leadhr_list, task_fhrs = vx_leadhr_list[hours_per_task+1:],vx_leadhr_list[:hours_per_task+1] settings['vx_leadhr_list'] = ', '.join(map(str,task_fhrs)) logger.debug(f"Task {i} will process lead hours: {settings['vx_leadhr_list']}") + logger.debug(f"{vx_leadhr_list=}") + logger.debug(f"{settings['vx_leadhr_list']=}") rendered = template.render(settings) with open(outconf,'w', encoding="utf-8") as f: f.write(rendered) + # Write additional lines to the end of the conf file + for k, v in extra.items(): + logger.debug(f"Adding extra line to conf file: '{k} = {v}'") + f.write(f"\n{k} = {v}") outconfs.append(outconf) else: #Remove task-specific suffixes if we're only using one task settings['metplus_log_fn'] = settings['metplus_log_fn'].rsplit('.',1)[0] settings['metplus_config_fn'] = settings['metplus_config_fn'].rsplit('.',1)[0] + if settings.get("staging_dir"): + settings['staging_dir'] = f"{str(settings['staging_dir']).rsplit('.',1)[0]}" outconf = f"{settings['output_dir']}/{settings['metplus_config_fn']}" logger.debug("Rendering conf file") logger.debug(f"metplus log file: {settings['metplus_log_fn']}") @@ -139,6 +157,10 @@ def render_metplus_confs(cfg,settings,template_fn,vx_leadhr_list,tasks): rendered = template.render(settings) with open(outconf,'w', encoding="utf-8") as f: f.write(rendered) + # Write additional lines to the end of the conf file + for k, v in extra.items(): + logger.debug(f"Adding extra line to conf file: '{k} = {v}'") + f.write(f"\n{k} = {v}") outconfs = [outconf] return outconfs diff --git a/ush/python_utils/misc.py b/ush/python_utils/misc.py index 853f2e60..6c19173d 100644 --- a/ush/python_utils/misc.py +++ b/ush/python_utils/misc.py @@ -2,31 +2,6 @@ import re - -def uppercase(s): - """Converts a given string to uppercase - - Args: - s (str): The string to change to uppercase - Returns: - Uppercased string - """ - - return s.upper() - - -def lowercase(s): - """Converts a given string to lowercase - - Args: - s (str): The string to change to lowercase - Returns: - Lowercased string - """ - - return s.lower() - - def find_pattern_in_str(pattern, source): """Finds a regular expression (regex) pattern in a string diff --git a/ush/select_validtime_obs.py b/ush/select_validtime_obs.py index eed96e24..12ddb9f1 100644 --- a/ush/select_validtime_obs.py +++ b/ush/select_validtime_obs.py @@ -37,11 +37,7 @@ def select_validtime_obs(valid_time, outdir, source, in_template, out_template, Returns: - string: The staged filename - - Raises: - FileNotFoundError: If no valid file was found within "window" seconds of the valid - time of the forecast + string: The staged filename, or a null string if none found """ @@ -84,6 +80,7 @@ def select_validtime_obs(valid_time, outdir, source, in_template, out_template, # Check to make sure closest file is within +/- window seconds of top of the hour difference = abs(closest_timestamp - valid) + target = '' if difference.total_seconds() <= window: filename1 = f"{template_start}{closest_timestamp.strftime(date_template)}{template_end}" filename2 = datetime.strftime(valid,out_template) @@ -95,9 +92,9 @@ def select_validtime_obs(valid_time, outdir, source, in_template, out_template, print(f"Moving file {origfile} to {target}") shutil.move(origfile,target) - else: - raise FileNotFoundError(f"Did not find a valid file within {window} seconds of {valid}") + print(f"WARNING: Did not find a valid file within {window} seconds of {valid}") + return target diff --git a/ush/set_job_env.sh b/ush/set_job_env.sh index beda5a8f..412a2c8f 100644 --- a/ush/set_job_env.sh +++ b/ush/set_job_env.sh @@ -17,3 +17,9 @@ if [ ! -z "${ACCUM_HH}" ]; then ACCUM_ARG="--accum_hh=${ACCUM_HH}" fi +# For tasks that need an ensemble member index, set ENSMEM_INDX +ENSMEM_ARG="" +if [ ! -z "${ENSMEM_INDX}" ]; then + ENSMEM_ARG="--ensmem_index=${ENSMEM_INDX}" +fi + diff --git a/ush/set_vx_params.py b/ush/set_vx_params.py index 62c597f1..60e22e56 100644 --- a/ush/set_vx_params.py +++ b/ush/set_vx_params.py @@ -20,14 +20,14 @@ def set_vx_params(obtype,field_group,accum_hh): fieldname_in_obs_in = field_group fieldname_in_fcst_in = field_group fieldname_in_met_out = field_group - fieldname_in_met_filedir_names = f"{field_group}{accum_hh:02}" + fieldname_in_met_filedir_names = f"{field_group}{accum_hh:02}h" case "NOHRSC": grid_or_point = "grid" if field_group == "ASNOW": fieldname_in_obs_in = field_group fieldname_in_fcst_in = field_group fieldname_in_met_out = field_group - fieldname_in_met_filedir_names = f"{field_group}{accum_hh:02}" + fieldname_in_met_filedir_names = f"{field_group}{accum_hh:02}h" case "MRMS": grid_or_point = "grid" diff --git a/ush/set_vx_params.sh b/ush/set_vx_params.sh deleted file mode 100644 index ca347546..00000000 --- a/ush/set_vx_params.sh +++ /dev/null @@ -1,327 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines a function that sets various parameters needed when -# performing verification. The way these parameters are set depends on -# the field group being verified and, if the field group consists of a -# set of cumulative fields (e.g. accumulated precipitation or accumulated -# snowfall), the accumulation interval (both of which are inputs to this -# function). -# -# The verification workflow is designed for the MET/METplus software -# (MET = Model Evaluation Tools) developed at the DTC (Developmental -# Testbed Center). -# -#----------------------------------------------------------------------- -# -function set_vx_params() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "obtype" \ - "field_group" \ - "accum_hh" \ - "outvarname_grid_or_point" \ - "outvarname_fieldname_in_obs_input" \ - "outvarname_fieldname_in_fcst_input" \ - "outvarname_fieldname_in_MET_output" \ - "outvarname_fieldname_in_MET_filedir_names" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# True. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# - local _grid_or_point_ \ - fieldname_in_obs_input \ - fieldname_in_fcst_input \ - fieldname_in_MET_output \ - fieldname_in_MET_filedir_names -# -#----------------------------------------------------------------------- -# -# Make sure that accum_hh is a 2-digit integer. -# -#----------------------------------------------------------------------- -# - if [ "${obtype}" = "CCPA" ] || [ "${obtype}" = "NOHRSC" ]; then - if [[ ! "${accum_hh}" =~ ^[0-9]{2}$ ]]; then - print_err_msg_exit "\ -For the given observation type (obtype), the accumulation (accum_hh) must -be a 2-digit integer: - obtype = \"${obtype}\" - accum_hh = \"${accum_hh}\"" - fi - fi -# -#----------------------------------------------------------------------- -# -# Set the parameters. Definitions: -# -# grid_or_point: -# String that is set to either "grid" or "point" depending on whether -# obs type containing the field group is gridded or point-based. -# -# fieldname_in_obs_input: -# If the field group represents a single field, this is the string used -# to search for that field in the input observation files read in by MET. -# If not, this is set to a null string. -# -# fieldname_in_fcst_input: -# If the field group represents a single field, this is the string used -# to search for that field in the input forecast files read in by MET. -# If not, this is set to a null string. -# -# fieldname_in_MET_output: -# String that will be used in naming arrays defined in MET output files -# (e.g. NetCDF, stat, etc). -# -# fieldname_in_MET_filedir_names: -# String that will be used in naming directories and files (e.g. NetCDF -# files, stat files, log files, staging directories) generated by MET -# or METplus. -# -#----------------------------------------------------------------------- -# - _grid_or_point_="" - fieldname_in_obs_input="" - fieldname_in_fcst_input="" - fieldname_in_MET_output="" - fieldname_in_MET_filedir_names="" - - case "${obtype}" in - - "CCPA") - - _grid_or_point_="grid" - case "${field_group}" in - - "APCP") - fieldname_in_obs_input="${field_group}" - fieldname_in_fcst_input="${field_group}" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}${accum_hh}h" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - "NOHRSC") - - _grid_or_point_="grid" - case "${field_group}" in - - "ASNOW") - fieldname_in_obs_input="${field_group}" - fieldname_in_fcst_input="${field_group}" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}${accum_hh}h" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - "MRMS") - - _grid_or_point_="grid" - case "${field_group}" in - - "REFC") - fieldname_in_obs_input="MergedReflectivityQCComposite" - fieldname_in_fcst_input="${field_group}" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}" - ;; - - "RETOP") - fieldname_in_obs_input="EchoTop18" - fieldname_in_fcst_input="${field_group}" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - "NDAS") - - _grid_or_point_="point" - case "${field_group}" in - - "SFC") - fieldname_in_obs_input="" - fieldname_in_fcst_input="" - fieldname_in_MET_output="ADP${field_group}" - fieldname_in_MET_filedir_names="ADP${field_group}" - ;; - - "UPA") - fieldname_in_obs_input="" - fieldname_in_fcst_input="" - fieldname_in_MET_output="ADP${field_group}" - fieldname_in_MET_filedir_names="ADP${field_group}" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - "AERONET") - - _grid_or_point_="point" - case "${field_group}" in - - "AOD") - fieldname_in_obs_input="${field_group}" - fieldname_in_fcst_input="AOTK" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - "AIRNOW") - - _grid_or_point_="point" - case "${field_group}" in - - "PM25") - fieldname_in_obs_input="${field_group}" - fieldname_in_fcst_input="MASSDEN" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}" - ;; - "PM10") - fieldname_in_obs_input="${field_group}" - fieldname_in_fcst_input="MASSDEN" - fieldname_in_MET_output="${field_group}" - fieldname_in_MET_filedir_names="${field_group}" - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype) and field_group (field_group) combination: - obtype = \"${obtype}\" - field_group = \"${field_group}\"" - ;; - - esac - ;; - - *) - print_err_msg_exit "\ -A method for setting verification parameters has not been specified for -this observation type (obtype): - obtype = \"${obtype}\"" - ;; - - esac -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${outvarname_grid_or_point}" ]; then - printf -v ${outvarname_grid_or_point} "%s" "${_grid_or_point_}" - fi - - if [ ! -z "${outvarname_fieldname_in_obs_input}" ]; then - printf -v ${outvarname_fieldname_in_obs_input} "%s" "${fieldname_in_obs_input}" - fi - - if [ ! -z "${outvarname_fieldname_in_fcst_input}" ]; then - printf -v ${outvarname_fieldname_in_fcst_input} "%s" "${fieldname_in_fcst_input}" - fi - - if [ ! -z "${outvarname_fieldname_in_MET_output}" ]; then - printf -v ${outvarname_fieldname_in_MET_output} "%s" "${fieldname_in_MET_output}" - fi - - if [ ! -z "${outvarname_fieldname_in_MET_filedir_names}" ]; then - printf -v ${outvarname_fieldname_in_MET_filedir_names} "%s" "${fieldname_in_MET_filedir_names}" - fi - -} diff --git a/ush/setup.py b/ush/setup.py index de99eb51..e07aa5d6 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -111,6 +111,11 @@ def load_config_for_setup(ushdir, default_config_path, user_config_path): taskgroups = default_config["workflow"]["taskgroups"] default_config["rocoto"]["tasks"] = {} for taskgroup in taskgroups: + if "verify_pre.yaml" in taskgroup: + logger.error("The `verify_pre.yaml` task group has been split into two task groups:") + logger.error("`get_obs.yaml` and `preprocessing.yaml`.") + logger.error("Update your config accordingly") + raise ValueError(f'{taskgroups=}') tasks = get_yaml_config(homedir / taskgroup) keep = {k: v for k, v in tasks.items() if not re.search(r"^default_*", k)} default_config["rocoto"]["tasks"].update(keep) @@ -132,37 +137,61 @@ def load_config_for_setup(ushdir, default_config_path, user_config_path): return default_config def check_bad_settings(cfg): + # pylint: disable=too-many-statements """Checks initial config for deprecated or incorrect settings""" logger = logging.getLogger(__name__) msg='' if bad:=cfg.get("global"): msg+=f"Config file contains invalid key `global`:\n{bad}\n" - msg+="The `global` section has been renamed to `ensemble`; " - msg+="update your config accordingly\n\n" - raise KeyError(msg) + msg+="The `global` section has been renamed to `ensemble`" if ex:=cfg.get("verification_resources").get("execution"): if bad:=ex.get("point2grid"): msg+=f"verification_resources:execution contains invalid key `point2grid`:\n{bad}\n" - msg+="these variables for this task have moved to top-level `point2grid` section;" - msg+="update your config accordingly\n\n" + msg+="these variables for this task have moved to top-level `point2grid` section" if bad:=ex.get("regriddataplane"): msg+=f"verification_resources:execution contains invalid key `regriddataplane`:\n{bad}" - msg+="these variables for this task moved to top-level `regriddataplane` section; " - msg+="update your config accordingly\n\n" + msg+="these variables for this task moved to top-level `regriddataplane` section \n" if bad:=ex.get("mode"): msg+=f"verification_resources:execution contains invalid key `mode`:\n{bad}\n" - msg+="these variables for this task have been moved to top-level `mode` section; " - msg+="update your config accordingly\n\n" + msg+="these variables for this task have been moved to top-level `mode` section\n" + if bad:=ex.get("pb2nc"): + msg+=f"verification_resources:execution contains invalid key `pb2nc`:\n{bad}\n" + msg+="these variables for this task have been moved to top-level `pb2nc` section\n" + if bad:=ex.get("deterministic"): + msg+=f"verification_resources:execution contains invalid key `deterministic`:\n{bad}\n" + if bad.get("gridstat"): + msg+="gridstat settings have been moved to top-level `gridstat` section\n" + if bad.get("pointstat"): + msg+="pointstat settings have been moved to top-level `pointstat` section\n" + else: + msg+="unknown key {bad}, see config_defaults.yaml for valid settings\n" + if bad:=ex.get("ensemble"): + if bad.get("genensprod"): + msg+="verification_resources:execution:ensemble contains invalid key `genensprod`\n" + msg+="These settings have been moved to top-level `genensprod` section\n" + if bad.get("ensemblestat"): + msg+="verification_resources:execution:ensemble has invalid key `ensemblestat`\n" + msg+="These settings have been moved to top-level `ensemblestat` section\n" + if bad.get("gridstat"): + msg+="verification_resources:execution:ensemble contains invalid key `gridstat`\n" + msg+="These settings have been moved to top-level `gridstat_ens` section\n" + if bad.get("pointstat"): + msg+="verification_resources:execution:ensemble contains invalid key `pointstat`\n" + msg+="These settings have been moved to top-level `pointstat_ens` section\n" + else: + msg+="unknown key {bad}, see config_defaults.yaml for valid settings" if bt:=cfg.get("platform").get("BEST_TRACK"): msg+=f"Config file contains invalid key `platform: BEST_TRACK`:\n{bt}\n" - msg+="The variable `BEST_TRACK` has been renamed to `BEST_TRACK_DIR`; " - msg+="update your config accordingly\n\n" - + msg+="The variable `BEST_TRACK` has been renamed to `BEST_TRACK_DIR`" + if vf:=cfg.get("verification"): + if bad:=vf.get("FCST_SUBDIR_TEMPLATE"): + msg+=f"verification: contains deprecated key `FCST_SUBDIR_TEMPLATE`:\n{bad}\n" + msg+="Prepend this value to `FCST_FN_TEMPLATE` to restore previous functionality" if msg: logger.critical("The following problems with your config must be fixed:") logger.critical(msg) - raise KeyError("Invalid keys found in config; see above messages for details") + raise KeyError("Invalid key(s) found in config.yaml file; see above messages for details") return cfg @@ -252,7 +281,6 @@ def setup(ushdir, user_config_fn="config.yaml", debug: bool = False): workflow_config["VERBOSE"] = True # The forecast length (in integer hours) cannot contain more than 3 characters. - # Thus, its maximum value is 999. fcst_len_hrs_max = 999 fcst_len_hrs = workflow_config["FCST_LEN_HRS"] if fcst_len_hrs > fcst_len_hrs_max: @@ -411,6 +439,39 @@ def _remove_tag(tasks, tag): # intervals from integer to datetime.timedelta objects. fcst_len_dt = datetime.timedelta(hours=fcst_len_hrs) vx_fcst_output_intvl_dt = datetime.timedelta(hours=vx_fcst_output_intvl_hrs) + + # Checks for consistency in ensemble settings + ens_config = expt_config["ensemble"] + do_ensemble = ens_config["DO_ENSEMBLE"] + + nummems = int(ens_config.get("NUM_ENS_MEMBERS")) + if nummems < 1: + raise ValueError(f"NUM_ENS_MEMBERS ({nummems}) must be > 0; set to 1 for deterministic vx") + if do_ensemble and nummems < 2: + raise ValueError(f"NUM_ENS_MEMBERS ({nummems}) must be > 1 if DO_ENSEMBLE=True") + if not do_ensemble and nummems > 1: + raise ValueError(f"NUM_ENS_MEMBERS ({nummems}) can not be > 1 if DO_ENSEMBLE=False") + # + # If FCST_FN_TEMPLATE is a string, make it a list of strings, one entry per ensemble member. + # If a list, ensure it's the same length as the specified ensemble size + # + fns = vx_config.get("FCST_FN_TEMPLATE") + if isinstance(fns, str): + fns = [fns] + if isinstance(fns, list): + # If list is len==1, make it len==nummems + if len(fns) == 1: + fns = fns * nummems + elif len(fns) != nummems: + logger.error(f"NUM_ENS_MEMBERS={nummems}\nFCST_FN_TEMPLATE={fns}") + raise ValueError("Number of entries in FCST_FN_TEMPLATE must equal number of members!") + else: + raise TypeError("Invalid FCST_FN_TEMPLATE entry type {type(fns)}: {fns}") + + # Update dictionary with modified settings + ens_config["NUM_ENS_MEMBERS"] = nummems + vx_config["FCST_FN_TEMPLATE"] = fns + # # ----------------------------------------------------------------------- # @@ -683,9 +744,6 @@ def _remove_tag(tasks, tag): ) - # Check to make sure that mandatory forecast variables are set. - ensemble_sect = expt_config["ensemble"] - # create experiment dir Path(exptdir).mkdir(parents=True) @@ -702,7 +760,6 @@ def _remove_tag(tasks, tag): # Get the value of the configuration flag for ensemble mode (DO_ENSEMBLE) # and ensure that it is set to True if ensemble vx tasks are included in # the workflow (or vice-versa). - do_ensemble = ensemble_sect["DO_ENSEMBLE"] if (not do_ensemble) and ens_vx_tasks: msg = dedent( f""" diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index 54bc20cb..2585a8ab 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -52,88 +52,6 @@ function source_util_funcs() { # #----------------------------------------------------------------------- # -# Source the file containing the functions that will echo given strings -# as uppercase or lowercase -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/change_case.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that checks for preexisting -# directories or files and handles them according to a specified method -# (which can be one of "delete", "rename", and "quit"). -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/check_for_preexist_dir_file.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that searches an array for a -# specified string. -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/is_element_of.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that gets the indices of those -# elements of an array that match a given string. -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/get_elem_inds.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that determines whether or not -# a specified variable is an array. -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/is_array.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that checks the validity of a -# variable's value (given a set of valid values). -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/check_var_valid_value.sh -# -#----------------------------------------------------------------------- -# -# -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/print_input_args.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that processes a list of argu- -# ments to a script or function, where the list is comprised of a set of -# argument name-value pairs, e.g. arg1="value1", ... -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/process_args.sh -# -#----------------------------------------------------------------------- -# -# Source the file containing the function that creates a symlink to a -# file (including performing checks). -# -#----------------------------------------------------------------------- -# - . ${bashutils_dir}/create_symlink_to_file.sh -# -#----------------------------------------------------------------------- -# # Source the file that sources YAML files as if they were bash # #-----------------------------------------------------------------------