From 10208f947f0e4aae04fc37f7cbcba5e702609cd1 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 18 Aug 2025 11:28:04 -0600 Subject: [PATCH 1/5] Create sub-directories for dump_points if they do not already exist. --- metplotpy/plots/line/line.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/metplotpy/plots/line/line.py b/metplotpy/plots/line/line.py index af32e90ae..98484609f 100644 --- a/metplotpy/plots/line/line.py +++ b/metplotpy/plots/line/line.py @@ -767,8 +767,12 @@ def write_output_file(self) -> None: filename = self.config_obj.points_path + os.path.sep + filename # save points - self._save_points(all_points_1, filename + ".points1") - self._save_points(all_points_2, filename + ".points2") + points_path = self.config_obj.points_path + os.makedirs(points_path, exist_ok=True) + # self._save_points(all_points_1, filename + ".points1") + self._save_points(all_points_1, os.path.join(points_path, filename + ".points1")) + # self._save_points(all_points_2, filename + ".points2") + self._save_points(all_points_2, os.path.join(points_path, filename + ".points2")) self.logger.info(f"Finished writing to output file: {datetime.now()}") From a2047be32961b1a85a21818b257f875b10189a3c Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 18 Aug 2025 12:02:09 -0600 Subject: [PATCH 2/5] Expected dump points values used to verify plot points match with plot points generated using env vars for FCST_VAR_VAL1/2 --- test/line/intermed_files/expected_var_stat_line.points1 | 3 +++ test/line/intermed_files/expected_var_stat_line.points2 | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 test/line/intermed_files/expected_var_stat_line.points1 create mode 100644 test/line/intermed_files/expected_var_stat_line.points2 diff --git a/test/line/intermed_files/expected_var_stat_line.points1 b/test/line/intermed_files/expected_var_stat_line.points1 new file mode 100644 index 000000000..bcd6edec8 --- /dev/null +++ b/test/line/intermed_files/expected_var_stat_line.points1 @@ -0,0 +1,3 @@ +14.714440 14.425071 15.003809 14.442990 14.182521 14.703459 0.216470 0.121860 0.311080 +14.615340 14.336420 14.894260 14.645345 14.338789 14.951901 0.022085 -0.056724 0.100894 +14.858610 14.548506 15.168714 14.981590 14.720549 15.242631 -0.076490 -0.170358 0.017378 diff --git a/test/line/intermed_files/expected_var_stat_line.points2 b/test/line/intermed_files/expected_var_stat_line.points2 new file mode 100644 index 000000000..2f82ef4a1 --- /dev/null +++ b/test/line/intermed_files/expected_var_stat_line.points2 @@ -0,0 +1,3 @@ +-1.311070 -1.454800 -1.167340 -1.262500 -1.414032 -1.110968 +-1.378120 -1.484110 -1.272130 -1.333030 -1.454853 -1.211207 +-1.371200 -1.520357 -1.222043 -1.398450 -1.545732 -1.251168 From d08ec497cb64e957c7b1c6cf3ce058e566150b84 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 18 Aug 2025 12:06:45 -0600 Subject: [PATCH 3/5] YAML file for testing env vars used for FCST_VAR_VAL1/2 and their corresponding stat values --- test/line/env_fcst_var.yaml | 231 ++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 test/line/env_fcst_var.yaml diff --git a/test/line/env_fcst_var.yaml b/test/line/env_fcst_var.yaml new file mode 100644 index 000000000..419c53d78 --- /dev/null +++ b/test/line/env_fcst_var.yaml @@ -0,0 +1,231 @@ +alpha: 0.05 +box_avg: 'False' +box_boxwex: 0.2 +box_notch: 'False' +box_outline: 'True' +box_pts: 'False' +caption_align: 0.0 +caption_col: '#333333' +caption_offset: 3.0 +caption_size: 0.8 +caption_weight: 1 +cex: 1 +colors: +- '#ff0000' +- '#55ff00' +- '#00aaff' +- '#0040ff' +- '#c3ff00' +con_series: +- 1 +- 1 +- 1 +- 1 +- 1 +create_html: 'False' +derived_series_1: +- - CONTROL RH MAE + - GTS RH MAE + - DIFF +derived_series_2: [] +dump_points_1: 'True' +dump_points_2: 'True' +event_equal: 'True' +fcst_var_val_1: + !ENV ${FCST_VAR_VAL1}: + - !ENV ${FCST_VAR_STAT1} +fcst_var_val_2: + !ENV ${FCST_VAR_VAL2}: + - !ENV ${FCST_VAR_STAT2} +fixed_vars_vals_input: + fcst_lev: + - Z02 +grid_col: '#cccccc' +grid_lty: 3 +grid_lwd: 1 +grid_on: 'True' +grid_x: listX +indy_label: +- '6' +- '12' +- '18' +indy_stagger_1: 'True' +indy_stagger_2: 'False' +indy_vals: +- '60000' +- '120000' +- '180000' +indy_var: fcst_lead +legend_box: o +legend_inset: + x: 0.0 + y: -0.25 +legend_ncol: 3 +legend_size: 0.8 +line_type: None +list_stat_1: +- MAE +list_stat_2: +- ME +list_static_val: + fcst_var: ME +mar: +- 8 +- 4 +- 5 +- 4 +method: bca +mgp: +- 1 +- 1 +- 0 +num_iterations: 1 +num_threads: -1 +plot_caption: Caption +plot_ci: +- std +- std +- std +- std +- std +plot_disp: +- 'True' +- 'True' +- 'True' +- 'True' +- 'True' +plot_filename: !ENV '${TEST_DIR}/env_fcst_var_stat_line.png' +plot_height: 8.5 +plot_res: 72 +plot_stat: median +plot_type: png16m +plot_units: in +plot_width: 11.0 + +# Optional, uncomment and set to directory to store the .points1 file +# that is used by METviewer (created when dump_points_1 is set to True) +# if dump_points_1 is True and this is uncommented, the points1 file +# will be saved in the default location (i.e. where the input data file is stored). +points_path: !ENV '${TEST_DIR}/intermed_files/env_fcst_var_stat' +random_seed: null +series_line_style: +- '-' +- '-' +- '-' +- '-' +- '-' +series_line_width: +- 1 +- 1 +- 1 +- 1 +- 1 +series_order: +- 1 +- 2 +- 3 +- 4 +- 5 +series_symbols: +- . +- . +- . +- . +- . +series_type: +- b +- b +- b +- b +- b +series_val_1: + model: + - CONTROL + - GTS +series_val_2: + model: + - CONTROL + - GTS +show_nstats: 'True' +show_signif: +- 'False' +- 'False' +- 'False' +- 'False' +- 'False' +stat_input: !ENV '${TEST_DIR}/line.data' +sync_yaxes: 'False' +title: test title +title_align: 0.5 +title_offset: -2 +title_size: 1.4 +title_weight: 2.0 +user_legend: [] +variance_inflation_factor: 'False' +vert_plot: 'False' +x2lab_align: 0.5 +x2lab_offset: -0.5 +x2lab_size: 0.8 +x2lab_weight: 1 +x2tlab_horiz: 0.5 +x2tlab_orient: 1 +x2tlab_perp: 1 +x2tlab_size: 0.8 +xaxis: test x_label +xaxis_reverse: 'False' +xlab_align: 0.5 +xlab_offset: 2 +xlab_size: 1 +xlab_weight: 1 +xlim: [] +xtlab_decim: 0 +xtlab_horiz: 0.5 +xtlab_orient: 1 +xtlab_perp: -0.75 +xtlab_size: 1 +y2lab_align: 0.5 +y2lab_offset: 1 +y2lab_size: 1 +y2lab_weight: 1 +y2lim: [] +y2tlab_horiz: 0.5 +y2tlab_orient: 1 +y2tlab_perp: 1 +y2tlab_size: 1.0 +yaxis_1: test y_label +yaxis_2: '' +ylab_align: 0.5 +ylab_offset: -2 +ylab_size: 1 +ylab_weight: 1 +ylim: [] +ytlab_horiz: 0.5 +ytlab_orient: 1 +ytlab_perp: 0.5 +ytlab_size: 1 + +lines: +- color: '#8000ff' + line_width: '2' + position: '11' + type: horiz_line + line_style: '--' +- line_style: '-' + color: '#000000' + line_width: 1 + position: "18" + type: vert_line + +# To save your log output to a file, specify a path and filename and uncomment the line below. Make sure you have +# permissions to the directory you specify. The default, as specified in the default config file is stdout. +#log_filename: ./line.log + +# To change the log level, specify a log level: debug, info, warning, error and uncomment the line below. +# Debug and info log level will produce more log output. +#log_level: WARNING +show_legend: +- 'True' +- 'True' +- 'True' +- 'True' +- 'True' From 42160303ca586ea7175509dcb208b4b56c9e5703 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 18 Aug 2025 12:07:45 -0600 Subject: [PATCH 4/5] Added test for using env vars for FCST_VAR_VAL1/2 and their corresponding stat values --- test/line/test_line_plot.py | 75 ++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/test/line/test_line_plot.py b/test/line/test_line_plot.py index bb4972dc7..c7830ec89 100644 --- a/test/line/test_line_plot.py +++ b/test/line/test_line_plot.py @@ -104,7 +104,7 @@ def test_no_nans_in_points_files(): custom_config_filename = f"{cwd}/custom_line2.yaml" l.main(custom_config_filename) - # Check for NaN's in the intermediate files, line.points1 and line.points2 + # Check for NaN's in the intermediate files,_line.points1 and line.points2 # Fail if there are any NaN's-this indicates something went wrong with the # line_series.py module's _create_series_points() method. nans_found = False @@ -322,6 +322,79 @@ def test_fixed_var_val(): # don't exist. Ignore. pass +def test_envs_fcst_var_stat(): + """ + Verify that the environment vars used for fcst_var_val1/2 and stat1/2 in + a config file creates the same data for plotting as a config file with the + fcst_var_val1 and stat hard-coded in a config file. + + """ + # Set up the METPLOTPY_BASE so that met_plot.py will correctly find + # the config directory containing all the default config files. + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/env_fcst_var.yaml" + + # Invoke the command to generate a line plot based on + # the custom config file. + os.environ['FCST_VAR_VAL1'] = "RH" + os.environ['FCST_VAR_STAT1'] = 'MAE' + os.environ['FCST_VAR_VAL2'] = "TMP" + os.environ['FCST_VAR_STAT2'] = 'ME' + + l.main(custom_config_filename) + + expected_points1 = f"{cwd}/intermed_files/expected_var_stat_line.points1" + expected_points2 = f"{cwd}/intermed_files/expected_var_stat_line.points2" + expected_df1 = pd.read_csv(f'{expected_points1}', + sep="\t", header=None) + expected_df2 = pd.read_csv( + f'{expected_points2}', sep="\t", header=None + ) + num_expected1_rows= expected_df1.shape[0] + num_expected1_cols = expected_df1.shape[1] + num_expected2_rows = expected_df2.shape[0] + num_expected2_cols = expected_df2.shape[1] + + + try: + + # Retrieve the .points1 files generated by METplotpy respectively + mpp_df1 = pd.read_csv(f'{cwd}/intermed_files/env_fcst_var_stat/line.points1', + sep="\t", header=None) + mpp_df2 = pd.read_csv( + f'{cwd}/intermed_files/env_fcst_var_stat/line.points2', sep="\t", header=None + ) + + # Verify that the values in the generated points1/2 files are identical + # to those in the rh_mae_tmp_me_line.points1/2 files. + + # First, verify that the points1 files have the same shape + num_mpp1_rows = mpp_df1.shape[0] + num_mpp1_cols = mpp_df1.shape[1] + num_mpp2_rows = mpp_df2.shape[0] + num_mpp2_cols = mpp_df2.shape[1] + + assert num_expected1_rows == num_mpp1_rows + assert num_expected1_cols == num_mpp1_cols + assert num_expected2_rows == num_mpp2_rows + assert num_expected2_cols == num_mpp2_cols + + assert mpp_df1.equals(expected_df1) + assert mpp_df2.equals(expected_df2) + + + + # Clean up the fbias.points1, fbias.points2, and .png files + os.remove(f'{cwd}/intermed_files/env_fcst_var_stat/line.points1') + os.remove(f'{cwd}/intermed_files/env_fcst_var_stat/line.points2') + os.rmdir(f'{cwd}/intermed_files/env_fcst_var_stat') + + except OSError as e: + # Typically when files have already been removed or + # don't exist. Ignore. + pass + @pytest.mark.skip("Image comparison for development only due to differences in hosts") def test_fixed_var_val_image_compare(): From 29d90290e6d21e99d27642bdc6faf162219a698f Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 18 Aug 2025 14:07:56 -0600 Subject: [PATCH 5/5] Use the get_params from metplotpy.plots.util module --- metplotpy/plots/scatter/scatter.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/metplotpy/plots/scatter/scatter.py b/metplotpy/plots/scatter/scatter.py index 40d8b9a6b..2f13b5888 100644 --- a/metplotpy/plots/scatter/scatter.py +++ b/metplotpy/plots/scatter/scatter.py @@ -21,6 +21,8 @@ from metplotpy.plots.base_plot import BasePlot from metplotpy.plots.scatter.scatter_config import ScatterConfig from metplotpy.plots import util +from metplotpy.plots.util import get_params +from metcalcpy.util.read_env_vars_in_config import parse_config class Scatter(BasePlot): """ @@ -199,16 +201,7 @@ def main(config_filename=None): Returns: None """ - - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) + docs = get_params(config_filename) try: plot = Scatter(docs)