Feature 539 env support

metplotpy/plots/line/line.py

@@ -767,8 +767,12 @@ def write_output_file(self) -> None:
                 filename = self.config_obj.points_path + os.path.sep + filename
 
             # save points
-            self._save_points(all_points_1, filename + ".points1")
-            self._save_points(all_points_2, filename + ".points2")
+            points_path = self.config_obj.points_path
+            os.makedirs(points_path, exist_ok=True)
+            # self._save_points(all_points_1, filename + ".points1")
+            self._save_points(all_points_1, os.path.join(points_path, filename + ".points1"))
+            # self._save_points(all_points_2, filename + ".points2")
+            self._save_points(all_points_2, os.path.join(points_path, filename + ".points2"))
 
             self.logger.info(f"Finished writing to output file: {datetime.now()}")
 

metplotpy/plots/scatter/scatter.py

@@ -21,6 +21,8 @@
 from metplotpy.plots.base_plot import BasePlot
 from metplotpy.plots.scatter.scatter_config import ScatterConfig
 from metplotpy.plots import util
+from metplotpy.plots.util import get_params
+from metcalcpy.util.read_env_vars_in_config import  parse_config
 
 class Scatter(BasePlot):
     """
@@ -199,16 +201,7 @@ def main(config_filename=None):
 
        Returns: None
     """
-
-    if not config_filename:
-        config_file = util.read_config_from_command_line()
-    else:
-        config_file = config_filename
-    with open(config_file, 'r') as stream:
-        try:
-            docs = yaml.load(stream, Loader=yaml.FullLoader)
-        except yaml.YAMLError as exc:
-            print(exc)
+    docs = get_params(config_filename)
 
     try:
         plot = Scatter(docs)

test/line/env_fcst_var.yaml

@@ -0,0 +1,231 @@
+alpha: 0.05
+box_avg: 'False'
+box_boxwex: 0.2
+box_notch: 'False'
+box_outline: 'True'
+box_pts: 'False'
+caption_align: 0.0
+caption_col: '#333333'
+caption_offset: 3.0
+caption_size: 0.8
+caption_weight: 1
+cex: 1
+colors:
+- '#ff0000'
+- '#55ff00'
+- '#00aaff'
+- '#0040ff'
+- '#c3ff00'
+con_series:
+- 1
+- 1
+- 1
+- 1
+- 1
+create_html: 'False'
+derived_series_1:
+- - CONTROL RH MAE
+  - GTS RH MAE
+  - DIFF
+derived_series_2: []
+dump_points_1: 'True'
+dump_points_2: 'True'
+event_equal: 'True'
+fcst_var_val_1:
+  !ENV ${FCST_VAR_VAL1}:
+    - !ENV ${FCST_VAR_STAT1}
+fcst_var_val_2:
+  !ENV ${FCST_VAR_VAL2}:
+   - !ENV ${FCST_VAR_STAT2}
+fixed_vars_vals_input:
+  fcst_lev:
+    - Z02
+grid_col: '#cccccc'
+grid_lty: 3
+grid_lwd: 1
+grid_on: 'True'
+grid_x: listX
+indy_label:
+- '6'
+- '12'
+- '18'
+indy_stagger_1: 'True'
+indy_stagger_2: 'False'
+indy_vals:
+- '60000'
+- '120000'
+- '180000'
+indy_var: fcst_lead
+legend_box: o
+legend_inset:
+  x: 0.0
+  y: -0.25
+legend_ncol: 3
+legend_size: 0.8
+line_type: None
+list_stat_1:
+- MAE
+list_stat_2:
+- ME
+list_static_val:
+  fcst_var: ME
+mar:
+- 8
+- 4
+- 5
+- 4
+method: bca
+mgp:
+- 1
+- 1
+- 0
+num_iterations: 1
+num_threads: -1
+plot_caption: Caption
+plot_ci:
+- std
+- std
+- std
+- std
+- std
+plot_disp:
+- 'True'
+- 'True'
+- 'True'
+- 'True'
+- 'True'
+plot_filename: !ENV '${TEST_DIR}/env_fcst_var_stat_line.png'
+plot_height: 8.5
+plot_res: 72
+plot_stat: median
+plot_type: png16m
+plot_units: in
+plot_width: 11.0
+
+# Optional, uncomment and set to directory to store the .points1 file
+# that is used by METviewer (created when dump_points_1 is set to True)
+# if dump_points_1 is True and this is uncommented, the points1 file
+# will be saved in the default location (i.e. where the input data file is stored).
+points_path:  !ENV '${TEST_DIR}/intermed_files/env_fcst_var_stat'
+random_seed: null
+series_line_style:
+- '-'
+- '-'
+- '-'
+- '-'
+- '-'
+series_line_width:
+- 1
+- 1
+- 1
+- 1
+- 1
+series_order:
+- 1
+- 2
+- 3
+- 4
+- 5
+series_symbols:
+- .
+- .
+- .
+- .
+- .
+series_type:
+- b
+- b
+- b
+- b
+- b
+series_val_1:
+  model:
+  - CONTROL
+  - GTS
+series_val_2:
+  model:
+  - CONTROL
+  - GTS
+show_nstats: 'True'
+show_signif:
+- 'False'
+- 'False'
+- 'False'
+- 'False'
+- 'False'
+stat_input: !ENV '${TEST_DIR}/line.data'
+sync_yaxes: 'False'
+title: test title
+title_align: 0.5
+title_offset: -2
+title_size: 1.4
+title_weight: 2.0
+user_legend: []
+variance_inflation_factor: 'False'
+vert_plot: 'False'
+x2lab_align: 0.5
+x2lab_offset: -0.5
+x2lab_size: 0.8
+x2lab_weight: 1
+x2tlab_horiz: 0.5
+x2tlab_orient: 1
+x2tlab_perp: 1
+x2tlab_size: 0.8
+xaxis: test x_label
+xaxis_reverse: 'False'
+xlab_align: 0.5
+xlab_offset: 2
+xlab_size: 1
+xlab_weight: 1
+xlim: []
+xtlab_decim: 0
+xtlab_horiz: 0.5
+xtlab_orient: 1
+xtlab_perp: -0.75
+xtlab_size: 1
+y2lab_align: 0.5
+y2lab_offset: 1
+y2lab_size: 1
+y2lab_weight: 1
+y2lim: []
+y2tlab_horiz: 0.5
+y2tlab_orient: 1
+y2tlab_perp: 1
+y2tlab_size: 1.0
+yaxis_1: test y_label
+yaxis_2: ''
+ylab_align: 0.5
+ylab_offset: -2
+ylab_size: 1
+ylab_weight: 1
+ylim: []
+ytlab_horiz: 0.5
+ytlab_orient: 1
+ytlab_perp: 0.5
+ytlab_size: 1
+
+lines:
+- color: '#8000ff'
+  line_width: '2'
+  position: '11'
+  type: horiz_line
+  line_style: '--'
+- line_style: '-'
+  color: '#000000'
+  line_width: 1
+  position: "18"
+  type: vert_line
+
+# To save your log output to a file, specify a path and filename and uncomment the line below.  Make sure you have
+# permissions to the directory you specify.  The default, as specified in the default config file is stdout.
+#log_filename: ./line.log
+
+# To change the log level, specify a log level: debug, info, warning, error and uncomment the line below.
+#  Debug and info log level will produce more log output.
+#log_level: WARNING
+show_legend:
+- 'True'
+- 'True'
+- 'True'
+- 'True'
+- 'True'

test/line/intermed_files/expected_var_stat_line.points1

@@ -0,0 +1,3 @@
+14.714440 14.425071 15.003809 14.442990 14.182521 14.703459 0.216470 0.121860 0.311080
+14.615340 14.336420 14.894260 14.645345 14.338789 14.951901 0.022085 -0.056724 0.100894
+14.858610 14.548506 15.168714 14.981590 14.720549 15.242631 -0.076490 -0.170358 0.017378

test/line/intermed_files/expected_var_stat_line.points2

@@ -0,0 +1,3 @@
+-1.311070 -1.454800 -1.167340 -1.262500 -1.414032 -1.110968
+-1.378120 -1.484110 -1.272130 -1.333030 -1.454853 -1.211207
+-1.371200 -1.520357 -1.222043 -1.398450 -1.545732 -1.251168

test/line/test_line_plot.py

@@ -104,7 +104,7 @@ def test_no_nans_in_points_files():
     custom_config_filename = f"{cwd}/custom_line2.yaml"
     l.main(custom_config_filename)
 
-    # Check for NaN's in the intermediate files, line.points1 and line.points2
+    # Check for NaN's in the intermediate files,_line.points1 and line.points2
     # Fail if there are any NaN's-this indicates something went wrong with the
     # line_series.py module's  _create_series_points() method.
     nans_found = False
@@ -322,6 +322,79 @@ def test_fixed_var_val():
         # don't exist.  Ignore.
         pass
 
+def test_envs_fcst_var_stat():
+    """
+        Verify that the environment vars used for fcst_var_val1/2 and stat1/2 in
+        a config file creates the same data for plotting as a config file with the
+         fcst_var_val1 and stat hard-coded in a config file.
+
+    """
+    # Set up the METPLOTPY_BASE so that met_plot.py will correctly find
+    # the config directory containing all the default config files.
+    os.environ['METPLOTPY_BASE'] = f"{cwd}/../../"
+    os.environ['TEST_DIR'] = cwd
+    custom_config_filename = f"{cwd}/env_fcst_var.yaml"
+
+    # Invoke the command to generate a line plot based on
+    # the custom config file.
+    os.environ['FCST_VAR_VAL1'] = "RH"
+    os.environ['FCST_VAR_STAT1'] = 'MAE'
+    os.environ['FCST_VAR_VAL2'] = "TMP"
+    os.environ['FCST_VAR_STAT2'] = 'ME'
+
+    l.main(custom_config_filename)
+
+    expected_points1 = f"{cwd}/intermed_files/expected_var_stat_line.points1"
+    expected_points2 = f"{cwd}/intermed_files/expected_var_stat_line.points2"
+    expected_df1 = pd.read_csv(f'{expected_points1}',
+                            sep="\t", header=None)
+    expected_df2 = pd.read_csv(
+        f'{expected_points2}', sep="\t", header=None
+        )
+    num_expected1_rows= expected_df1.shape[0]
+    num_expected1_cols = expected_df1.shape[1]
+    num_expected2_rows = expected_df2.shape[0]
+    num_expected2_cols = expected_df2.shape[1]
+
+
+    try:
+
+        # Retrieve the .points1 files generated by METplotpy respectively
+        mpp_df1 = pd.read_csv(f'{cwd}/intermed_files/env_fcst_var_stat/line.points1',
+                            sep="\t", header=None)
+        mpp_df2 = pd.read_csv(
+            f'{cwd}/intermed_files/env_fcst_var_stat/line.points2', sep="\t", header=None
+            )
+
+        # Verify that the values in the generated points1/2 files are identical
+        # to those in the rh_mae_tmp_me_line.points1/2 files.
+
+        # First, verify that the points1 files have the same shape
+        num_mpp1_rows = mpp_df1.shape[0]
+        num_mpp1_cols = mpp_df1.shape[1]
+        num_mpp2_rows = mpp_df2.shape[0]
+        num_mpp2_cols = mpp_df2.shape[1]
+
+        assert num_expected1_rows == num_mpp1_rows
+        assert num_expected1_cols == num_mpp1_cols
+        assert num_expected2_rows == num_mpp2_rows
+        assert num_expected2_cols == num_mpp2_cols
+
+        assert mpp_df1.equals(expected_df1)
+        assert mpp_df2.equals(expected_df2)
+
+
+
+        # Clean up the fbias.points1,  fbias.points2, and .png files
+        os.remove(f'{cwd}/intermed_files/env_fcst_var_stat/line.points1')
+        os.remove(f'{cwd}/intermed_files/env_fcst_var_stat/line.points2')
+        os.rmdir(f'{cwd}/intermed_files/env_fcst_var_stat')
+
+    except OSError as e:
+        # Typically when files have already been removed or
+        # don't exist.  Ignore.
+        pass
+
 
 @pytest.mark.skip("Image comparison for development only due to differences in hosts")
 def test_fixed_var_val_image_compare():