Feature #3304 power_spectra

.github/jobs/set_job_controls.sh

@@ -6,6 +6,7 @@
 run_diff=false
 run_update_truth=false
 met_base_repo=met-base
+# TODO: Change 3.4 back to 3.5 after solving TC-Diag failure
 met_base_tag=3.4-latest
 input_data_version=develop
 truth_data_version=develop

.github/workflows/testing.yml

@@ -148,7 +148,7 @@ jobs:
           - jobid: 'job1' 
             tests: 'ascii2nc'
           - jobid: 'job2'
-            tests: 'pb2nc madis2nc pcp_combine gen_ens_prod regrid'
+            tests: 'pb2nc madis2nc pcp_combine gen_ens_prod regrid winds'
       fail-fast: false
     steps:
       - uses: actions/checkout@v6

data/config/ConfigConstants

@@ -22,6 +22,18 @@ output_precision = 5;
 // MET_TMP_DIR.
 tmp_dir = "/tmp";
 
+////////////////////////////////////////////////////////////////////////////////
+//
+// Settings to define wind vector metadata.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+// Comma-separated lists of wind variable names for rotating and deriving
+u_wind_field_name = "UGRD,U";
+v_wind_field_name = "VGRD,V";
+wind_speed_field_name = "WIND,SP";
+wind_direction_field_name = "WDIR,DD";
+
 ////////////////////////////////////////////////////////////////////////////////
 //
 // Standard unit conversion functions.
@@ -202,3 +214,7 @@ CLIMO_STD_ANOM = 3;
 FCST_ANOM      = 4;
 FCST_STD_ANOM  = 5;
 
+// Power spectrum missing flag types
+NONE  = 1;
+MEAN  = 2;
+VALUE = 3;

data/config/GridDiagConfig_default

@@ -32,6 +32,15 @@ regrid = {
 censor_thresh = [];
 censor_val    = [];
 
+//
+// Specify the handling of missing data for power spectra
+//
+power_spectrum = {
+   missing_flag  = NONE;
+   missing_value = 0.0;
+   vld_thresh    = 0.5;
+}
+
 //
 // Data fields
 //
@@ -68,9 +77,10 @@ mask = {
 // Diagnostic output types
 //
 output_flag = {
-   histogram_1d = TRUE;
-   histogram_2d = TRUE;
-   info_theory  = FALSE;
+   histogram_1d   = TRUE;
+   histogram_2d   = TRUE;
+   info_theory    = FALSE;
+   power_spectrum = FALSE;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

docs/Users_Guide/config_options.rst

@@ -661,6 +661,41 @@ Some tools override the temporary directory by the command line argument
 A description of the use of temporary files in MET can be found in
 :numref:`Contributor's Guide Section %s <tmp_file_use>`.
 
+.. _config_wind_field_names:
+
+wind_field_names
+----------------
+
+As described in :numref:`PS_wind_rotation_derivation`, MET can automatically
+derive wind variables and also rotate them from being grid-relative to earth-relative.
+Logic is provided for each supported input file type to identify input wind components
+and determine whether they are defined as being grid-relative or earth-relative.
+
+The following configuration options can be set to refine the metadata, when needed:
+
+.. code-block:: none
+
+  is_u_wind         = boolean;
+  is_v_wind         = boolean;
+  is_grid_relative  = boolean;
+  is_wind_speed     = boolean;
+  is_wind_direction = boolean;
+  is_kinetic_energy = boolean;
+
+When deriving and rotating winds, MET searches the input file for matching wind
+components. That search is driven by the following configuration options:
+
+.. code-block:: none
+
+  u_wind_field_name = "UGRD,U";
+  v_wind_field_name = "VGRD,V";
+  wind_speed_field_name = "WIND,SP";
+  wind_direction_field_name = "WDIR,DD";
+
+Each is a comma-separated list of wind variable names to be searched. Users can
+explicity set these options to configure what data should be used in the wind
+derivation and rotation logic.
+
 message_type_group_map
 ----------------------
 
@@ -1163,6 +1198,7 @@ below:
   is_grid_relative     = boolean;
   is_wind_speed        = boolean;
   is_wind_direction    = boolean;
+  is_kinetic_energy    = boolean;
   is_prob              = boolean;
 
 

docs/Users_Guide/grid-diag.rst

@@ -7,9 +7,13 @@ Grid-Diag Tool
 Introduction
 ============
 
-The Grid-Diag tool creates histograms (probability distributions when normalized) for an arbitrary collection of data fields and levels. Joint histograms are created for all possible pairs of variables. If no masking region is specified to subset the data fields spatially, then all points in the input domain are used. However, an arbitrary number of masking regions can be specified and output is created for each one. The histograms are accumulated over all of the input data files. Typically this tool is run with a time series of input data files, similar to Series-Analysis.
+Unlike the Grid-Stat and Point-Stat tools, which compute verification statistics for matched pairs of forecast and observation values, the Grid-Diag tool provides diagnostic information based on the distribution of values found in gridded input files.
 
-The Grid-Diag tool also uses the histograms to derive information theory statistics. Entropy is derived from each 1-dimensional histogram, and joint entropy and mutual information are derived from each 2-dimensional joint histogram. These statistics are defined using log base 2, rather than the natural logarithm which is also commonly used.
+Grid-Diag creates histograms (probability distributions when normalized) for an arbitrary collection of data fields and levels. Joint histograms are created for all possible pairs of variables. If no masking region is specified to subset the data fields spatially, then all points in the input domain are used. However, an arbitrary number of masking regions can be specified and output is created for each one. The histograms are accumulated over all of the input data files. Typically this tool is run with a time series of input data files, similar to Series-Analysis.
+
+Grid-Diag also uses the histograms to derive information theory statistics. Entropy is derived from each 1-dimensional histogram, and joint entropy and mutual information are derived from each 2-dimensional joint histogram. These statistics are defined using log base 2, rather than the natural logarithm which is also commonly used.
+
+Finally, Grid-Diag also derives power spectra for each input field. However, power spectra are computed over the full model domain rather than being subset by spatial masking regions. Note that special logic is applied when processing kinetic energy. Grid-Diag reads the corresponding U and V-wind vector components and applies a discrete cosine transform to them separately prior to copmuting the power spectrum.
 
 Practical Information
 =====================
@@ -77,6 +81,20 @@ The configuration options listed above are common to many MET tools and are desc
 
 _____________________
 
+.. code-block:: none
+
+  power_spectrum = {
+     missing_flag  = NONE;
+     missing_value = 0.0;
+     vld_thresh    = 0.5;
+  }
+
+The **power_spectrum** dictionary defines options for computing power spectra and can be specified separately for each **data.field** entry below.
+
+The **missing_flag** and **missing_value** entries define how bad data values should be handled. For all other output types, bad data values are ignored but they are problematic for power spectra. Set **missing_flag** to **NONE** (default) to skip power spectrum when bad data is present, to **MEAN** to replace bad data with the mean of each input field, or to **VALUE** to replace bad data with the constant numeric value specified by **missing_value**. Set **vld_thresh** to a number beween 0 and 1 to define the required ratio of valid data to be present to compute power spectra output for that field.
+
+_____________________
+
 .. code-block:: none
 
   data = {
@@ -105,9 +123,10 @@ _____________________
 .. code-block:: none
 
    output_flag = {
-      histogram_1d = TRUE;
-      histogram_2d = TRUE;
-      info_theory  = FALSE;
+      histogram_1d   = TRUE;
+      histogram_2d   = TRUE;
+      info_theory    = FALSE;
+      power_spectrum = FALSE;
    }
 
 The **output_flag** dictionary controls the type of output that the Grid-Diag tool generates. Each flag should be set to **TRUE** or **FALSE** to enable the computation and writing of one or more variables to the output NetCDF file, as described below:
@@ -118,6 +137,8 @@ The **output_flag** dictionary controls the type of output that the Grid-Diag to
 
 3. **info_theory** for information theory metrics, including entropy for each **data.field** entry and mutual information and joint entropy for each pair of entries.
 
+4. **power_spectrum** for power spectrum output.
+
 grid_diag Output File
 ---------------------
 

docs/Users_Guide/point-stat.rst

@@ -117,6 +117,33 @@ _______________________
 
 The forecast value at P is chosen as the grid point inside the interpolation area whose value most closely matches the observation value.
 
+.. _PS_wind_rotation_derivation:
+
+Wind Rotation and Derivation
+----------------------------
+
+Numerical weather prediction model output often defines winds relative to the orientation of the model grid rather than true north-south and east-west directions on the earth.
+However point observations typically define winds relative to true earth directions. Prior to comparing them, the model wind data must be rotated from grid-relative to
+to earth-relative. While the degree of grid-to-earth rotation varies based on the projection type and location, failing to rotate the winds has a significant impact on
+the verification results.
+
+For simplicity, the MET library code attempts to rotate all wind components from being grid-relative to earth-relative regardless of whether they are being compared to point
+observations or gridded analyses. Running the MET tools at verbosity level 3 (-v 3) prints log messages to describing the wind rotation process.
+
+Wind fields which must be rotated include wind direction and both the U and V components of the wind vector. While wind direction can be rotated by itself, both U and V are
+required to rotate either component. When processing U-wind data, MET attempts to read corresponding V-wind data and vice-versa.
+
+The configuration options for wind rotation and derivation in MET are described in section :numref:`config_wind_field_names`.
+
+When reading V-wind data to rotate U-wind or U-wind data to rotate V-wind, MET first searches using the same field name, but with both upper and lowercase U's and V's
+swapped (e.g. for "U_PL" search for "V_PL"). If the result is unsuccesful, it searches other common field names specified by the **u_wind_field_name** and **v_wind_field_name**
+configuration options. If needed, users should set these configuration options to indicate how the U-wind and V-wind data should be paired.
+
+In addition to rotating winds, MET can also derive them. If U-wind and V-wind are present in the input file, request field names of **WDIR**, **WIND**, or **KENG**
+to derive wind direction, wind speed, and kinetic engery from the U and V components, respectively. If wind speed and direction are present in the input file, request field
+names of **UGRD** or **VGRD** for MET to derive the U and V components from them. Depending on the wind field naming conventions, the configuration options described in
+:numref:`config_wind_field_names` may be required to configure this derivation logic.
+
 .. _PS_HiRA_framework:
 
 HiRA Framework

docs/Users_Guide/release-notes.rst

@@ -182,6 +182,14 @@ MET Version 13.0.0 Upgrade Instructions
      * The "ensemble_flag.eas" and "ensemble_flag.eas_width" entries are added to enable the
        writing of EAS output fields.
 
+   * ConfigConstanst configuration file
+
+     * The "u_wind_field_name", "v_wind_field_name", "wind_speed_field_name", and "wind_direction_field_name"
+       entries are comma-separated lists of field names to search when read wind data needed to derive winds
+       or rotate them from being grid-relative to earth-relative. See :numref:`PS_wind_rotation_derivation`
+       for a description of wind rotation and derivation and :numref:`config_wind_field_names` for the
+       corresponding configuration options.
+
 .. dropdown:: Output format changes - NONE
 
 .. dropdown:: Output format changes - NONE

internal/test_unit/bin/unit_test.sh

@@ -40,6 +40,7 @@ UNIT_XML="unit_ascii2nc.xml \
           unit_gen_ens_prod.xml \
           unit_pcp_combine.xml \
           unit_regrid.xml \
+          unit_winds.xml \
           unit_wwmca_regrid.xml \
           unit_point_stat.xml \
           unit_stat_analysis_ps.xml \

internal/test_unit/config/GridDiagConfig

@@ -32,6 +32,15 @@ regrid = {
 censor_thresh = [];
 censor_val    = [];
 
+//
+// Specify the handling of missing data for power spectra
+//
+power_spectrum = {
+   missing_flag  = NONE;
+   missing_value = 0.0;
+   vld_thresh    = 0.5;
+}
+
 //
 // Data fields
 //

internal/test_unit/config/GridDiagConfig_APCP_06_FCST_OBS → internal/test_unit/config/GridDiagConfig_FCST_OBS

@@ -32,15 +32,19 @@ regrid = {
 censor_thresh = [];
 censor_val    = [];
 
+//
+// Specify the handling of missing data for power spectra
+//
+power_spectrum = {
+   missing_flag  = VALUE;
+   missing_value = 0.0;
+   vld_thresh    = 0.5;
+}
+
 //
 // Data fields
 //
 data = {
-   name   = "APCP";
-   level  = ["A06"];
-   n_bins = 50;
-   range  = [0,25];
-
    field = [
       { ${FIELD1} },
       { ${FIELD2} }
@@ -62,10 +66,6 @@ mask = {
 //
 // Diagnostic output types
 //
-output_flag = {
-   histogram_1d = TRUE;
-   histogram_2d = TRUE;
-   info_theory  = FALSE;
-}
+output_flag = TRUE;
 
 ////////////////////////////////////////////////////////////////////////////////

internal/test_unit/config/GridDiagConfig_TMP

@@ -32,6 +32,15 @@ regrid = {
 censor_thresh = [];
 censor_val    = [];
 
+//
+// Specify the handling of missing data for power spectra
+//
+power_spectrum = {
+   missing_flag  = NONE;
+   missing_value = 0.0;
+   vld_thresh    = 0.5;
+}
+
 //
 // Data fields
 //
@@ -74,9 +83,10 @@ mask = {
 // Diagnostic output types
 //
 output_flag = {
-   histogram_1d = TRUE;
-   histogram_2d = TRUE;
-   info_theory  = FALSE;
+   histogram_1d   = TRUE;
+   histogram_2d   = TRUE;
+   info_theory    = FALSE;
+   power_spectrum = FALSE;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

internal/test_unit/xml/unit_grib_tables.xml

@@ -36,8 +36,8 @@
         <param> \
             &DATA_DIR_MODEL;/grib2/um_raw/PS.557WW_SC.U_DI.F_GP.GALWEM-GD_GR.C17KM_AR.GLOBAL_DD.20160424_CY.00_FH.000_DF.GR2 \
             &OUTPUT_DIR;/grib_tables/um_raw/PS.557WW_SC.U_DI.F_GP.GALWEM-GD_GR.C17KM_AR.GLOBAL_DD.20160424_CY.00_FH.000_DF.ps \
-            'name = "TCDC"; level = "L0";' \
-            -title "GRIB2 um_raw L0 TCDC" \
+            'name = "TCC"; level = "L0";' \
+            -title "GRIB2 um_raw L0 TCC" \
             -v 1
         </param>
         <output>