From 781957f55f76fb9997ab838fb17212fc37365496 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Fri, 22 Aug 2025 16:51:07 +0000 Subject: [PATCH 01/98] Pkg by pkg --- environment.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/environment.yml b/environment.yml index cef291f..3dddd5c 100644 --- a/environment.yml +++ b/environment.yml @@ -1,16 +1,20 @@ name: pygraf channels: - conda-forge - - defaults + - nodefaults dependencies: - - python=3.7* - - basemap=1.2* - - basemap-data-hires=1.2* + - python + - basemap + - basemap-data-hires - pynio=1.5.5 - - matplotlib=3.2* + - notebook + - matplotlib - metpy=0.12.1 - - pylint=2.4* - - pytest=6.1* - - pyyaml=5.3* + - numpy=1.21.* + - pint=0.10.* + - pylint + - pytest + - pyyaml + - setuptools=59.8.* - xarray=0.15* - dask From cc75b2c806fd49954e2ceb1472955a8a9326edfc Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 15 Oct 2025 14:44:32 +0000 Subject: [PATCH 02/98] Adding cfgrib entries to default_specs.yml --- adb_graphics/default_specs.yml | 187 ++++++++++++++++++++++++++++++++- 1 file changed, 185 insertions(+), 2 deletions(-) diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 6ccfa36..bad5a86 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -79,6 +79,10 @@ 1hsnw: # 1 hr Accumulated Snow Using 10:1 Ratio sfc: + cfgrib: + shortName: sdwe + level: 0 + typeOfLevel: surface clevs: [0.03, 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 6, 7, 8] cmap: gist_ncar colors: snow_colors @@ -94,6 +98,9 @@ unit: in 1ref: # Reflectivity at 1 km AGL 1000m: &refl + cfgrib: + shortName: rare + typeOfLevel: heightAboveGround clevs: !!python/object/apply:numpy.arange [5, 76, 5] cmap: NWSReflectivity colors: cref_colors @@ -114,10 +121,22 @@ acfrozr: # Run Total Graupel acfrzr: # Run Total Freezing Rain sfc: <<: *graupel + cfgrib: + shortName: frzr + level: 0 + typeOfLevel: surface + stepType: accum + stepRange: # startswith("0-") ncl_name: FRZR_P8_L1_GLC0_acc title: Run Total Freezing Rain acpcp: # Accumulated run total precipitation sfc: + cfgrib: + shortName: tp + level: 0 + typeOfLevel: surface + stepRange: # startswith("0-") + stepType: accum clevs: [0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 15, 20, 40] cmap: gist_ncar colors: rainbow12_colors @@ -220,6 +239,11 @@ acpcpens6: # ensemble probability of precipitation, 6h title: Probability of 6-hr Precipitation >= 0.5 in within 40km acsnod: # Accumulated snow sfc: &snow + cfgrib: + level: 0 + typeOfLevel: surface + stepType: accum + parameterName: Total snowfall clevs: [0.01, 0.1, 1, 2, 3, 4, 6, 8, 10, 12, 18, 24] cmap: gist_ncar colors: snow_colors @@ -230,6 +254,10 @@ acsnod: # Accumulated snow unit: in acsnw: # Run Total Accumulated Snow Using 10:1 Ratio sfc: + cfgrib: + shortName: sdwe + level: 0 + typeOfLevel: surface clevs: [0.01, 0.1, 1, 2, 3, 4, 6, 8, 10, 12, 18, 24] cmap: gist_ncar colors: snow_colors @@ -311,6 +339,10 @@ bc2: # Black Carbon 2 transform: [] cape: mu: &cape # Most Unstable CAPE + cfgrib: + shortName: cape + level: 18000 + typeOfLevel: pressureFromGroundLayer clevs: [1, 100, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000] cmap: gist_ncar colors: vort_colors @@ -335,6 +367,10 @@ cape: vertical_index: 2 mul: # Most Unstable Layer CAPE <<: *cape + cfgrib: + shortName: cape + level: 18000 + typeOfLevel: pressureFromGroundLayer contours: cape: colors: white @@ -354,6 +390,10 @@ cape: vertical_index: 1 mx90mb: # Lowest 90 mb Mixed Layer CAPE <<: *cape + cfgrib: + shortName: cape + level: 9000 + typeOfLevel: pressureFromGroundLayer contours: cape: colors: white @@ -373,6 +413,10 @@ cape: title: Lowest 90 mb Mixed Layer CAPE sfc: <<: *cape + cfgrib: + shortName: cape + level: 0 + typeOfLevel: surface contours: cin: colors: 'k' @@ -387,14 +431,26 @@ cape: title: Surface CAPE cell: # Storm cell motion ua: + cfgrib: + shortName: ustm + typeOfLevel: heightAboveGroundLayer + level: 6000 ncl_name: USTM_P0_2L103_{grid} transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - field2: VSTM_P0_2L103_{grid} + field2: + cfgrib: + shortName: ustm + typeOfLevel: heightAboveGroundLayer + level: 6000 unit: kt ceil: # Ceiling ua: &ceil + cfgrib: + shortName: gh + typeOfLevel: cloudCeiling + level: 0 clevs: [0, 0.1, 0.3, 0.5, 1, 2, 3, 5, 10, 15, 20, 30, 52] cmap: gist_ncar colors: ceil_colors @@ -410,12 +466,20 @@ ceil: # Ceiling ceilexp: # Ceiling - experimental ua: <<: *ceil + cfgrib: + shortName: ceil + typeOfLevel: cloudCeiling + level: 0 ncl_name: regional_mpas: CEIL_P0_L215_{grid} title: Ceiling (exp) ceilexp2: # Ceiling - experimental no.2 ua: <<: *ceil + cfgrib: + shortName: gh + typeOfLevel: cloudBase + level: 0 ncl_name: # Note that the "HGT_P0_L2_{grid}" ncl_names in RAPv5/HRRRv4 (below) # seemingly correspond to cloud-base height. This is intentional; i.e., @@ -431,18 +495,33 @@ ceilexp2: # Ceiling - experimental no.2 cloudbase: # Cloud-base height ua: <<: *ceil + cfgrib: + shortName: gh + typeOfLevel: cloudBase + level: 0 ncl_name: rrfs: HGT_P0_L2_{grid} regional_mpas: HGT_P0_L2_{grid} title: Cloud-Base Height cfrzr: # Categorical Freezing Rain sfc: + cfgrib: + shortName: cfrzr + typeOfLevel: surface ncl_name: CFRZR_P0_L1_{grid} cicep: # Categorical Ice Pellets sfc: + cfgrib: + shortName: cicep + typeOfLevel: surface + stepType: accum ncl_name: CICEP_P0_L1_{grid} cin: # Surface Convective Inhibition mu: + cfgrib: + shortName: cin + level: 25500 + typeOfLevel: pressureFromGroundLayer clevs: [-300, -200, -150, -100, -75, -50, -40, -30, -20, -10, -1] cmap: gist_ncar colors: cin_colors @@ -450,10 +529,18 @@ cin: # Surface Convective Inhibition unit: J/kg vertical_index: 2 mx90mb: # Lowest 90 mb Mixed Layer CIN + cfgrib: + shortName: cin + level: 9000 + typeOfLevel: pressureFromGroundLayer ncl_name: CIN_P0_2L108_{grid} title: 'ML CIN < -50' vertical_index: 0 sfc: &sfc_cin + cfgrib: + shortName: cin + level: 0 + typeOfLevel: surface clevs: [-300, -200, -150, -100, -75, -50, -40, -30, -20, -10, -1] cmap: gist_ncar colors: cin_colors @@ -467,6 +554,10 @@ cin: # Surface Convective Inhibition title: Surface CIN < -50 cloudcover: bndylay: &cld_cover # PBL ... 1 km Cloud Cover + cfgrib: + shortName: tcc + typeOfLevel: boundaryLayerCloudLayer + level: 0 clevs: [2, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95] cmap: gist_ncar colors: cldcov_colors @@ -476,6 +567,10 @@ cloudcover: unit: '%' high: <<: *cld_cover + cfgrib: + shortName: hcc + typeOfLevel: highCloudLayer + level: 0 clevs: [2, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95] cmap: gist_ncar colors: cldcov_colors @@ -485,18 +580,34 @@ cloudcover: unit: '%' low: <<: *cld_cover + cfgrib: + shortName: lcc + typeOfLevel: lowCloudLayer + level: 0 ncl_name: LCDC_P0_L214_{grid} title: Low-Level Cloud Cover mid: <<: *cld_cover + cfgrib: + shortName: mcc + typeOfLevel: middleCloudLayer + level: 0 ncl_name: MCDC_P0_L224_{grid} title: Mid-Level Cloud Cover total: <<: *cld_cover + cfgrib: + shortName: tcc + typeOfLevel: atmosphereSingleLayer + stepType: instant ncl_name: TCDC_P0_L{level_type}_{grid} title: Total Cloud Cover clwmr: # Cloud water Mixing Ratio ua: + cfgrib: + prs: + shortName: clwmr + typeOfLevel: isobaricInhPa ncl_name: nat: CLWMR_P0_L105_{grid} prs: CLWMR_P0_L100_{grid} @@ -587,6 +698,9 @@ cpofp: # Frozen Precipitation Percentage unit: '%' crain: # Categorical Rain sfc: + cfgrib: + shortName: crain + typeOfLevel: surface ncl_name: CRAIN_P0_L1_{grid} cref: # Composite reflectivity esbl: @@ -615,9 +729,15 @@ crefmax: # Comp reflectivity (max over forecast) unit: dBZ csnow: # Categorical Snow sfc: + cfgrib: + shortName: csnow + typeOfLevel: surface ncl_name: CSNOW_P0_L1_{grid} ctop: # Cloud top height ua: + cfgrib: + shortName: gh + typeOfLevel: cloudTop clevs: !!python/object/apply:numpy.arange [0, 61, 5] cmap: gist_ncar colors: ceil_colors @@ -626,8 +746,12 @@ ctop: # Cloud top height title: Cloud Top Height transform: conversions.m_to_kft unit: kft asl -dewp: # Dew point temperature +dewp: # Dew point temperaeure 2m: + cfgrib: + shortName: 2d + level: 2 + typeOfLevel: heightAboveGround clevs: !!python/object/apply:numpy.arange [-50, 141, 10] cmap: gist_ncar colors: tsfc_colors @@ -638,6 +762,9 @@ dewp: # Dew point temperature wind: 10m dlwrf: # Downward Longwave Radiation Flux sfc: &radiation_flux + cfgrib: + shortName: sdlwrf + typeOfLevel: surface clevs: !!python/object/apply:numpy.arange [200, 501, 12] cmap: gist_ncar colors: radiation_colors @@ -656,6 +783,9 @@ dlwrf: # Downward Longwave Radiation Flux dlwrfavg: # Downward Longwave Radiation Flux Average sfc: <<: *radiation_flux + cfgrib: + shortName: avg_sdlwrf + typeOfLevel: surface clevs: !!python/object/apply:numpy.arange [200, 501, 12] cmap: gist_ncar colors: radiation_colors @@ -666,6 +796,9 @@ dlwrfavg: # Downward Longwave Radiation Flux Average dswrf: # Downward Shortwave Radiation Flux sfc: <<: *radiation_flux + cfgrib: + shortName: sdswrf + typeOfLevel: surface clevs: [0, 50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] colors: rainbow12_colors ncl_name: DSWRF_P0_L1_{grid} @@ -682,6 +815,9 @@ dswrf: # Downward Shortwave Radiation Flux dswrfavg: # Downward Shortwave Radiation Flux Average sfc: <<: *radiation_flux + cfgrib: + shortName: avg_sdswrf + typeOfLevel: surface clevs: [0, 50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] colors: rainbow12_colors ncl_name: DSWRF_P8_L1_{grid}_avg6h @@ -792,6 +928,9 @@ flru: # Aviation Flight Rules unit: "category" G113bt: # GOES-W Brightness temperature, water vapor (Ch 3) sat: &goes_sat + cfgrib: + parameterNumber: 242 + typeOfLevel: atmosphere clevs: !!python/object/apply:numpy.arange [-80, 41, 1] cmap: WVCIMSS_r colors: goes_colors @@ -819,6 +958,9 @@ G124bt: # GOES-E Brightness temperature, infrared (Ch 4) unit: ch 4 gh: # Geopotential height 5mb: &ua_gh + cfgrib: + shortName: gh + typeOfLevel: isobaricInhPa clevs: !!python/object/apply:numpy.arange [6, 4680, 6] cmap: rainbow colors: terrain_colors @@ -852,6 +994,9 @@ gh: # Geopotential height clevs: !!python/object/apply:numpy.arange [500, 600, 10] sfc: <<: *ua_gh + cfgrib: + shortName: orog + typeOfLevel: surface clevs: !!python/object/apply:numpy.arange [0, 5000, 250] cmap: gist_earth ncl_name: HGT_P0_L1_{grid} @@ -862,6 +1007,10 @@ gh: # Geopotential height <<: *ua_gh ghtfl: # Ground Heat Flux sfc: + cfgrib: + shortName: gflux + typeOfLevel: surface + stepType: instant cmap: magma_r clevs: [-200, -150, -100, -50, -25, 0, 25, 50, 100, 150, 200, 300] cmap: PuOr @@ -872,11 +1021,18 @@ ghtfl: # Ground Heat Flux unit: W/m$^{2}$ grle: # Graupel ua: + cfgrib: + prs: + shortName: grle + typeOfLevel: isobaricInhPa ncl_name: nat: GRLE_P0_L105_{grid} prs: GRLE_P0_L100_{grid} gust: 10m: + cfgrib: + shortName: gust + typeOfLevel: surface clevs: !!python/object/apply:numpy.arange [5, 95, 5] cmap: gist_ncar colors: wind_colors @@ -897,6 +1053,10 @@ gust: unit: in hail: # Max 1h Hail diameter maxsfc: &hail # surface + cfgrib: + shortName: hail + typeOfLevel: surface + stepType: max clevs: [0.10, 0.25, 0.50, 0.75, 1.0, 1.5, 2.0, 2.5, 3.0] cmap: gist_ncar colors: hail_colors @@ -1262,6 +1422,10 @@ PM25: # PM25, global chem title: PM25 pres: sfc: + cfgrib: + shortName: sp + level: 0 + typeOfLevel: surface clevs: !!python/object/apply:numpy.arange [650, 1051, 4] cmap: gist_ncar colors: ps_colors @@ -1270,6 +1434,10 @@ pres: transform: conversions.pa_to_hpa unit: hPa msl: + cfgrib: + shortName: mslet + typeOfLevel: meanSea + level: 0 clevs: !!python/object/apply:numpy.arange [976, 1051, 4] cmap: Spectral_r colors: pmsl_colors @@ -1296,6 +1464,10 @@ presmin: msl: accumulate: True annotate: True + cfgrib: + shortName: mslet + typeOfLevel: meanSea + level: 0 clevs: !!python/object/apply:numpy.arange [860, 1001, 10] cmap: NWSReflectivity colors: cref_colors @@ -1746,6 +1918,10 @@ temp: # Temperature wind: False 2m: &sfc_temp annotate: True + cfgrib: + shortName: 2t + level: 2 + typeOfLevel: heightAboveGround clevs: !!python/object/apply:numpy.arange [-50, 141, 10] cmap: gist_ncar colors: tsfc_colors @@ -1755,6 +1931,9 @@ temp: # Temperature unit: F wind: 10m 500mb: &ua_temp + cfgrib: + shortName: t + typeOfLevel: isobaricInhPa clevs: !!python/object/apply:numpy.arange [-40, 40, 2.5] cmap: jet colors: ua_temp_colors @@ -1813,6 +1992,10 @@ temp: # Temperature levels: [0, 925] sfc: <<: *sfc_temp + cfgrib: + shortName: 2t + level: 0 + typeOfLevel: surface ncl_name: TMP_P0_L1_{grid} title: Skin Temperature wind: False From 6626b01bfe5b9204d1ecf78b0bf0e7798e779bd6 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 15 Oct 2025 14:50:46 +0000 Subject: [PATCH 03/98] A few more cfgrib entries. --- adb_graphics/default_specs.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index bad5a86..2cc52af 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -1199,6 +1199,9 @@ hlcytot: transform: run_max hpbl: # Height of Planetary Boundary Layer sfc: + cfgrib: + shortName: gh + typeOfLevel: planetaryBoundaryLayer clevs: [25, 50, 100, 200, 300, 500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000] cmap: ir_rgbv colors: pbl_colors @@ -1208,6 +1211,10 @@ hpbl: # Height of Planetary Boundary Layer unit: m icmr: # Ice Water Mixing Ratio ua: + cfgrib: + prs: + shortName: icmr + typeOfLevel: isobaricInhPa ncl_name: nat: - ICMR_P0_L105_{grid} From b954aa089095911f1dfb835d9a69c6712e333ace Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 15 Oct 2025 11:46:08 -0600 Subject: [PATCH 04/98] Best effort with RRFS pressure level file. --- adb_graphics/default_specs.yml | 329 +++++++++++++++++++++++++++++++++ 1 file changed, 329 insertions(+) diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 2cc52af..465fdac 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -1282,6 +1282,9 @@ icsev: # Icing Severity variable: icsev lcl: # Lifted condensation level sfc: &lcl + cfgrib: + shortName: gh + typeOfLevel: adiabaticCondensation clevs: !!python/object/apply:numpy.arange [0, 5000, 250] cmap: rainbow colors: lcl_colors @@ -1299,6 +1302,9 @@ lcl: # Lifted condensation level unit: m lhtfl: # Latent Heat Net Flux sfc: &lhtflsfc + cfgrib: + shortName: slhtf + typeOfLevel: surface cmap: magma_r clevs: [-100, -50, -25, -10, 0, 10, 25, 50, 100, 150, 200, 250, 300, 400, 500, 750] cmap: BrBG @@ -1310,10 +1316,17 @@ lhtfl: # Latent Heat Net Flux lhtflavg: sfc: <<: *lhtflsfc + cfgrib: + shortName: avg_slhtf + typeOfLevel: surface ncl_name: LHTFL_P8_L1_{grid}_avg6h title: Latent Heat Net Flux 6h Avg li: # Lifted Index best: &lifted_index + cfgrib: + shortName: 4lftx + typeOfLevel: pressureFromGroundLayer + level: 18000 clevs: !!python/object/apply:numpy.arange [-15, 16] cmap: Spectral colors: lifted_index_colors @@ -1323,10 +1336,18 @@ li: # Lifted Index unit: C sfc: <<: *lifted_index + cfgrib: + shortName: lftx + typeOfLevel: isobaricLayer + level: 500 ncl_name: LFTX_P0_2L100_{grid} title: Surface Lifted Index lpl: # Lifted parcel level agl: + cfgrib: + shortName: plpl + typeOfLevel: pressureFromGroundLayer + level: 25500 ncl_name: PLPL_P0_2L108_{grid} title: Lifted Parcel Level AGL >50 transform: @@ -1336,6 +1357,10 @@ lpl: # Lifted parcel level level2: sfc unit: hPa ua: + cfgrib: + shortName: plpl + typeOfLevel: pressureFromGroundLayer + level: 25500 ncl_name: PLPL_P0_2L108_{grid} transform: conversions.pa_to_hpa unit: hPa @@ -1350,6 +1375,9 @@ ltg3: # Lightning Threat (LTG1 ... LTG2) unit: flashes / km$^{2}$ / 5 min ltng: # Lightning sfc: + cfgrib: + shortName: ltng + typeOfLevel: atmosphere clevs: !!python/object/apply:numpy.arange [5, 91, 5] cmap: jet colors: graupel_colors @@ -1358,6 +1386,11 @@ ltng: # Lightning unit: strikes / hr lwtp: # Lightning with total precip sfc: + cfgrib: + shortName: tp + typeOfLevel: surface + stepType: accum + stepRange: # startswith fhr - 1 clevs: [0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar colors: pcp_colors_high @@ -1466,6 +1499,9 @@ pres: unit: hPa wind: 10m ua: + cfgrib: + shortName: + typeOfLevel: ncl_name: PRES_P0_L105_{grid} presmin: msl: @@ -1496,6 +1532,9 @@ presmin: # wind: 10m ptmp: # Potential temperature 2m: + cfgrib: + shortName: pt + typeOfLevel: surface clevs: !!python/object/apply:numpy.arange [210, 350, 5] cmap: jet colors: t_colors @@ -1505,6 +1544,11 @@ ptmp: # Potential temperature wind: 10m ptyp: # Hourly total precipitation sfc: + cfgrib: + shortName: tp + typeOfLevel: surface + stepType: accum + stepRange: # startswith fhr - 1 clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar colors: pcp_colors @@ -1560,6 +1604,9 @@ ptyp: # Hourly total precipitation unit: in pwtr: # Precipitable water sfc: + cfgrib: + shortName: pwat + typeOfLevel: atmosphereSingleLayer clevs: !!python/object/apply:numpy.arange [4, 81, 4] cmap: gist_ncar colors: pw_colors @@ -1577,6 +1624,10 @@ ref: # Maximum reflectivity for past hour at 1 km AGL title: Max 1h -10C Isothermal Reflectivity rh: # Relative Humidity 2m: &rh + cfgrib: + shortName: 2r + typeOfLevel: heightAboveGround + level: 2 clevs: [10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 100, 105] cmap: gist_ncar colors: rainbow12_colors @@ -1585,6 +1636,9 @@ rh: # Relative Humidity unit: '%' 500mb: &rh_ua <<: *rh + cfgrib: + shortName: r + typeOfLevel: isobaricInhPa ncl_name: RH_P0_L100_{grid} contours: pres_sfc: @@ -1666,6 +1720,9 @@ rh: # Relative Humidity title: Relative Humidity wrt Precipitable Water rvil: # Radar-derived Vertically Integrated Liquid sfc: &vert_int_liq + cfgrib: + shortName: veril + typeOfLevel: atmosphereSingleLayer clevs: [0.05, 0.15, 0.76, 3.47, 6.92, 12, 31.6, 35, 40, 45, 50, 55, 60, 65, 70] cmap: NWSReflectivity colors: cref_colors @@ -1681,6 +1738,10 @@ rvil: # Radar-derived Vertically Integrated Liquid unit: kg/m$^{2}$ rwmr: # Rain Mixing Ratio ua: + cfgrib: + prs: + shortName: rwmr + typeOfLevel: isobaricInhPa ncl_name: nat: RWMR_P0_L105_{grid} prs: RWMR_P0_L100_{grid} @@ -1696,6 +1757,11 @@ seasalt: # Fine dust, global chem unit: $\mu g/m^3$ shear: 01km: &shear # 0-1 km + cfgrib: + shortName: vucsh + typeOfLevel: heightAboveGroundLayer + topLevel: 0 + bottomLevel: 1000 clevs: [5, 10, 20, 30, 40, 50, 60] cmap: gist_ncar colors: wind_colors_high @@ -1705,6 +1771,11 @@ shear: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: field2: VVCSH_P0_2L103_{grid} + cfgrib: + shortName: vvcsh + typeOfLevel: heightAboveGroundLayer + topLevel: 0 + bottomLevel: 1000 one_lev: True split: True level: 01km @@ -1713,18 +1784,32 @@ shear: unit: kt 06km: # 0-6 km <<: *shear + cfgrib: + shortName: vucsh + typeOfLevel: heightAboveGroundLayer + topLevel: 0 + bottomLevel: 6000 clevs: [20, 30, 40, 50, 60, 70, 80, 90, 100] split: True transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: field2: VVCSH_P0_2L103_{grid} + cfgrib: + shortName: vvcsh + typeOfLevel: heightAboveGroundLayer + topLevel: 0 + bottomLevel: 6000 one_lev: True split: True level: 06km title: 0–6 km Bulk Shear shtfl: # Sensible Heat Net Flux sfc: &shtflsfc + cfgrib: + shortName: ishf + typeOfLevel: surface + stepType: instant cmap: magma_r clevs: [-100, -50, -25, -10, 0, 10, 25, 50, 100, 150, 200, 250, 300, 400, 500, 750] cmap: RdYlBu_r @@ -1736,6 +1821,10 @@ shtfl: # Sensible Heat Net Flux shtflavg: sfc: <<: *shtflsfc + cfgrib: + shortName: avg_ishf + typeOfLevel: surface + stepType: avg ncl_name: SHTFL_P8_L1_{grid}_avg6h title: Sensible Heat Net Flux 6h Avg sipd: # Supercooled Large Droplet Icing @@ -1779,12 +1868,19 @@ slw: # Supercooled Liquid Water unit: kg/m$^{2}$ snmr: # Snow Mixing Ratio ua: + cfgrib: + prs: + shortName: snmr + typeOfLevel: isobaricInhPa ncl_name: nat: SNMR_P0_L105_{grid} prs: SNMR_P0_L100_{grid} snod: # Snow Depth sfc: <<: *snow + cfgrib: + shortName: sde + typeOfLevel: surface ncl_name: SNOD_P0_L1_{grid} title: Cycled Snow Depth soilm: # Soil Moisture Availability @@ -1804,6 +1900,11 @@ soilm: # Soil Moisture Availability unit: "%" soilt: # Soil Temperature 0cm: &soilt_levs + cfgrib: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 0 + scaledValueOfSecondFixedSurface: 0 clevs: !!python/object/apply:numpy.arange [235, 331, 5] cmap: gist_ncar colors: tsfc_colors @@ -1813,33 +1914,78 @@ soilt: # Soil Temperature unit: K 1cm: <<: *soilt_levs + cfgrib: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 1 + scaledValueOfSecondFixedSurface: 1 title: Soil Temperature at 1cm 4cm: <<: *soilt_levs + cfgrib: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 4 + scaledValueOfSecondFixedSurface: 4 title: Soil Temperature at 4cm 10cm: <<: *soilt_levs + cfgrib: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 10 + scaledValueOfSecondFixedSurface: 10 title: Soil Temperature at 10cm 30cm: <<: *soilt_levs + cfgrib: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 30 + scaledValueOfSecondFixedSurface: 30 title: Soil Temperature at 30cm 40cm: <<: *soilt_levs title: Soil Temperature at 30cm 60cm: <<: *soilt_levs + cfgrib: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 60 + scaledValueOfSecondFixedSurface: 60 title: Soil Temperature at 60cm 1m: <<: *soilt_levs + cfgrib: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 100 + scaledValueOfSecondFixedSurface: 100 title: Soil Temperature at 1m 1.6m: <<: *soilt_levs + cfgrib: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 160 + scaledValueOfSecondFixedSurface: 160 title: Soil Temperature at 1.6m 3m: <<: *soilt_levs + cfgrib: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 300 + scaledValueOfSecondFixedSurface: 300 title: Soil Temperature at 3m soilw: # Soil Moisture 0cm: &soilw_levs + cfgrib: + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 0 + scaledValueOfSecondFixedSurface: 0 clevs: [0, 0.01, 0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50] cmap: jet_r colors: soilw_colors @@ -1849,33 +1995,81 @@ soilw: # Soil Moisture unit: fraction 1cm: <<: *soilw_levs + cfgrib: + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 1 + scaledValueOfSecondFixedSurface: 1 title: Soil Moisture at 1cm 4cm: <<: *soilw_levs + cfgrib: + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 4 + scaledValueOfSecondFixedSurface: 4 title: Soil Moisture at 4cm 10cm: <<: *soilw_levs + cfgrib: + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 10 + scaledValueOfSecondFixedSurface: 10 title: Soil Moisture at 10cm 30cm: <<: *soilw_levs + cfgrib: + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 30 + scaledValueOfSecondFixedSurface: 30 title: Soil Moisture at 30cm 40cm: <<: *soilw_levs + cfgrib: + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 40 + scaledValueOfSecondFixedSurface: 40 title: Soil Moisture at 40cm 60cm: <<: *soilw_levs + cfgrib: + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 60 + scaledValueOfSecondFixedSurface: 60 title: Soil Moisture at 60cm 1m: <<: *soilw_levs + cfgrib: + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 100 + scaledValueOfSecondFixedSurface: 100 title: Soil Moisture at 1m 1.6m: <<: *soilw_levs + cfgrib: + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 160 + scaledValueOfSecondFixedSurface: 160 title: Soil Moisture at 1.6m 3m: <<: *soilw_levs + cfgrib: + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 300 + scaledValueOfSecondFixedSurface: 300 title: Soil Moisture at 3m solar: # Incoming Solar Radiation sfc: &incoming_radiation + cfgrib: + shortName: csdsf + typeOfLevel: surface clevs: [50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100] cmap: gist_ncar colors: vort_colors @@ -1885,11 +2079,21 @@ solar: # Incoming Solar Radiation unit: W/m$^{2}$ sphum: # Specific humidity 2m: + cfgrib: + shortName: 2sh + typeOfLevel: heightAboveGround ncl_name: SPFH_P0_L103_{grid} ua: + cfgrib: + shortName: q + typeOfLevel: isobaricInhPa ncl_name: SPFH_P0_L105_{grid} ssrun: # Storm Surface Runoff sfc: &precip + cfgrib: + shortname: ssrun + typeOfLevel: surface + stepType: accum clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar colors: pcp_colors @@ -1910,6 +2114,10 @@ sulf: # Sulfate, global chem unit: $\mu g/m^3$ temp: # Temperature 2ds: # 2m - Sfc + cfgrib: + shortName: 2t + level: 2 + typeOfLevel: heightAboveGround clevs: !!python/object/apply:numpy.arange [-32, 33, 2] cmap: Spectral_r colors: centered_diff @@ -2020,6 +2228,11 @@ thick: totp: # Hourly total precipitation sfc: <<: *precip + cfgrib: + shortName: tp + typeOfLevel: surface + stepRange: # startswith fhr - 1 + stepType: accum contours: pres_msl: colors: red @@ -2035,6 +2248,11 @@ totp: # Hourly total precipitation totp6h: # 6-hourly total precipitation sfc: <<: *precip + cfgrib: + shortName: tp + typeOfLevel: surface + stepRange: # startswith 0 + stepType: accum contours: pres_msl: colors: red @@ -2094,12 +2312,31 @@ trc1: wind: 10m u: 10m: &agl_uwind + cfgrib: + shortName: 10u + level: 10 + typeOfLevel: heightAboveGround ncl_name: UGRD_P0_L103_{grid} transform: conversions.ms_to_kt 80m: *agl_uwind + cfgrib: + shortName: u + level: 80 + typeOfLevel: heightAboveGround 160m: *agl_uwind + cfgrib: + shortName: u + level: 160 + typeOfLevel: heightAboveGround 320m: *agl_uwind + cfgrib: + shortName: u + level: 320 + typeOfLevel: heightAboveGround 5mb: &ua_uwind + cfgrib: + shortName: u + typeOfLevel: isobaricInhPa ncl_name: prs: UGRD_P0_L100_{grid} nat: UGRD_P0_L105_{grid} @@ -2119,6 +2356,9 @@ u: <<: *nat_uwind vertical_index: 11 max: + cfgrib: + shortName: u + typeOfLevel: maxWind ncl_name: MAXUW_P8_L103_{grid}_max1h transform: conversions.ms_to_kt ua: @@ -2126,6 +2366,10 @@ u: ulwrf: # Upward Longwave Radiation Flux sfc: <<: *radiation_flux + cfgrib: + shortName: sulwrf + typeOfLevel: surface + stepType: instant clevs: !!python/object/apply:numpy.arange [350, 601, 10] cmap: gist_ncar colors: radiation_colors @@ -2134,6 +2378,10 @@ ulwrf: # Upward Longwave Radiation Flux title: Upward LW Radiation Flux, Surface unit: W/m$^{2}$ top: # Nominal top of atmosphere + cfgrib: + parameterName: Upward long-wave radiation flux + typeOfLevel: nominalTop + stepType: instant clevs: !!python/object/apply:numpy.arange [80, 341, 2] cmap: ir_rgbv_r colors: radiation_mix_colors @@ -2144,6 +2392,10 @@ ulwrf: # Upward Longwave Radiation Flux ulwrfavg: # Upward Longwave Radiation Flux sfc: <<: *radiation_flux + cfgrib: + shortName: sulwrf + typeOfLevel: surface + stepType: avg clevs: !!python/object/apply:numpy.arange [350, 601, 10] cmap: gist_ncar colors: radiation_colors @@ -2152,6 +2404,10 @@ ulwrfavg: # Upward Longwave Radiation Flux title: Upward LW Radiation Flux, Surface unit: W/m$^{2}$ top: # Nominal top of atmosphere + cfgrib: + parameterName: Upward long-wave radiation flux + typeOfLevel: nominalTop + stepType: avg clevs: !!python/object/apply:numpy.arange [80, 341, 2] cmap: ir_rgbv_r colors: radiation_mix_colors @@ -2162,6 +2418,10 @@ ulwrfavg: # Upward Longwave Radiation Flux uswrf: # Upward Shortwave Radiation Flux sfc: <<: *radiation_flux + cfgrib: + shortName: suswrf + typeOfLevel: surface + stepType: instant clevs: [0, 50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] colors: rainbow12_colors ncl_name: USWRF_P0_L1_{grid} @@ -2178,6 +2438,10 @@ uswrf: # Upward Shortwave Radiation Flux uswrfavg: # Upward Shortwave Radiation Flux Average sfc: <<: *radiation_flux + cfgrib: + shortName: suswrf + typeOfLevel: surface + stepType: avg clevs: [0, 50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] colors: rainbow12_colors ncl_name: USWRF_P8_L1_{grid}_avg6h @@ -2193,12 +2457,23 @@ uswrfavg: # Upward Shortwave Radiation Flux Average title: Outgoing SW Radiation Flux 6h Avg, TOA v: 10m: &agl_wind + cfgrib: + shortName: 10v + level: 10 + typeOfLevel: heightAboveGround ncl_name: VGRD_P0_L103_{grid} transform: conversions.ms_to_kt 80m: *agl_wind + cfgrib: + shortName: v + typeOfLevel: heightAboveGround 160m: *agl_wind + typeOfLevel: heightAboveGround 320m: *agl_wind 5mb: &ua_vwind + cfgrib: + shortName: v + typeOfLevel: isobaricInhPa ncl_name: prs: VGRD_P0_L100_{grid} nat: VGRD_P0_L105_{grid} @@ -2218,6 +2493,9 @@ v: <<: *nat_vwind vertical_index: 11 max: + cfgrib: + shortName: v + typeOfLevel: maxWind ncl_name: MAXVW_P8_L103_{grid}_max1h transform: conversions.ms_to_kt ua: @@ -2225,11 +2503,17 @@ v: vbdsf: # Incoming Direct Radiation sfc: <<: *incoming_radiation + cfgrib: + shortName: vbdsf + typeOfLevel: surface ncl_name: VBDSF_P0_L1_{grid} title: Incoming Direct Radiation vddsf: # Incoming Diffuse Radiation sfc: <<: *incoming_radiation + cfgrib: + shortName: vddsf + typeOfLevel: surface ncl_name: VDDSF_P0_L1_{grid} title: Incoming Diffuse Radiation vig: # Vertically-integrated graupel @@ -2248,6 +2532,9 @@ vil: # Vertically Integrated Liquid title: Vertically Integrated Liquid vis: # Visibility sfc: &vis + cfgrib: + shortName: vis + typeOfLevel: surface # see the description provided in adb_graphics/utils.py, for the join_ranges method. clevs: !join_ranges [[0, 10, 0.1], [10, 51, 1.0]] cmap: gist_ncar @@ -2264,6 +2551,9 @@ visbsn: # Visibility incl. blowing snow title: Sfc Visibility incl. blowing snow (exp) vort: # Absolute vorticity 500mb: + cfgrib: + shortName: absv + typeOfLevel: isobaricInhPa clevs: !!python/object/apply:numpy.arange [6, 29, 2] cmap: gist_ncar colors: vort_colors @@ -2276,6 +2566,9 @@ vort: # Absolute vorticity unit: 1E-5/s vvel: # Vertical velocity 700mb: + cfgrib: + shortName: wz + typeOfLevel: isobaricInhPa clevs: !!python/object/apply:numpy.arange [-17, 34, 5] cmap: gist_ncar colors: vvel_colors @@ -2296,6 +2589,10 @@ vvel: # Vertical velocity unit: m/s vvort: # Vertical vorticity mx01: &vvort # Hourly maximum of vertical vorticity over 0-2 km layer + cfgrib: + shortName: max_vo + typeOfLevel: heightAboveGroundLayer + level: 1000 clevs: !!python/object/apply:numpy.arange [0.0025, 0.0301, 0.0025] cmap: gist_ncar colors: vort_colors @@ -2305,9 +2602,16 @@ vvort: # Vertical vorticity unit: 1/s mx02: # Hourly maximum of vertical vorticity over 0-2 km layer <<: *vvort + cfgrib: + shortName: max_vo + typeOfLevel: heightAboveGroundLayer + level: 2000 title: 0-2km Max Vertical Vorticity (over prev hour) weasd: # Water equivalent of accumulated snow depth sfc: + cfgrib: + shortName: sdwe + typeOfLevel: surface clevs: [0.01, 0.1, 0.3, 0.5, 1, 2, 3, 4, 5, 7.5, 10, 20] cmap: gist_ncar colors: snow_colors @@ -2329,6 +2633,10 @@ snoliqr: # Snow-liquid ratio (from U. Utah diagnostic in UPP) windmax: 10m: accumulate: True + cfgrib: + shortName: max_10si + typeOfLevel: heightAboveGround + level: 10 clevs: !!python/object/apply:numpy.arange [5, 95, 5] cmap: gist_ncar colors: wind_colors @@ -2339,6 +2647,10 @@ windmax: unit: kt wspeed: # Wind Speed 10m: &ua_wspeed + cfgrib: + shortName: 10u + level: 10 + typeOfLevel: heightAboveGround clevs: !!python/object/apply:numpy.arange [5, 95, 5] cmap: gist_ncar colors: wind_colors @@ -2349,9 +2661,16 @@ wspeed: # Wind Speed funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: field2: VGRD_P0_L103_{grid} + cfgrib: + shortName: 10v + level: 10 + typeOfLevel: heightAboveGround unit: kt wind: True 5mb: &ua_wspeed_high + cfgrib: + shortName: u + typeOfLevel: isobaricInhPa clevs: [20, 40, 60, 70, 80, 90, 100, 110, 120, 140, 160, 180, 200] cmap: gist_ncar colors: wind_colors_high @@ -2366,6 +2685,9 @@ wspeed: # Wind Speed funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: field2: VGRD_P0_L100_{grid} + cfgrib: + shortName: v + typeOfLevel: isobaricInhPa unit: kt wind: True 10mb: @@ -2398,8 +2720,15 @@ wspeed: # Wind Speed funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: field2: VGRD_P0_L100_{grid} + cfgrib: + shortName: v + typeOfLevel: isobaricInhPa max: # Hourly Maximum 10m Wind <<: *ua_wspeed + cfgrib: + shortName: max_10si + typeOfLevel: heightAboveGround + level: 10 ncl_name: WIND_P8_L103_{grid}_max1h title: Max 10m Wind (over prev hour) transform: conversions.ms_to_kt From cea64b5fa26894eddd92088b486e6d9b06eface1 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 15 Oct 2025 16:59:49 -0600 Subject: [PATCH 05/98] Can plot t2m field. --- adb_graphics/datahandler/gribdata.py | 155 +++++------ adb_graphics/datahandler/gribfile.py | 9 +- adb_graphics/default_specs.yml | 25 +- adb_graphics/figure_builders.py | 21 +- create_graphics.py | 14 +- image_lists/rrfs_subset.yml | 378 +++++++++++++-------------- 6 files changed, 309 insertions(+), 293 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index af24425..6490d6f 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -5,7 +5,7 @@ ''' import abc -import datetime +from datetime import datetime, timedelta from functools import lru_cache from string import digits, ascii_letters @@ -40,6 +40,7 @@ def __init__(self, ds, short_name, **kwargs): config = kwargs.get('config', 'adb_graphics/default_specs.yml') self.model = kwargs.get('model') self.filetype = kwargs.get('filetype', 'prs') + self.grib_path = kwargs.get("grib_path") specs.VarSpec.__init__(self, config) @@ -53,12 +54,12 @@ def __init__(self, ds, short_name, **kwargs): self.ds = ds @property - def anl_dt(self) -> datetime.datetime: + def anl_dt(self) -> datetime: ''' Returns the initial time of the grib file as a datetime object from the grib file.''' - return datetime.datetime.strptime(self.field.initial_time, '%m/%d/%Y (%H:%M)') + return datetime.fromisoformat(str(self.field.time.values).split(".")[0]) @property def clevs(self) -> np.ndarray: @@ -98,7 +99,7 @@ def field(self): ''' Wrapper that calls get_field method for the current variable. Returns the NioVariable object ''' - return self._get_field(self.ncl_name(self.vspec)) + return self.ds.__getattr__([x for x in self.ds.data_vars][0]) def field_column_max(self, values, variable, level, **kwargs): @@ -167,15 +168,11 @@ def _get_data_levels(self, vertical_dim): ret.append(self.ds[dim].sel(**selector).values) return ret - def _get_field(self, ncl_name): + def _get_field(self, spec): ''' Given an ncl_name, return the NioVariable object. ''' - try: - field = self.ds[ncl_name.format(level_type=self.level_type)] - except KeyError: - raise errors.GribReadError(f'{ncl_name}') - return field + return gribfile.GribFile(self.grib_path, spec).contents def _get_level(self, field, level, spec, **kwargs): @@ -428,12 +425,12 @@ def opposite(values, **kwargs): return - values @property - def valid_dt(self) -> datetime.datetime: + def valid_dt(self) -> datetime: ''' Returns a datetime object corresponding to the forecast hour's valid time as set in the Grib file. ''' - fh = datetime.timedelta(hours=int(self.fhr)) + fh = timedelta(hours=int(self.fhr)) return self.anl_dt + fh @abc.abstractmethod @@ -619,9 +616,12 @@ def grid_info(self): ''' Returns a dict that includes the grid info for the full grid. ''' # Keys are grib names, values are Basemap argument names - ncl_to_basemap = dict( + keys_to_basemap = dict( CenterLon='lon_0', CenterLat='lat_0', + GRIB_Latin2InDegrees='lat_1', + GRIB_Latin1InDegrees='lat_2', + GRIB_LoVInDegrees='lon_0', Latin2='lat_1', Latin1='lat_2', Lov='lon_0', @@ -635,39 +635,43 @@ def grid_info(self): lat_var = [var for var in self.field.coords if 'lat' in var][0] # Get the latitude variable - lat = self.ds[lat_var] grid_info = {} - if self.model != 'hrrrhi': - grid_info['corners'] = self.corners - if self.grid_suffix in ['GLC0']: - attrs = ['Latin1', 'Latin2', 'Lov'] + var_info = self.field + grid_def = var_info.attrs["GRIB_gridDefinitionDescription"].lower() + if 'lambert' in grid_def: + attrs = ["GRIB_Latin1InDegrees", "GRIB_Latin2InDegrees", "GRIB_LoVInDegrees"] grid_info['projection'] = 'lcc' grid_info['lat_0'] = 39.0 - elif self.grid_suffix == 'GST0': - attrs = ['Lov'] - grid_info['projection'] = 'stere' - grid_info['lat_0'] = 90 - elif self.grid_suffix == 'GLL0': - attrs = [] - grid_info['projection'] = 'cyl' - else: - attrs = [] - grid_info['projection'] = 'rotpole' - # CenterLon in RAP and Longitude_of_southern_pole in RRFS - lon_0 = lat.attrs.get('CenterLon', lat.attrs.get('Longitude_of_southern_pole')) - grid_info['lon_0'] = lon_0[0] - 360 - - # CenterLat in RAP and Latitude_of_southern_pole in RRFS - center_lat = lat.attrs.get('CenterLat', lat.attrs.get('Latitude_of_southern_pole')) - grid_info['o_lat_p'] = - center_lat[0] if center_lat[0] < 0 else 90 - center_lat[0] - - grid_info['o_lon_p'] = 180 + if self.model != 'hrrrhi': + grid_info['corners'] = self.corners + #if self.grid_suffix in ['GLC0']: + # attrs = ['Latin1', 'Latin2', 'Lov'] + #elif self.grid_suffix == 'GST0': + # attrs = ['Lov'] + # grid_info['projection'] = 'stere' + # grid_info['lat_0'] = 90 + #elif self.grid_suffix == 'GLL0': + # attrs = [] + # grid_info['projection'] = 'cyl' + #else: + # attrs = [] + # grid_info['projection'] = 'rotpole' + + # # CenterLon in RAP and Longitude_of_southern_pole in RRFS + # lon_0 = lat.attrs.get('CenterLon', lat.attrs.get('Longitude_of_southern_pole')) + # grid_info['lon_0'] = lon_0[0] - 360 + + # # CenterLat in RAP and Latitude_of_southern_pole in RRFS + # center_lat = lat.attrs.get('CenterLat', lat.attrs.get('Latitude_of_southern_pole')) + # grid_info['o_lat_p'] = - center_lat[0] if center_lat[0] < 0 else 90 - center_lat[0] + + # grid_info['o_lon_p'] = 180 for attr in attrs: - bm_arg = ncl_to_basemap[attr] - val = lat.attrs[attr] + bm_arg = keys_to_basemap[attr] + val = var_info.attrs[attr] val = val[0] if isinstance(val, np.ndarray) else val grid_info[bm_arg] = val del val @@ -678,8 +682,6 @@ def grid_info(self): grid_info['width'] = 2000000 grid_info['height'] = 2000000 - del lat - return grid_info def icing_adjust_trace(self, values, **kwargs): @@ -811,17 +813,18 @@ def values(self, level=None, name=None, **kwargs): vertical_index the index (int) of the desired vertical level ''' - level = level if level else self.level + level = level or self.level + vals = self.field - one_lev = kwargs.get('one_lev', True) - vertical_index = kwargs.get('vertical_index') + #one_lev = kwargs.get('one_lev', True) + #vertical_index = kwargs.get('vertical_index') - ncl_name = kwargs.get('ncl_name', '') - ncl_name = ncl_name.format(fhr=self.fhr, grid=self.grid_suffix) + #ncl_name = kwargs.get('ncl_name', '') + #ncl_name = ncl_name.format(fhr=self.fhr, grid=self.grid_suffix) do_transform = kwargs.get('do_transform', True) - if name is None and not ncl_name: + if name is None: # Use field and spec from the current object field = self.field @@ -833,41 +836,41 @@ def values(self, level=None, name=None, **kwargs): spec = self.spec.get(name, {}).get(level, {}) if not spec and name is not None: raise errors.NoGraphicsDefinitionForVariable(name, level) - field = self._get_field(ncl_name or self.ncl_name(spec)) + field = self._get_field(spec["cfgrib"]) - lev = vertical_index - vals = field - if one_lev: + #lev = vertical_index + #vals = field + #if one_lev: - # Check if it's a 3D variable (lv in any dimension field) - dim_name = self.vertical_dim(field) + # # Check if it's a 3D variable (lv in any dimension field) + # dim_name = self.vertical_dim(field) - if dim_name: # Field has a vertical dimension + # if dim_name: # Field has a vertical dimension - # Use vertical_index if provided in kwargs - lev = vertical_index if vertical_index is not None else \ - self._get_level(field, level, spec) + # # Use vertical_index if provided in kwargs + # lev = vertical_index if vertical_index is not None else \ + # self._get_level(field, level, spec) - if lev is None or dim_name is None: - print(f'ERROR: Could not find dim_name ({dim_name}) or' \ - f'lev {lev} for {vals}') - raise ValueError + # if lev is None or dim_name is None: + # print(f'ERROR: Could not find dim_name ({dim_name}) or' \ + # f'lev {lev} for {vals}') + # raise ValueError - try: - vals = vals.isel(**{dim_name: lev}) - except: - print(f'Error for {vals.name} : {dim_name} {lev} \ - {level} {spec}') - raise + # try: + # vals = vals.isel(**{dim_name: lev}) + # except: + # print(f'Error for {vals.name} : {dim_name} {lev} \ + # {level} {spec}') + # raise - if self.mem is not None: - vals = vals.isel(**{'ens_mem': self.mem}) + #if self.mem is not None: + # vals = vals.isel(**{'ens_mem': self.mem}) - # Select a single forecast hour (only if there are many) - if not spec.get('accumulate', False): - if 'fcst_hr' in vals.dims: - fcst_hr = 0 if self.ds.sizes['fcst_hr'] <= 1 else int(self.fhr) - vals = vals.sel(**{'fcst_hr': fcst_hr}) + ## Select a single forecast hour (only if there are many) + #if not spec.get('accumulate', False): + # if 'fcst_hr' in vals.dims: + # fcst_hr = 0 if self.ds.sizes['fcst_hr'] <= 1 else int(self.fhr) + # vals = vals.sel(**{'fcst_hr': fcst_hr}) transforms = spec.get('transform') if transforms and do_transform: @@ -930,10 +933,10 @@ def wind(self, level) -> [np.ndarray, np.ndarray]: fhr=self.fhr, level=level, short_name=var, - ) + ).field u, v = [field_lambda(self.ds, level, var) for var in ['u', 'v']] - return [component.values() for component in [u, v]] + return [u, v] class profileData(UPPData): diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 66140a6..ac2d68f 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -8,13 +8,14 @@ class GribFile(): - ''' Wrappers and helper functions for interfacing with pyNIO.''' + ''' Wrappers and helper functions for interfacing with cfgrib.''' - def __init__(self, filename, **kwargs): + def __init__(self, filename, var_config, **kwargs): # pylint: disable=unused-argument self.filename = filename + self.var_config = var_config self.contents = self._load() def _load(self): @@ -23,9 +24,9 @@ def _load(self): iterator. ''' return xr.open_dataset(self.filename, - engine='pynio', + engine='cfgrib', lock=False, - backend_kwargs=dict(format="grib2"), + backend_kwargs=({"filter_by_keys": self.var_config}), ) class GribFiles(): diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 465fdac..f3fce82 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -2144,7 +2144,7 @@ temp: # Temperature ticks: 10 transform: conversions.k_to_f unit: F - wind: 10m + #wind: 10m 500mb: &ua_temp cfgrib: shortName: t @@ -2318,17 +2318,20 @@ u: typeOfLevel: heightAboveGround ncl_name: UGRD_P0_L103_{grid} transform: conversions.ms_to_kt - 80m: *agl_uwind + 80m: + <<: *agl_uwind cfgrib: shortName: u level: 80 typeOfLevel: heightAboveGround - 160m: *agl_uwind + 160m: + <<: *agl_uwind cfgrib: shortName: u level: 160 typeOfLevel: heightAboveGround - 320m: *agl_uwind + 320m: + <<: *agl_uwind cfgrib: shortName: u level: 320 @@ -2463,13 +2466,21 @@ v: typeOfLevel: heightAboveGround ncl_name: VGRD_P0_L103_{grid} transform: conversions.ms_to_kt - 80m: *agl_wind + 80m: + <<: *agl_wind + cfgrib: + shortName: v + typeOfLevel: heightAboveGround + 160m: + <<: *agl_wind cfgrib: shortName: v typeOfLevel: heightAboveGround - 160m: *agl_wind + 320m: + <<: *agl_wind + cfgrib: + shortName: v typeOfLevel: heightAboveGround - 320m: *agl_wind 5mb: &ua_vwind cfgrib: shortName: v diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index bbefc30..d22ae5a 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -51,8 +51,8 @@ def add_obs_panel(ax, model_name, obs_file, proj_info, short_name, tile): # Draw the map dm.draw(show=True) -def parallel_maps(cla, fhr, ds, level, model, spec, variable, workdir, - tile='full', ds2=None): +def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, + tile='full', dp2=None): # pylint: disable=too-many-arguments,too-many-locals # pylint: disable=too-many-branches,too-many-statements @@ -64,7 +64,7 @@ def parallel_maps(cla, fhr, ds, level, model, spec, variable, workdir, Input: fhr forecast hour - ds xarray dataset from the grib file + grib_path path to grib file level the vertical level of the variable to be plotted corresponding to a key in the specs file model model name: rap, hrrr, hrrre, rrfs, rtma @@ -72,12 +72,18 @@ def parallel_maps(cla, fhr, ds, level, model, spec, variable, workdir, and level variable the name of the variable section in the specs file workdir output directory + tile Optional: tile the label of the tile being plotted + dp2 path to a second grib file ''' fig, axes = set_figure(cla.model_name, cla.graphic_type, tile) + ds = gribfile.GribFile(grib_path, spec["cfgrib"]).contents + + if dp2: + ds2 = gribfile.GribFile(dp2, spec["cfgrib"]).contents # set last_panel to send into DataMap for colorbar control last_panel = False @@ -117,15 +123,10 @@ def parallel_maps(cla, fhr, ds, level, model, spec, variable, workdir, member=mem, model=model, short_name=variable, - config=cla.specs['file'] + config=cla.specs['file'], + grib_path=dp2, ) - try: - field.field - except errors.GribReadError: - print(f'Cannot find grib2 variable for {variable} at {level}. Skipping.') - return - if cla.graphic_type == "diff": field2 = gribdata.fieldData( diff --git a/create_graphics.py b/create_graphics.py index 9b07010..d4e67d2 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -70,7 +70,7 @@ def create_skewt(cla, fhr, grib_path, workdir): with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_skewt, args) -def create_maps(cla, fhr, grib_contents, workdir, grib_contents2=None): +def create_maps(cla, fhr, grib_path, workdir, grib_path2=None): ''' Generate arguments for parallel processing of plan-view maps and generate a pool of workers to complete the task. ''' @@ -88,12 +88,13 @@ def create_maps(cla, fhr, grib_contents, workdir, grib_contents2=None): msg = f'graphics: {variable} {level}' raise errors.NoGraphicsDefinitionForVariable(msg) - args.append((cla, fhr, grib_contents, level, model, spec, - variable, workdir, tile, grib_contents2)) + args.append((cla, fhr, grib_path, level, model, spec, + variable, workdir, tile, grib_path2)) print(f'Queueing {len(args)} maps') - with Pool(processes=cla.nprocs) as pool: - pool.starmap(parallel_maps, args) + parallel_maps(*args[0]) + #with Pool(processes=cla.nprocs) as pool: + # pool.starmap(parallel_maps, args) def gather_gribfiles(cla, fhr, filename, gribfiles): @@ -615,10 +616,9 @@ def graphics_driver(cla): if cla.graphic_type == 'skewts': create_skewt(cla, fhr, grib_path, workdir) elif cla.graphic_type == 'maps': - gribfiles = gather_gribfiles(cla, fhr, grib_path, gribfiles) create_maps(cla, fhr=fhr, - grib_contents=gribfiles.contents, + grib_path=grib_path, workdir=workdir, ) elif cla.graphic_type == 'diff': diff --git a/image_lists/rrfs_subset.yml b/image_lists/rrfs_subset.yml index 358d666..b627b20 100644 --- a/image_lists/rrfs_subset.yml +++ b/image_lists/rrfs_subset.yml @@ -1,194 +1,194 @@ hourly: model: rrfs variables: - 1hsnw: - - sfc - 1ref: - - 1000m - acfrozr: - - sfc - acfrzr: - - sfc - acpcp: - - sfc - acsnod: - - sfc - acsnw: - - sfc - cape: - - mu - - mul - - mx90mb - - sfc - ceil: - - ua - ceilexp: - - ua - ceilexp2: - - ua - cin: - - sfc - cloudbase: - - ua - cloudcover: - - bndylay - - high - - low - - mid - - total - coarsedust: - - sfc - cpofp: - - sfc - cref: - - sfc - ctop: - - ua - dewp: - - 2m - echotop: - - sfc - emissions: - - sfc - finedust: - - sfc - firewx: - - sfc - flru: - - sfc - fullintdust: - - int - G113bt: - - sat - G114bt: - - sat - G123bt: - - sat - G124bt: - - sat - ghtfl: - - sfc - gust: - - 10m - hailcast: - - maxsfc - hlcy: - - in25 - - mn03 - - mn25 - - mx03 - - mx25 - - sr01 - - sr03 - hlcytot: - - mn03 - - mn25 - - mx03 - - mx25 - hpbl: - - sfc - lcl: - - sfc - lhtfl: - - sfc - li: - - best - - sfc - ltg3: - - sfc - ltng: - - sfc - lwtp: - - sfc - mref: - - sfc - pres: - - msl - - sfc - ptmp: - - 2m - ptyp: - - sfc - pwtr: - - sfc - ref: - - m10 - - maxm10 - rh: - - 2m - - 850mb - - mean - - pw - rvil: - - sfc - shear: - - 01km - - 06km - shtfl: - - sfc - snod: - - sfc - soilm: - - sfc - soilt: &soilt_levs - - 0cm - - 1cm - - 4cm - - 10cm - - 30cm - - 60cm - - 1m - - 1.6m - - 3m - soilw: *soilt_levs - solar: - - sfc - ssrun: - - sfc +# 1hsnw: +# - sfc +# 1ref: +# - 1000m +# acfrozr: +# - sfc +# acfrzr: +# - sfc +# acpcp: +# - sfc +# acsnod: +# - sfc +# acsnw: +# - sfc +# cape: +# - mu +# - mul +# - mx90mb +# - sfc +# ceil: +# - ua +# ceilexp: +# - ua +# ceilexp2: +# - ua +# cin: +# - sfc +# cloudbase: +# - ua +# cloudcover: +# - bndylay +# - high +# - low +# - mid +# - total +# coarsedust: +# - sfc +# cpofp: +# - sfc +# cref: +# - sfc +# ctop: +# - ua +# dewp: +# - 2m +# echotop: +# - sfc +# emissions: +# - sfc +# finedust: +# - sfc +# firewx: +# - sfc +# flru: +# - sfc +# fullintdust: +# - int +# G113bt: +# - sat +# G114bt: +# - sat +# G123bt: +# - sat +# G124bt: +# - sat +# ghtfl: +# - sfc +# gust: +# - 10m +# hailcast: +# - maxsfc +# hlcy: +# - in25 +# - mn03 +# - mn25 +# - mx03 +# - mx25 +# - sr01 +# - sr03 +# hlcytot: +# - mn03 +# - mn25 +# - mx03 +# - mx25 +# hpbl: +# - sfc +# lcl: +# - sfc +# lhtfl: +# - sfc +# li: +# - best +# - sfc +# ltg3: +# - sfc +# ltng: +# - sfc +# lwtp: +# - sfc +# mref: +# - sfc +# pres: +# - msl +# - sfc +# ptmp: +# - 2m +# ptyp: +# - sfc +# pwtr: +# - sfc +# ref: +# - m10 +# - maxm10 +# rh: +# - 2m +# - 850mb +# - mean +# - pw +# rvil: +# - sfc +# shear: +# - 01km +# - 06km +# shtfl: +# - sfc +# snod: +# - sfc +# soilm: +# - sfc +# soilt: &soilt_levs +# - 0cm +# - 1cm +# - 4cm +# - 10cm +# - 30cm +# - 60cm +# - 1m +# - 1.6m +# - 3m +# soilw: *soilt_levs +# solar: +# - sfc +# ssrun: +# - sfc temp: - - 2ds +# - 2ds - 2m - - 500mb - - 700mb - - 850mb - - 925mb - - sfc - totp: - - sfc - trc1: - - int - - sfc - ulwrf: - - sfc - - top - uswrf: - - sfc - - top - vbdsf: - - sfc - vddsf: - - sfc - vig: - - sfc - vis: - - sfc - visbsn: - - sfc - vort: - - 500mb - vvel: - - 700mb - - mean - vvort: - - mx01 - - mx02 - weasd: - - sfc - wspeed: - - 10m - - 80m - - 250mb - - 850mb - - max - - mdn - - mup +# - 500mb +# - 700mb +# - 850mb +# - 925mb +# - sfc +# totp: +# - sfc +# trc1: +# - int +# - sfc +# ulwrf: +# - sfc +# - top +# uswrf: +# - sfc +# - top +# vbdsf: +# - sfc +# vddsf: +# - sfc +# vig: +# - sfc +# vis: +# - sfc +# visbsn: +# - sfc +# vort: +# - 500mb +# vvel: +# - 700mb +# - mean +# vvort: +# - mx01 +# - mx02 +# weasd: +# - sfc +# wspeed: +# - 10m +# - 80m +# - 250mb +# - 850mb +# - max +# - mdn +# - mup From 969826908580cec58b496df650652796f03eb3a2 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 17 Oct 2025 11:00:42 -0600 Subject: [PATCH 06/98] Tested all graphics that show up on web for RRFS. --- adb_graphics/conversions.py | 4 +- adb_graphics/datahandler/gribdata.py | 130 +++------- adb_graphics/default_specs.yml | 159 +++++++++--- adb_graphics/figure_builders.py | 52 +--- adb_graphics/figures/maps.py | 130 +++++++--- adb_graphics/specs.py | 5 - adb_graphics/utils.py | 35 +++ create_graphics.py | 4 +- image_lists/rrfs_subset.yml | 350 ++++++++++++--------------- 9 files changed, 478 insertions(+), 391 deletions(-) diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index 1917a1f..e6b9121 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -5,7 +5,7 @@ converted values as output. ''' -import numpy as np +from xarray.ufuncs import sqrt, square def k_to_c(field, **kwargs): @@ -29,7 +29,7 @@ def magnitude(a, b, **kwargs): ''' Return the magnitude of vector components ''' - return np.sqrt(np.square(a) + np.square(b)) + return sqrt(square(a) + square(b)) def m_to_dm(field, **kwargs): diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 6490d6f..0a8dc15 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -12,7 +12,7 @@ from matplotlib import cm import numpy as np import xarray as xr - +from adb_graphics.datahandler import gribfile from .. import conversions from .. import errors from .. import specs @@ -33,19 +33,15 @@ class UPPData(specs.VarSpec): model: string describing the model type ''' - def __init__(self, ds, short_name, **kwargs): + def __init__(self, ds, short_name, spec, **kwargs): # Parse kwargs first - config = kwargs.get('config', 'adb_graphics/default_specs.yml') self.model = kwargs.get('model') - self.filetype = kwargs.get('filetype', 'prs') self.grib_path = kwargs.get("grib_path") - specs.VarSpec.__init__(self, config) - - self.spec = self.yml + self.spec = spec self.short_name = short_name self.level = 'ua' @@ -142,15 +138,8 @@ def field_mean(self, values, variable, levels, global_levels, **kwargs): ''' Returns the mean of the values. ''' - fsum = np.zeros_like(values) - - chosen_levels = global_levels if 'global' in self.model else levels - for level in global_levels: - val_lev = self.values(name=variable, level=level) - fsum = fsum + val_lev - val_lev.close() - - return fsum / len(chosen_levels) + levs = [int(x[:-2]) for x in levels] + return values.sel(isobaricInhPa=levs).mean("isobaricInhPa") def _get_data_levels(self, vertical_dim): @@ -172,7 +161,8 @@ def _get_field(self, spec): ''' Given an ncl_name, return the NioVariable object. ''' - return gribfile.GribFile(self.grib_path, spec).contents + ds = gribfile.GribFile(self.grib_path, spec).contents + return ds.__getattr__([x for x in ds.data_vars][0]) def _get_level(self, field, level, spec, **kwargs): @@ -331,57 +321,6 @@ def lev_descriptor(self): return self.field.level_type - @property - def level_type(self): - - ''' Returns a Grib2 code for type of level. 10 is used for - entire atmosphere in HRRR, while 200 is used in RRFS. ''' - - if self.filetype == 'prs': - if self.model == 'rrfs' or self.model == 'regional_mpas': - return 200 - return 10 - return 105 - - def ncl_name(self, spec: dict): - - ''' Get the ncl_name from the specified spec dict. ''' - - name = spec.get('ncl_name') - - if isinstance(name, dict): - if self.model in name.keys(): - name = name.get(self.model) - else: - name = name.get(self.filetype) - - if name is None: - print(f"Cannot find ncl_name for: ") - for key, value in spec.items(): - print(f'{key}: {value}') - raise KeyError - - # The level_type for the entire atmosphere could be L10 or L200. Thanks - # Grib2! Handle that in "try" statement when reading file. - - name = name if isinstance(name, list) else [name] - - try_name = '' - for try_name in name: - try_name = try_name.format(fhr=self.fhr, - grid=self.grid_suffix, - level_type=self.level_type) - - try: - self._get_field(try_name) - except errors.GribReadError: - continue - else: - return try_name - - msg = f'Could not find any of {try_name} in input file' - raise errors.GribReadError(msg) - def numeric_level(self, index_match=True, level=None, split=None): ''' @@ -495,7 +434,7 @@ def aviation_flight_rules(self, values, **kwargs): Generates a field of Aviation Flight Rules from Ceil and Vis ''' - ceil = values + ceil = values.to_dataarray().squeeze() vis = self.values(name='vis', level='sfc') flru = np.where((ceil > 1.) & (ceil < 3.), 1.01, 0.0) @@ -631,11 +570,6 @@ def grid_info(self): Lo2='lon_2', ) - # Last coordinate listed should be latitude or longitude - lat_var = [var for var in self.field.coords if 'lat' in var][0] - - # Get the latitude variable - grid_info = {} var_info = self.field grid_def = var_info.attrs["GRIB_gridDefinitionDescription"].lower() @@ -814,7 +748,7 @@ def values(self, level=None, name=None, **kwargs): ''' level = level or self.level - vals = self.field + vals = self.ds #one_lev = kwargs.get('one_lev', True) #vertical_index = kwargs.get('vertical_index') @@ -836,7 +770,7 @@ def values(self, level=None, name=None, **kwargs): spec = self.spec.get(name, {}).get(level, {}) if not spec and name is not None: raise errors.NoGraphicsDefinitionForVariable(name, level) - field = self._get_field(spec["cfgrib"]) + vals = self._get_field(spec["cfgrib"]) #lev = vertical_index #vals = field @@ -876,9 +810,11 @@ def values(self, level=None, name=None, **kwargs): if transforms and do_transform: vals = self.get_transform(transforms, vals) + if isinstance(vals, xr.Dataset): + return vals.to_dataarray().squeeze() return vals - def vector_magnitude(self, field1, field2, level=None, vertical_index=None, **kwargs): + def vector_magnitude(self, field1, cfkeys=None, field2_id=None, level=None, vertical_index=None, **kwargs): # pylint: disable=unused-argument @@ -888,23 +824,29 @@ def vector_magnitude(self, field1, field2, level=None, vertical_index=None, **kw first layer of a variable is returned if none is provided. ''' - if isinstance(field1, str): - field1 = self.values( - level=level, - ncl_name=field1, - vertical_index=vertical_index, - **kwargs, - ) - - if isinstance(field2, str): - field2 = self.values( - level=level, - ncl_name=field2, - vertical_index=vertical_index, - **kwargs, - ) - - mag = conversions.magnitude(field1, field2) + if cfkeys: + if cfkeys.get("level") is None: + cfkeys["level"] = utils.numeric_level(level=self.level, index_match=False)[0] + field2_spec = {"cfgrib": cfkeys} + else: + var, lev = field2_id.split(".") + field2_spec = self.spec + for key in (var, lev): + field2_spec = field2_spec[key] + + ds = gribfile.GribFile(self.grib_path, field2_spec["cfgrib"]).contents + args = { + "ds": ds, + "fhr": self.fhr, + "level": self.level, + "model": self.model, + "short_name": self.short_name, + "spec": self.spec, + "grib_path": self.grib_path, + } + field2 = fieldData(**args).ds + + mag = conversions.magnitude(field1.to_dataarray().squeeze(), field2.to_dataarray().squeeze()) field1.close() field2.close() diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index f3fce82..0b7e764 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -101,6 +101,7 @@ cfgrib: shortName: rare typeOfLevel: heightAboveGround + level: 1000 clevs: !!python/object/apply:numpy.arange [5, 76, 5] cmap: NWSReflectivity colors: cref_colors @@ -110,6 +111,10 @@ unit: dBZ acfrozr: # Run Total Graupel sfc: &graupel + cfgrib: + parameterNumber: 227 + stepRange: 0-16 + typeOfLevel: surface clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 2] cmap: gist_ncar colors: pcp_colors @@ -126,7 +131,7 @@ acfrzr: # Run Total Freezing Rain level: 0 typeOfLevel: surface stepType: accum - stepRange: # startswith("0-") + #stepRange: # startswith("0-") ncl_name: FRZR_P8_L1_GLC0_acc title: Run Total Freezing Rain acpcp: # Accumulated run total precipitation @@ -135,7 +140,7 @@ acpcp: # Accumulated run total precipitation shortName: tp level: 0 typeOfLevel: surface - stepRange: # startswith("0-") + #stepRange: # startswith("0-") stepType: accum clevs: [0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 15, 20, 40] cmap: gist_ncar @@ -439,11 +444,10 @@ cell: # Storm cell motion transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - field2: - cfgrib: - shortName: ustm - typeOfLevel: heightAboveGroundLayer - level: 6000 + cfkeys: + shortName: vstm + typeOfLevel: heightAboveGroundLayer + level: 6000 unit: kt ceil: # Ceiling ua: &ceil @@ -514,7 +518,8 @@ cicep: # Categorical Ice Pellets cfgrib: shortName: cicep typeOfLevel: surface - stepType: accum + level: 0 + stepType: instant ncl_name: CICEP_P0_L1_{grid} cin: # Surface Convective Inhibition mu: @@ -600,6 +605,7 @@ cloudcover: shortName: tcc typeOfLevel: atmosphereSingleLayer stepType: instant + level: 0 ncl_name: TCDC_P0_L{level_type}_{grid} title: Total Cloud Cover clwmr: # Cloud water Mixing Ratio @@ -689,6 +695,9 @@ colto: # Column Total Ozone unit: DU cpofp: # Frozen Precipitation Percentage sfc: + cfgrib: + shortName: cpofp + typeOfLevel: surface clevs: [-0.1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] cmap: gist_ncar colors: frzn_colors @@ -713,6 +722,11 @@ cref: # Composite reflectivity title: Observed Composite Reflectivity sfc: <<: *refl + cfgrib: + parameterNumber: 5 + parameterCategory: 16 + typeOfLevel: atmosphereSingleLayer + level: 0 ncl_name: REFC_P0_L{level_type}_{grid} title: Composite Reflectivity include_obs: True @@ -732,12 +746,14 @@ csnow: # Categorical Snow cfgrib: shortName: csnow typeOfLevel: surface + level: 0 ncl_name: CSNOW_P0_L1_{grid} ctop: # Cloud top height ua: cfgrib: shortName: gh typeOfLevel: cloudTop + level: 0 clevs: !!python/object/apply:numpy.arange [0, 61, 5] cmap: gist_ncar colors: ceil_colors @@ -855,6 +871,11 @@ fullintdust: # Full vertically integrated dust (Fine dust + Coarse dust) level2: int echotop: # Echo Top sfc: + cfgrib: + parameterNumber: 3 + parameterCategory: 16 + typeOfLevel: atmosphereSingleLayer + level: 0 clevs: !!python/object/apply:numpy.arange [4, 50, 3] cmap: NWSReflectivity colors: cref_colors @@ -905,6 +926,10 @@ ffldro: # Ensemble flash flood runoff title: Prob of 6-hr Precip > RFC flash flood guidance w/in 40 km firewx: # Fire Weather Index sfc: + cfgrib: + parameterNumber: 26 + parameterCategory: 4 + typeOfLevel: surface clevs: [10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 100] cmap: gist_ncar colors: rainbow11_colors @@ -914,6 +939,10 @@ firewx: # Fire Weather Index unit: "%" flru: # Aviation Flight Rules sfc: + cfgrib: + shortName: gh + typeOfLevel: cloudCeiling + level: 0 clevs: [0.0, 1.0, 2.0, 3.0, 4.0] cmap: gist_ncar colors: flru_colors @@ -931,6 +960,7 @@ G113bt: # GOES-W Brightness temperature, water vapor (Ch 3) cfgrib: parameterNumber: 242 typeOfLevel: atmosphere + level: 0 clevs: !!python/object/apply:numpy.arange [-80, 41, 1] cmap: WVCIMSS_r colors: goes_colors @@ -1033,6 +1063,7 @@ gust: cfgrib: shortName: gust typeOfLevel: surface + level: 0 clevs: !!python/object/apply:numpy.arange [5, 95, 5] cmap: gist_ncar colors: wind_colors @@ -1057,6 +1088,7 @@ hail: # Max 1h Hail diameter shortName: hail typeOfLevel: surface stepType: max + level: 0 clevs: [0.10, 0.25, 0.50, 0.75, 1.0, 1.5, 2.0, 2.5, 3.0] cmap: gist_ncar colors: hail_colors @@ -1086,6 +1118,11 @@ hlcy: # Helicity unit: $m^2 / s^2$ in25: # Hourly updraft helicity over 2-5 km layer <<: *hlcy + cfgrib: + parameterNumber: 15 + parameterCategory: 7 + topLevel: 5000 + bottomLevel: 2000 clevs: !!python/object/apply:numpy.arange [25, 301, 25] ncl_name: UPHL_P0_2L103_{grid} split: True @@ -1101,12 +1138,22 @@ hlcy: # Helicity unit: $m^2 / s^2$ mn03: &hlcy_mn03 # Hourly minimum of updraft helicity over 0-3 km layer <<: *hlcy_mn + cfgrib: + parameterNumber: 200 + typeOfLevel: heightAboveGroundLayer + topLevel: 3000 + bottomLevel: 0 title: 0-3km Min Updraft Helicity (over prv hr) mn16: &hlcy_mn16 # Hourly minimum of updraft helicity over 1-6 km layer <<: *hlcy_mn title: 1-6km Min Updraft Helicity (over prv hr) mn25: &hlcy_mn25 # Hourly minimum of updraft helicity over 2-5 km layer <<: *hlcy_mn + cfgrib: + parameterNumber: 200 + typeOfLevel: heightAboveGroundLayer + topLevel: 5000 + bottomLevel: 2000 title: 2-5km Min Updraft Helicity (over prv hr) mx02: &hlcy_mx02 # Hourly maximum of updraft helicity over 0-2 km layer <<: *hlcy @@ -1118,6 +1165,11 @@ hlcy: # Helicity title: 0-2km Max Updraft Helicity (over prv hr) mx03: &hlcy_mx03 # Hourly maximum of updraft helicity over 0-3 km layer <<: *hlcy + cfgrib: + parameterNumber: 199 + typeOfLevel: heightAboveGroundLayer + topLevel: 3000 + bottomLevel: 0 clevs: !join_ranges [[12.5, 87.6, 12.5], [100, 301, 25]] colors: rainbow16_colors ncl_name: MXUPHL_P8_2L103_{grid}_max1h @@ -1134,6 +1186,11 @@ hlcy: # Helicity title: 1-6km Max Updraft Helicity (over prv hr) mx25: &hlcy_mx25 # Hourly maximum of updraft helicity over 2-5 km layer <<: *hlcy + cfgrib: + parameterNumber: 199 + typeOfLevel: heightAboveGroundLayer + topLevel: 5000 + bottomLevel: 2000 clevs: !join_ranges [[25, 176, 25], [200, 601, 50]] colors: rainbow16_colors ncl_name: MXUPHL_P8_2L103_{grid}_max1h @@ -1142,6 +1199,11 @@ hlcy: # Helicity title: 2-5km Max Updraft Helicity (over prv hr) sr01: # 0-1 km Storm Relative Helicity <<: *hlcy + cfgrib: + shortName: hlcy + typeOfLevel: heightAboveGroundLayer + topLevel: 1000 + bottomLevel: 0 clevs: [25, 50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800] ncl_name: HLCY_P0_2L103_{grid} unit: $m^2 / s^2$ @@ -1150,6 +1212,11 @@ hlcy: # Helicity title: 0-1 km Storm Relative Helicity sr03: # 0-3 km Storm Relative Helicity <<: *hlcy + cfgrib: + shortName: hlcy + typeOfLevel: heightAboveGroundLayer + topLevel: 3000 + bottomLevel: 0 clevs: [25, 50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800] ncl_name: HLCY_P0_2L103_{grid} unit: $m^2 / s^2$ @@ -1202,6 +1269,7 @@ hpbl: # Height of Planetary Boundary Layer cfgrib: shortName: gh typeOfLevel: planetaryBoundaryLayer + level: 0 clevs: [25, 50, 100, 200, 300, 500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000] cmap: ir_rgbv colors: pbl_colors @@ -1285,6 +1353,7 @@ lcl: # Lifted condensation level cfgrib: shortName: gh typeOfLevel: adiabaticCondensation + level: 0 clevs: !!python/object/apply:numpy.arange [0, 5000, 250] cmap: rainbow colors: lcl_colors @@ -1326,7 +1395,7 @@ li: # Lifted Index cfgrib: shortName: 4lftx typeOfLevel: pressureFromGroundLayer - level: 18000 + topLevel: 18000 clevs: !!python/object/apply:numpy.arange [-15, 16] cmap: Spectral colors: lifted_index_colors @@ -1378,6 +1447,7 @@ ltng: # Lightning cfgrib: shortName: ltng typeOfLevel: atmosphere + level: 0 clevs: !!python/object/apply:numpy.arange [5, 91, 5] cmap: jet colors: graupel_colors @@ -1390,7 +1460,7 @@ lwtp: # Lightning with total precip shortName: tp typeOfLevel: surface stepType: accum - stepRange: # startswith fhr - 1 + #stepRange: # startswith fhr - 1 clevs: [0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar colors: pcp_colors_high @@ -1416,6 +1486,11 @@ mfrp: # Fire radiative power mref: # Maximum reflectivity for past hour at 1 km AGL sfc: <<: *refl + cfgrib: + parameterNumber: 198 + typeOfLevel: heightAboveGround + stepType: max + level: 1000 ncl_name: MAXREF_P8_L103_{grid}_max1h title: Max 1km agl Reflectivity (over prev hr) oc: # Organic Carbon @@ -1535,6 +1610,7 @@ ptmp: # Potential temperature cfgrib: shortName: pt typeOfLevel: surface + level: 0 clevs: !!python/object/apply:numpy.arange [210, 350, 5] cmap: jet colors: t_colors @@ -1547,8 +1623,9 @@ ptyp: # Hourly total precipitation cfgrib: shortName: tp typeOfLevel: surface + level: 0 stepType: accum - stepRange: # startswith fhr - 1 + #stepRange: # startswith fhr - 1 clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar colors: pcp_colors @@ -1616,6 +1693,10 @@ pwtr: # Precipitable water ref: # Maximum reflectivity for past hour at 1 km AGL m10: <<: *refl + cfgrib: + shortName: rare + typeOfLevel: isothermal + level: 263 ncl_name: REFD_P0_L20_{grid} title: -10C Isothermal Reflectivity maxm10: @@ -1704,6 +1785,9 @@ rh: # Relative Humidity variable: rh mean: # Mean RH 850mb to 500mb <<: *rh + cfgrib: + shortName: r + typeOfLevel: isobaricInhPa print_units: false title: Mean 850-500mb RH (%, shaded), 700mb Wind (kt) transform: @@ -1716,6 +1800,10 @@ rh: # Relative Humidity wind: 700mb pw: # RH wrt Precipitable Water <<: *rh + cfgrib: + parameterNumber: 242 + typeOfLevel: atmosphere + level: 0 ncl_name: RHPW_P0_L10_{grid} title: Relative Humidity wrt Precipitable Water rvil: # Radar-derived Vertically Integrated Liquid @@ -1723,6 +1811,7 @@ rvil: # Radar-derived Vertically Integrated Liquid cfgrib: shortName: veril typeOfLevel: atmosphereSingleLayer + level: 0 clevs: [0.05, 0.15, 0.76, 3.47, 6.92, 12, 31.6, 35, 40, 45, 50, 55, 60, 65, 70] cmap: NWSReflectivity colors: cref_colors @@ -1760,6 +1849,7 @@ shear: cfgrib: shortName: vucsh typeOfLevel: heightAboveGroundLayer + level: 0 topLevel: 0 bottomLevel: 1000 clevs: [5, 10, 20, 30, 40, 50, 60] @@ -1770,11 +1860,11 @@ shear: transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - field2: VVCSH_P0_2L103_{grid} - cfgrib: + cfkeys: shortName: vvcsh typeOfLevel: heightAboveGroundLayer topLevel: 0 + level: 0 bottomLevel: 1000 one_lev: True split: True @@ -1787,6 +1877,7 @@ shear: cfgrib: shortName: vucsh typeOfLevel: heightAboveGroundLayer + level: 0 topLevel: 0 bottomLevel: 6000 clevs: [20, 30, 40, 50, 60, 70, 80, 90, 100] @@ -1794,10 +1885,10 @@ shear: transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - field2: VVCSH_P0_2L103_{grid} - cfgrib: + cfkeys: shortName: vvcsh typeOfLevel: heightAboveGroundLayer + level: 0 topLevel: 0 bottomLevel: 6000 one_lev: True @@ -2069,6 +2160,7 @@ solar: # Incoming Solar Radiation sfc: &incoming_radiation cfgrib: shortName: csdsf + level: 0 typeOfLevel: surface clevs: [50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100] cmap: gist_ncar @@ -2091,8 +2183,9 @@ sphum: # Specific humidity ssrun: # Storm Surface Runoff sfc: &precip cfgrib: - shortname: ssrun + shortName: ssrun typeOfLevel: surface + level: 0 stepType: accum clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar @@ -2144,7 +2237,7 @@ temp: # Temperature ticks: 10 transform: conversions.k_to_f unit: F - #wind: 10m + wind: 10m 500mb: &ua_temp cfgrib: shortName: t @@ -2208,7 +2301,7 @@ temp: # Temperature sfc: <<: *sfc_temp cfgrib: - shortName: 2t + shortName: t level: 0 typeOfLevel: surface ncl_name: TMP_P0_L1_{grid} @@ -2231,7 +2324,7 @@ totp: # Hourly total precipitation cfgrib: shortName: tp typeOfLevel: surface - stepRange: # startswith fhr - 1 + #stepRange: # startswith fhr - 1 stepType: accum contours: pres_msl: @@ -2251,7 +2344,7 @@ totp6h: # 6-hourly total precipitation cfgrib: shortName: tp typeOfLevel: surface - stepRange: # startswith 0 + #stepRange: # startswith 0 stepType: accum contours: pres_msl: @@ -2384,7 +2477,8 @@ ulwrf: # Upward Longwave Radiation Flux cfgrib: parameterName: Upward long-wave radiation flux typeOfLevel: nominalTop - stepType: instant + level: 0 + stepType: avg clevs: !!python/object/apply:numpy.arange [80, 341, 2] cmap: ir_rgbv_r colors: radiation_mix_colors @@ -2432,6 +2526,9 @@ uswrf: # Upward Shortwave Radiation Flux title: Upward SW Radiation Flux, Surface top: # Nominal top of atmosphere <<: *radiation_flux + cfgrib: + parameterNumber: 8 + typeOfLevel: nominalTop clevs: !!python/object/apply:numpy.arange [50, 851, 10] cmap: Greys_r colors: radiation_bw_colors @@ -2671,8 +2768,7 @@ wspeed: # Wind Speed transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - field2: VGRD_P0_L103_{grid} - cfgrib: + cfkeys: shortName: 10v level: 10 typeOfLevel: heightAboveGround @@ -2695,8 +2791,7 @@ wspeed: # Wind Speed transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - field2: VGRD_P0_L100_{grid} - cfgrib: + cfkeys: shortName: v typeOfLevel: isobaricInhPa unit: kt @@ -2730,8 +2825,7 @@ wspeed: # Wind Speed transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - field2: VGRD_P0_L100_{grid} - cfgrib: + cfkeys: shortName: v typeOfLevel: isobaricInhPa max: # Hourly Maximum 10m Wind @@ -2743,7 +2837,13 @@ wspeed: # Wind Speed ncl_name: WIND_P8_L103_{grid}_max1h title: Max 10m Wind (over prev hour) transform: conversions.ms_to_kt + wind: 10m mdn: # Hourly Maximum Downdraft Velocity + cfgrib: + parameterNumber: 221 + typeOfLevel: isobaricLayer + topLevel: 100 + bottomLevel: 1000 clevs: [-40, -35, -30, -25, -22.5, -20, -17.5, -15, -12.5, -10, -7.5, -5, -2.5, -2, -1.5, -1, -0.5] cmap: jet colors: mdn_colors @@ -2752,6 +2852,11 @@ wspeed: # Wind Speed title: Max Downdraft Velocity (over prev hour) unit: m/s mup: # Hourly Maximum Updraft Velocity + cfgrib: + parameterNumber: 220 + typeOfLevel: isobaricLayer + topLevel: 100 + bottomLevel: 1000 clevs: [0.5, 1, 1.5, 2, 2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20, 22.5, 25, 30, 35, 40] cmap: jet colors: mup_colors diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index d22ae5a..84b0092 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -9,12 +9,14 @@ import matplotlib.pyplot as plt import numpy as np +import yaml from adb_graphics.datahandler import gribfile from adb_graphics.datahandler import gribdata import adb_graphics.errors as errors from adb_graphics.figures import maps from adb_graphics.figures import skewt +from adb_graphics.utils import numeric_level AIRPORTS = 'static/Airports_locs.txt' @@ -80,10 +82,6 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, ''' fig, axes = set_figure(cla.model_name, cla.graphic_type, tile) - ds = gribfile.GribFile(grib_path, spec["cfgrib"]).contents - - if dp2: - ds2 = gribfile.GribFile(dp2, spec["cfgrib"]).contents # set last_panel to send into DataMap for colorbar control last_panel = False @@ -114,45 +112,14 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, mem = mem if mem < 4 else index - 1 mem = mem if mem < 8 else index - 2 - # Object to be plotted on the map in filled contours. - field = gribdata.fieldData( - ds=ds, + # Create an object that holds all the fields for this map + map_fields = maps.MapFields( + grib_path=grib_path, + grib_path2=dp2, fhr=fhr, - filetype=cla.file_type, + fields_spec=cla.specs, level=level, - member=mem, - model=model, - short_name=variable, - config=cla.specs['file'], - grib_path=dp2, - ) - - if cla.graphic_type == "diff": - - field2 = gribdata.fieldData( - ds=ds2, - fhr=fhr, - filetype=cla.file_type, - level=level, - member=mem, - model=model, - short_name=variable, - config=cla.specs['file'] - ) - - try: - field2.field - except errors.GribReadError: - print((f'Cannot find grib2 variable for {variable} at {level} in', - '2nd set of files. Skipping.')) - return - - field.data = field.values() - field2.values() - - - map_fields = maps.MapFields( - fields_spec=spec, - main_field=field, + name=variable, map_type=cla.graphic_type, model=model, tile=tile, @@ -162,7 +129,7 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, m = maps.Map( airport_fn=AIRPORTS, ax=current_ax, - grid_info=field.grid_info(), + grid_info=map_fields.shaded.grid_info(), model=model, plot_airports=spec.get('plot_airports', True), tile=tile, @@ -225,7 +192,6 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, plt.clf() # Closes all the figure windows. plt.close('all') - del field del m gc.collect() diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 149c9ca..4fd1c26 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -18,6 +18,9 @@ from mpl_toolkits.basemap import shiftgrid import numpy as np +from adb_graphics.datahandler import gribdata, gribfile +from adb_graphics.utils import numeric_level + # FULL_TILES is a list of strings that includes the labels GSL attaches to some of # the wgrib2 cutouts used for larger domains like RAP, RRFS NA, and global. FULL_TILES = [ @@ -277,13 +280,17 @@ class DataMap(): #pylint: disable=unused-argument def __init__(self, map_fields, map_, model_name=None, **kwargs): - self.field = map_fields.main_field + self.field = map_fields.shaded self.contour_fields = map_fields.contours self.hatch_fields = map_fields.hatches + self.map_fields = map_fields self.map = map_ self.model_name = model_name self.plot_scatter = map_fields.fields_spec.get('plot_scatter', False) + def wind_fields(self, level): + return self.map_fields.wind_fields(level) + @staticmethod def add_logo(ax): @@ -327,11 +334,11 @@ def _colorbar(self, cc, ax): ) if self.field.short_name == 'flru': - ticks = [label.rjust(30) for label in ['VFR', 'MVFR', 'IFR', 'LIFR']] + ticks = [label.rjust(30) for label in ['VFR', 'MVFR', 'IFR', 'LIFR', ""]] # this step is done to allow proper order of icing severity levels (trace before light) if self.field.short_name == 'icsev': - ticks = [label.rjust(30) for label in ['TRACE', 'LIGHT', 'MODERATE', 'HEAVY']] + ticks = [label.rjust(30) for label in ['TRACE', 'LIGHT', 'MODERATE', 'HEAVY', ""]] cbar.ax.set_xticklabels(ticks, fontsize=12) @@ -556,12 +563,10 @@ def _draw_hatches(self, ax): func=self.map.m.contourf, **field.contour_kwargs, ) - # For each level, we set the color of its hatch - for collection in cf.collections: - collection.set_edgecolor(colors) - collection.set_facecolor(['None']) - collection.set_linewidth(linewidths) + cf.set_edgecolor(colors) + cf.set_facecolor("None") + cf.set_linewidth(linewidths) # Create legend for precip type field if self.field.short_name == 'ptyp': @@ -627,7 +632,7 @@ def _title(self): else: level = level if not isinstance(level, list) else level[0] title = f'{level} {lev_unit} {f.field.long_name} {units}' - plt.title(f"{title}", position=(0.5, 1.10), fontsize=18) + plt.title(f"{title}", loc="center", y=1.10, fontsize=18) # Two lines for hatched data (top), and contoured data (bottom) on the right contoured = self._set_overlay_string() @@ -643,7 +648,8 @@ def _wind_barbs(self, level): by lat,lon so the stride is set in the TILE_DEFS. For the globalCONUS subdomains, further dividing by 2.5 works well. ''' - u, v = self.field.wind(level) + lev = level if not isinstance(level, bool) else self.field.level + u, v = [f.data for f in self.wind_fields(lev)] tile = self.map.tile @@ -891,15 +897,55 @@ class MapFields(): contours, hatched spaces, and overlayed contours needed for a full product. ''' - def __init__(self, main_field, fields_spec=None, map_type=None, + def __init__(self, fhr, fields_spec, grib_path, level, name, map_type=None, **kwargs): - self.main_field = main_field - self.fields_spec = fields_spec if fields_spec is not None else {} + self.grib_path = grib_path + self.fhr = fhr + self.fields_spec = fields_spec + self.level = level self.map_type = map_type self.model = kwargs.get('model') + self.name = name self.tile = kwargs.get('tile', 'full') + self.map_spec = self.fields_spec[self.name][self.level] + self.set_level(self.level, self.map_spec) + # Required if map_type is "diff" + self.grib_path2 = kwargs.get("grib_path2") + + @staticmethod + def set_level(level, spec): + + nlevel, _ = numeric_level(level=level, index_match=False) + if nlevel and spec["cfgrib"].get("level") is None: + spec["cfgrib"]["level"] = nlevel + #if spec["cfgrib"].get("level") is None and not spec["cfgrib"].get("stepRange") and not \ + # spec["cfgrib"].get("topLevel") and not \ + # spec["cfgrib"].get("typeOfLevel") == "surface" and not \ + # spec["cfgrib"].get("scaledValueOfFirstFixedSurface"): + + @property + def shaded(self): + ds = gribfile.GribFile(self.grib_path, self.map_spec["cfgrib"]).contents + args = { + "ds": ds, + "fhr": self.fhr, + "level": self.level, + "model": self.model, + "short_name": self.name, + "spec": self.fields_spec, + "grib_path": self.grib_path, + } + field = gribdata.fieldData(**args) + if self.map_type == "diff": + args["ds"] = gribfile.GribFile(self.grib_path2, self.map_spec["cfgrib"]).contents + args["grib_path"] == self.grib_path2 + field2 = gribdata.fieldData(**args) + field.data = field.values() - field2.values() + + return field + @property def contours(self): ''' Return the list of contour fieldData objects''' @@ -920,25 +966,51 @@ def hatches(self): return self._overlay_fields('hatches') + def wind_fields(self, level=None): + ''' Return u, v tuple of wind fields ''' + + lev = level or self.level + winds = [] + for var in ("u", "v"): + wind_spec = self.fields_spec[var][lev] + self.set_level(lev, wind_spec) + args = { + "ds": gribfile.GribFile(self.grib_path, wind_spec["cfgrib"]).contents, + "fhr": self.fhr, + "level": lev, + "model": self.model, + "short_name": var, + "spec": self.fields_spec, + "grib_path": self.grib_path, + } + winds.append(gribdata.fieldData(**args)) + return winds + def _overlay_fields(self, spec_sect): ''' Generate a list of fieldData objects for the specified type of overlay -- hatches or contours ''' - overlays = self.fields_spec.get(spec_sect) + overlay_fields = [] - if overlays is not None: - for overlay, overlay_kwargs in overlays.items(): - if '_' in overlay: - var, lev = overlay.split('_') - else: - var, lev = overlay, self.main_field.level - - # Make a copy of the main object, and change the - # attributes to match the overlay field - overlay_obj = copy.deepcopy(self.main_field) - overlay_obj.contour_kwargs = overlay_kwargs - overlay_obj.short_name = var - overlay_obj.level = lev - - overlay_fields.append(overlay_obj) + for overlay, overlay_kwargs in self.map_spec.get(spec_sect, {}).items(): + if '_' in overlay: + var, lev = overlay.split('_') + else: + var, lev = overlay, self.level + + overlay_spec = self.fields_spec[var][lev] + self.set_level(lev, overlay_spec) + args = { + "ds": gribfile.GribFile(self.grib_path, overlay_spec["cfgrib"]).contents, + "fhr": self.fhr, + "level": lev, + "model": self.model, + "short_name": var, + "spec": self.fields_spec, + "grib_path": self.grib_path, + } + overlay_obj = gribdata.fieldData(**args) + # Set the attributes for the overlay field + overlay_obj.contour_kwargs = overlay_kwargs + overlay_fields.append(overlay_obj) return overlay_fields diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index a4088e1..4b331f4 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -21,11 +21,6 @@ class VarSpec(abc.ABC): the config file. ''' - def __init__(self, config): - - with open(config, 'r') as cfg: - self.yml = yaml.load(cfg, Loader=yaml.Loader) - @property def aod_colors(self) -> np.ndarray: diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index f91c812..f86ffa3 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -10,6 +10,7 @@ import importlib as il from math import atan2, degrees from multiprocessing import Process +from string import digits, ascii_letters import os import subprocess import sys @@ -291,6 +292,40 @@ def load_specs(arg): return specs +def numeric_level(index_match=True, level=None, split=None): + + ''' + Split the numeric level and unit associated with the level key. + + A blank string is returned for lev_val for levels that do not contain a + numeric, e.g., 'sfc' or 'ua'. + ''' + + level = level if level is not None else 0 + + # Gather all the numbers in the string + lev_val = ''.join([c for c in level if (c in digits or c == '.')]) + + # Convert the numbers to a list, and make integers or floats + if lev_val: + if split is not None: + lev_val = [int(lev) for lev in lev_val] + else: + lev_val = [float(lev_val) if '.' in lev_val else int(lev_val)] + + # Gather all the letters + lev_unit = ''.join([c for c in level if c in ascii_letters]) + + if index_match: + if lev_unit == 'cm': + lev_val = [val / 100. for val in lev_val] + if lev_unit in ['mb', 'mxmb']: + lev_val = [val * 100. for val in lev_val] + if lev_unit in ['in', 'km', 'mn', 'mx', 'sr']: + lev_val = [val * 1000. for val in lev_val] + + return lev_val, lev_unit + def old_enough(age, file_path): ''' diff --git a/create_graphics.py b/create_graphics.py index d4e67d2..d87753c 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -91,8 +91,8 @@ def create_maps(cla, fhr, grib_path, workdir, grib_path2=None): args.append((cla, fhr, grib_path, level, model, spec, variable, workdir, tile, grib_path2)) - print(f'Queueing {len(args)} maps') - parallel_maps(*args[0]) + #print(f'Queueing {len(args)} maps') + parallel_maps(*args[-1]) #with Pool(processes=cla.nprocs) as pool: # pool.starmap(parallel_maps, args) diff --git a/image_lists/rrfs_subset.yml b/image_lists/rrfs_subset.yml index b627b20..fe7c7b6 100644 --- a/image_lists/rrfs_subset.yml +++ b/image_lists/rrfs_subset.yml @@ -1,194 +1,166 @@ hourly: model: rrfs variables: -# 1hsnw: -# - sfc -# 1ref: -# - 1000m -# acfrozr: -# - sfc -# acfrzr: -# - sfc -# acpcp: -# - sfc -# acsnod: -# - sfc -# acsnw: -# - sfc -# cape: -# - mu -# - mul -# - mx90mb -# - sfc -# ceil: -# - ua -# ceilexp: -# - ua -# ceilexp2: -# - ua -# cin: -# - sfc -# cloudbase: -# - ua -# cloudcover: -# - bndylay -# - high -# - low -# - mid -# - total -# coarsedust: -# - sfc -# cpofp: -# - sfc -# cref: -# - sfc -# ctop: -# - ua -# dewp: -# - 2m -# echotop: -# - sfc -# emissions: -# - sfc -# finedust: -# - sfc -# firewx: -# - sfc -# flru: -# - sfc -# fullintdust: -# - int -# G113bt: -# - sat -# G114bt: -# - sat -# G123bt: -# - sat -# G124bt: -# - sat -# ghtfl: -# - sfc -# gust: -# - 10m -# hailcast: -# - maxsfc -# hlcy: -# - in25 -# - mn03 -# - mn25 -# - mx03 -# - mx25 -# - sr01 -# - sr03 -# hlcytot: -# - mn03 -# - mn25 -# - mx03 -# - mx25 -# hpbl: -# - sfc -# lcl: -# - sfc -# lhtfl: -# - sfc -# li: -# - best -# - sfc -# ltg3: -# - sfc -# ltng: -# - sfc -# lwtp: -# - sfc -# mref: -# - sfc -# pres: -# - msl -# - sfc -# ptmp: -# - 2m -# ptyp: -# - sfc -# pwtr: -# - sfc -# ref: -# - m10 -# - maxm10 -# rh: -# - 2m -# - 850mb -# - mean -# - pw -# rvil: -# - sfc -# shear: -# - 01km -# - 06km -# shtfl: -# - sfc -# snod: -# - sfc -# soilm: -# - sfc -# soilt: &soilt_levs -# - 0cm -# - 1cm -# - 4cm -# - 10cm -# - 30cm -# - 60cm -# - 1m -# - 1.6m -# - 3m -# soilw: *soilt_levs -# solar: -# - sfc -# ssrun: -# - sfc + 1hsnw: + - sfc + 1ref: + - 1000m + acfrozr: + - sfc + acfrzr: + - sfc + acpcp: + - sfc + acsnod: + - sfc + acsnw: + - sfc + cape: + - mu + - mul + - mx90mb + - sfc + ceil: + - ua + ceilexp: + - ua + ceilexp2: + - ua + cin: + - sfc + cloudbase: + - ua + cloudcover: + - bndylay + - high + - low + - mid + - total + cpofp: + - sfc + cref: + - sfc + ctop: + - ua + dewp: + - 2m + echotop: + - sfc + firewx: + - sfc + flru: + - sfc + ghtfl: + - sfc + gust: + - 10m + hailcast: + - maxsfc + hlcy: + - in25 + - mn03 + - mn25 + - mx03 + - mx25 + - sr01 + - sr03 + hlcytot: + - mn03 + - mn25 + - mx03 + - mx25 + hpbl: + - sfc + lcl: + - sfc + lhtfl: + - sfc + li: + - best + ltng: + - sfc + lwtp: + - sfc + mref: + - sfc + ptmp: + - 2m + pres: + - msl + - sfc + ptyp: + - sfc + pwtr: + - sfc + ref: + - m10 + rh: + - 2m + - 850mb + - mean + - pw + rvil: + - sfc + shear: + - 01km + - 06km + shtfl: + - sfc + snod: + - sfc + soilt: &soilt_levs + - 0cm + - 1cm + - 4cm + - 10cm + - 30cm + - 60cm + - 1m + - 1.6m + - 3m + soilw: *soilt_levs + solar: + - sfc + ssrun: + - sfc temp: -# - 2ds + - 2ds - 2m -# - 500mb -# - 700mb -# - 850mb -# - 925mb -# - sfc -# totp: -# - sfc -# trc1: -# - int -# - sfc -# ulwrf: -# - sfc -# - top -# uswrf: -# - sfc -# - top -# vbdsf: -# - sfc -# vddsf: -# - sfc -# vig: -# - sfc -# vis: -# - sfc -# visbsn: -# - sfc -# vort: -# - 500mb -# vvel: -# - 700mb -# - mean -# vvort: -# - mx01 -# - mx02 -# weasd: -# - sfc -# wspeed: -# - 10m -# - 80m -# - 250mb -# - 850mb -# - max -# - mdn -# - mup + - 500mb + - 700mb + - 850mb + - 925mb + - sfc + + totp: + - sfc + ulwrf: + - sfc + - top + uswrf: + - sfc + - top + vbdsf: + - sfc + vddsf: + - sfc + vis: + - sfc + vort: + - 500mb + vvel: + - 700mb + - mean + vvort: + - mx01 + - mx02 + weasd: + - sfc + wspeed: + - 10m + - 80m + - 250mb + - 850mb + - max + - mdn + - mup From 51e962f656d8c845948475b779f503a68684402c Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 20 Oct 2025 15:03:42 -0600 Subject: [PATCH 07/98] All complete. Working through diffs in graphics. --- adb_graphics/datahandler/gribdata.py | 29 ++- adb_graphics/default_specs.yml | 287 ++++++++++++++++----- adb_graphics/figures/maps.py | 31 +-- adb_graphics/specs.py | 6 +- adb_graphics/utils.py | 6 + image_lists/hrrr_subset.yml | 356 +++++++++++++-------------- image_lists/rrfs_subset.yml | 15 +- 7 files changed, 455 insertions(+), 275 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 0a8dc15..3afa734 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -13,6 +13,7 @@ import numpy as np import xarray as xr from adb_graphics.datahandler import gribfile +from adb_graphics.utils import cfgrib_spec from .. import conversions from .. import errors from .. import specs @@ -508,13 +509,27 @@ def fire_weather_index(self, values, **kwargs): ''' + def _load_field(level, short_name): + spec = cfgrib_spec(self.spec[short_name][level]["cfgrib"], self.model) + ds = gribfile.GribFile(self.grib_path, spec).contents + args = { + "ds": ds, + "fhr": self.fhr, + "level": level, + "model": self.model, + "short_name": short_name, + "spec": self.spec, + "grib_path": self.grib_path, + } + return fieldData(**args).values(do_transform=False) + # Gather fields from the input - veg = values # Chose this value as the main one in the default_specs - temp = self.values(name='temp', level='2m', do_transform=False) - dewpt = self.values(name='dewp', level='2m', do_transform=False) - weasd = self.values(name='weasd', level='sfc', do_transform=False) - gust = self.values(name='gust', level='10m', do_transform=False) - soilm = self.values(name='soilm', level='sfc', do_transform=False) + veg = values.to_dataarray().squeeze() # Chose this value as the main one in the default_specs + temp = _load_field(level="2m", short_name="temp") + dewpt = _load_field(level='2m', short_name='dewp') + weasd = _load_field(level='sfc', short_name='weasd') + gust = _load_field(level='10m', short_name='gust') + soilm = _load_field(level='sfc', short_name='soilm') # A few derived fields dewpt_depression = temp - dewpt @@ -770,7 +785,7 @@ def values(self, level=None, name=None, **kwargs): spec = self.spec.get(name, {}).get(level, {}) if not spec and name is not None: raise errors.NoGraphicsDefinitionForVariable(name, level) - vals = self._get_field(spec["cfgrib"]) + vals = self._get_field(utils.cfgrib_spec(spec["cfgrib"], self.model)) #lev = vertical_index #vals = field diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 0b7e764..836fcfa 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -80,9 +80,16 @@ 1hsnw: # 1 hr Accumulated Snow Using 10:1 Ratio sfc: cfgrib: - shortName: sdwe - level: 0 - typeOfLevel: surface + rrfs: + shortName: sdwe + level: 0 + typeOfLevel: surface + hrrr: + shortName: sdwe + level: 0 + typeOfLevel: surface + stepType: accum + stepRange: "0-16" clevs: [0.03, 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 6, 7, 8] cmap: gist_ncar colors: snow_colors @@ -99,9 +106,14 @@ 1ref: # Reflectivity at 1 km AGL 1000m: &refl cfgrib: - shortName: rare - typeOfLevel: heightAboveGround - level: 1000 + hrrr: + shortName: refd + typeOfLevel: heightAboveGround + level: 1000 + rrfs: + shortName: rare + typeOfLevel: heightAboveGround + level: 1000 clevs: !!python/object/apply:numpy.arange [5, 76, 5] cmap: NWSReflectivity colors: cref_colors @@ -131,7 +143,6 @@ acfrzr: # Run Total Freezing Rain level: 0 typeOfLevel: surface stepType: accum - #stepRange: # startswith("0-") ncl_name: FRZR_P8_L1_GLC0_acc title: Run Total Freezing Rain acpcp: # Accumulated run total precipitation @@ -260,9 +271,16 @@ acsnod: # Accumulated snow acsnw: # Run Total Accumulated Snow Using 10:1 Ratio sfc: cfgrib: - shortName: sdwe - level: 0 - typeOfLevel: surface + hrrr: + shortName: sdwe + level: 0 + typeOfLevel: surface + stepType: accum + stepRange: "0-16" + rrfs: + shortName: sdwe + level: 0 + typeOfLevel: surface clevs: [0.01, 0.1, 1, 2, 3, 4, 6, 8, 10, 12, 18, 24] cmap: gist_ncar colors: snow_colors @@ -602,10 +620,16 @@ cloudcover: total: <<: *cld_cover cfgrib: - shortName: tcc - typeOfLevel: atmosphereSingleLayer - stepType: instant - level: 0 + hrrr: + shortName: tcc + typeOfLevel: atmosphere + stepType: instant + level: 0 + rrfs: + shortName: tcc + typeOfLevel: atmosphereSingleLayer + stepType: instant + level: 0 ncl_name: TCDC_P0_L{level_type}_{grid} title: Total Cloud Cover clwmr: # Cloud water Mixing Ratio @@ -723,10 +747,15 @@ cref: # Composite reflectivity sfc: <<: *refl cfgrib: - parameterNumber: 5 - parameterCategory: 16 - typeOfLevel: atmosphereSingleLayer - level: 0 + hrrr: + shortName: refc + typeOfLevel: atmosphere + level: 0 + rrfs: + parameterNumber: 5 + parameterCategory: 16 + typeOfLevel: atmosphereSingleLayer + level: 0 ncl_name: REFC_P0_L{level_type}_{grid} title: Composite Reflectivity include_obs: True @@ -872,10 +901,16 @@ fullintdust: # Full vertically integrated dust (Fine dust + Coarse dust) echotop: # Echo Top sfc: cfgrib: - parameterNumber: 3 - parameterCategory: 16 - typeOfLevel: atmosphereSingleLayer - level: 0 + hrrr: + parameterNumber: 3 + parameterCategory: 16 + typeOfLevel: cloudTop + level: 0 + rrfs: + parameterNumber: 3 + parameterCategory: 16 + typeOfLevel: atmosphereSingleLayer + level: 0 clevs: !!python/object/apply:numpy.arange [4, 50, 3] cmap: NWSReflectivity colors: cref_colors @@ -925,11 +960,12 @@ ffldro: # Ensemble flash flood runoff ncl_name: FFLDRO_P9_L1_{grid}_acc6h title: Prob of 6-hr Precip > RFC flash flood guidance w/in 40 km firewx: # Fire Weather Index - sfc: + sfc: &firewx cfgrib: - parameterNumber: 26 - parameterCategory: 4 - typeOfLevel: surface + rrfs: + parameterNumber: 26 + parameterCategory: 4 + typeOfLevel: surface clevs: [10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 100] cmap: gist_ncar colors: rainbow11_colors @@ -937,6 +973,14 @@ firewx: # Fire Weather Index ticks: 0 title: Hourly Wildfire Potential unit: "%" +firewx_transform: + sfc: + <<: *firewx + cfgrib: + shortName: vgtyp + typeOfLevel: surface + level: 0 + transform: fire_weather_index flru: # Aviation Flight Rules sfc: cfgrib: @@ -958,9 +1002,14 @@ flru: # Aviation Flight Rules G113bt: # GOES-W Brightness temperature, water vapor (Ch 3) sat: &goes_sat cfgrib: - parameterNumber: 242 - typeOfLevel: atmosphere - level: 0 + hrrr: + shortName: SBT113 + typeOfLevel: nominalTop + level: 0 + rrfs: + parameterNumber: 242 + typeOfLevel: atmosphere + level: 0 clevs: !!python/object/apply:numpy.arange [-80, 41, 1] cmap: WVCIMSS_r colors: goes_colors @@ -972,17 +1021,30 @@ G113bt: # GOES-W Brightness temperature, water vapor (Ch 3) G114bt: # GOES-W Brightness temperature, infrared (Ch 4) sat: <<: *goes_sat + cfgrib: + shortName: SBT114 + typeOfLevel: nominalTop + level: 0 ncl_name: SBT114_P0_L8_{grid} title: GOES-W Brightness Temperature, Infrared unit: ch 4 G123bt: # GOES-E Brightness temperature, water vapor (Ch 3) sat: <<: *goes_sat + cfgrib: + shortName: SBT123 + typeOfLevel: nominalTop + level: 0 ncl_name: SBT123_P0_L8_{grid} title: GOES-E Brightness Temperature, Water Vapor G124bt: # GOES-E Brightness temperature, infrared (Ch 4) sat: <<: *goes_sat + cfgrib: + shortName: SBT124 + typeOfLevel: nominalTop + level: 0 + ncl_name: SBT124_P0_L8_{grid} title: GOES-E Brightness Temperature, Infrared unit: ch 4 @@ -1085,10 +1147,12 @@ gust: hail: # Max 1h Hail diameter maxsfc: &hail # surface cfgrib: - shortName: hail - typeOfLevel: surface - stepType: max - level: 0 + hrrr: + shortName: hail + typeOfLevel: atmosphere + rrfs: + shortName: hail + typeOfLevel: surface clevs: [0.10, 0.25, 0.50, 0.75, 1.0, 1.5, 2.0, 2.5, 3.0] cmap: gist_ncar colors: hail_colors @@ -1120,7 +1184,6 @@ hlcy: # Helicity <<: *hlcy cfgrib: parameterNumber: 15 - parameterCategory: 7 topLevel: 5000 bottomLevel: 2000 clevs: !!python/object/apply:numpy.arange [25, 301, 25] @@ -1128,6 +1191,11 @@ hlcy: # Helicity split: True title: 2-5km Updraft Helicity mn02: &hlcy_mn # Hourly minimum of updraft helicity over 0-2 km layer + cfgrib: + parameterNumber: 200 + typeOfLevel: heightAboveGroundLayer + topLevel: 2000 + bottomLevel: 0 clevs: !!python/object/apply:numpy.arange [-300, -24, 25] cmap: gist_ncar colors: rainbow12_reverse @@ -1157,6 +1225,11 @@ hlcy: # Helicity title: 2-5km Min Updraft Helicity (over prv hr) mx02: &hlcy_mx02 # Hourly maximum of updraft helicity over 0-2 km layer <<: *hlcy + cfgrib: + parameterNumber: 199 + typeOfLevel: heightAboveGroundLayer + topLevel: 2000 + bottomLevel: 0 clevs: !join_ranges [[12.5, 87.6, 12.5], [100, 301, 25]] colors: rainbow16_colors ncl_name: MXUPHL_P8_2L103_{grid}_max1h @@ -1267,9 +1340,13 @@ hlcytot: hpbl: # Height of Planetary Boundary Layer sfc: cfgrib: - shortName: gh - typeOfLevel: planetaryBoundaryLayer - level: 0 + hrrr: + shortName: blh + typeOfLevel: surface + rrfs: + shortName: gh + typeOfLevel: planetaryBoundaryLayer + level: 0 clevs: [25, 50, 100, 200, 300, 500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000] cmap: ir_rgbv colors: pbl_colors @@ -1435,6 +1512,10 @@ lpl: # Lifted parcel level unit: hPa ltg3: # Lightning Threat (LTG1 ... LTG2) sfc: + cfgrib: + shortName: ltng + typeOfLevel: atmosphere + level: 0 clevs: [0.02, 0.5, 1.0, 1.5, 2.0, 2.5, 3, 4, 5, 6, 7, 8, 10, 12] cmap: NWSReflectivity colors: cref_colors @@ -1475,6 +1556,9 @@ lwtp: # Lightning with total precip unit: in mfrp: # Fire radiative power sfc: + cfgrib: + shortName: cfnsf + typeOfLevel: surface clevs: [0, 10, 25, 50, 100, 250] colors: fire_power_colors ncl_name: CFNSF_P0_L1_{grid} @@ -1550,9 +1634,14 @@ pres: unit: hPa msl: cfgrib: - shortName: mslet - typeOfLevel: meanSea - level: 0 + hrrr: + shortName: mslma + typeOfLevel: meanSea + level: 0 + rrfs: + shortName: mslet + typeOfLevel: meanSea + level: 0 clevs: !!python/object/apply:numpy.arange [976, 1051, 4] cmap: Spectral_r colors: pmsl_colors @@ -1608,9 +1697,14 @@ presmin: ptmp: # Potential temperature 2m: cfgrib: - shortName: pt - typeOfLevel: surface - level: 0 + hrrr: + shortName: pt + typeOfLevel: heightAboveGround + level: 2 + rrfs: + shortName: pt + typeOfLevel: surface + level: 0 clevs: !!python/object/apply:numpy.arange [210, 350, 5] cmap: jet colors: t_colors @@ -1694,13 +1788,25 @@ ref: # Maximum reflectivity for past hour at 1 km AGL m10: <<: *refl cfgrib: - shortName: rare - typeOfLevel: isothermal - level: 263 + hrrr: + shortName: refd + typeOfLevel: isothermal + level: 263 + stepType: instant + rrfs: + shortName: rare + typeOfLevel: isothermal + level: 263 ncl_name: REFD_P0_L20_{grid} title: -10C Isothermal Reflectivity maxm10: <<: *refl + cfgrib: + hrrr: + shortName: refd + typeOfLevel: isothermal + level: 263 + stepType: max ncl_name: REFD_P8_L20_{grid}_max1h title: Max 1h -10C Isothermal Reflectivity rh: # Relative Humidity @@ -1809,6 +1915,11 @@ rh: # Relative Humidity rvil: # Radar-derived Vertically Integrated Liquid sfc: &vert_int_liq cfgrib: + hrrr: + shortName: veril + typeOfLevel: atmosphere + level: 0 + rrfs: shortName: veril typeOfLevel: atmosphereSingleLayer level: 0 @@ -1976,6 +2087,10 @@ snod: # Snow Depth title: Cycled Snow Depth soilm: # Soil Moisture Availability sfc: + cfgrib: + shortName: mstav + typeOfLevel: depthBelowLand + level: 0 clevs: [0, 5, 15, 25, 35, 45, 55, 65, 75, 85, 95] cmap: Spectral colors: soilm_colors @@ -2159,9 +2274,14 @@ soilw: # Soil Moisture solar: # Incoming Solar Radiation sfc: &incoming_radiation cfgrib: - shortName: csdsf - level: 0 - typeOfLevel: surface + hrrr: + shortName: sdswrf + level: 0 + typeOfLevel: surface + rrfs: + shortName: csdsf + level: 0 + typeOfLevel: surface clevs: [50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100] cmap: gist_ncar colors: vort_colors @@ -2360,6 +2480,10 @@ totp6h: # 6-hourly total precipitation title: 6 hr Total Precip trc1: int: + cfgrib: + parameterNumber: 1 + typeOfLevel: atmosphereSingleLayer + level: 0 clevs: [1, 4, 7, 11, 15, 20, 25, 30, 40, 50, 75, 150, 250, 500] colors: smoke_colors ncl_name: @@ -2389,6 +2513,10 @@ trc1: title: 6000ft AGL Smoke vertical_index: 11 sfc: + cfgrib: + parameterNumber: 0 + typeOfLevel: heightAboveGround + level: 8 clevs: [1, 2, 4, 6, 8, 12, 16, 20, 25, 30, 40, 60, 100, 200] colors: smoke_colors ncl_name: @@ -2475,10 +2603,15 @@ ulwrf: # Upward Longwave Radiation Flux unit: W/m$^{2}$ top: # Nominal top of atmosphere cfgrib: - parameterName: Upward long-wave radiation flux - typeOfLevel: nominalTop - level: 0 - stepType: avg + hrrr: + parameterNumber: 4 + typeOfLevel: nominalTop + level: 0 + rrfs: + parameterName: 4 + typeOfLevel: nominalTop + level: 0 + stepType: avg clevs: !!python/object/apply:numpy.arange [80, 341, 2] cmap: ir_rgbv_r colors: radiation_mix_colors @@ -2526,9 +2659,12 @@ uswrf: # Upward Shortwave Radiation Flux title: Upward SW Radiation Flux, Surface top: # Nominal top of atmosphere <<: *radiation_flux + cfgrib: cfgrib: parameterNumber: 8 + parameterCategory: 4 typeOfLevel: nominalTop + level: 0 clevs: !!python/object/apply:numpy.arange [50, 851, 10] cmap: Greys_r colors: radiation_bw_colors @@ -2626,6 +2762,10 @@ vddsf: # Incoming Diffuse Radiation title: Incoming Diffuse Radiation vig: # Vertically-integrated graupel sfc: + cfgrib: + parameterNumber: 74 + typeOfLevel: atmosphereSingleLayer + level: 0 clevs: !!python/object/apply:numpy.arange [5, 91, 5] cmap: jet colors: graupel_colors @@ -2675,8 +2815,12 @@ vort: # Absolute vorticity vvel: # Vertical velocity 700mb: cfgrib: - shortName: wz - typeOfLevel: isobaricInhPa + hrrr: + shortName: w + typeOfLevel: isobaricInhPa + rrfs: + shortName: wz + typeOfLevel: isobaricInhPa clevs: !!python/object/apply:numpy.arange [-17, 34, 5] cmap: gist_ncar colors: vvel_colors @@ -2688,6 +2832,10 @@ vvel: # Vertical velocity transform: conversions.vvel_scale unit: -Pa/s * 10 mean: # Mean Vertical Velocity + cfgrib: + shortName: wz + typeOfLevel: sigmaLayer + level: 1 clevs: [-20, -15, -10, -5, -1, -0.75, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 0.75, 1, 5, 10, 15, 20] cmap: Spectral_r colors: mean_vvel_colors @@ -2720,6 +2868,7 @@ weasd: # Water equivalent of accumulated snow depth cfgrib: shortName: sdwe typeOfLevel: surface + stepType: instant clevs: [0.01, 0.1, 0.3, 0.5, 1, 2, 3, 4, 5, 7.5, 10, 20] cmap: gist_ncar colors: snow_colors @@ -2840,10 +2989,16 @@ wspeed: # Wind Speed wind: 10m mdn: # Hourly Maximum Downdraft Velocity cfgrib: - parameterNumber: 221 - typeOfLevel: isobaricLayer - topLevel: 100 - bottomLevel: 1000 + hrrr: + parameterNumber: 221 + typeOfLevel: pressureFromGroundLayer + topLevel: 10000 + bottomLevel: 100000 + rrfs: + parameterNumber: 221 + typeOfLevel: isobaricLayer + topLevel: 100 + bottomLevel: 1000 clevs: [-40, -35, -30, -25, -22.5, -20, -17.5, -15, -12.5, -10, -7.5, -5, -2.5, -2, -1.5, -1, -0.5] cmap: jet colors: mdn_colors @@ -2853,10 +3008,16 @@ wspeed: # Wind Speed unit: m/s mup: # Hourly Maximum Updraft Velocity cfgrib: - parameterNumber: 220 - typeOfLevel: isobaricLayer - topLevel: 100 - bottomLevel: 1000 + hrrr: + parameterNumber: 220 + typeOfLevel: pressureFromGroundLayer + topLevel: 10000 + bottomLevel: 100000 + rrfs: + parameterNumber: 220 + typeOfLevel: isobaricLayer + topLevel: 100 + bottomLevel: 1000 clevs: [0.5, 1, 1.5, 2, 2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20, 22.5, 25, 30, 35, 40] cmap: jet colors: mup_colors diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 4fd1c26..621bba9 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -19,7 +19,7 @@ import numpy as np from adb_graphics.datahandler import gribdata, gribfile -from adb_graphics.utils import numeric_level +from adb_graphics.utils import cfgrib_spec, numeric_level # FULL_TILES is a list of strings that includes the labels GSL attaches to some of # the wgrib2 cutouts used for larger domains like RAP, RRFS NA, and global. @@ -186,9 +186,8 @@ def draw_airports(self): lats = self.airports[:, 0] lons = 360 + self.airports[:, 1] # Convert to positive longitude x, y = self.m(lons, lats) - self.m.plot(x, y, 'ko', + self.m.plot(x, y, 'wo', ax=self.ax, - color='w', fillstyle='full', markeredgecolor='k', markeredgewidth=0.5, @@ -358,7 +357,7 @@ def draw(self, show=False): # Create a pop-up to display the figure, if show=True if show: plt.tight_layout() - plt.show() + #plt.show() self.add_logo(self.map.ax) @@ -650,7 +649,6 @@ def _wind_barbs(self, level): lev = level if not isinstance(level, bool) else self.field.level u, v = [f.data for f in self.wind_fields(lev)] - tile = self.map.tile full_tile = tile in FULL_TILES @@ -823,7 +821,7 @@ def draw(self, show=False): # Create a pop-up to display the figure, if show=True if show: plt.tight_layout() - plt.show() + #plt.show() return cf @@ -914,12 +912,15 @@ def __init__(self, fhr, fields_spec, grib_path, level, name, map_type=None, # Required if map_type is "diff" self.grib_path2 = kwargs.get("grib_path2") - @staticmethod - def set_level(level, spec): + def set_level(self, level, spec): nlevel, _ = numeric_level(level=level, index_match=False) - if nlevel and spec["cfgrib"].get("level") is None: - spec["cfgrib"]["level"] = nlevel + level_info = any(x for x in cfgrib_spec(spec["cfgrib"], self.model) for l in ("level", "top", "bottom", "Surface") if l in x) + if nlevel and not level_info: + if spec["cfgrib"].get(self.model): + spec["cfgrib"][self.model]["level"] = nlevel + else: + spec["cfgrib"]["level"] = nlevel #if spec["cfgrib"].get("level") is None and not spec["cfgrib"].get("stepRange") and not \ # spec["cfgrib"].get("topLevel") and not \ # spec["cfgrib"].get("typeOfLevel") == "surface" and not \ @@ -927,7 +928,8 @@ def set_level(level, spec): @property def shaded(self): - ds = gribfile.GribFile(self.grib_path, self.map_spec["cfgrib"]).contents + cf = cfgrib_spec(self.map_spec["cfgrib"], self.model) + ds = gribfile.GribFile(self.grib_path, cf).contents args = { "ds": ds, "fhr": self.fhr, @@ -939,7 +941,7 @@ def shaded(self): } field = gribdata.fieldData(**args) if self.map_type == "diff": - args["ds"] = gribfile.GribFile(self.grib_path2, self.map_spec["cfgrib"]).contents + args["ds"] = gribfile.GribFile(self.grib_path2, cf).contents args["grib_path"] == self.grib_path2 field2 = gribdata.fieldData(**args) field.data = field.values() - field2.values() @@ -974,8 +976,9 @@ def wind_fields(self, level=None): for var in ("u", "v"): wind_spec = self.fields_spec[var][lev] self.set_level(lev, wind_spec) + ds = gribfile.GribFile(self.grib_path, cfgrib_spec(wind_spec["cfgrib"], self.model)).contents args = { - "ds": gribfile.GribFile(self.grib_path, wind_spec["cfgrib"]).contents, + "ds": ds, "fhr": self.fhr, "level": lev, "model": self.model, @@ -1001,7 +1004,7 @@ def _overlay_fields(self, spec_sect): overlay_spec = self.fields_spec[var][lev] self.set_level(lev, overlay_spec) args = { - "ds": gribfile.GribFile(self.grib_path, overlay_spec["cfgrib"]).contents, + "ds": gribfile.GribFile(self.grib_path, cfgrib_spec(overlay_spec["cfgrib"], self.model)).contents, "fhr": self.fhr, "level": lev, "model": self.model, diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index 4b331f4..e3c687b 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -27,7 +27,7 @@ def aod_colors(self) -> np.ndarray: ''' Default color map for AOD products and chem products ''' grays = cm.get_cmap('Greys', 2)([0]) - others = cm.get_cmap(self.vspec.get('cmap'), 15)(range(1, 15, 1), alpha=0.6) + others = cm.get_cmap(self.vspec.get('cmap'), 15)(range(1, 15, 1)) return np.concatenate((grays, others)) def centered_diff(self, cmap=None, nlev=None): @@ -259,7 +259,7 @@ def mdn_colors(self) -> np.ndarray: ''' Default color map for Max Downdraft ''' grays = cm.get_cmap('Greys', 2)([0]) - others = cm.get_cmap(self.vspec.get('cmap'), 18)(range(18, 1, -1), alpha=0.6) + others = cm.get_cmap(self.vspec.get('cmap'), 18)(range(18, 1, -1)) return np.concatenate((others, grays)) @property @@ -277,7 +277,7 @@ def mup_colors(self) -> np.ndarray: ''' Default color map for Max Updraft ''' grays = cm.get_cmap('Greys', 2)([0]) - others = cm.get_cmap(self.vspec.get('cmap'), 18)(range(1, 18, 1), alpha=0.6) + others = cm.get_cmap(self.vspec.get('cmap'), 18)(range(1, 18, 1)) return np.concatenate((grays, others)) @property diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index f86ffa3..113cf92 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -19,6 +19,12 @@ import numpy as np import yaml +def cfgrib_spec(config, model): + + if (spec := config.get(model)): + return spec + return config + def create_zip(files_to_zip, zipf): diff --git a/image_lists/hrrr_subset.yml b/image_lists/hrrr_subset.yml index a9d7e3a..d16a4eb 100644 --- a/image_lists/hrrr_subset.yml +++ b/image_lists/hrrr_subset.yml @@ -1,185 +1,181 @@ hourly: model: hrrr variables: - 1hsnw: - - sfc - 1ref: - - 1000m - acfrozr: - - sfc - acfrzr: - - sfc - acpcp: - - sfc - acsnod: - - sfc - acsnw: - - sfc - snoliqr: - - sfc - cape: - - mu - - mul - - mx90mb - - sfc - ceil: - - ua - ceilexp: - - ua - ceilexp2: - - ua - cin: - - sfc - cloudcover: - - bndylay - - high - - low - - mid - - total - cpofp: - - sfc - cref: - - sfc - ctop: - - ua - dewp: - - 2m - echotop: - - sfc - firewx: - - sfc - flru: - - sfc - G113bt: - - sat - G114bt: - - sat - G123bt: - - sat - G124bt: - - sat - gust: - - 10m - hail: - - max - - maxsfc - hlcy: - - in25 - - mn02 - - mn03 - - mn25 - - mx02 - - mx03 - - mx25 - - sr01 - - sr03 - hlcytot: - - mn02 - - mn03 - - mn25 - - mx02 - - mx03 - - mx25 - hpbl: - - sfc - lcl: - - sfc - lhtfl: - - sfc - li: - - best - - sfc - ltg3: - - sfc - mfrp: - - sfc - mref: - - sfc - pres: - - msl - - sfc - ptmp: - - 2m - ptyp: - - sfc - pwtr: - - sfc - ref: - - m10 - - maxm10 - rh: - - 2m - - 850mb - - mean - - pw - rvil: - - sfc - shear: - - 01km - - 06km - shtfl: - - sfc - snod: - - sfc - soilm: - - sfc - soilt: &soilt_levs - - 0cm - - 1cm - - 4cm - - 10cm - - 30cm - - 60cm - - 1m - - 1.6m - - 3m - soilw: *soilt_levs - solar: - - sfc - ssrun: - - sfc - temp: - - 2ds - - 2m - - 500mb - - 700mb - - 850mb - - 925mb - - sfc - totp: - - sfc - trc1: - - sfc - - int - ulwrf: - - sfc - - top - uswrf: - - sfc - - top - vbdsf: - - sfc - vddsf: - - sfc - vig: - - sfc - vis: - - sfc - vort: - - 500mb - vvel: - - 700mb - - mean - vvort: - - mx01 - - mx02 - weasd: - - sfc +# 1hsnw: +# - sfc +# 1ref: +# - 1000m +# acfrozr: +# - sfc +# acfrzr: +# - sfc +# acpcp: +# - sfc +# acsnod: +# - sfc +# acsnw: +# - sfc +# cape: +# - mu +# - mul +# - mx90mb +# - sfc +# ceil: +# - ua +# ceilexp2: +# - ua +# cin: +# - sfc +# cloudcover: +# - high +# - low +# - mid +# - total +# cpofp: +# - sfc +# cref: +# - sfc +# ctop: +# - ua +# dewp: +# - 2m +# echotop: +# - sfc +# firewx_transform: +# - sfc +# flru: +# - sfc +# G113bt: +# - sat +# G114bt: +# - sat +# G123bt: +# - sat +# G124bt: +# - sat +# gust: +# - 10m +# hail: +# - max +# - maxsfc +# hlcy: +# - mn02 +# - mn03 +# - mn25 +# - mx02 +# - mx03 +# - mx25 +# - sr01 +# - sr03 +# hlcytot: +# - mn02 +# - mn03 +# - mn25 +# - mx02 +# - mx03 +# - mx25 +# hpbl: +# - sfc +# lcl: +# - sfc +# lhtfl: +# - sfc +# li: +# - best +# - sfc +# ltg3: +# - sfc +# mfrp: +# - sfc +# mref: +# - sfc +# pres: +# - msl +# - sfc +# ptmp: +# - 2m +# ptyp: +# - sfc +# pwtr: +# - sfc +# ref: +# - m10 +# - maxm10 +# rh: +# - 2m +# - 850mb +# - mean +# - pw +# rvil: +# - sfc +# shear: +# - 01km +# - 06km +# shtfl: +# - sfc +# slw: +# - sfc +# snod: +# - sfc +# soilm: +# - sfc +# soilt: &soilt_levs +# - 0cm +# - 1cm +# - 4cm +# - 10cm +# - 30cm +# - 60cm +# - 1m +# - 1.6m +# - 3m +# soilw: *soilt_levs +# solar: +# - sfc +# ssrun: +# - sfc +# temp: +# - 2ds +# - 2m +# - 500mb +# - 700mb +# - 850mb +# - 925mb +# - sfc +# totp: +# - sfc +# trc1: +# - sfc +# - int +# ulwrf: +# - sfc +# - top +# uswrf: +# - sfc +# - top +# vbdsf: +# - sfc +# vddsf: +# - sfc +# vig: +# - sfc +# vis: +# - sfc +# vort: +# - 500mb +# vvel: +# - 700mb +# - mean +# vvort: +# - mx01 +# - mx02 +# weasd: +# - sfc wspeed: - - 10m - - 80m +# - 10m +# - 80m - 250mb - - 850mb - - max - - mdn - - mup +# - 850mb +# - max +# - mdn +# - mup diff --git a/image_lists/rrfs_subset.yml b/image_lists/rrfs_subset.yml index fe7c7b6..2497d50 100644 --- a/image_lists/rrfs_subset.yml +++ b/image_lists/rrfs_subset.yml @@ -48,8 +48,8 @@ hourly: - sfc firewx: - sfc - flru: - - sfc + flru: + - sfc ghtfl: - sfc gust: @@ -64,11 +64,11 @@ hourly: - mx25 - sr01 - sr03 - hlcytot: - - mn03 - - mn25 - - mx03 - - mx25 +# hlcytot: +# - mn03 +# - mn25 +# - mx03 +# - mx25 hpbl: - sfc lcl: @@ -150,7 +150,6 @@ hourly: - 500mb vvel: - 700mb - - mean vvort: - mx01 - mx02 From 24069a6f86c167b81db3b82568a96431c000c633 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 21 Oct 2025 13:51:05 -0600 Subject: [PATCH 08/98] Matching all RRFS/HRRR graphics. --- adb_graphics/conversions.py | 2 +- adb_graphics/datahandler/gribdata.py | 13 +- adb_graphics/default_specs.yml | 70 ++++-- adb_graphics/figures/maps.py | 14 +- create_graphics.py | 8 +- image_lists/hrrr_subset.yml | 336 +++++++++++++-------------- image_lists/rrfs_subset.yml | 2 + 7 files changed, 239 insertions(+), 206 deletions(-) diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index e6b9121..e17ab72 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -98,7 +98,7 @@ def vort_scale(field, **kwargs): def weasd_to_1hsnw(field, **kwargs): - ''' Conversion from snow wter equiv to snow (10:1 ratio) ''' + ''' Conversion from snow water equiv to snow (10:1 ratio) ''' return field * 10. diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 3afa734..209a92a 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -5,6 +5,7 @@ ''' import abc +from copy import deepcopy from datetime import datetime, timedelta from functools import lru_cache from string import digits, ascii_letters @@ -140,7 +141,8 @@ def field_mean(self, values, variable, levels, global_levels, **kwargs): ''' Returns the mean of the values. ''' levs = [int(x[:-2]) for x in levels] - return values.sel(isobaricInhPa=levs).mean("isobaricInhPa") + ret = values.sel(isobaricInhPa=levs).mean("isobaricInhPa") + return ret def _get_data_levels(self, vertical_dim): @@ -782,10 +784,15 @@ def values(self, level=None, name=None, **kwargs): else: # Get the spec dict and ncl_name for the given variable name - spec = self.spec.get(name, {}).get(level, {}) + spec = deepcopy(self.spec.get(name, {}).get(level, {})) if not spec and name is not None: raise errors.NoGraphicsDefinitionForVariable(name, level) - vals = self._get_field(utils.cfgrib_spec(spec["cfgrib"], self.model)) + cfkeys = utils.cfgrib_spec(spec["cfgrib"], self.model) + nlevel = utils.numeric_level(level=level, index_match=False)[0] + level_info = any(x for x in utils.cfgrib_spec(spec["cfgrib"], self.model) for l in ("level", "top", "bottom", "Surface") if l in x) + if nlevel and not level_info: + cfkeys["level"] = utils.numeric_level(level=level, index_match=False)[0] + vals = self._get_field(cfkeys) #lev = vertical_index #vals = field diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 836fcfa..6daa68e 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -80,16 +80,18 @@ 1hsnw: # 1 hr Accumulated Snow Using 10:1 Ratio sfc: cfgrib: - rrfs: + hrrr: shortName: sdwe level: 0 typeOfLevel: surface - hrrr: - shortName: sdwe + stepType: accum + stepRange: "15-16" + rrfs: + parameterNumber: 50 level: 0 typeOfLevel: surface stepType: accum - stepRange: "0-16" + stepRange: "15-16" clevs: [0.03, 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 6, 7, 8] cmap: gist_ncar colors: snow_colors @@ -151,7 +153,7 @@ acpcp: # Accumulated run total precipitation shortName: tp level: 0 typeOfLevel: surface - #stepRange: # startswith("0-") + stepRange: 0-16 stepType: accum clevs: [0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 15, 20, 40] cmap: gist_ncar @@ -278,9 +280,10 @@ acsnw: # Run Total Accumulated Snow Using 10:1 Ratio stepType: accum stepRange: "0-16" rrfs: - shortName: sdwe + parameterNumber: 50 level: 0 typeOfLevel: surface + stepRange: "0-16" clevs: [0.01, 0.1, 1, 2, 3, 4, 6, 8, 10, 12, 18, 24] cmap: gist_ncar colors: snow_colors @@ -364,7 +367,7 @@ cape: mu: &cape # Most Unstable CAPE cfgrib: shortName: cape - level: 18000 + level: 25500 typeOfLevel: pressureFromGroundLayer clevs: [1, 100, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000] cmap: gist_ncar @@ -802,6 +805,7 @@ dewp: # Dew point temperaeure colors: tsfc_colors ncl_name: DPT_P0_L103_{grid} ticks: 10 + title: 2m Dewpoint Temperature transform: conversions.k_to_f unit: F wind: 10m @@ -1089,6 +1093,7 @@ gh: # Geopotential height cfgrib: shortName: orog typeOfLevel: surface + level: 0 clevs: !!python/object/apply:numpy.arange [0, 5000, 250] cmap: gist_earth ncl_name: HGT_P0_L1_{grid} @@ -1149,7 +1154,8 @@ hail: # Max 1h Hail diameter cfgrib: hrrr: shortName: hail - typeOfLevel: atmosphere + typeOfLevel: sigma + level: 0 rrfs: shortName: hail typeOfLevel: surface @@ -1163,6 +1169,14 @@ hail: # Max 1h Hail diameter unit: in max: # total atmosphere <<: *hail + cfgrib: + hrrr: + shortName: hail + typeOfLevel: atmosphere + level: 0 + rrfs: + shortName: hail + typeOfLevel: surface ncl_name: HAIL_P8_L10_{grid}_max1h title: Max 1h Hail/Graupel Diameter, Entire Column hailcast: # Max 1h Hail diameter @@ -1340,13 +1354,8 @@ hlcytot: hpbl: # Height of Planetary Boundary Layer sfc: cfgrib: - hrrr: - shortName: blh - typeOfLevel: surface - rrfs: - shortName: gh - typeOfLevel: planetaryBoundaryLayer - level: 0 + shortName: blh + typeOfLevel: surface clevs: [25, 50, 100, 200, 300, 500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000] cmap: ir_rgbv colors: pbl_colors @@ -1630,6 +1639,7 @@ pres: colors: ps_colors ncl_name: PRES_P0_L1_{grid} ticks: 20 + title: Surface Pressure transform: conversions.pa_to_hpa unit: hPa msl: @@ -1719,7 +1729,7 @@ ptyp: # Hourly total precipitation typeOfLevel: surface level: 0 stepType: accum - #stepRange: # startswith fhr - 1 + stepRange: 15-16 clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar colors: pcp_colors @@ -1819,6 +1829,7 @@ rh: # Relative Humidity cmap: gist_ncar colors: rainbow12_colors ncl_name: RH_P0_L103_{grid} + title: 2 m Relative Humidity ticks: 10 unit: '%' 500mb: &rh_ua @@ -1920,9 +1931,9 @@ rvil: # Radar-derived Vertically Integrated Liquid typeOfLevel: atmosphere level: 0 rrfs: - shortName: veril - typeOfLevel: atmosphereSingleLayer - level: 0 + shortName: veril + typeOfLevel: atmosphereSingleLayer + level: 0 clevs: [0.05, 0.15, 0.76, 3.47, 6.92, 12, 31.6, 35, 40, 45, 50, 55, 60, 65, 70] cmap: NWSReflectivity colors: cref_colors @@ -2058,8 +2069,11 @@ sipd: # Supercooled Large Droplet Icing kwargs: level: 1000ft variable: sipd -slw: # Supercooled Liquid Water +slw: # Supercooled Liquid Water -- requires nat data sfc: + cfgrib: + shortName: t + typeOfLevel: surface clevs: [0.05, 0.1, 0.2, .3, .4, .5, .75, 1., 1.25, 1.5, 2., 3., 4., 5., 6.] cmap: gist_ncar colors: slw_colors @@ -2355,6 +2369,7 @@ temp: # Temperature colors: tsfc_colors ncl_name: TMP_P0_L103_{grid} ticks: 10 + title: 2m Temperature transform: conversions.k_to_f unit: F wind: 10m @@ -2444,7 +2459,7 @@ totp: # Hourly total precipitation cfgrib: shortName: tp typeOfLevel: surface - #stepRange: # startswith fhr - 1 + stepRange: 15-16 stepType: accum contours: pres_msl: @@ -2497,7 +2512,8 @@ trc1: title: Vertically Integrated Smoke transform: conversions.to_micro unit: $mg/m^2$ - 1000ft: &tracer1 + 1000ft: &tracer1 + # requires nat data clevs: [1, 2, 4, 6, 8, 12, 16, 20, 25, 30, 40, 60, 100, 200] colors: smoke_colors ncl_name: MASSDEN_P0_L105_{grid} @@ -2509,6 +2525,7 @@ trc1: unit: $\mu g/m^3$ vertical_index: 4 6000ft: + # requires nat data <<: *tracer1 title: 6000ft AGL Smoke vertical_index: 11 @@ -2608,7 +2625,7 @@ ulwrf: # Upward Longwave Radiation Flux typeOfLevel: nominalTop level: 0 rrfs: - parameterName: 4 + parameterNumber: 4 typeOfLevel: nominalTop level: 0 stepType: avg @@ -2962,9 +2979,16 @@ wspeed: # Wind Speed title: 320m Wind 80m: <<: *ua_wspeed + cfgrib: + shortName: u + typeOfLevel: heightAboveGround + level: 80 title: 80m Wind 850mb: <<: *ua_wspeed + cfgrib: + shortName: u + typeOfLevel: isobaricInhPa contours: gh: colors: white diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 621bba9..801ea28 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -8,7 +8,7 @@ barbs, and descriptive annotation. ''' -import copy +from copy import copy, deepcopy from math import isnan import matplotlib.pyplot as plt import matplotlib.image as mpimg @@ -525,7 +525,7 @@ def _draw_field_values(self, ax): if self.map.corners is None: return data_values = self.field.data - crnrs = copy.copy(self.map.corners) + crnrs = copy(self.map.corners) if crnrs[2] < 0: crnrs[2] = 360 + crnrs[2] if crnrs[3] < 0: @@ -900,14 +900,14 @@ def __init__(self, fhr, fields_spec, grib_path, level, name, map_type=None, self.grib_path = grib_path self.fhr = fhr - self.fields_spec = fields_spec + self.fields_spec = deepcopy(fields_spec) self.level = level self.map_type = map_type self.model = kwargs.get('model') self.name = name self.tile = kwargs.get('tile', 'full') - self.map_spec = self.fields_spec[self.name][self.level] + self.map_spec = deepcopy(self.fields_spec[self.name][self.level]) self.set_level(self.level, self.map_spec) # Required if map_type is "diff" self.grib_path2 = kwargs.get("grib_path2") @@ -940,6 +940,7 @@ def shaded(self): "grib_path": self.grib_path, } field = gribdata.fieldData(**args) + if self.map_type == "diff": args["ds"] = gribfile.GribFile(self.grib_path2, cf).contents args["grib_path"] == self.grib_path2 @@ -1001,10 +1002,11 @@ def _overlay_fields(self, spec_sect): else: var, lev = overlay, self.level - overlay_spec = self.fields_spec[var][lev] + overlay_spec = deepcopy(self.fields_spec[var][lev]) self.set_level(lev, overlay_spec) + ds = gribfile.GribFile(self.grib_path, cfgrib_spec(overlay_spec["cfgrib"], self.model)).contents args = { - "ds": gribfile.GribFile(self.grib_path, cfgrib_spec(overlay_spec["cfgrib"], self.model)).contents, + "ds": ds, "fhr": self.fhr, "level": lev, "model": self.model, diff --git a/create_graphics.py b/create_graphics.py index d87753c..152455c 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -91,10 +91,10 @@ def create_maps(cla, fhr, grib_path, workdir, grib_path2=None): args.append((cla, fhr, grib_path, level, model, spec, variable, workdir, tile, grib_path2)) - #print(f'Queueing {len(args)} maps') - parallel_maps(*args[-1]) - #with Pool(processes=cla.nprocs) as pool: - # pool.starmap(parallel_maps, args) + print(f'Queueing {len(args)} maps') + # parallel_maps(*args[-1]) + with Pool(processes=cla.nprocs) as pool: + pool.starmap(parallel_maps, args) def gather_gribfiles(cla, fhr, filename, gribfiles): diff --git a/image_lists/hrrr_subset.yml b/image_lists/hrrr_subset.yml index d16a4eb..1f9fad5 100644 --- a/image_lists/hrrr_subset.yml +++ b/image_lists/hrrr_subset.yml @@ -1,72 +1,72 @@ hourly: model: hrrr variables: -# 1hsnw: -# - sfc -# 1ref: -# - 1000m -# acfrozr: -# - sfc -# acfrzr: -# - sfc -# acpcp: -# - sfc -# acsnod: -# - sfc -# acsnw: -# - sfc -# cape: -# - mu -# - mul -# - mx90mb -# - sfc -# ceil: -# - ua -# ceilexp2: -# - ua -# cin: -# - sfc -# cloudcover: -# - high -# - low -# - mid -# - total -# cpofp: -# - sfc -# cref: -# - sfc -# ctop: -# - ua -# dewp: -# - 2m -# echotop: -# - sfc -# firewx_transform: -# - sfc -# flru: -# - sfc -# G113bt: -# - sat -# G114bt: -# - sat -# G123bt: -# - sat -# G124bt: -# - sat -# gust: -# - 10m -# hail: -# - max -# - maxsfc -# hlcy: -# - mn02 -# - mn03 -# - mn25 -# - mx02 -# - mx03 -# - mx25 -# - sr01 -# - sr03 + 1hsnw: + - sfc + 1ref: + - 1000m + acfrozr: + - sfc + acfrzr: + - sfc + acpcp: + - sfc + acsnod: + - sfc + acsnw: + - sfc + cape: + - mu + - mul + - mx90mb + - sfc + ceil: + - ua + ceilexp2: + - ua + cin: + - sfc + cloudcover: + - high + - low + - mid + - total + cpofp: + - sfc + cref: + - sfc + ctop: + - ua + dewp: + - 2m + echotop: + - sfc + firewx_transform: + - sfc + flru: + - sfc + G113bt: + - sat + G114bt: + - sat + G123bt: + - sat + G124bt: + - sat + gust: + - 10m + hail: + - max + - maxsfc + hlcy: + - mn02 + - mn03 + - mn25 + - mx02 + - mx03 + - mx25 + - sr01 + - sr03 # hlcytot: # - mn02 # - mn03 @@ -74,108 +74,106 @@ hourly: # - mx02 # - mx03 # - mx25 -# hpbl: -# - sfc -# lcl: -# - sfc -# lhtfl: -# - sfc -# li: -# - best -# - sfc -# ltg3: -# - sfc -# mfrp: -# - sfc -# mref: -# - sfc -# pres: -# - msl -# - sfc -# ptmp: -# - 2m -# ptyp: -# - sfc -# pwtr: -# - sfc -# ref: -# - m10 -# - maxm10 -# rh: -# - 2m -# - 850mb -# - mean -# - pw -# rvil: -# - sfc -# shear: -# - 01km -# - 06km -# shtfl: -# - sfc -# slw: -# - sfc -# snod: -# - sfc -# soilm: -# - sfc -# soilt: &soilt_levs -# - 0cm -# - 1cm -# - 4cm -# - 10cm -# - 30cm -# - 60cm -# - 1m -# - 1.6m -# - 3m -# soilw: *soilt_levs -# solar: -# - sfc -# ssrun: -# - sfc -# temp: -# - 2ds -# - 2m -# - 500mb -# - 700mb -# - 850mb -# - 925mb -# - sfc -# totp: -# - sfc -# trc1: -# - sfc -# - int -# ulwrf: -# - sfc -# - top -# uswrf: -# - sfc -# - top -# vbdsf: -# - sfc -# vddsf: -# - sfc -# vig: -# - sfc -# vis: -# - sfc -# vort: -# - 500mb -# vvel: -# - 700mb -# - mean -# vvort: -# - mx01 -# - mx02 -# weasd: -# - sfc + hpbl: + - sfc + lcl: + - sfc + lhtfl: + - sfc + li: + - best + - sfc + ltg3: + - sfc + mfrp: + - sfc + mref: + - sfc + pres: + - msl + - sfc + ptmp: + - 2m + ptyp: + - sfc + pwtr: + - sfc + ref: + - m10 + - maxm10 + rh: + - 2m + - 850mb + - mean + - pw + rvil: + - sfc + shear: + - 01km + - 06km + shtfl: + - sfc + snod: + - sfc + soilm: + - sfc + soilt: &soilt_levs + - 0cm + - 1cm + - 4cm + - 10cm + - 30cm + - 60cm + - 1m + - 1.6m + - 3m + soilw: *soilt_levs + solar: + - sfc + ssrun: + - sfc + temp: + - 2ds + - 2m + - 500mb + - 700mb + - 850mb + - 925mb + - sfc + totp: + - sfc + trc1: + - sfc + - int + ulwrf: + - sfc + - top + uswrf: + - sfc + - top + vbdsf: + - sfc + vddsf: + - sfc + vig: + - sfc + vis: + - sfc + vort: + - 500mb + vvel: + - 700mb + - mean + vvort: + - mx01 + - mx02 + weasd: + - sfc wspeed: -# - 10m -# - 80m + - 10m + - 80m - 250mb -# - 850mb -# - max -# - mdn -# - mup + - 850mb + - max + - mdn + - mup diff --git a/image_lists/rrfs_subset.yml b/image_lists/rrfs_subset.yml index 2497d50..6c752c8 100644 --- a/image_lists/rrfs_subset.yml +++ b/image_lists/rrfs_subset.yml @@ -108,6 +108,8 @@ hourly: - sfc snod: - sfc + soilm: + - sfc soilt: &soilt_levs - 0cm - 1cm From 413b5b1b6b408da7180ebaf358c6ef2456b7fac8 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 21 Oct 2025 14:03:06 -0600 Subject: [PATCH 09/98] Adding some tooling for testing. --- Makefile | 20 +++++++++++++++ format | 5 ++++ recipe/run_test.sh | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 Makefile create mode 100755 format create mode 100644 recipe/run_test.sh diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ec1342c --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +TARGETS = format lint test typecheck unittest + +.PHONY: $(TARGETS) + + +format: + @./format + +lint: + recipe/run_test.sh lint + +test: + recipe/run_test.sh + +typecheck: + recipe/run_test.sh typecheck + +unittest: + recipe/run_test.sh unittest + diff --git a/format b/format new file mode 100755 index 0000000..7e75b41 --- /dev/null +++ b/format @@ -0,0 +1,5 @@ +echo "=> Formatting code" +ruff format . + +echo "=> Sorting imports" +ruff check --select I --fix . diff --git a/recipe/run_test.sh b/recipe/run_test.sh new file mode 100644 index 0000000..e95e14b --- /dev/null +++ b/recipe/run_test.sh @@ -0,0 +1,63 @@ +#!/bin/bash -eu + +# This script is called 1. By conda-build, to test code at package-build time, and 2. By "make test" +# to test code in a live development shell. Its name is dictated by conda-build (see URL below for +# details), but its contents derive from https://github.com/maddenp/condev/tree/main/demo. If run +# with no arguments, it executes all test types: linting, typechecking, unit testing, and basic CLI- +# tool verification. It can also be run with "lint", "typecheck", "unittest", or "cli" arguments to +# run only a single test type. (The "make lint", "make typecheck", and "make unittest" targets in +# the root-directory Makefile run the first test types individually; CLI tools can be hand-tested.) + +# https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#test-section + +cli() { + msg Testing CLI + ( + set -eux + uw --version + ) + msg OK +} + +lint() { + msg Running linter + ( + set -eux + ruff check . + ) + msg OK +} + +msg() { + echo "=> $@" +} + +typecheck() { + msg Running typechecker + ( + set -eux + mypy --install-types --non-interactive . + ) + msg OK +} + +unittest() { + msg Running unit tests + ( + set -eux + pytest --cov=uwtools -n 4 . + ) + msg OK +} + +test "${CONDEV_SHELL:-}" = 1 && cd $(dirname $0)/../src || cd ../test_files +if [[ -n "${1:-}" ]]; then + # Run single specified code-quality tool. + $1 +else + # Run all code-quality tools. + lint + typecheck + unittest + cli +fi From 30b6cb451d8e6ad5828e440963aaea8bac64ade5 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 21 Oct 2025 14:03:23 -0600 Subject: [PATCH 10/98] Formatting existing code. --- adb_graphics/conversions.py | 74 +-- adb_graphics/datahandler/gribdata.py | 575 +++++++++--------- adb_graphics/datahandler/gribfile.py | 284 ++++----- adb_graphics/errors.py | 18 +- adb_graphics/figure_builders.py | 132 ++--- adb_graphics/figures/maps.py | 845 +++++++++++++++------------ adb_graphics/figures/skewt.py | 716 ++++++++++++----------- adb_graphics/specs.py | 508 ++++++++-------- adb_graphics/utils.py | 236 ++++---- conftest.py | 33 +- create_graphics.py | 639 ++++++++++---------- tests/test_common.py | 390 +++++++------ tests/test_grib.py | 78 +-- tests/test_hrrr_maps.py | 79 ++- 14 files changed, 2391 insertions(+), 2216 deletions(-) diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index e17ab72..6ce4498 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -1,109 +1,109 @@ # pylint: disable=unused-argument,invalid-name -''' +""" This module contains functions for converting the units of a field. The interface requires a single atmospheric field in a Numpy array, and returns the converted values as output. -''' +""" from xarray.ufuncs import sqrt, square -def k_to_c(field, **kwargs): - ''' Conversion from Kelvin to Celsius ''' +def k_to_c(field, **kwargs): + """Conversion from Kelvin to Celsius""" return field - 273.15 + def k_to_f(field, **kwargs): + """Conversion from Kelvin to Farenheit""" - ''' Conversion from Kelvin to Farenheit ''' + return (field - 273.15) * 9 / 5 + 32 - return (field - 273.15) * 9/5 + 32 def kgm2_to_in(field, **kwargs): - - ''' Conversion from kg per m^2 to inches ''' + """Conversion from kg per m^2 to inches""" return field * 0.03937 -def magnitude(a, b, **kwargs): - ''' Return the magnitude of vector components ''' +def magnitude(a, b, **kwargs): + """Return the magnitude of vector components""" return sqrt(square(a) + square(b)) + def m_to_dm(field, **kwargs): + """Conversion from meters to decameters""" - ''' Conversion from meters to decameters ''' + return field / 10.0 - return field / 10. def m_to_in(field, **kwargs): - - ''' Conversion from meters to inches ''' + """Conversion from meters to inches""" return field * 39.3701 -def m_to_kft(field, **kwargs): - ''' Conversion from meters to kilofeet ''' +def m_to_kft(field, **kwargs): + """Conversion from meters to kilofeet""" return field / 304.8 -def m_to_mi(field, **kwargs): - ''' Conversion from meters to miles ''' +def m_to_mi(field, **kwargs): + """Conversion from meters to miles""" return field / 1609.344 -def ms_to_kt(field, **kwargs): - ''' Conversion from m s-1 to knots ''' +def ms_to_kt(field, **kwargs): + """Conversion from m s-1 to knots""" return field * 1.9438 + def pa_to_hpa(field, **kwargs): + """Conversion from Pascals to hectopascals""" - ''' Conversion from Pascals to hectopascals ''' + return field / 100.0 - return field / 100. def percent(field, **kwargs): + """Conversion from values between 0 - 1 to percent""" - ''' Conversion from values between 0 - 1 to percent ''' + return field * 100.0 - return field * 100. def to_micro(field, **kwargs): + """Convert field to micro""" - ''' Convert field to micro ''' + return field * 1e6 - return field * 1E6 def to_micrograms_per_m3(field, **kwargs): + """Convert field to micrograms per cubic meter""" - ''' Convert field to micrograms per cubic meter ''' + return field * 1e9 - return field * 1E9 def vvel_scale(field, **kwargs): - - ''' Scale vertical velocity for plotting ''' + """Scale vertical velocity for plotting""" return field * -10 + def vort_scale(field, **kwargs): + """Scale vorticity for plotting""" - ''' Scale vorticity for plotting ''' + return field / 1e-05 - return field / 1E-05 def weasd_to_1hsnw(field, **kwargs): + """Conversion from snow water equiv to snow (10:1 ratio)""" - ''' Conversion from snow water equiv to snow (10:1 ratio) ''' + return field * 10.0 - return field * 10. def sden_to_slr(field, **kwargs): + """Convert snow density (kg m-3) to snow-liquid ratio""" - ''' Convert snow density (kg m-3) to snow-liquid ratio ''' - - return 1000. / field + return 1000.0 / field diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 209a92a..1b0bcc1 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -1,28 +1,27 @@ # pylint: disable=invalid-name, too-many-public-methods, too-many-lines -''' +""" Classes that handle the specifics of grib files from UPP. -''' +""" import abc from copy import deepcopy from datetime import datetime, timedelta from functools import lru_cache -from string import digits, ascii_letters +from string import ascii_letters, digits -from matplotlib import cm import numpy as np import xarray as xr +from matplotlib import cm + from adb_graphics.datahandler import gribfile from adb_graphics.utils import cfgrib_spec -from .. import conversions -from .. import errors -from .. import specs -from .. import utils -class UPPData(specs.VarSpec): +from .. import conversions, errors, specs, utils + - ''' +class UPPData(specs.VarSpec): + """ Class provides interface for accessing field data from UPP in Grib2 format. @@ -33,44 +32,39 @@ class UPPData(specs.VarSpec): Keyword Arguments: config: path to a user-specified configuration file model: string describing the model type - ''' + """ def __init__(self, ds, short_name, spec, **kwargs): - - # Parse kwargs first - self.model = kwargs.get('model') + self.model = kwargs.get("model") self.grib_path = kwargs.get("grib_path") - self.spec = spec self.short_name = short_name - self.level = 'ua' + self.level = "ua" - self.fhr = str(kwargs['fhr']) + self.fhr = str(kwargs["fhr"]) self.ds = ds @property def anl_dt(self) -> datetime: - - ''' Returns the initial time of the grib file as a datetime object from - the grib file.''' + """Returns the initial time of the grib file as a datetime object from + the grib file.""" return datetime.fromisoformat(str(self.field.time.values).split(".")[0]) @property def clevs(self) -> np.ndarray: - - ''' + """ Uses the information contained in the yaml config file to determine the set of levels to be contoured. Returns the list of levels. The yaml file "clevs" key may contain a list, a range, or a call to a function. The logic to parse those options is included here. - ''' + """ - clev = np.asarray(self.vspec.get('clevs', [])) + clev = np.asarray(self.vspec.get("clevs", [])) # Is clevs a list? if isinstance(clev, (list, np.ndarray)): @@ -80,30 +74,29 @@ def clevs(self) -> np.ndarray: try: return utils.get_func(clev)() except ImportError: - print(f'Check yaml file definition of CLEVS for {self.short_name}. ', - 'Must be a list, range, or function call!') + print( + f"Check yaml file definition of CLEVS for {self.short_name}. ", + "Must be a list, range, or function call!", + ) @staticmethod def date_to_str(date: datetime) -> str: + """Returns a formatted string (for graphic title) from a datetime + object""" - ''' Returns a formatted string (for graphic title) from a datetime - object''' - - return date.strftime('%Y%m%d %H UTC') + return date.strftime("%Y%m%d %H UTC") @property def field(self): - - ''' Wrapper that calls get_field method for the current variable. - Returns the NioVariable object ''' + """Wrapper that calls get_field method for the current variable. + Returns the NioVariable object""" return self.ds.__getattr__([x for x in self.ds.data_vars][0]) def field_column_max(self, values, variable, level, **kwargs): - # pylint: disable=unused-argument - ''' Returns the column max of the values. ''' + """Returns the column max of the values.""" vals = self.values(name=variable, level=level, one_lev=False) maxvals = vals.max(axis=0) @@ -111,10 +104,9 @@ def field_column_max(self, values, variable, level, **kwargs): return maxvals def field_sum(self, values, variable2, level2, **kwargs): - # pylint: disable=unused-argument - ''' Return the sum of the values. ''' + """Return the sum of the values.""" value2 = self.values(name=variable2, level=level2) sum2 = values + value2 @@ -123,10 +115,9 @@ def field_sum(self, values, variable2, level2, **kwargs): return sum2 def field_diff(self, values, variable2, level2, **kwargs): - # pylint: disable=unused-argument - ''' Subtracts the values from variable2 from self.field. ''' + """Subtracts the values from variable2 from self.field.""" value2 = self.values(name=variable2, level=level2) diff = values - value2 @@ -135,41 +126,35 @@ def field_diff(self, values, variable2, level2, **kwargs): return diff def field_mean(self, values, variable, levels, global_levels, **kwargs): - # pylint: disable=unused-argument - ''' Returns the mean of the values. ''' + """Returns the mean of the values.""" levs = [int(x[:-2]) for x in levels] - ret = values.sel(isobaricInhPa=levs).mean("isobaricInhPa") + ret = values.sel(isobaricInhPa=levs).mean("isobaricInhPa") return ret def _get_data_levels(self, vertical_dim): + """Return a list of vertical dimension values corresponding to the + requested vertical dimension to get the values of those dimensions""" - ''' Return a list of vertical dimension values corresponding to the - requested vertical dimension to get the values of those dimensions ''' - - fcst_hr = 0 if self.ds.sizes.get('fcst_hr', 0) <= 1 else int(self.fhr) + fcst_hr = 0 if self.ds.sizes.get("fcst_hr", 0) <= 1 else int(self.fhr) ret = [] - for dim in [var for var in self.ds.variables \ - if vertical_dim in var]: - + for dim in [var for var in self.ds.variables if vertical_dim in var]: # Get the current forecast hour slice, if it's in the dataset - selector = {'fcst_hr': fcst_hr} if 'fcst_hr' in self.ds[dim].dims else {} + selector = {"fcst_hr": fcst_hr} if "fcst_hr" in self.ds[dim].dims else {} ret.append(self.ds[dim].sel(**selector).values) return ret def _get_field(self, spec): - - ''' Given an ncl_name, return the NioVariable object. ''' + """Given an ncl_name, return the NioVariable object.""" ds = gribfile.GribFile(self.grib_path, spec).contents return ds.__getattr__([x for x in ds.data_vars][0]) def _get_level(self, field, level, spec, **kwargs): - - ''' Returns the value of the level to for a 3D array + """Returns the value of the level to for a 3D array Arguments: @@ -189,10 +174,10 @@ def _get_level(self, field, level, spec, **kwargs): Integer value corresponding to the array index for the atmospheric level. - ''' + """ # The index of the requested level - lev = spec.get('vertical_index') + lev = spec.get("vertical_index") if lev is not None: return lev @@ -200,9 +185,10 @@ def _get_level(self, field, level, spec, **kwargs): # numeric_level returns a list of length 1 (e.g. [500] for 500 mb) or of # length 2 when split=True and it's like 0-6 km, so returns [0, 6000] - requested_level, _ = self.numeric_level(level=level, - split=kwargs.get('split', spec.get('split')), - ) + requested_level, _ = self.numeric_level( + level=level, + split=kwargs.get("split", spec.get("split")), + ) # data_levels contains a list of vertical dimension values data_levels = self._get_data_levels(vertical_dim) @@ -224,21 +210,24 @@ def _get_level(self, field, level, spec, **kwargs): lev = int(lev[0]) return lev except ValueError: - print(f'BAD LEVEL is {lev} for {field.name}') + print(f"BAD LEVEL is {lev} for {field.name}") - print(f"Could not find a level for {field.name} at requested \ + print( + f"Could not find a level for {field.name} at requested \ level = {requested_level} for variable levels = {data_levels}. Index \ - was {lev}.") + was {lev}." + ) # If neither of those cases worked out appropriately, raise an error. - msg = f'Length of requested_level ({len(requested_level)}) or '\ - f'data_levels ({len(data_levels)}) bad!' \ - f' {level} {field.name}' + msg = ( + f"Length of requested_level ({len(requested_level)}) or " + f"data_levels ({len(data_levels)}) bad!" + f" {level} {field.name}" + ) raise ValueError(msg) def get_transform(self, transforms, val): - - ''' Applies a set of one or more transforms to an np.array of + """Applies a set of one or more transforms to an np.array of data values. Input: @@ -251,21 +240,21 @@ def get_transform(self, transforms, val): val: updated values after transforms have been applied - ''' + """ transform_kwargs = {} if isinstance(transforms, dict): - transform_list = transforms.get('funcs') + transform_list = transforms.get("funcs") if not isinstance(transform_list, list): transform_list = [transform_list] - transform_kwargs = transforms.get('kwargs') + transform_kwargs = transforms.get("kwargs") elif isinstance(transforms, str): transform_list = [transforms] else: transform_list = transforms for transform in transform_list: - if len(transform.split('.')) == 1: + if len(transform.split(".")) == 1: val = self.__getattribute__(transform)(val, **transform_kwargs) else: val = utils.get_func(transform)(val, **transform_kwargs) @@ -273,11 +262,10 @@ def get_transform(self, transforms, val): @lru_cache() def get_xypoint(self, site_lat, site_lon) -> tuple: - - ''' + """ Return the X, Y grid point corresponding to the site location. No interpolation is used. - ''' + """ lats, lons = self.latlons() adjust = 360 if np.any(lons < 0) else 0 @@ -286,118 +274,110 @@ def get_xypoint(self, site_lat, site_lon) -> tuple: # Numpy magic to grab the X, Y grid point nearest the profile site # pylint: disable=unbalanced-tuple-unpacking - x, y = np.unravel_index((np.abs(lats - site_lat) \ - + np.abs(lons - site_lon)).argmin(), lats.shape) + x, y = np.unravel_index( + (np.abs(lats - site_lat) + np.abs(lons - site_lon)).argmin(), lats.shape + ) # pylint: enable=unbalanced-tuple-unpacking if x <= 0 or y <= 0 or x >= max_x or y >= max_y: - print(f'site location is outside your domain! {site_lat} {site_lon}') - return(-1, -1) + print(f"site location is outside your domain! {site_lat} {site_lon}") + return (-1, -1) return (x, y) @property def grid_suffix(self): - - ''' Return the suffix of the first variable with 4 sections (split on _) - in the file. This should correspond to the grid tag. ''' + """Return the suffix of the first variable with 4 sections (split on _) + in the file. This should correspond to the grid tag.""" for var in self.ds.keys(): - vsplit = var.split('_') + vsplit = var.split("_") if len(vsplit) == 4: return vsplit[-1] - return 'GRID NOT FOUND' - + return "GRID NOT FOUND" def latlons(self): + """Returns the set of latitudes and longitudes""" - ''' Returns the set of latitudes and longitudes ''' - - coords = sorted([c for c in list(self.ds.coords) if - any(ele in c for ele in ['lat', 'lon'])]) + coords = sorted( + [c for c in list(self.ds.coords) if any(ele in c for ele in ["lat", "lon"])] + ) return [self.ds.coords[c].values for c in coords] @property def lev_descriptor(self): - - ''' Returns the descriptor for the variable's level type. ''' + """Returns the descriptor for the variable's level type.""" return self.field.level_type def numeric_level(self, index_match=True, level=None, split=None): - - ''' + """ Split the numeric level and unit associated with the level key. A blank string is returned for lev_val for levels that do not contain a numeric, e.g., 'sfc' or 'ua'. - ''' + """ level = level if level else self.level # Gather all the numbers in the string - lev_val = ''.join([c for c in level if (c in digits or c == '.')]) + lev_val = "".join([c for c in level if (c in digits or c == ".")]) # Convert the numbers to a list, and make integers or floats if lev_val: if split is not None: lev_val = [int(lev) for lev in lev_val] else: - lev_val = [float(lev_val) if '.' in lev_val else int(lev_val)] + lev_val = [float(lev_val) if "." in lev_val else int(lev_val)] # Gather all the letters - lev_unit = ''.join([c for c in level if c in ascii_letters]) + lev_unit = "".join([c for c in level if c in ascii_letters]) if index_match: - if lev_unit == 'cm': - lev_val = [val / 100. for val in lev_val] - if lev_unit in ['mb', 'mxmb']: - lev_val = [val * 100. for val in lev_val] - if lev_unit in ['in', 'km', 'mn', 'mx', 'sr']: - lev_val = [val * 1000. for val in lev_val] + if lev_unit == "cm": + lev_val = [val / 100.0 for val in lev_val] + if lev_unit in ["mb", "mxmb"]: + lev_val = [val * 100.0 for val in lev_val] + if lev_unit in ["in", "km", "mn", "mx", "sr"]: + lev_val = [val * 1000.0 for val in lev_val] return lev_val, lev_unit @staticmethod def opposite(values, **kwargs): - # pylint: disable=unused-argument + # pylint: disable=unused-argument - ''' Returns the opposite of input values ''' + """Returns the opposite of input values""" - return - values + return -values @property def valid_dt(self) -> datetime: - - ''' Returns a datetime object corresponding to the forecast hour's valid - time as set in the Grib file. ''' + """Returns a datetime object corresponding to the forecast hour's valid + time as set in the Grib file.""" fh = timedelta(hours=int(self.fhr)) return self.anl_dt + fh @abc.abstractmethod def values(self, level=None, name=None, **kwargs): - - ''' Returns the values of a given variable. ''' + """Returns the values of a given variable.""" ... @staticmethod def vertical_dim(field): - - ''' Determine the vertical dimension of the variable by looking through + """Determine the vertical dimension of the variable by looking through the field's dimensions for one that includes "lv". Return the first - matching instance. ''' + matching instance.""" - vert_dim = [dim for dim in field.dims if ('lv' in dim or 'probability' in dim)] + vert_dim = [dim for dim in field.dims if ("lv" in dim or "probability" in dim)] if vert_dim: return vert_dim[0] - return '' - + return "" @property def vspec(self): - - ''' Return the graphics specification for a given level. ''' + """Return the graphics specification for a given level.""" vspec = self.spec.get(self.short_name, {}).get(self.level) if not vspec: @@ -406,8 +386,7 @@ def vspec(self): class fieldData(UPPData): - - ''' + """ Class provides interface for accessing field (2D plan view) data from UPP in Grib2 format. @@ -420,32 +399,31 @@ class fieldData(UPPData): config: path to a user-specified configuration file member: integer describing the ensemble member number to grab data for - ''' + """ def __init__(self, ds, level, short_name, **kwargs): - super().__init__(ds, short_name, **kwargs) self.level = level - self.contour_kwargs = kwargs.get('contour_kwargs', {}) - self.mem = kwargs.get('member', None) + self.contour_kwargs = kwargs.get("contour_kwargs", {}) + self.mem = kwargs.get("member", None) def aviation_flight_rules(self, values, **kwargs): # pylint: disable=unused-argument - ''' + """ Generates a field of Aviation Flight Rules from Ceil and Vis - ''' + """ ceil = values.to_dataarray().squeeze() - vis = self.values(name='vis', level='sfc') + vis = self.values(name="vis", level="sfc") - flru = np.where((ceil > 1.) & (ceil < 3.), 1.01, 0.0) - flru = np.where((vis > 3.) & (vis < 5.), 1.01, flru) - flru = np.where((ceil > 0.5) & (ceil < 1.), 2.01, flru) - flru = np.where((vis > 1.) & (vis < 3.), 2.01, flru) + flru = np.where((ceil > 1.0) & (ceil < 3.0), 1.01, 0.0) + flru = np.where((vis > 3.0) & (vis < 5.0), 1.01, flru) + flru = np.where((ceil > 0.5) & (ceil < 1.0), 2.01, flru) + flru = np.where((vis > 1.0) & (vis < 3.0), 2.01, flru) flru = np.where((ceil > 0.0) & (ceil < 0.5), 3.01, flru) - flru = np.where((vis < 1.), 3.01, flru) + flru = np.where((vis < 1.0), 3.01, flru) vis.close() @@ -453,23 +431,21 @@ def aviation_flight_rules(self, values, **kwargs): @property def cmap(self): + """Returns the LinearSegmentedColormap specified by the config key + "cmap" """ - ''' Returns the LinearSegmentedColormap specified by the config key - "cmap" ''' - - return cm.get_cmap(self.vspec['cmap']) + return cm.get_cmap(self.vspec["cmap"]) @property def colors(self) -> np.ndarray: - - ''' + """ Returns a list of colors, specified by the config key "colors". The yaml file "colors" key may contain a list or a function to be called. - ''' + """ - color_spec = self.vspec.get('colors') + color_spec = self.vspec.get("colors") if isinstance(color_spec, (list, np.ndarray)): return np.asarray(color_spec) @@ -483,16 +459,15 @@ def colors(self) -> np.ndarray: @property def corners(self) -> list: - - ''' + """ Returns lat and lon of lower left (ll) and upper right(ur) corners: ll_lat, ur_lat, ll_lon, ur_lon - ''' + """ lat, lon = self.latlons() - if self.model in ['global', 'hfip', 'obs']: + if self.model in ["global", "hfip", "obs"]: ret = [lat[-1], lat[0], lon[0], lon[-1]] - elif self.model == 'global_mpas': + elif self.model == "global_mpas": ret = [lat[0], lat[-1], lon[0], lon[-1]] else: ret = [lat[0, 0], lat[-1, -1], lon[0, 0], lon[-1, -1]] @@ -500,38 +475,39 @@ def corners(self) -> list: return ret def fire_weather_index(self, values, **kwargs): - # pylint: disable=unused-argument - ''' + """ Generates a field of Fire Weather Index This method uses wrfprs data to find regions where weather conditions are most likely to lead to wildfires. - ''' + """ def _load_field(level, short_name): spec = cfgrib_spec(self.spec[short_name][level]["cfgrib"], self.model) ds = gribfile.GribFile(self.grib_path, spec).contents args = { - "ds": ds, - "fhr": self.fhr, - "level": level, - "model": self.model, - "short_name": short_name, - "spec": self.spec, - "grib_path": self.grib_path, - } + "ds": ds, + "fhr": self.fhr, + "level": level, + "model": self.model, + "short_name": short_name, + "spec": self.spec, + "grib_path": self.grib_path, + } return fieldData(**args).values(do_transform=False) # Gather fields from the input - veg = values.to_dataarray().squeeze() # Chose this value as the main one in the default_specs + veg = ( + values.to_dataarray().squeeze() + ) # Chose this value as the main one in the default_specs temp = _load_field(level="2m", short_name="temp") - dewpt = _load_field(level='2m', short_name='dewp') - weasd = _load_field(level='sfc', short_name='weasd') - gust = _load_field(level='10m', short_name='gust') - soilm = _load_field(level='sfc', short_name='soilm') + dewpt = _load_field(level="2m", short_name="dewp") + weasd = _load_field(level="sfc", short_name="weasd") + gust = _load_field(level="10m", short_name="gust") + soilm = _load_field(level="sfc", short_name="soilm") # A few derived fields dewpt_depression = temp - dewpt @@ -542,7 +518,7 @@ def _load_field(level, short_name): snowc = (25.0 - weasd) / 25.0 snowc = np.where(snowc > 0.0, snowc, 0.0) - mois = 0.01*(100.0 - soilm) + mois = 0.01 * (100.0 - soilm) # Set urban (13), snow/ice (15), barren (16), and water (17) to 0. for vegtype in [13, 15, 16, 17]: @@ -551,11 +527,9 @@ def _load_field(level, short_name): # Set all others vegetation types to 1 veg = np.where(veg > 0, 1, veg) - fwi = veg * (2.37 * - (gust_max ** 1.11) * - (dewpt_depression ** 0.92) * - (mois ** 6.95) * - snowc) + fwi = veg * ( + 2.37 * (gust_max**1.11) * (dewpt_depression**0.92) * (mois**6.95) * snowc + ) fwi = fwi / 10.0 @@ -568,45 +542,48 @@ def _load_field(level, short_name): return fwi def grid_info(self): - - ''' Returns a dict that includes the grid info for the full grid. ''' + """Returns a dict that includes the grid info for the full grid.""" # Keys are grib names, values are Basemap argument names keys_to_basemap = dict( - CenterLon='lon_0', - CenterLat='lat_0', - GRIB_Latin2InDegrees='lat_1', - GRIB_Latin1InDegrees='lat_2', - GRIB_LoVInDegrees='lon_0', - Latin2='lat_1', - Latin1='lat_2', - Lov='lon_0', - La1='lat_0', - La2='lat_2', - Lo1='lon_1', - Lo2='lon_2', - ) + CenterLon="lon_0", + CenterLat="lat_0", + GRIB_Latin2InDegrees="lat_1", + GRIB_Latin1InDegrees="lat_2", + GRIB_LoVInDegrees="lon_0", + Latin2="lat_1", + Latin1="lat_2", + Lov="lon_0", + La1="lat_0", + La2="lat_2", + Lo1="lon_1", + Lo2="lon_2", + ) grid_info = {} var_info = self.field grid_def = var_info.attrs["GRIB_gridDefinitionDescription"].lower() - if 'lambert' in grid_def: - attrs = ["GRIB_Latin1InDegrees", "GRIB_Latin2InDegrees", "GRIB_LoVInDegrees"] - grid_info['projection'] = 'lcc' - grid_info['lat_0'] = 39.0 - - if self.model != 'hrrrhi': - grid_info['corners'] = self.corners - #if self.grid_suffix in ['GLC0']: + if "lambert" in grid_def: + attrs = [ + "GRIB_Latin1InDegrees", + "GRIB_Latin2InDegrees", + "GRIB_LoVInDegrees", + ] + grid_info["projection"] = "lcc" + grid_info["lat_0"] = 39.0 + + if self.model != "hrrrhi": + grid_info["corners"] = self.corners + # if self.grid_suffix in ['GLC0']: # attrs = ['Latin1', 'Latin2', 'Lov'] - #elif self.grid_suffix == 'GST0': + # elif self.grid_suffix == 'GST0': # attrs = ['Lov'] # grid_info['projection'] = 'stere' # grid_info['lat_0'] = 90 - #elif self.grid_suffix == 'GLL0': + # elif self.grid_suffix == 'GLL0': # attrs = [] # grid_info['projection'] = 'cyl' - #else: + # else: # attrs = [] # grid_info['projection'] = 'rotpole' @@ -627,53 +604,48 @@ def grid_info(self): grid_info[bm_arg] = val del val - if self.model == 'hrrrhi': - grid_info['lat_0'] = 20.44 - grid_info['lon_0'] = 202.54 - grid_info['width'] = 2000000 - grid_info['height'] = 2000000 + if self.model == "hrrrhi": + grid_info["lat_0"] = 20.44 + grid_info["lon_0"] = 202.54 + grid_info["width"] = 2000000 + grid_info["height"] = 2000000 return grid_info def icing_adjust_trace(self, values, **kwargs): - # pylint: disable=unused-argument,no-self-use - ''' Changes the value of ICSEV trace from 4.0 to 0.5, to maintain ascending order ''' + """Changes the value of ICSEV trace from 4.0 to 0.5, to maintain ascending order""" vals = np.where(values == 4.0, 0.5, values) return vals def run_max(self, values, **kwargs): - - ''' Finds the max hourly value over all the forecast lead times available. ''' + """Finds the max hourly value over all the forecast lead times available.""" # pylint: disable=unused-argument,no-self-use - return values.max(dim='fcst_hr') + return values.max(dim="fcst_hr") def run_min(self, values, **kwargs): - - ''' Finds the min hourly value over all the forecast lead times available. ''' + """Finds the min hourly value over all the forecast lead times available.""" # pylint: disable=unused-argument,no-self-use - return values.min(dim='fcst_hr') + return values.min(dim="fcst_hr") def run_total(self, values, **kwargs): - - ''' Sums over all the forecast lead times available. ''' + """Sums over all the forecast lead times available.""" # pylint: disable=unused-argument,no-self-use - return values.sum(dim='fcst_hr') + return values.sum(dim="fcst_hr") def supercooled_liquid_water(self, values, **kwargs): - # pylint: disable=unused-argument - ''' + """ Generates a field of Supercooled Liquid Water This method uses wrfnat data to find regions where @@ -686,28 +658,33 @@ def supercooled_liquid_water(self, values, **kwargs): next sigma level. The process is iterative to the topof the atmosphere. - ''' + """ - pres_sfc = self.values(name='pres', level='sfc') * 100. # convert back to Pa - pres_nat_lev = self.values(name='pres', level='ua', one_lev=False) - temp = self.values(name='temp', level='ua', one_lev=False) - cloud_mixing_ratio = self.values(name='clwmr', level='ua', one_lev=False) - rain_mixing_ratio = self.values(name='rwmr', level='ua', one_lev=False) + pres_sfc = self.values(name="pres", level="sfc") * 100.0 # convert back to Pa + pres_nat_lev = self.values(name="pres", level="ua", one_lev=False) + temp = self.values(name="temp", level="ua", one_lev=False) + cloud_mixing_ratio = self.values(name="clwmr", level="ua", one_lev=False) + rain_mixing_ratio = self.values(name="rwmr", level="ua", one_lev=False) gravity = 9.81 - slw = pres_sfc * 0. # start with array of zero values + slw = pres_sfc * 0.0 # start with array of zero values - nlevs = np.shape(pres_nat_lev)[0] # determine number of vertical levels + nlevs = np.shape(pres_nat_lev)[0] # determine number of vertical levels for n in range(nlevs): if n == 0: pres_layer = 2 * (pres_sfc[:, :] - pres_nat_lev[n, :, :]) # layer depth - pres_sigma = pres_sfc - pres_layer # pressure at next sigma level + pres_sigma = pres_sfc - pres_layer # pressure at next sigma level else: - pres_layer = 2 * (pres_sigma[:, :] - pres_nat_lev[n, :, :]) # layer depth - pres_sigma = pres_sigma - pres_layer # pressure at next sigma level + pres_layer = 2 * ( + pres_sigma[:, :] - pres_nat_lev[n, :, :] + ) # layer depth + pres_sigma = pres_sigma - pres_layer # pressure at next sigma level # compute supercooled water in layer and add to previous values - supercool_locs = np.where((temp[n, :, :] < 0.0), \ - cloud_mixing_ratio[n, :, :]+rain_mixing_ratio[n, :, :], 0.0) + supercool_locs = np.where( + (temp[n, :, :] < 0.0), + cloud_mixing_ratio[n, :, :] + rain_mixing_ratio[n, :, :], + 0.0, + ) slw = slw + pres_layer / gravity * supercool_locs pres_sfc.close() @@ -719,25 +696,23 @@ def supercooled_liquid_water(self, values, **kwargs): @property def ticks(self) -> int: + """Returns the number of color bar tick marks from the yaml config + settings.""" - ''' Returns the number of color bar tick marks from the yaml config - settings. ''' - - return self.vspec.get('ticks', 10) + return self.vspec.get("ticks", 10) @property def units(self) -> str: + """Returns the variable unit from the yaml config, if available. If not + specified in the yaml file, returns the value set in the Grib file.""" - ''' Returns the variable unit from the yaml config, if available. If not - specified in the yaml file, returns the value set in the Grib file. ''' - - return self.vspec.get('unit', self.field.units) + return self.vspec.get("unit", self.field.units) @property def data(self): - ''' Sets the data property on the object for use when we need to update - the values associated with a given object -- helpful for differences.''' - if not hasattr(self, '_data'): + """Sets the data property on the object for use when we need to update + the values associated with a given object -- helpful for differences.""" + if not hasattr(self, "_data"): return self.values() return self._data @@ -746,8 +721,7 @@ def data(self, value): self._data = value def values(self, level=None, name=None, **kwargs): - - ''' + """ Returns the numpy array of values at the requested level for the variable after applying any unit conversion to the original data. @@ -762,41 +736,44 @@ def values(self, level=None, name=None, **kwargs): one_lev bool flag. if True, get the single level of the variable (default: True) vertical_index the index (int) of the desired vertical level - ''' + """ level = level or self.level vals = self.ds - #one_lev = kwargs.get('one_lev', True) - #vertical_index = kwargs.get('vertical_index') + # one_lev = kwargs.get('one_lev', True) + # vertical_index = kwargs.get('vertical_index') - #ncl_name = kwargs.get('ncl_name', '') - #ncl_name = ncl_name.format(fhr=self.fhr, grid=self.grid_suffix) + # ncl_name = kwargs.get('ncl_name', '') + # ncl_name = ncl_name.format(fhr=self.fhr, grid=self.grid_suffix) - do_transform = kwargs.get('do_transform', True) + do_transform = kwargs.get("do_transform", True) if name is None: - # Use field and spec from the current object field = self.field spec = self.vspec else: - # Get the spec dict and ncl_name for the given variable name spec = deepcopy(self.spec.get(name, {}).get(level, {})) if not spec and name is not None: raise errors.NoGraphicsDefinitionForVariable(name, level) cfkeys = utils.cfgrib_spec(spec["cfgrib"], self.model) nlevel = utils.numeric_level(level=level, index_match=False)[0] - level_info = any(x for x in utils.cfgrib_spec(spec["cfgrib"], self.model) for l in ("level", "top", "bottom", "Surface") if l in x) + level_info = any( + x + for x in utils.cfgrib_spec(spec["cfgrib"], self.model) + for l in ("level", "top", "bottom", "Surface") + if l in x + ) if nlevel and not level_info: cfkeys["level"] = utils.numeric_level(level=level, index_match=False)[0] vals = self._get_field(cfkeys) - #lev = vertical_index - #vals = field - #if one_lev: + # lev = vertical_index + # vals = field + # if one_lev: # # Check if it's a 3D variable (lv in any dimension field) # dim_name = self.vertical_dim(field) @@ -819,16 +796,16 @@ def values(self, level=None, name=None, **kwargs): # {level} {spec}') # raise - #if self.mem is not None: + # if self.mem is not None: # vals = vals.isel(**{'ens_mem': self.mem}) ## Select a single forecast hour (only if there are many) - #if not spec.get('accumulate', False): + # if not spec.get('accumulate', False): # if 'fcst_hr' in vals.dims: # fcst_hr = 0 if self.ds.sizes['fcst_hr'] <= 1 else int(self.fhr) # vals = vals.sel(**{'fcst_hr': fcst_hr}) - transforms = spec.get('transform') + transforms = spec.get("transform") if transforms and do_transform: vals = self.get_transform(transforms, vals) @@ -836,19 +813,28 @@ def values(self, level=None, name=None, **kwargs): return vals.to_dataarray().squeeze() return vals - def vector_magnitude(self, field1, cfkeys=None, field2_id=None, level=None, vertical_index=None, **kwargs): - + def vector_magnitude( + self, + field1, + cfkeys=None, + field2_id=None, + level=None, + vertical_index=None, + **kwargs, + ): # pylint: disable=unused-argument - ''' + """ Returns the vector magnitude of two component vector fields. The input fields can be either NCL names (string) or full data fields. The first layer of a variable is returned if none is provided. - ''' + """ if cfkeys: if cfkeys.get("level") is None: - cfkeys["level"] = utils.numeric_level(level=self.level, index_match=False)[0] + cfkeys["level"] = utils.numeric_level( + level=self.level, index_match=False + )[0] field2_spec = {"cfgrib": cfkeys} else: var, lev = field2_id.split(".") @@ -858,32 +844,33 @@ def vector_magnitude(self, field1, cfkeys=None, field2_id=None, level=None, vert ds = gribfile.GribFile(self.grib_path, field2_spec["cfgrib"]).contents args = { - "ds": ds, - "fhr": self.fhr, - "level": self.level, - "model": self.model, - "short_name": self.short_name, - "spec": self.spec, - "grib_path": self.grib_path, - } + "ds": ds, + "fhr": self.fhr, + "level": self.level, + "model": self.model, + "short_name": self.short_name, + "spec": self.spec, + "grib_path": self.grib_path, + } field2 = fieldData(**args).ds - mag = conversions.magnitude(field1.to_dataarray().squeeze(), field2.to_dataarray().squeeze()) + mag = conversions.magnitude( + field1.to_dataarray().squeeze(), field2.to_dataarray().squeeze() + ) field1.close() field2.close() return mag def wind(self, level) -> [np.ndarray, np.ndarray]: - - ''' + """ Returns the u, v wind components as a list (length 2) of arrays. Input: level bool or level key. If True, use same level as self, if a string level key is provided, use wind at that level. - ''' + """ level = self.level if level and isinstance(level, bool) else level @@ -897,15 +884,14 @@ def wind(self, level) -> [np.ndarray, np.ndarray]: fhr=self.fhr, level=level, short_name=var, - ).field - u, v = [field_lambda(self.ds, level, var) for var in ['u', 'v']] + ).field + u, v = [field_lambda(self.ds, level, var) for var in ["u", "v"]] return [u, v] class profileData(UPPData): - - ''' + """ Class provides methods for getting profiles from a specific lat/lon location from a grib file. @@ -921,15 +907,13 @@ class profileData(UPPData): Only used for base classes. - ''' + """ def __init__(self, ds, loc, short_name, **kwargs): - super().__init__(ds, short_name, **kwargs) # The first 31 columns are space delimted - self.site_code, _, self.site_num, lat, lon = \ - loc[:31].split() + self.site_code, _, self.site_num, lat, lon = loc[:31].split() # The variable lenght site name is included past column 37 self.site_name = loc[37:].rstrip() @@ -941,13 +925,14 @@ def __init__(self, ds, loc, short_name, **kwargs): # minus sign to convert the longitude to deg East, and then need to # adjust to the 0 to 360 system. self.site_lat = float(lat) - self.site_lon = -float(lon) # lons are -180 but without minus sign in input file + self.site_lon = -float( + lon + ) # lons are -180 but without minus sign in input file if self.site_lon < 0: self.site_lon = self.site_lon + 360.0 def values(self, level=None, name=None, **kwargs): - - ''' + """ Returns the numpy array of values at the object's x, y location for the requested variable. Transforms are performed in the child class. @@ -962,18 +947,18 @@ def values(self, level=None, name=None, **kwargs): split bool flag. if True, level string numbers are split into a list, e.g. used to get [0, 6000] from 06km vertical_index the index of the required level - ''' + """ # Set the defaults here since this is an instance of an abstract method # level refers to the level key in the specs file. - level = level if level is not None else 'ua' + level = level if level is not None else "ua" if not name: name = self.short_name - one_lev = kwargs.get('one_lev', False) - vertical_index = kwargs.get('vertical_index') - split = kwargs.get('split') + one_lev = kwargs.get("one_lev", False) + vertical_index = kwargs.get("vertical_index") + split = kwargs.get("split") # Retrive the location for the profile x, y = self.get_xypoint(self.site_lat, self.site_lon) @@ -982,14 +967,14 @@ def values(self, level=None, name=None, **kwargs): var_spec = self.spec.get(name, {}).get(level, {}) # Set the NCL name from the specs section, unless otherwise specified - ncl_name = kwargs.get('ncl_name') or self.ncl_name(var_spec) + ncl_name = kwargs.get("ncl_name") or self.ncl_name(var_spec) ncl_name = ncl_name.format(fhr=self.fhr, grid=self.grid_suffix) if not ncl_name: raise errors.NoGraphicsDefinitionForVariable( name, - 'ua', - ) + "ua", + ) # Get the full 2- or 3-D field field = self.ds[ncl_name] @@ -1008,16 +993,16 @@ def values(self, level=None, name=None, **kwargs): profile = profile[:, x, y] return profile - def vector_magnitude(self, field1, field2, level='ua', vertical_index=None, **kwargs): - - ''' + def vector_magnitude( + self, field1, field2, level="ua", vertical_index=None, **kwargs + ): + """ Returns the vector magnitude of two component vector profiles. The input fields can be either NCL names (string) or full data fields. If no layer or level is provided, the default 'ua' will be used in self.values. - ''' - + """ if isinstance(field1, str): field1 = self.values( @@ -1025,7 +1010,7 @@ def vector_magnitude(self, field1, field2, level='ua', vertical_index=None, **kw ncl_name=field1, vertical_index=vertical_index, **kwargs, - ) + ) if isinstance(field2, str): field2 = self.values( @@ -1033,6 +1018,6 @@ def vector_magnitude(self, field1, field2, level='ua', vertical_index=None, **kw ncl_name=field2, vertical_index=vertical_index, **kwargs, - ) + ) return conversions.magnitude(field1, field2) diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index ac2d68f..13b5692 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -1,17 +1,16 @@ # pylint: disable=invalid-name,too-few-public-methods,too-many-locals,too-many-branches,too-many-statements -''' +""" Classes that load grib files. -''' +""" import xarray as xr -class GribFile(): - ''' Wrappers and helper functions for interfacing with cfgrib.''' +class GribFile: + """Wrappers and helper functions for interfacing with cfgrib.""" def __init__(self, filename, var_config, **kwargs): - # pylint: disable=unused-argument self.filename = filename @@ -19,24 +18,23 @@ def __init__(self, filename, var_config, **kwargs): self.contents = self._load() def _load(self): + """Internal method that opens the grib file. Returns a grib message + iterator.""" - ''' Internal method that opens the grib file. Returns a grib message - iterator. ''' - - return xr.open_dataset(self.filename, - engine='cfgrib', - lock=False, - backend_kwargs=({"filter_by_keys": self.var_config}), - ) + return xr.open_dataset( + self.filename, + engine="cfgrib", + lock=False, + backend_kwargs=({"filter_by_keys": self.var_config}), + ) -class GribFiles(): - ''' Class for loading in a set of grib files and combining them over - forecast hours. ''' +class GribFiles: + """Class for loading in a set of grib files and combining them over + forecast hours.""" def __init__(self, coord_dims, filenames, filetype, **kwargs): - - ''' + """ Arguments: coord_dims dict containing the name of the dimension to @@ -49,9 +47,9 @@ def __init__(self, coord_dims, filenames, filetype, **kwargs): Keyword Arguments: model string describing the model type - ''' + """ - self.model = kwargs.get('model', '') + self.model = kwargs.get("model", "") self.filenames = filenames self.filetype = filetype @@ -59,132 +57,149 @@ def __init__(self, coord_dims, filenames, filetype, **kwargs): self.grid_suffix = self._get_grid_suffix(filenames) self.contents = self._load() - def append(self, filenames): - - ''' Add a single new slice to existing data set. Must match coord_dims - and filetype of original dataset. Updates current contents of Object''' + """Add a single new slice to existing data set. Must match coord_dims + and filetype of original dataset. Updates current contents of Object""" self.contents = self._load(filenames) def free_fcst_names(self, ds, fcst_type): - - ''' Given an opened dataset, return a dict of original variable names - (key) and the desired name (value) ''' + """Given an opened dataset, return a dict of original variable names + (key) and the desired name (value)""" ret = {} - fhr = self.coord_dims['fcst_hr'][-1] + fhr = self.coord_dims["fcst_hr"][-1] - special_suffixes = ['max', 'min', 'acc', 'avg'] + special_suffixes = ["max", "min", "acc", "avg"] for var in ds.variables: - suffix = var.split('_')[-1] + suffix = var.split("_")[-1] # Keeping lists of misbehaving "accumulated" variables here because # there doesn't seem to be another way to know.... - if fcst_type == '01fcst': + if fcst_type == "01fcst": # Don't rename these variables at early hours odd_variables = [ - 'ASNOW', - 'FROZR', - 'FRZR', - 'LRGHR', - ] - if self.model == 'rrfs': - odd_variables.append('WEASD') - if self.model != 'rrfs': - odd_variables.extend([ - 'CDLYR', - 'TCDC', - ]) - needs_renaming = var.split('_')[0] not in odd_variables + "ASNOW", + "FROZR", + "FRZR", + "LRGHR", + ] + if self.model == "rrfs": + odd_variables.append("WEASD") + if self.model != "rrfs": + odd_variables.extend( + [ + "CDLYR", + "TCDC", + ] + ) + needs_renaming = var.split("_")[0] not in odd_variables if suffix in special_suffixes and needs_renaming: - if 'global' not in self.model or self.model == 'global_mpas': - new_suffix = f'{suffix}1h' + if "global" not in self.model or self.model == "global_mpas": + new_suffix = f"{suffix}1h" else: - new_suffix = f'{suffix}6h' + new_suffix = f"{suffix}6h" ret[var] = var.replace(suffix, new_suffix) # MASSDEN is a special case when ending in "avg_1'" - if var.split('_')[0] == 'MASSDEN' and var.split('_')[-2] == 'avg': - print(f'Special change to MASSDEN avg_1 name to avg1h_1') - ret[var] = var.replace('avg', 'avg1h') + if var.split("_")[0] == "MASSDEN" and var.split("_")[-2] == "avg": + print(f"Special change to MASSDEN avg_1 name to avg1h_1") + ret[var] = var.replace("avg", "avg1h") else: # Only rename these variables at late hours odd_variables = [ - 'APCP', - 'CDLYR', - 'FROZR', - 'FRZR', - 'LRGHR', - 'TCDC', - 'TSNOWP', - 'WEASD', - ] - if self.model == 'rrfs': - odd_variables.remove('WEASD') - variable = var.split('_')[0] + "APCP", + "CDLYR", + "FROZR", + "FRZR", + "LRGHR", + "TCDC", + "TSNOWP", + "WEASD", + ] + if self.model == "rrfs": + odd_variables.remove("WEASD") + variable = var.split("_")[0] needs_renaming = variable in odd_variables contains_suffix = [] for suf in special_suffixes: - # The LRGHR variable behaves differently in RRFS than in all # others! At 7 hours, it starts averaging since 6h. From 0-6 # h it's named with suffix avg, after its named avg1h, # avg2h, etc. - if self.model == 'rrfs' and \ - variable == 'LRGHR' and \ - suffix == f'{suf}1h': + if ( + self.model == "rrfs" + and variable == "LRGHR" + and suffix == f"{suf}1h" + ): contains_suffix.append(suf) # RRFS_A has fields that have the suffix 'acc0h' but we don't # want those. Drop them if they come up. - bad_0h_vars = ['APCP_P8_L1_GLL0_acc0h', \ - 'FROZR_P8_L1_GLC0_acc0h', 'FRZR_P8_L1_GLC0_acc0h', \ - 'CDLYR_P8_L200_GLC0_avg0h', 'TCDC_P8_L200_GLC0_avg0h', \ - 'APCP_P8_L1_GLC0_acc0h', 'APCP_P8_L1_GST0_acc0h'] + bad_0h_vars = [ + "APCP_P8_L1_GLL0_acc0h", + "FROZR_P8_L1_GLC0_acc0h", + "FRZR_P8_L1_GLC0_acc0h", + "CDLYR_P8_L200_GLC0_avg0h", + "TCDC_P8_L200_GLC0_avg0h", + "APCP_P8_L1_GLC0_acc0h", + "APCP_P8_L1_GST0_acc0h", + ] if fhr != 0 and var in bad_0h_vars: - print(f'dropping {var}') + print(f"dropping {var}") ds.drop(var) continue # mpas_global has fields that have the suffix 'acc1h' but we don't # want those since the output is 6h. Drop them if they come up. - bad_1h_vars = ['APCP_P8_L1_GLL0_acc1h', \ - 'FROZR_P8_L1_GLL0_acc1h', 'FRZR_P8_L1_GLL0_acc1h', \ - 'CDLYR_P8_L200_GLL0_avg1h', 'TCDC_P8_L200_GLL0_avg1h', \ - 'APCP_P8_L1_GLL0_acc1h', 'APCP_P8_L1_GST0_acc1h', \ - 'WEASD_P8_L1_GLL0_acc1h'] - if self.model == 'global_mpas' and fhr != 0 and var in bad_1h_vars: - print(f'dropping {var}') + bad_1h_vars = [ + "APCP_P8_L1_GLL0_acc1h", + "FROZR_P8_L1_GLL0_acc1h", + "FRZR_P8_L1_GLL0_acc1h", + "CDLYR_P8_L200_GLL0_avg1h", + "TCDC_P8_L200_GLL0_avg1h", + "APCP_P8_L1_GLL0_acc1h", + "APCP_P8_L1_GST0_acc1h", + "WEASD_P8_L1_GLL0_acc1h", + ] + if self.model == "global_mpas" and fhr != 0 and var in bad_1h_vars: + print(f"dropping {var}") ds.drop(var) continue # For the RAP CONUS and AK domains, the APCP, WEASD, and FROZR # variables all have 3h accumulation fields in addition to # the 1h accumulation fields. This causes problems with the # renaming, so just drop those fields from the dataset. - bad_3h_vars = ['APCP_P8_L1_GLC0_acc3h', \ - 'WEASD_P8_L1_GLC0_acc3h', 'FROZR_P8_L1_GLC0_acc3h', \ - 'APCP_P8_L1_GST0_acc3h', 'WEASD_P8_L1_GST0_acc3h', \ - 'FROZR_P8_L1_GST0_acc3h'] - if self.model == 'rap' and fhr != 3 and var in bad_3h_vars: - print(f'dropping {var}') + bad_3h_vars = [ + "APCP_P8_L1_GLC0_acc3h", + "WEASD_P8_L1_GLC0_acc3h", + "FROZR_P8_L1_GLC0_acc3h", + "APCP_P8_L1_GST0_acc3h", + "WEASD_P8_L1_GST0_acc3h", + "FROZR_P8_L1_GST0_acc3h", + ] + if self.model == "rap" and fhr != 3 and var in bad_3h_vars: + print(f"dropping {var}") ds.drop(var) continue # Some global models will start producing 12h accumulations at # lead times past 246h. These cause problems with the renaming, # so we can drop those fields. - bad_12h_vars = ['APCP_P8_L1_GLL0_acc12h', \ - 'APCP_P8_L1_GLC0_acc12h', 'APCP_P8_L1_GST0_acc12h'] + bad_12h_vars = [ + "APCP_P8_L1_GLL0_acc12h", + "APCP_P8_L1_GLC0_acc12h", + "APCP_P8_L1_GST0_acc12h", + ] if fhr != 12 and var in bad_12h_vars: - print(f'dropping {var}') + print(f"dropping {var}") ds.drop(var) continue # All the variables that need to be renamed. In most cases, # exclude the "1h" ("6h" for global) accumulated variables - accum_freq = 6 if 'global' in self.model else 1 - if suf in suffix and suffix != f'{suf}{accum_freq}h': + accum_freq = 6 if "global" in self.model else 1 + if suf in suffix and suffix != f"{suf}{accum_freq}h": contains_suffix.append(suf) if contains_suffix and needs_renaming: @@ -194,28 +209,27 @@ def free_fcst_names(self, ds, fcst_type): @staticmethod def _get_grid_suffix(filenames): - - ''' Return the suffix of the first variable with 4 sections (split on _) - in the file. This should correspond to the grid tag. ''' + """Return the suffix of the first variable with 4 sections (split on _) + in the file. This should correspond to the grid tag.""" for files in filenames.values(): if files: - gfile = xr.open_dataset(files[0], - cache=False, - engine='pynio', - lock=False, - backend_kwargs=dict(format="grib2"), - ) + gfile = xr.open_dataset( + files[0], + cache=False, + engine="pynio", + lock=False, + backend_kwargs=dict(format="grib2"), + ) for var in gfile.keys(): - vsplit = var.split('_') + vsplit = var.split("_") if len(vsplit) == 4: gfile.close() return vsplit[-1] - return 'GRID NOT FOUND' + return "GRID NOT FOUND" def _load(self, filenames=None): - - ''' Load the set of files into a single XArray structure. ''' + """Load the set of files into a single XArray structure.""" all_leads = [] if filenames is None else [self.contents] filenames = self.filenames if filenames is None else filenames @@ -223,23 +237,22 @@ def _load(self, filenames=None): # 0h and 1h accumulated forecast variables are named differently than # the rest of the forecast hours. Rename those accumulated variables if # needed. - for fcst_type in ['01fcst', 'free_fcst']: - + for fcst_type in ["01fcst", "free_fcst"]: if filenames.get(fcst_type): for filename in filenames.get(fcst_type): - print(f'Loading grib2 file: {fcst_type}, {filename}') + print(f"Loading grib2 file: {fcst_type}, {filename}") # Rename variables to match free forecast variables dataset = xr.open_mfdataset( filenames[fcst_type], **self.open_kwargs, - ) + ) renaming = self.free_fcst_names(dataset, fcst_type) - if renaming and self.model not in ['hrrre', 'rrfse']: - print(f'RENAMING VARIABLES:') + if renaming and self.model not in ["hrrre", "rrfse"]: + print(f"RENAMING VARIABLES:") for old_name, new_name in renaming.items(): - print(f' {old_name:>30s} -> {new_name}') + print(f" {old_name:>30s} -> {new_name}") dataset = dataset.rename_vars(renaming) if len(all_leads) == 1: @@ -249,46 +262,47 @@ def _load(self, filenames=None): # update "in place" og_ds = all_leads[0] bad_vars = [ - 'APCP_P8_L1_{grid}_acc', - 'ACPCP_P8_L1_{grid}_acc', - 'FROZR_P8_L1_{grid}_acc', - 'NCPCP_P8_L1_{grid}_acc', - 'WEASD_P8_L1_{grid}_acc', - ] - bad_vars = [v.format(grid=self.grid_suffix) for v in \ - bad_vars] + "APCP_P8_L1_{grid}_acc", + "ACPCP_P8_L1_{grid}_acc", + "FROZR_P8_L1_{grid}_acc", + "NCPCP_P8_L1_{grid}_acc", + "WEASD_P8_L1_{grid}_acc", + ] + bad_vars = [v.format(grid=self.grid_suffix) for v in bad_vars] for bad_var in bad_vars: # Check to see if the bad variable is in the current # dataset and NOT in the original dataset. - if bad_var not in og_ds.variables and \ - dataset.get(bad_var) is not None: - print(f'Adding {bad_var} to og ds') + if ( + bad_var not in og_ds.variables + and dataset.get(bad_var) is not None + ): + print(f"Adding {bad_var} to og ds") # Duplicate the accumulated variable with the # required name - og_ds[bad_var] = og_ds.get(f'{bad_var}1h') + og_ds[bad_var] = og_ds.get(f"{bad_var}1h") all_leads.append(dataset) - ret = xr.combine_nested(all_leads, - compat='override', - concat_dim=list(self.coord_dims.keys())[0], - coords='minimal', - data_vars='all', - ) + ret = xr.combine_nested( + all_leads, + compat="override", + concat_dim=list(self.coord_dims.keys())[0], + coords="minimal", + data_vars="all", + ) return ret @property def open_kwargs(self): - - ''' Defines the key word arguments used by the various calls to XArray - open_mfdataset ''' + """Defines the key word arguments used by the various calls to XArray + open_mfdataset""" return dict( backend_kwargs=dict(format="grib2"), cache=False, - combine='nested', - compat='override', + combine="nested", + compat="override", concat_dim=list(self.coord_dims.keys())[0], - coords='minimal', - engine='pynio', + coords="minimal", + engine="pynio", lock=False, - ) + ) diff --git a/adb_graphics/errors.py b/adb_graphics/errors.py index f3532d9..e90f9be 100644 --- a/adb_graphics/errors.py +++ b/adb_graphics/errors.py @@ -1,15 +1,16 @@ -''' Errors specific to the ADB Graphics package. ''' +"""Errors specific to the ADB Graphics package.""" class Error(Exception): - '''Base class for handling errors''' + """Base class for handling errors""" class FieldNotUnique(Error): - '''Exception raised when multiple Grib fields are found with input parameters''' + """Exception raised when multiple Grib fields are found with input parameters""" + class GribReadError(Error): - '''Exception raised when there is an error reading the grib file.''' + """Exception raised when there is an error reading the grib file.""" def __init__(self, name, message="was not found"): self.name = name @@ -20,11 +21,14 @@ def __init__(self, name, message="was not found"): def __str__(self): return f'"{self.name}" {self.message}' + class NoGraphicsDefinitionForVariable(Error): - '''Exception raised when there is no configuration for the variable.''' + """Exception raised when there is no configuration for the variable.""" + class LevelNotFound(Error): - '''Exception raised when there is no configuration for the variable.''' + """Exception raised when there is no configuration for the variable.""" + class OutsideDomain(Error): - '''Exception raised when there is no configuration for the variable.''' + """Exception raised when there is no configuration for the variable.""" diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 84b0092..18f80a7 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -1,8 +1,8 @@ # pylint: disable=invalid-name -''' +""" This module is where pieces of the figures are put together. Data is compbined with maps and skewts to provide the final product. -''' +""" import gc import os @@ -11,55 +11,54 @@ import numpy as np import yaml -from adb_graphics.datahandler import gribfile -from adb_graphics.datahandler import gribdata import adb_graphics.errors as errors -from adb_graphics.figures import maps -from adb_graphics.figures import skewt +from adb_graphics.datahandler import gribdata, gribfile +from adb_graphics.figures import maps, skewt from adb_graphics.utils import numeric_level -AIRPORTS = 'static/Airports_locs.txt' +AIRPORTS = "static/Airports_locs.txt" -def add_obs_panel(ax, model_name, obs_file, proj_info, short_name, tile): +def add_obs_panel(ax, model_name, obs_file, proj_info, short_name, tile): # pylint: disable=too-many-arguments - ''' Plot observation data provided by the obs_file - path using the assigned projection. ''' + """Plot observation data provided by the obs_file + path using the assigned projection.""" gribobs = gribfile.GribFile(filename=obs_file) - ax.axis('on') + ax.axis("on") field = gribdata.fieldData( ds=gribobs.contents, fhr=0, - level='obs', - model='obs', + level="obs", + model="obs", short_name=short_name, - ) + ) map_fields = maps.MapFields(main_field=field) m = maps.Map( airport_fn=AIRPORTS, ax=ax, grid_info=proj_info, - model='obs', + model="obs", tile=tile, - ) + ) dm = maps.MultiPanelDataMap( map_fields=map_fields, map_=m, - member='obs', + member="obs", model_name=model_name, - ) + ) # Draw the map dm.draw(show=True) -def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, - tile='full', dp2=None): +def parallel_maps( + cla, fhr, grib_path, level, model, spec, variable, workdir, tile="full", dp2=None +): # pylint: disable=too-many-arguments,too-many-locals # pylint: disable=too-many-branches,too-many-statements - ''' + """ Function that creates plan-view maps, either a single panel, or multipanel for a forecast ensemble. Can be used in parallel. @@ -79,7 +78,7 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, Optional: tile the label of the tile being plotted dp2 path to a second grib file - ''' + """ fig, axes = set_figure(cla.model_name, cla.graphic_type, tile) @@ -90,18 +89,17 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, map_classes = { "enspanel": maps.MultiPanelDataMap, "diff": maps.DiffMap, - } + } map_class = map_classes.get(cla.graphic_type, maps.DataMap) for index, current_ax in enumerate(axes): - if current_ax is axes[-1] or index == cla.ens_size: last_panel = True mem = None - if cla.graphic_type == 'enspanel': + if cla.graphic_type == "enspanel": # Don't put data in the top left or bottom left panels. if index in (0, 8): - current_ax.axis('off') + current_ax.axis("off") # If we have less than 10 members, skip the remaining panels. if index > cla.ens_size: @@ -123,7 +121,7 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, map_type=cla.graphic_type, model=model, tile=tile, - ) + ) # Generate a map object m = maps.Map( @@ -131,9 +129,9 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, ax=current_ax, grid_info=map_fields.shaded.grid_info(), model=model, - plot_airports=spec.get('plot_airports', True), + plot_airports=spec.get("plot_airports", True), tile=tile, - ) + ) # Send all objects (map_field, contours, hatches) to a DataMap object dm = map_class( @@ -142,15 +140,15 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, member=mem, model_name=cla.model_name, last_panel=last_panel, - ) + ) # Draw the map - if cla.graphic_type == 'enspanel': + if cla.graphic_type == "enspanel": if index == 0: dm.title() dm.add_logo(current_ax) elif index == 8: - if spec.get('include_obs', False) and cla.obs_file_path: + if spec.get("include_obs", False) and cla.obs_file_path: # Add observation panel to lower left. Currently only # supported for composite reflectivity. add_obs_panel( @@ -160,30 +158,30 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, proj_info=field.grid_info(), short_name=variable, tile=tile, - ) + ) else: dm.draw(show=True) else: dm.draw(show=True) # Build the output path - png_file = f'{variable}_{tile}_{level}_f{fhr:03d}.png' + png_file = f"{variable}_{tile}_{level}_f{fhr:03d}.png" png_file = png_file.replace("__", "_") png_path = os.path.join(workdir, png_file) - print('*' * 120) + print("*" * 120) print(f"Creating image file: {png_path}") - print('*' * 120) + print("*" * 120) # Save the png file to disk plt.savefig( png_path, - bbox_inches='tight', + bbox_inches="tight", dpi=cla.img_res, - format='png', - orientation='landscape', - pil_kwargs={'optimize': True}, - ) + format="png", + orientation="landscape", + pil_kwargs={"optimize": True}, + ) fig.clear() # Clear the current axes. @@ -191,13 +189,13 @@ def parallel_maps(cla, fhr, grib_path, level, model, spec, variable, workdir, # Clear the current figure. plt.clf() # Closes all the figure windows. - plt.close('all') + plt.close("all") del m gc.collect() -def parallel_skewt(cla, fhr, ds, site, workdir): - ''' +def parallel_skewt(cla, fhr, ds, site, workdir): + """ Function that creates a single SkewT plot. Can be used in parallel. Input: @@ -206,7 +204,7 @@ def parallel_skewt(cla, fhr, ds, site, workdir): fhr the forecast hour integer site the string representation of the site from the sites file workdir output directory - ''' + """ skew = skewt.SkewTDiagram( ds=ds, @@ -215,38 +213,38 @@ def parallel_skewt(cla, fhr, ds, site, workdir): loc=site, max_plev=cla.max_plev, model_name=cla.model_name, - ) + ) skew.create_diagram() outfile = f"{skew.site_code}_{skew.site_num}_skewt_f{fhr:03d}.png" png_path = os.path.join(workdir, outfile) - print('*' * 80) + print("*" * 80) print(f"Creating image file: {png_path}") - print('*' * 80) + print("*" * 80) # pylint: disable=duplicate-code plt.savefig( png_path, - bbox_inches='tight', + bbox_inches="tight", dpi=cla.img_res, - format='png', - orientation='landscape', - ) + format="png", + orientation="landscape", + ) - start_time = cla.start_time.strftime('%Y%m%d%H') + start_time = cla.start_time.strftime("%Y%m%d%H") csvfile = f"{skew.site_code}.{skew.site_num}.skewt.{start_time}_f{fhr:03d}.csv" csv_path = os.path.join(workdir, csvfile) - print('*' * 80) + print("*" * 80) print(f"Creating csv file: {csv_path}") - print('*' * 80) + print("*" * 80) skew.create_csv(csv_path) plt.close() -def set_figure(model_name, graphic_type, tile): - ''' Create the figure and subplots appropriate for the model and - graphics type. Return the figure handle and list of axes. ''' +def set_figure(model_name, graphic_type, tile): + """Create the figure and subplots appropriate for the model and + graphics type. Return the figure handle and list of axes.""" if model_name == "HRRR-HI": inches = 12.2 @@ -259,27 +257,29 @@ def set_figure(model_name, graphic_type, tile): nrows = 1 ncols = 1 - if graphic_type == 'enspanel': + if graphic_type == "enspanel": nrows = 3 ncols = 4 inches = 20 # Most rough-square subdomains can use the 0.8 y_aspect y_aspect = 0.8 x_aspect = 1 - if tile in ['full', 'NW']: + if tile in ["full", "NW"]: # Horizontal rectangle subdomains, and CONUS need more # squashed horizontal rectangles y_aspect = 0.5 - if tile in ['SE']: + if tile in ["SE"]: # Vertical rectangle subdomains can use a bit more height # than the others y_aspect = 0.95 - fig, ax = plt.subplots(nrows, ncols, - figsize=(x_aspect*inches, y_aspect*inches), - sharex=True, - sharey=True, - ) + fig, ax = plt.subplots( + nrows, + ncols, + figsize=(x_aspect * inches, y_aspect * inches), + sharex=True, + sharey=True, + ) # Flatten the 2D array and number panel axes from top left to bottom right # sequentially ax = ax.flatten() if isinstance(ax, np.ndarray) else [ax] diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 801ea28..e0175d9 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -1,22 +1,22 @@ # pylint: disable=invalid-name,too-few-public-methods -''' +""" Module contains classes relevant to plotting maps. The Map class handles all the functionality related to a Basemap, and adding airports to a blank map. The DataMap class takes as input a Map object and a DataHandler object (e.g., UPPData object) and creates a standard plot with shaded fields, contours, wind barbs, and descriptive annotation. -''' +""" from copy import copy, deepcopy from math import isnan -import matplotlib.pyplot as plt + import matplotlib.image as mpimg import matplotlib.offsetbox as mpob import matplotlib.patches as mpatches -from mpl_toolkits.basemap import Basemap -from mpl_toolkits.basemap import shiftgrid +import matplotlib.pyplot as plt import numpy as np +from mpl_toolkits.basemap import Basemap, shiftgrid from adb_graphics.datahandler import gribdata, gribfile from adb_graphics.utils import cfgrib_spec, numeric_level @@ -31,59 +31,85 @@ "hrrr", "hrrrak", "NHemi", - ] +] # TILE_DEFS is a dict of dicts with predefined tiles specifying the corners of the grid # to be plotted, and the stride and length of the wind barbs. # Order for corners: [lower left lat, upper right lat, lower left lon, upper right lon] TILE_DEFS = { - 'NC': {'corners': [36, 51, -109, -85], 'stride': 10, 'length': 4}, - 'NE': {'corners': [36, 48, -91, -62], 'stride': 10, 'length': 4}, - 'NW': {'corners': [35, 52, -126, -102], 'stride': 10, 'length': 4}, - 'SC': {'corners': [24, 41, -107, -86], 'stride': 10, 'length': 4}, - 'SE': {'corners': [22, 37, -93.5, -72], 'stride': 10, 'length': 4}, - 'SW': {'corners': [24.5, 45, -122, -103], 'stride': 10, 'length': 4}, - 'Africa': {'corners': [-40, 40, -40, 60], 'stride': 7, 'length': 5}, - 'AKZoom': {'corners': [52, 73, -162, -132], 'stride': 4, 'length': 5}, - 'AKZoom2': {'corners': [37.9, 80.8, 180, -105.7], 'stride': 8, 'length': 5}, - 'AKRange': {'corners': [62.0, 67.0, -152.0, -143.0], 'stride': 4, 'length': 4}, - 'Anchorage': {'corners': [58.59, 62.776, -152.749, -146.218], 'stride': 4, 'length': 4}, - 'ATL': {'corners': [31.2, 35.8, -87.4, -79.8], 'stride': 4, 'length': 4}, - 'Beijing': {'corners': [25, 53, 102, 133], 'stride': 3, 'length': 5}, - 'CA-NV': {'corners': [30, 45, -124, -114], 'stride': 10, 'length': 4}, - 'Cambodia': {'corners': [0, 24, 90, 118], 'stride': 3, 'length': 5}, - 'CentralCA': {'corners': [34.5, 40.5, -124, -118], 'stride': 4, 'length': 4}, - 'CHI-DET': {'corners': [39, 44, -92, -83], 'stride': 4, 'length': 4}, - 'DCArea': {'corners': [36.7, 40, -81, -72], 'stride': 4, 'length': 4}, - 'EastCO': {'corners': [36.5, 41.5, -108, -101.8], 'stride': 4, 'length': 4}, - 'EPacific': {'corners': [0, 60, 180, 300], 'stride': 10, 'length': 5}, - 'Europe': {'corners': [15, 75, -30, 75], 'stride': 10, 'length': 5}, - 'Florida': {'corners': [19.2305, 29.521, -86.1119, -73.8189], 'stride': 10, 'length': 5}, - 'GreatLakes': {'corners': [37, 50, -96, -70], 'stride': 10, 'length': 4}, - 'HI': {'corners': [16.6, 24.6, -157.6, -157.5], 'stride': 1, 'length': 4}, - 'HI-zoom': {'corners': None, 'width': 800000, 'height': 800000, 'stride': 4, 'length': 4}, - 'HFIP': {'corners': [8.35, 51.6, 244., 336.], 'stride': 30, 'length': 4}, - 'Hurr-Car': {'corners': [21, 28, -96, -69], 'stride': 10, 'length': 4}, - 'Juneau': {'corners': [55.741, 59.629, -140.247, -129.274], 'stride': 4, 'length': 4}, - 'NW-large': {'corners': [29.5787, 52.6127, -121.666, -96.5617], 'stride': 15, 'length': 4}, - 'NYC-BOS': {'corners': [39, 43.5, -77, -66.5], 'stride': 4, 'length': 4}, - 'PuertoRico': {'corners': [15.5257, 24.0976, -74.6703, -61.848], 'stride': 10, 'length': 5}, - 'SEA-POR': {'corners': [43, 50, -125, -119], 'stride': 4, 'length': 4}, - 'SouthCA': {'corners': [31, 37, -120, -114], 'stride': 4, 'length': 4}, - 'SouthFL': {'corners': [24, 28.5, -84, -77], 'stride': 4, 'length': 4}, - 'Taiwan': {'corners': [19, 28, 116, 126], 'stride': 1, 'length': 5}, - 'VortexSE': {'corners': [30, 37, -92.5, -82], 'stride': 4, 'length': 4}, - 'WAtlantic': {'corners': [-0.25, 50.25, 261.75, 330.25], 'stride': 5, 'length': 5}, - 'WFIP3-d01': {'corners': [33.66, 46.86, -78.83, -61.01], 'stride': 10, 'length': 4}, - 'WFIP3-d02': {'corners': [37.84, 43.22, -74.77, -66.50], 'stride': 5, 'length': 5}, - 'WPacific': {'corners': [-40, 50, 90, 240], 'stride': 10, 'length': 5}, + "NC": {"corners": [36, 51, -109, -85], "stride": 10, "length": 4}, + "NE": {"corners": [36, 48, -91, -62], "stride": 10, "length": 4}, + "NW": {"corners": [35, 52, -126, -102], "stride": 10, "length": 4}, + "SC": {"corners": [24, 41, -107, -86], "stride": 10, "length": 4}, + "SE": {"corners": [22, 37, -93.5, -72], "stride": 10, "length": 4}, + "SW": {"corners": [24.5, 45, -122, -103], "stride": 10, "length": 4}, + "Africa": {"corners": [-40, 40, -40, 60], "stride": 7, "length": 5}, + "AKZoom": {"corners": [52, 73, -162, -132], "stride": 4, "length": 5}, + "AKZoom2": {"corners": [37.9, 80.8, 180, -105.7], "stride": 8, "length": 5}, + "AKRange": {"corners": [62.0, 67.0, -152.0, -143.0], "stride": 4, "length": 4}, + "Anchorage": { + "corners": [58.59, 62.776, -152.749, -146.218], + "stride": 4, + "length": 4, + }, + "ATL": {"corners": [31.2, 35.8, -87.4, -79.8], "stride": 4, "length": 4}, + "Beijing": {"corners": [25, 53, 102, 133], "stride": 3, "length": 5}, + "CA-NV": {"corners": [30, 45, -124, -114], "stride": 10, "length": 4}, + "Cambodia": {"corners": [0, 24, 90, 118], "stride": 3, "length": 5}, + "CentralCA": {"corners": [34.5, 40.5, -124, -118], "stride": 4, "length": 4}, + "CHI-DET": {"corners": [39, 44, -92, -83], "stride": 4, "length": 4}, + "DCArea": {"corners": [36.7, 40, -81, -72], "stride": 4, "length": 4}, + "EastCO": {"corners": [36.5, 41.5, -108, -101.8], "stride": 4, "length": 4}, + "EPacific": {"corners": [0, 60, 180, 300], "stride": 10, "length": 5}, + "Europe": {"corners": [15, 75, -30, 75], "stride": 10, "length": 5}, + "Florida": { + "corners": [19.2305, 29.521, -86.1119, -73.8189], + "stride": 10, + "length": 5, + }, + "GreatLakes": {"corners": [37, 50, -96, -70], "stride": 10, "length": 4}, + "HI": {"corners": [16.6, 24.6, -157.6, -157.5], "stride": 1, "length": 4}, + "HI-zoom": { + "corners": None, + "width": 800000, + "height": 800000, + "stride": 4, + "length": 4, + }, + "HFIP": {"corners": [8.35, 51.6, 244.0, 336.0], "stride": 30, "length": 4}, + "Hurr-Car": {"corners": [21, 28, -96, -69], "stride": 10, "length": 4}, + "Juneau": { + "corners": [55.741, 59.629, -140.247, -129.274], + "stride": 4, + "length": 4, + }, + "NW-large": { + "corners": [29.5787, 52.6127, -121.666, -96.5617], + "stride": 15, + "length": 4, + }, + "NYC-BOS": {"corners": [39, 43.5, -77, -66.5], "stride": 4, "length": 4}, + "PuertoRico": { + "corners": [15.5257, 24.0976, -74.6703, -61.848], + "stride": 10, + "length": 5, + }, + "SEA-POR": {"corners": [43, 50, -125, -119], "stride": 4, "length": 4}, + "SouthCA": {"corners": [31, 37, -120, -114], "stride": 4, "length": 4}, + "SouthFL": {"corners": [24, 28.5, -84, -77], "stride": 4, "length": 4}, + "Taiwan": {"corners": [19, 28, 116, 126], "stride": 1, "length": 5}, + "VortexSE": {"corners": [30, 37, -92.5, -82], "stride": 4, "length": 4}, + "WAtlantic": {"corners": [-0.25, 50.25, 261.75, 330.25], "stride": 5, "length": 5}, + "WFIP3-d01": {"corners": [33.66, 46.86, -78.83, -61.01], "stride": 10, "length": 4}, + "WFIP3-d02": {"corners": [37.84, 43.22, -74.77, -66.50], "stride": 5, "length": 5}, + "WPacific": {"corners": [-40, 50, 90, 240], "stride": 10, "length": 5}, } -class Map(): +class Map: # pylint: disable=too-many-instance-attributes - ''' + """ Class includes utilities needed to create a Basemap object, add airport locations, and draw the blank map. @@ -106,164 +132,170 @@ class Map(): certain plots, default is True tile a string corresponding to a pre-defined tile in the TILE_DEFS dictionary - ''' + """ def __init__(self, airport_fn, ax, **kwargs): - self.ax = ax - self.grid_info = kwargs.get('grid_info', {}) - self.model = kwargs.get('model') - self.plot_airports = kwargs.get('plot_airports', True) - self.tile = kwargs.get('tile', 'full') + self.grid_info = kwargs.get("grid_info", {}) + self.model = kwargs.get("model") + self.plot_airports = kwargs.get("plot_airports", True) + self.tile = kwargs.get("tile", "full") self.airports = self.load_airports(airport_fn) - if self.model == 'hrrr' and 'WFIP3' in self.tile: - self.grid_info.update({'lat_1': 40.6, 'lat_2': 40.6, 'lon_0': 289.2}) - if self.model != 'hrrrhi': + if self.model == "hrrr" and "WFIP3" in self.tile: + self.grid_info.update({"lat_1": 40.6, "lat_2": 40.6, "lon_0": 289.2}) + if self.model != "hrrrhi": if self.tile in FULL_TILES: - self.corners = self.grid_info.pop('corners') + self.corners = self.grid_info.pop("corners") else: self.corners = self.get_corners() - self.grid_info.pop('corners') + self.grid_info.pop("corners") else: self.corners = None if self.tile in FULL_TILES: - self.width = self.grid_info.pop('width') - self.height = self.grid_info.pop('height') + self.width = self.grid_info.pop("width") + self.height = self.grid_info.pop("height") else: self.width = self.get_width() - self.grid_info.pop('width') + self.grid_info.pop("width") self.height = self.get_height() - self.grid_info.pop('height') + self.grid_info.pop("height") # Some of Hawaii's smaller islands and islands in the Caribbean don't # show up with a larger threshold. area_thresh = 1000 - if self.tile in ['HI', 'Florida', 'PuertoRico'] or self.model in ['hrrrhi', 'hrrrcar']: + if self.tile in ["HI", "Florida", "PuertoRico"] or self.model in [ + "hrrrhi", + "hrrrcar", + ]: area_thresh = 100 self.m = self._get_basemap(area_thresh=area_thresh, **self.grid_info) - if self.model == 'hrrrhi': - parallels = np.arange(0., 81, 5.) + if self.model == "hrrrhi": + parallels = np.arange(0.0, 81, 5.0) self.m.drawparallels(parallels, labels=[False, True, True, False]) - meridians = np.arange(10., 351., 5.) + meridians = np.arange(10.0, 351.0, 5.0) self.m.drawmeridians(meridians, labels=[True, False, False, True]) def boundaries(self): - - ''' Draws map boundaries - coasts, states, countries. ''' + """Draws map boundaries - coasts, states, countries.""" try: self.m.drawcoastlines(linewidth=0.5) except ValueError: - self.m.drawcounties(color='gray', - linewidth=0.4, - zorder=2, - ) + self.m.drawcounties( + color="gray", + linewidth=0.4, + zorder=2, + ) else: - if self.model not in ['global', 'hfip'] and self.tile not in FULL_TILES: - self.m.drawcounties(antialiased=False, - color='gray', - linewidth=0.1, - zorder=2, - ) + if self.model not in ["global", "hfip"] and self.tile not in FULL_TILES: + self.m.drawcounties( + antialiased=False, + color="gray", + linewidth=0.1, + zorder=2, + ) self.m.drawstates() self.m.drawcountries() def draw(self): - - ''' Draw a map with political boundaries and airports only. ''' + """Draw a map with political boundaries and airports only.""" self.boundaries() - if self.plot_airports and 'global' not in self.model: # airports are too dense in global + if ( + self.plot_airports and "global" not in self.model + ): # airports are too dense in global self.draw_airports() def draw_airports(self): - - ''' Plot each of the airport locations on the map. ''' + """Plot each of the airport locations on the map.""" lats = self.airports[:, 0] - lons = 360 + self.airports[:, 1] # Convert to positive longitude + lons = 360 + self.airports[:, 1] # Convert to positive longitude x, y = self.m(lons, lats) - self.m.plot(x, y, 'wo', - ax=self.ax, - fillstyle='full', - markeredgecolor='k', - markeredgewidth=0.5, - markersize=4, - ) + self.m.plot( + x, + y, + "wo", + ax=self.ax, + fillstyle="full", + markeredgecolor="k", + markeredgewidth=0.5, + markersize=4, + ) del x del y def _get_basemap(self, **get_basemap_kwargs): - - ''' Wrapper around basemap creation ''' + """Wrapper around basemap creation""" basemap_args = dict( ax=self.ax, - resolution='i', - ) + resolution="i", + ) if self.corners is not None: corners = self.corners - basemap_args.update(dict( - llcrnrlat=corners[0], - llcrnrlon=corners[2], - urcrnrlat=corners[1], - urcrnrlon=corners[3], - )) + basemap_args.update( + dict( + llcrnrlat=corners[0], + llcrnrlon=corners[2], + urcrnrlat=corners[1], + urcrnrlon=corners[3], + ) + ) else: - basemap_args.update(dict( - width=self.width, - height=self.height, - )) + basemap_args.update( + dict( + width=self.width, + height=self.height, + ) + ) basemap_args.update(get_basemap_kwargs) return Basemap(**basemap_args) def get_corners(self): - - ''' + """ Gather the corners for a specific tile. Corners are supplied in the following format: lat and lon of lower left (ll) and upper right(ur) corners: ll_lat, ur_lat, ll_lon, ur_lon - ''' + """ return TILE_DEFS[self.tile]["corners"] def get_width(self): - - ''' + """ Gather the width for a specific tile. - ''' + """ return TILE_DEFS[self.tile]["width"] def get_height(self): - - ''' + """ Gather the height for a specific tile. - ''' + """ return TILE_DEFS[self.tile]["height"] @staticmethod def load_airports(fn): + """Load lat, lon pairs from a text file, return a list of lists.""" - ''' Load lat, lon pairs from a text file, return a list of lists. ''' - - with open(fn, 'r') as f: + with open(fn, "r") as f: data = f.readlines() - return np.array([l.strip().split(',') for l in data], dtype=float) + return np.array([l.strip().split(",") for l in data], dtype=float) + -class DataMap(): - #pylint: disable=too-many-arguments +class DataMap: + # pylint: disable=too-many-arguments - ''' + """ Class that combines the input data and the chosen map to plot both together. Input: @@ -274,28 +306,26 @@ class DataMap(): fields map maps object - ''' + """ - #pylint: disable=unused-argument + # pylint: disable=unused-argument def __init__(self, map_fields, map_, model_name=None, **kwargs): - self.field = map_fields.shaded self.contour_fields = map_fields.contours self.hatch_fields = map_fields.hatches self.map_fields = map_fields self.map = map_ self.model_name = model_name - self.plot_scatter = map_fields.fields_spec.get('plot_scatter', False) + self.plot_scatter = map_fields.fields_spec.get("plot_scatter", False) def wind_fields(self, level): return self.map_fields.wind_fields(level) @staticmethod def add_logo(ax): + """Puts the NOAA logo at the bottom left of the matplotlib axes.""" - ''' Puts the NOAA logo at the bottom left of the matplotlib axes. ''' - - logo = mpimg.imread('static/noaa-logo-50x50.png') + logo = mpimg.imread("static/noaa-logo-50x50.png") imagebox = mpob.OffsetImage(logo) ab = mpob.AnnotationBbox( @@ -303,48 +333,51 @@ def add_logo(ax): (0, 0), box_alignment=(-0.2, -0.2), frameon=False, - xycoords='axes points', - ) + xycoords="axes points", + ) ax.add_artist(ab) - def _colorbar(self, cc, ax): - - ''' Internal method that plots the color bar for a contourf field. - If ticks is set to zero, use a user-defined list of clevs from default_specs - If ticks is less than zero, use abs(ticks) as the step for labeling clevs ''' + """Internal method that plots the color bar for a contourf field. + If ticks is set to zero, use a user-defined list of clevs from default_specs + If ticks is less than zero, use abs(ticks) as the step for labeling clevs""" if self.field.ticks > 0: - ticks = np.arange(np.amin(self.field.clevs), - np.amax(self.field.clevs+1), self.field.ticks) + ticks = np.arange( + np.amin(self.field.clevs), + np.amax(self.field.clevs + 1), + self.field.ticks, + ) elif self.field.ticks == 0: ticks = self.field.clevs else: - ticks = self.field.clevs[0:len(self.field.clevs):-self.field.ticks] + ticks = self.field.clevs[0 : len(self.field.clevs) : -self.field.ticks] ticks = np.around(ticks, 4) - cbar = plt.colorbar(cc, - ax=ax, - orientation='horizontal', - pad=0.02, - shrink=1.0, - ticks=ticks, - ) + cbar = plt.colorbar( + cc, + ax=ax, + orientation="horizontal", + pad=0.02, + shrink=1.0, + ticks=ticks, + ) - if self.field.short_name == 'flru': - ticks = [label.rjust(30) for label in ['VFR', 'MVFR', 'IFR', 'LIFR', ""]] + if self.field.short_name == "flru": + ticks = [label.rjust(30) for label in ["VFR", "MVFR", "IFR", "LIFR", ""]] # this step is done to allow proper order of icing severity levels (trace before light) - if self.field.short_name == 'icsev': - ticks = [label.rjust(30) for label in ['TRACE', 'LIGHT', 'MODERATE', 'HEAVY', ""]] + if self.field.short_name == "icsev": + ticks = [ + label.rjust(30) for label in ["TRACE", "LIGHT", "MODERATE", "HEAVY", ""] + ] cbar.ax.set_xticklabels(ticks, fontsize=12) def draw(self, show=False): - - ''' Main method for creating the plot. Set show=True to display the - figure from the command line. ''' + """Main method for creating the plot. Set show=True to display the + figure from the command line.""" cf = self._draw_panel() @@ -357,23 +390,23 @@ def draw(self, show=False): # Create a pop-up to display the figure, if show=True if show: plt.tight_layout() - #plt.show() + # plt.show() self.add_logo(self.map.ax) - def _draw_panel(self, wind_barbs=True): # pylint: disable=too-many-locals, too-many-branches - + def _draw_panel(self, wind_barbs=True): # pylint: disable=too-many-locals, too-many-branches ax = self.map.ax # Draw a map and add the shaded field self.map.draw() - cf = self._draw_field(ax=ax, - colors=self.field.colors, - extend='both', - field=self.field, - func=self.map.m.contourf, - levels=self.field.clevs, - ) + cf = self._draw_field( + ax=ax, + colors=self.field.colors, + extend="both", + field=self.field, + func=self.map.m.contourf, + levels=self.field.clevs, + ) not_labeled = [self.field.short_name] if self.hatch_fields: @@ -388,16 +421,16 @@ def _draw_panel(self, wind_barbs=True): # pylint: disable=too-many-locals, too-m self._draw_hatches(ax) # Add wind barbs, if requested - add_wind = self.field.vspec.get('wind', False) + add_wind = self.field.vspec.get("wind", False) if add_wind and wind_barbs: self._wind_barbs(add_wind) # Add field values at airports - annotate = self.field.vspec.get('annotate', False) + annotate = self.field.vspec.get("annotate", False) model_name = self.model_name - if annotate and 'global' not in self.map.model: # too dense in global - if model_name not in ['RRFS NA 3km']: # too dense in full RRFS domain - if model_name == 'RAP-NCEP' and self.map.tile not in ['full']: + if annotate and "global" not in self.map.model: # too dense in global + if model_name not in ["RRFS NA 3km"]: # too dense in full RRFS domain + if model_name == "RAP-NCEP" and self.map.tile not in ["full"]: self._draw_field_values(ax) # Add scatter plot, if requested @@ -407,56 +440,66 @@ def _draw_panel(self, wind_barbs=True): # pylint: disable=too-many-locals, too-m return cf def _draw_contours(self, ax, not_labeled): - - ''' Draw the contour fields requested. ''' + """Draw the contour fields requested.""" model_name = self.model_name main_field = self.field.short_name for contour_field in self.contour_fields: - levels = contour_field.contour_kwargs.pop('levels', - contour_field.clevs) + levels = contour_field.contour_kwargs.pop("levels", contour_field.clevs) if model_name in ["RAP-NCEP", "RRFS-NCEP", "RRFS NA 3km"]: - if main_field == "totp" and contour_field.short_name == "pres" and \ - self.map.tile == "full": + if ( + main_field == "totp" + and contour_field.short_name == "pres" + and self.map.tile == "full" + ): levels = np.arange(650, 1051, 8) - cc = self._draw_field(ax=ax, - field=contour_field, - func=self.map.m.contour, - levels=levels, - **contour_field.contour_kwargs, - ) + cc = self._draw_field( + ax=ax, + field=contour_field, + func=self.map.m.contour, + levels=levels, + **contour_field.contour_kwargs, + ) if contour_field.short_name not in not_labeled: try: - plt.clabel(cc, levels[::4], - colors='k', - fmt='%1.0f', - fontsize=10, - inline=1, - ) + plt.clabel( + cc, + levels[::4], + colors="k", + fmt="%1.0f", + fontsize=10, + inline=1, + ) except ValueError: - print(f'Cannot add contour labels to map for {self.field.short_name} \ - {self.field.level}') + print( + f"Cannot add contour labels to map for {self.field.short_name} \ + {self.field.level}" + ) def _draw_scatter(self, ax): - - ''' Plot dots at locations on the map that meet a threshold. ''' + """Plot dots at locations on the map that meet a threshold.""" field = self.field levels = self.field.clevs colors = self.field.colors vals = self.field.values() - value_to_color = np.full_like(vals, colors[0], dtype='object') + value_to_color = np.full_like(vals, colors[0], dtype="object") num_levels = len(levels) for i in range(num_levels): if i != num_levels - 1: - value_to_color = np.where((vals > levels[i]) & \ - (vals <= levels[i+1]), colors[i+1], value_to_color) + value_to_color = np.where( + (vals > levels[i]) & (vals <= levels[i + 1]), + colors[i + 1], + value_to_color, + ) else: - value_to_color = np.where(vals > levels[i], colors[i+1], value_to_color) + value_to_color = np.where( + vals > levels[i], colors[i + 1], value_to_color + ) vtc1d = np.ravel(value_to_color) @@ -464,17 +507,17 @@ def _draw_scatter(self, ax): # without altering the colors we just set. field.data = np.log10(field.values()) * 20 - self._draw_field(ax=ax, - field=field, - alpha=1.0, - c=vtc1d, - func=self.map.m.scatter, - **field.contour_kwargs, - ) + self._draw_field( + ax=ax, + field=field, + alpha=1.0, + c=vtc1d, + func=self.map.m.scatter, + **field.contour_kwargs, + ) def _draw_field(self, ax, field, func, **kwargs): - - ''' + """ Internal implementation that calls a matplotlib function. Input args: @@ -488,23 +531,26 @@ def _draw_field(self, ax, field, func, **kwargs): Return: The return from the function called. - ''' + """ x, y = self._xy_mesh(field) vals = field.data # For global lat-lon models, make 2D arrays for x and y # Shift the map and data if needed - if self.map.model in ['global', 'global_mpas', 'hfip']: + if self.map.model in ["global", "global_mpas", "hfip"]: tile = self.map.tile - if tile in ['Africa', 'Europe']: - vals, x = shiftgrid(180., vals, x, start=False) - y, x = np.meshgrid(y, x, sparse=False, indexing='ij') - - ret = func(x, y, vals, - ax=ax, - **kwargs, - ) + if tile in ["Africa", "Europe"]: + vals, x = shiftgrid(180.0, vals, x, start=False) + y, x = np.meshgrid(y, x, sparse=False, indexing="ij") + + ret = func( + x, + y, + vals, + ax=ax, + **kwargs, + ) del x del y @@ -512,13 +558,12 @@ def _draw_field(self, ax, field, func, **kwargs): vals.close() except AttributeError: del vals - print(f'CLOSE ERROR: {field.short_name} {field.level}') + print(f"CLOSE ERROR: {field.short_name} {field.level}") return ret def _draw_field_values(self, ax): - - ''' Add the text value of the field at airport locations. ''' - annotate_decimal = self.field.vspec.get('annotate_decimal', 0) + """Add the text value of the field at airport locations.""" + annotate_decimal = self.field.vspec.get("annotate_decimal", 0) lats = self.map.airports[:, 0] lons = 360 + self.map.airports[:, 1] x, y = self.map.m(lons, lats) @@ -531,50 +576,57 @@ def _draw_field_values(self, ax): if crnrs[3] < 0: crnrs[3] = 360 + crnrs[3] for i, lat in enumerate(lats): - if crnrs[1] > lat > crnrs[0] and \ - crnrs[3] > lons[i] > crnrs[2]: + if crnrs[1] > lat > crnrs[0] and crnrs[3] > lons[i] > crnrs[2]: xgrid, ygrid = self.field.get_xypoint(lat, lons[i]) data_value = data_values[xgrid, ygrid].values.item() if xgrid > 0 and ygrid > 0: - if (not isnan(data_value)) and (data_value != 0.): - ax.annotate(f"{data_value:.{annotate_decimal}f}", \ - xy=(x[i], y[i]), fontsize=10) + if (not isnan(data_value)) and (data_value != 0.0): + ax.annotate( + f"{data_value:.{annotate_decimal}f}", + xy=(x[i], y[i]), + fontsize=10, + ) data_values.close() def _draw_hatches(self, ax): - - ''' Draw the hatched regions requested. ''' + """Draw the hatched regions requested.""" # Levels should be included in the settings dict here since they don't # correspond to a full field of contours. handles = [] for field in self.hatch_fields: - colors = field.contour_kwargs.get('colors', 'k') - hatches = field.contour_kwargs.get('hatches', '----') - labels = field.contour_kwargs.get('labels', 'XXXX') - linewidths = field.contour_kwargs.get('linewidths', 0.1) - handles.append(mpatches.Patch(edgecolor=colors[-1], facecolor='lightgrey', \ - label=labels, hatch=hatches[-1])) - - cf = self._draw_field(ax=ax, - extend='both', - field=field, - func=self.map.m.contourf, - **field.contour_kwargs, - ) + colors = field.contour_kwargs.get("colors", "k") + hatches = field.contour_kwargs.get("hatches", "----") + labels = field.contour_kwargs.get("labels", "XXXX") + linewidths = field.contour_kwargs.get("linewidths", 0.1) + handles.append( + mpatches.Patch( + edgecolor=colors[-1], + facecolor="lightgrey", + label=labels, + hatch=hatches[-1], + ) + ) + + cf = self._draw_field( + ax=ax, + extend="both", + field=field, + func=self.map.m.contourf, + **field.contour_kwargs, + ) # For each level, we set the color of its hatch cf.set_edgecolor(colors) cf.set_facecolor("None") cf.set_linewidth(linewidths) # Create legend for precip type field - if self.field.short_name == 'ptyp': + if self.field.short_name == "ptyp": plt.legend(handles=handles, loc=[0.25, 0.03]) def _set_overlay_string(self): - - ''' Creates the main title of the plot with select hatched and - contoured fields defined. ''' + """Creates the main title of the plot with select hatched and + contoured fields defined.""" f = self.field @@ -585,67 +637,67 @@ def _set_overlay_string(self): if self.hatch_fields: cf = self.hatch_fields[0] not_labeled.extend([h.short_name for h in self.hatch_fields]) - if not any(list(set(cf.short_name).intersection(['pres']))): - title = cf.vspec.get('title', cf.field.long_name) - contoured.append(f'{title} ({cf.units}, hatched)') + if not any(list(set(cf.short_name).intersection(["pres"]))): + title = cf.vspec.get("title", cf.field.long_name) + contoured.append(f"{title} ({cf.units}, hatched)") # Add descriptor string for the important contoured fields if self.contour_fields: for cf in self.contour_fields: if cf.short_name not in not_labeled: - title = cf.vspec.get('title', cf.field.long_name) + title = cf.vspec.get("title", cf.field.long_name) title = title.replace("Geopotential", "Geop.") - contoured.append(f'{title}') - contoured_units.append(f'{cf.units}') + contoured.append(f"{title}") + contoured_units.append(f"{cf.units}") - contoured = '\n'.join(contoured) # Make 'contoured' a string with linefeeds + contoured = "\n".join(contoured) # Make 'contoured' a string with linefeeds if contoured_units: contoured = f"{contoured} ({', '.join(contoured_units)}, contoured)" return contoured def _title(self): - - ''' Draw the title for a map. ''' + """Draw the title for a map.""" f = self.field atime = f.date_to_str(f.anl_dt) vtime = f.date_to_str(f.valid_dt) # Analysis time (top) and forecast hour with Valid Time (bottom) on the left - plt.title(f"{self.model_name}: {atime}\nFcst Hr: {f.fhr}, Valid Time {vtime}", - alpha=None, - fontsize=14, - loc='left', - ) + plt.title( + f"{self.model_name}: {atime}\nFcst Hr: {f.fhr}, Valid Time {vtime}", + alpha=None, + fontsize=14, + loc="left", + ) level, lev_unit = f.numeric_level(index_match=False) - if f.vspec.get('print_units', True): - units = f'({f.units}, shaded)' + if f.vspec.get("print_units", True): + units = f"({f.units}, shaded)" else: - units = f'' + units = f"" # Title or Atmospheric level and unit in the high center - if f.vspec.get('title'): + if f.vspec.get("title"): title = f"{f.vspec.get('title')} {units}" else: level = level if not isinstance(level, list) else level[0] - title = f'{level} {lev_unit} {f.field.long_name} {units}' + title = f"{level} {lev_unit} {f.field.long_name} {units}" plt.title(f"{title}", loc="center", y=1.10, fontsize=18) # Two lines for hatched data (top), and contoured data (bottom) on the right contoured = self._set_overlay_string() - plt.title(f"{contoured}", - loc='right', - fontsize=14, - ) + plt.title( + f"{contoured}", + loc="right", + fontsize=14, + ) def _wind_barbs(self, level): - - ''' Draws the wind barbs. A decent stride can be found if you divide the - number of grid points on the shorter side by 35. Subdomains are defined - by lat,lon so the stride is set in the TILE_DEFS. For the globalCONUS - subdomains, further dividing by 2.5 works well. ''' + """Draws the wind barbs. A decent stride can be found if you divide the + number of grid points on the shorter side by 35. Subdomains are defined + by lat,lon so the stride is set in the TILE_DEFS. For the globalCONUS + subdomains, further dividing by 2.5 works well.""" lev = level if not isinstance(level, bool) else self.field.level u, v = [f.data for f in self.wind_fields(lev)] @@ -663,14 +715,20 @@ def _wind_barbs(self, level): else: stride = TILE_DEFS[tile]["stride"] length = TILE_DEFS[tile]["length"] - if self.map.model == 'globalCONUS': + if self.map.model == "globalCONUS": stride = int(round(stride / 2.5)) length = 5 - if self.map.model == 'hrrr' and self.model_name == 'WFIP3-FULL' and \ - tile == 'WFIP3-d02': + if ( + self.map.model == "hrrr" + and self.model_name == "WFIP3-FULL" + and tile == "WFIP3-d02" + ): stride = 6 - if self.map.model == 'hrrr' and self.model_name == 'WFIP3-NEST' and \ - tile == 'WFIP3-d02': + if ( + self.map.model == "hrrr" + and self.model_name == "WFIP3-NEST" + and tile == "WFIP3-d02" + ): stride = 17 mask = np.ones_like(u) @@ -680,54 +738,56 @@ def _wind_barbs(self, level): # For global lat-lon models, make 2D arrays for x and y # Shift the map and data if needed - if self.map.m.projection == 'cyl': - if tile in ['Africa', 'Europe']: + if self.map.m.projection == "cyl": + if tile in ["Africa", "Europe"]: savex = x - u, x = shiftgrid(180., u, x, start=False) - v, savex = shiftgrid(180., v, savex, start=False) - y, x = np.meshgrid(y, x, sparse=False, indexing='ij') + u, x = shiftgrid(180.0, u, x, start=False) + v, savex = shiftgrid(180.0, v, savex, start=False) + y, x = np.meshgrid(y, x, sparse=False, indexing="ij") mu, mv = [np.ma.masked_array(c, mask=mask) for c in [u, v]] - self.map.m.barbs(x, y, mu, mv, - barbcolor='k', - flagcolor='k', - length=length, - linewidth=0.2, - sizes={'spacing': 0.25}, - ) + self.map.m.barbs( + x, + y, + mu, + mv, + barbcolor="k", + flagcolor="k", + length=length, + linewidth=0.2, + sizes={"spacing": 0.25}, + ) def _xy_mesh(self, field): - - ''' Helper function to create mesh for various plot. ''' + """Helper function to create mesh for various plot.""" lat, lon = field.latlons() - if self.map.model == 'obs': - lat, lon = np.meshgrid(lat, lon, sparse=False, indexing='ij') + if self.map.model == "obs": + lat, lon = np.meshgrid(lat, lon, sparse=False, indexing="ij") adjust = 360 if np.any(lon < 0) else 0 return self.map.m(adjust + lon, lat) + class DiffMap(DataMap): - ''' + """ Extends DataMap for handling difference plots, which need different titles, and will not plot overlays and such. - ''' + """ def _colorbar(self, cc, ax): - - ''' Set the colorbar for a difference field. ''' + """Set the colorbar for a difference field.""" plt.colorbar( cc, ax=ax, - orientation='horizontal', + orientation="horizontal", pad=0.02, shrink=1.0, - ) + ) def _draw_panel(self, wind_barbs=False): - - ''' Draw a map of the difference field. ''' + """Draw a map of the difference field.""" ax = self.map.ax @@ -737,18 +797,19 @@ def _draw_panel(self, wind_barbs=False): # The number of levels (nlev) here, should be the same number as is used # in the linspace call in self._eq_contours. 21 seems reasonable, but is # arbitrary. - colors = self.field.centered_diff(cmap='Spectral_r', nlev=21) - cf = self._draw_field(ax=ax, - colors=colors, - extend='both', - field=self.field, - func=self.map.m.contourf, - levels=self._eq_contours(), - ) + colors = self.field.centered_diff(cmap="Spectral_r", nlev=21) + cf = self._draw_field( + ax=ax, + colors=colors, + extend="both", + field=self.field, + func=self.map.m.contourf, + levels=self._eq_contours(), + ) return cf def _eq_contours(self): - ''' Center the contours based on the data min/max ''' + """Center the contours based on the data min/max""" minval = np.amin(self.field.data) maxval = np.amax(self.field.data) @@ -758,54 +819,52 @@ def _eq_contours(self): return np.linspace(-maxval, maxval, 21) def _title(self): - ''' Draw the title for a map. ''' + """Draw the title for a map.""" f = self.field atime = f.date_to_str(f.anl_dt) vtime = f.date_to_str(f.valid_dt) # Analysis time (top) and forecast hour with Valid Time (bottom) on the left - plt.title(f"{self.model_name}: {atime}\nFcst Hr: {f.fhr}, Valid Time {vtime}", - alpha=None, - fontsize=14, - loc='left', - ) + plt.title( + f"{self.model_name}: {atime}\nFcst Hr: {f.fhr}, Valid Time {vtime}", + alpha=None, + fontsize=14, + loc="left", + ) level, lev_unit = f.numeric_level(index_match=False) - if f.vspec.get('print_units', True): - units = f'({f.units}, shaded)' + if f.vspec.get("print_units", True): + units = f"({f.units}, shaded)" else: - units = f'' + units = f"" # Title or Atmospheric level and unit in the high center - if f.vspec.get('title'): + if f.vspec.get("title"): title = f"Diff: {f.vspec.get('title')} {units}" else: level = level if not isinstance(level, list) else level[0] - title = f'Diff: {level} {lev_unit} {f.field.long_name} {units}' + title = f"Diff: {level} {lev_unit} {f.field.long_name} {units}" plt.title(f"{title}", position=(0.5, 1.08), fontsize=18) - class MultiPanelDataMap(DataMap): - ''' + """ Class that extends a DataMap for handling multiple panels. Keyword arguments: last_panel flag for multipanel plots to designate last panel drawn - ''' + """ def __init__(self, map_fields, map_, member, model_name=None, **kwargs): - super().__init__(map_fields, map_, model_name=model_name) - self.last_panel = kwargs.get('last_panel', False) + self.last_panel = kwargs.get("last_panel", False) self.member = str(member) def draw(self, show=False): - - ''' Main method for creating the plot. Set show=True to display the - figure from the command line. ''' + """Main method for creating the plot. Set show=True to display the + figure from the command line.""" cf = self._draw_panel(wind_barbs=False) @@ -816,32 +875,31 @@ def draw(self, show=False): if self.last_panel: cax = plt.axes([0.0, 0.0, 1.0, 0.2]) self._colorbar(ax=cax, cc=cf) - cax.axis('off') + cax.axis("off") # Create a pop-up to display the figure, if show=True if show: plt.tight_layout() - #plt.show() + # plt.show() return cf def _label_member(self): - - ''' Add the member label to the top left of the plot ''' + """Add the member label to the top left of the plot""" ax = self.map.ax ax.text( - 0.05, 0.90, + 0.05, + 0.90, self.member, fontsize=18, - fontweight='bold', + fontweight="bold", backgroundcolor="white", transform=ax.transAxes, - ) + ) def title(self): - - ''' Draw the title for a map. ''' + """Draw the title for a map.""" f = self.field atime = f.date_to_str(f.anl_dt) @@ -849,63 +907,69 @@ def title(self): ax = self.map.ax # Analysis time (top) and forecast hour with Valid Time (bottom) on the left - ax.text(0.0, 0.5, - f"{self.model_name}: {atime}\nFcst Hr: {f.fhr}, Valid Time {vtime}", - alpha=None, - fontsize=14, - horizontalalignment='left', - verticalalignment='top', - transform=ax.transAxes, - ) + ax.text( + 0.0, + 0.5, + f"{self.model_name}: {atime}\nFcst Hr: {f.fhr}, Valid Time {vtime}", + alpha=None, + fontsize=14, + horizontalalignment="left", + verticalalignment="top", + transform=ax.transAxes, + ) level, lev_unit = f.numeric_level(index_match=False) - if f.vspec.get('print_units', True): - units = f'({f.units}, shaded)' + if f.vspec.get("print_units", True): + units = f"({f.units}, shaded)" else: - units = f'' + units = f"" # Title or Atmospheric level and unit in the high center - if f.vspec.get('title'): + if f.vspec.get("title"): title = f"{f.vspec.get('title')} {units}" else: level = level if not isinstance(level, list) else level[0] - title = f'{level} {lev_unit} {f.field.long_name} {units}' - ax.text(0, 0.7, - f"{title}", - horizontalalignment='left', - verticalalignment='top', - fontsize=16, - transform=ax.transAxes, - ) + title = f"{level} {lev_unit} {f.field.long_name} {units}" + ax.text( + 0, + 0.7, + f"{title}", + horizontalalignment="left", + verticalalignment="top", + fontsize=16, + transform=ax.transAxes, + ) # Two lines for hatched data (top), and contoured data (bottom) on the right contoured = self._set_overlay_string() - ax.text(0, 0.6, - f"{contoured}", - horizontalalignment='left', - verticalalignment='top', - fontsize=14, - transform=ax.transAxes, - ) + ax.text( + 0, + 0.6, + f"{contoured}", + horizontalalignment="left", + verticalalignment="top", + fontsize=14, + transform=ax.transAxes, + ) -class MapFields(): - ''' Class that packages all the field objects need for producing +class MapFields: + """Class that packages all the field objects need for producing desired map content, i.e. an object that contains all filled contours, hatched spaces, and overlayed contours needed for a full - product. ''' - - def __init__(self, fhr, fields_spec, grib_path, level, name, map_type=None, - **kwargs): + product.""" + def __init__( + self, fhr, fields_spec, grib_path, level, name, map_type=None, **kwargs + ): self.grib_path = grib_path self.fhr = fhr self.fields_spec = deepcopy(fields_spec) self.level = level self.map_type = map_type - self.model = kwargs.get('model') + self.model = kwargs.get("model") self.name = name - self.tile = kwargs.get('tile', 'full') + self.tile = kwargs.get("tile", "full") self.map_spec = deepcopy(self.fields_spec[self.name][self.level]) self.set_level(self.level, self.map_spec) @@ -913,15 +977,19 @@ def __init__(self, fhr, fields_spec, grib_path, level, name, map_type=None, self.grib_path2 = kwargs.get("grib_path2") def set_level(self, level, spec): - nlevel, _ = numeric_level(level=level, index_match=False) - level_info = any(x for x in cfgrib_spec(spec["cfgrib"], self.model) for l in ("level", "top", "bottom", "Surface") if l in x) + level_info = any( + x + for x in cfgrib_spec(spec["cfgrib"], self.model) + for l in ("level", "top", "bottom", "Surface") + if l in x + ) if nlevel and not level_info: if spec["cfgrib"].get(self.model): spec["cfgrib"][self.model]["level"] = nlevel else: spec["cfgrib"]["level"] = nlevel - #if spec["cfgrib"].get("level") is None and not spec["cfgrib"].get("stepRange") and not \ + # if spec["cfgrib"].get("level") is None and not spec["cfgrib"].get("stepRange") and not \ # spec["cfgrib"].get("topLevel") and not \ # spec["cfgrib"].get("typeOfLevel") == "surface" and not \ # spec["cfgrib"].get("scaledValueOfFirstFixedSurface"): @@ -951,33 +1019,35 @@ def shaded(self): @property def contours(self): - ''' Return the list of contour fieldData objects''' + """Return the list of contour fieldData objects""" # We won't plot contours on multipanel plots, or full global # plots. - if self.map_type == 'enspanel': + if self.map_type == "enspanel": return [] - if 'global' in self.model and self.tile in ['full']: + if "global" in self.model and self.tile in ["full"]: return [] - return self._overlay_fields('contours') + return self._overlay_fields("contours") @property def hatches(self): - ''' Return the list of hatch fieldData objects''' + """Return the list of hatch fieldData objects""" - return self._overlay_fields('hatches') + return self._overlay_fields("hatches") def wind_fields(self, level=None): - ''' Return u, v tuple of wind fields ''' + """Return u, v tuple of wind fields""" lev = level or self.level winds = [] for var in ("u", "v"): wind_spec = self.fields_spec[var][lev] self.set_level(lev, wind_spec) - ds = gribfile.GribFile(self.grib_path, cfgrib_spec(wind_spec["cfgrib"], self.model)).contents + ds = gribfile.GribFile( + self.grib_path, cfgrib_spec(wind_spec["cfgrib"], self.model) + ).contents args = { "ds": ds, "fhr": self.fhr, @@ -991,20 +1061,21 @@ def wind_fields(self, level=None): return winds def _overlay_fields(self, spec_sect): - - ''' Generate a list of fieldData objects for the specified type - of overlay -- hatches or contours ''' + """Generate a list of fieldData objects for the specified type + of overlay -- hatches or contours""" overlay_fields = [] for overlay, overlay_kwargs in self.map_spec.get(spec_sect, {}).items(): - if '_' in overlay: - var, lev = overlay.split('_') + if "_" in overlay: + var, lev = overlay.split("_") else: var, lev = overlay, self.level overlay_spec = deepcopy(self.fields_spec[var][lev]) self.set_level(lev, overlay_spec) - ds = gribfile.GribFile(self.grib_path, cfgrib_spec(overlay_spec["cfgrib"], self.model)).contents + ds = gribfile.GribFile( + self.grib_path, cfgrib_spec(overlay_spec["cfgrib"], self.model) + ).contents args = { "ds": ds, "fhr": self.fhr, diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 56bafbd..32474a2 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -1,31 +1,31 @@ # pylint: disable=invalid-name -''' +""" The module the contains the SkewTDiagram class responsible for creating a Skew-T Log-P diagram using MetPy. -''' +""" from collections import OrderedDict from functools import lru_cache -import numpy as np import matplotlib.font_manager as fm -import matplotlib.pyplot as plt -from matplotlib.ticker import FixedLocator import matplotlib.lines as mlines -from matplotlib.lines import Line2D +import matplotlib.pyplot as plt import metpy.calc as mpcalc +import numpy as np +import pandas as pd +from matplotlib.lines import Line2D +from matplotlib.ticker import FixedLocator from metpy.plots import Hodograph, SkewT from metpy.units import units from mpl_toolkits.axes_grid1.inset_locator import inset_axes -import pandas as pd import adb_graphics.datahandler.gribdata as gribdata import adb_graphics.errors as errors import adb_graphics.utils as utils -class SkewTDiagram(gribdata.profileData): - ''' The class responsible for gathering all data needed from a grib file to +class SkewTDiagram(gribdata.profileData): + """The class responsible for gathering all data needed from a grib file to produce a Skew-T Log-P diagram. Input: @@ -41,197 +41,224 @@ class SkewTDiagram(gribdata.profileData): Additional keyword arguments for the gribdata.profileData base class should also be included. - ''' + """ def __init__(self, ds, loc, **kwargs): - # Initialize on the temperature field since we need to gather # field-specific data from this object, e.g. dates, lat, lon, etc. - super().__init__(ds=ds, - loc=loc, - short_name='temp', - **kwargs, - ) + super().__init__( + ds=ds, + loc=loc, + short_name="temp", + **kwargs, + ) - self.max_plev = kwargs.get('max_plev', 0) - self.model_name = kwargs.get('model_name', 'Analysis') + self.max_plev = kwargs.get("max_plev", 0) + self.model_name = kwargs.get("model_name", "Analysis") def _add_hydrometeors(self, hydro_subplot): - # pylint: disable=too-many-locals - mixing_ratios = OrderedDict({ - 'clwmr': { - 'color': 'blue', - 'label': 'CWAT', - 'marker': 's', - 'scale': 1.0, - 'units': 'g/m2' - }, - 'icmr': { - 'color': 'red', - 'label': 'CICE', - 'marker': '^', - 'scale': 10.0, - 'units': 'g/m2' - }, - 'rwmr': { - 'color': 'cyan', - 'label': 'RAIN', - 'marker': 'o', - 'scale': 1.0, - 'units': 'g/m2' - }, - 'snmr': { - 'color': 'purple', - 'label': 'SNOW', - 'marker': '*', - 'scale': 1.0, - 'units': 'g/m2' - }, - 'grle': { - 'color': 'orange', - 'label': 'GRPL', - 'marker': 'D', - 'scale': 1.0, - 'units': 'g/m2' - }, - }) - - profiles = self.atmo_profiles # dictionary - pres = profiles.get('pres').get('data') - temp = profiles.get('temp').get('data') - nlevs = len(pres) # determine number of vertical levels - pres_sfc = pres[0] # need correct surface pressure value! + # pylint: disable=too-many-locals + mixing_ratios = OrderedDict( + { + "clwmr": { + "color": "blue", + "label": "CWAT", + "marker": "s", + "scale": 1.0, + "units": "g/m2", + }, + "icmr": { + "color": "red", + "label": "CICE", + "marker": "^", + "scale": 10.0, + "units": "g/m2", + }, + "rwmr": { + "color": "cyan", + "label": "RAIN", + "marker": "o", + "scale": 1.0, + "units": "g/m2", + }, + "snmr": { + "color": "purple", + "label": "SNOW", + "marker": "*", + "scale": 1.0, + "units": "g/m2", + }, + "grle": { + "color": "orange", + "label": "GRPL", + "marker": "D", + "scale": 1.0, + "units": "g/m2", + }, + } + ) + + profiles = self.atmo_profiles # dictionary + pres = profiles.get("pres").get("data") + temp = profiles.get("temp").get("data") + nlevs = len(pres) # determine number of vertical levels + pres_sfc = pres[0] # need correct surface pressure value! handles = [] - gravity = 9.81 # m/s^2 + gravity = 9.81 # m/s^2 - lines = ['Vert. Integrated Amt\n(Resolved, Total)\n'\ - +'(supercool layers shaded,\nwith filled markers)'] + lines = [ + "Vert. Integrated Amt\n(Resolved, Total)\n" + + "(supercool layers shaded,\nwith filled markers)" + ] for mixr, settings in mixing_ratios.items(): # Get the profile values - scale = settings.get('scale') + scale = settings.get("scale") try: - profile = np.asarray(self.values(name=mixr)) * 1000. * scale + profile = np.asarray(self.values(name=mixr)) * 1000.0 * scale except errors.GribReadError: - print(f'missing {mixr} for hydrometeor plot, skipping that field.') + print(f"missing {mixr} for hydrometeor plot, skipping that field.") continue - mixr_total = 0. + mixr_total = 0.0 for n in range(nlevs): if n == 0: pres_layer = 2 * (pres_sfc - pres[n]) # layer depth - pres_sigma = pres_sfc - pres_layer # pressure at next sigma level + pres_sigma = pres_sfc - pres_layer # pressure at next sigma level else: - pres_layer = 2 * (pres_sigma - pres[n]) # layer depth - pres_sigma = pres_sigma - pres_layer # pressure at next sigma level + pres_layer = 2 * (pres_sigma - pres[n]) # layer depth + pres_sigma = pres_sigma - pres_layer # pressure at next sigma level mixr_total = mixr_total + pres_layer / gravity * profile[n] # limit values to upper and lower values of lotting range - profile = np.where((profile > 0.) & (profile < 1.e-4), 1.e-4, profile) - profile = np.where((profile > 10.), 10., profile) + profile = np.where((profile > 0.0) & (profile < 1.0e-4), 1.0e-4, profile) + profile = np.where((profile > 10.0), 10.0, profile) # plot line - hydro_subplot.plot(profile, pres, - settings.get('color'), - fillstyle='none', - linewidth=0.5, - marker=settings.get('marker'), - markersize=6, - ) - if mixr in ['clwmr', 'rwmr']: - hydro_subplot.plot(profile[temp.magnitude < 32.0], pres[temp.magnitude < 32.0], - settings.get('color'), - fillstyle='full', - linewidth=0.5, - marker=settings.get('marker'), - markersize=6, - ) + hydro_subplot.plot( + profile, + pres, + settings.get("color"), + fillstyle="none", + linewidth=0.5, + marker=settings.get("marker"), + markersize=6, + ) + if mixr in ["clwmr", "rwmr"]: + hydro_subplot.plot( + profile[temp.magnitude < 32.0], + pres[temp.magnitude < 32.0], + settings.get("color"), + fillstyle="full", + linewidth=0.5, + marker=settings.get("marker"), + markersize=6, + ) layer = False for i, profile_lev in enumerate(profile): - if ((profile_lev > 0.0 and temp[i].magnitude < 32.0) and not layer): + if (profile_lev > 0.0 and temp[i].magnitude < 32.0) and not layer: layer = True p_base = pres[i].magnitude - elif ((profile_lev <= 0.0 or temp[i].magnitude > 32.0) and layer): + elif (profile_lev <= 0.0 or temp[i].magnitude > 32.0) and layer: # Shade the supercooled water depth - p_top = pres[i-1].magnitude - rect = plt.Rectangle((0, p_top), 100, (p_base-p_top),\ - facecolor=settings.get('color'), alpha=0.1) + p_top = pres[i - 1].magnitude + rect = plt.Rectangle( + (0, p_top), + 100, + (p_base - p_top), + facecolor=settings.get("color"), + alpha=0.1, + ) hydro_subplot.add_patch(rect) layer = False # compute vertically integrated amount and add legend line - line = f"{settings.get('label'):<7s} {mixr_total.magnitude:>10.3f} "\ - f"{settings.get('units')}" + line = ( + f"{settings.get('label'):<7s} {mixr_total.magnitude:>10.3f} " + f"{settings.get('units')}" + ) if scale != 1.0: - line = f"{settings.get('label'):<5s}(x{scale}) {mixr_total.magnitude:.3f} "\ - f"{settings.get('units')}" + line = ( + f"{settings.get('label'):<5s}(x{scale}) {mixr_total.magnitude:.3f} " + f"{settings.get('units')}" + ) lines.append(line) label = f"{settings.get('label'):<7s}" if scale != 1.0: label = f"{settings.get('label'):<5s}(x{scale})" - handles.append(mlines.Line2D([], [], - color=settings.get('color'), - fillstyle='none', - label=label, - linewidth=1.0, - marker=settings.get('marker'), - markersize=8, - ) - ) + handles.append( + mlines.Line2D( + [], + [], + color=settings.get("color"), + fillstyle="none", + label=label, + linewidth=1.0, + marker=settings.get("marker"), + markersize=8, + ) + ) hydro_subplot.legend(handles=handles, loc=[0.05, 0.65]) - contents = '\n'.join(lines) + contents = "\n".join(lines) # Draw the vertically integrated amounts box - hydro_subplot.text(0.02, 0.95, contents, - bbox=dict(facecolor='white', edgecolor='black', alpha=0.7), - fontproperties=fm.FontProperties(family='monospace'), - size=8, - transform=hydro_subplot.transAxes, - verticalalignment='top', - ) + hydro_subplot.text( + 0.02, + 0.95, + contents, + bbox=dict(facecolor="white", edgecolor="black", alpha=0.7), + fontproperties=fm.FontProperties(family="monospace"), + size=8, + transform=hydro_subplot.transAxes, + verticalalignment="top", + ) def _add_thermo_inset(self, skew): - # Build up the text that goes in the thermo-dyniamics box lines = [] for name, items in self.thermo_variables.items(): - # Magic to get the desired number of decimals to appear. - decimals = items.get('decimals', 0) - value = items['data'] - if value != '--': - value = int(value) if decimals == 0 else value.round(decimals=decimals).values + decimals = items.get("decimals", 0) + value = items["data"] + if value != "--": + value = ( + int(value) + if decimals == 0 + else value.round(decimals=decimals).values + ) # Sure would have been nice to use a variable in the f string to # denote the format per variable. line = f"{name.upper():<7s} {str(value):>6} {items['units']}" lines.append(line) - contents = '\n'.join(lines) + contents = "\n".join(lines) # Draw the text box - skew.ax.text(0.75, 0.98, contents, - bbox=dict(facecolor='white', edgecolor='black', alpha=0.7), - fontproperties=fm.FontProperties(family='monospace'), - size=8, - transform=skew.ax.transAxes, - verticalalignment='top', - ) + skew.ax.text( + 0.75, + 0.98, + contents, + bbox=dict(facecolor="white", edgecolor="black", alpha=0.7), + fontproperties=fm.FontProperties(family="monospace"), + size=8, + transform=skew.ax.transAxes, + verticalalignment="top", + ) @property @lru_cache() def atmo_profiles(self): - - ''' + """ Return a dictionary of atmospheric data profiles for each variable needed by the skewT. Each of these variables must be have units set appropriately for use with MetPy SkewT. Handle those units and conversions here since it differs from the requirements of other graphics units/transforms. - ''' + """ # OrderedDict because we need to get pressure profile first. Entries in # the dict are as follows: @@ -240,54 +267,54 @@ def atmo_profiles(self): # transform: units string to pass to MetPy's to() function # units: the end unit of the field (after transform, # if applicable). - atmo_vars = OrderedDict({ - 'pres': { - 'transform': 'hectoPa', - 'units': units.Pa, + atmo_vars = OrderedDict( + { + "pres": { + "transform": "hectoPa", + "units": units.Pa, }, - 'gh': { - 'units': units.gpm, + "gh": { + "units": units.gpm, }, - 'sphum': { - 'units': units.dimensionless, + "sphum": { + "units": units.dimensionless, }, - 'temp': { - 'transform': 'degF', - 'units': units.degK, + "temp": { + "transform": "degF", + "units": units.degK, }, - 'u': { - 'transform': 'knots', - 'units': units.meter_per_second, + "u": { + "transform": "knots", + "units": units.meter_per_second, }, - 'v': { - 'transform': 'knots', - 'units': units.meter_per_second, + "v": { + "transform": "knots", + "units": units.meter_per_second, }, - }) + } + ) top = None for var, items in atmo_vars.items(): - # Get the profile values and attach MetPy units - tmp = np.asarray(self.values(name=var)) * items['units'] + tmp = np.asarray(self.values(name=var)) * items["units"] # Apply any needed transdecimals - transform = items.get('transform') + transform = items.get("transform") if transform: tmp = tmp.to(transform) # Only return values up to the maximum pressure level requested - if var == 'pres' and top is None: + if var == "pres" and top is None: top = np.sum(np.where(tmp.magnitude >= self.max_plev)) - 1 - atmo_vars[var]['data'] = tmp[:top] + atmo_vars[var]["data"] = tmp[:top] return atmo_vars def create_diagram(self): - - ''' Calls the private methods for creating each component of the SkewT - Diagram. ''' + """Calls the private methods for creating each component of the SkewT + Diagram.""" skew, hydro_subplot = self._setup_diagram() self._title() @@ -300,14 +327,11 @@ def create_diagram(self): self._add_hydrometeors(hydro_subplot) def create_csv(self, csv_path): - - ''' Calls the private methods for writing each of the SkewT Data. ''' + """Calls the private methods for writing each of the SkewT Data.""" self._write_profile(csv_path) def _plot_hodograph(self, skew): - - # Create an array that indicates which layer (10-3, 3-1, 0-1 km) the # wind belongs to. The array, agl, will be set to the height # corresponding to the top of the layer. The resulting array will look @@ -318,119 +342,122 @@ def _plot_hodograph(self, skew): # Where the values above 10 km are unchanged, and there are three levels # in each of the 3 layers of interest. # - agl = np.copy(self.atmo_profiles.get('gh', {}).get('data')).to('km') + agl = np.copy(self.atmo_profiles.get("gh", {}).get("data")).to("km") # Retrieve the wind data profiles - u_wind = self.atmo_profiles.get('u', {}).get('data') - v_wind = self.atmo_profiles.get('v', {}).get('data') + u_wind = self.atmo_profiles.get("u", {}).get("data") + v_wind = self.atmo_profiles.get("v", {}).get("data") # Create an inset axes object that is 28% width and height of the # figure and put it in the upper left hand corner. - ax = inset_axes(skew.ax, '25%', '25%', loc=2) - h = Hodograph(ax, component_range=80.) + ax = inset_axes(skew.ax, "25%", "25%", loc=2) + h = Hodograph(ax, component_range=80.0) h.add_grid(increment=20, linewidth=0.5) intervals = [0, 1, 3, 10] * agl.units - colors = ['xkcd:salmon', 'xkcd:aquamarine', 'xkcd:navy blue'] + colors = ["xkcd:salmon", "xkcd:aquamarine", "xkcd:navy blue"] line_width = 1.5 # Plot the line colored by height AGL only up to the 10km level - lines = h.plot_colormapped(u_wind, v_wind, agl, - colors=colors, - intervals=intervals, - linewidth=line_width, - ) + lines = h.plot_colormapped( + u_wind, + v_wind, + agl, + colors=colors, + intervals=intervals, + linewidth=line_width, + ) # Local function to create a proxy line object for creating a legend on # a LineCollection returned from plot_colormapped. Using lines and # colors from outside scope. def make_proxy(zval, idx=None, **kwargs): - color = colors[idx] if idx < len(colors) else lines.cmap(zval-1) + color = colors[idx] if idx < len(colors) else lines.cmap(zval - 1) return Line2D([0, 1], [0, 1], color=color, linewidth=line_width, **kwargs) # Make a list of proxies - proxies = [make_proxy(item, idx=i) for i, item in - enumerate(intervals.magnitude)] + proxies = [ + make_proxy(item, idx=i) for i, item in enumerate(intervals.magnitude) + ] # Draw the legend - ax.legend(proxies[:-1], - ['0-1 km', '1-3 km', '3-10 km', ''], - fontsize='small', - loc='lower left', - ) + ax.legend( + proxies[:-1], + ["0-1 km", "1-3 km", "3-10 km", ""], + fontsize="small", + loc="lower left", + ) @staticmethod def _plot_labels(skew): - - skew.ax.set_xlabel('Temperature (F)') - skew.ax.set_ylabel('Pressure (hPa)') + skew.ax.set_xlabel("Temperature (F)") + skew.ax.set_ylabel("Pressure (hPa)") def _write_profile(self, csv_path): - - profiles = self.atmo_profiles # dictionary - pres = profiles.get('pres').get('data') - u = profiles.get('u').get('data') - v = profiles.get('v').get('data') - temp = profiles.get('temp').get('data').to('degC') - sphum = profiles.get('sphum').get('data') - - dewpt = np.array(mpcalc.dewpoint_from_specific_humidity( - sphum, temp, pres).to('degC')) + profiles = self.atmo_profiles # dictionary + pres = profiles.get("pres").get("data") + u = profiles.get("u").get("data") + v = profiles.get("v").get("data") + temp = profiles.get("temp").get("data").to("degC") + sphum = profiles.get("sphum").get("data") + + dewpt = np.array( + mpcalc.dewpoint_from_specific_humidity(sphum, temp, pres).to("degC") + ) wspd = np.array(mpcalc.wind_speed(u, v)) wdir = np.array(mpcalc.wind_direction(u, v)) pres = np.array(pres) temp = np.array(temp) - profile = pd.DataFrame({ - 'LEVEL': pres, - 'TEMP': temp, - 'DWPT': dewpt, - 'WDIR': wdir, - 'WSPD': wspd, - }) + profile = pd.DataFrame( + { + "LEVEL": pres, + "TEMP": temp, + "DWPT": dewpt, + "WDIR": wdir, + "WSPD": wspd, + } + ) profile.to_csv(csv_path, index=False, float_format="%10.2f") def _plot_profile(self, skew): + profiles = self.atmo_profiles # dictionary + pres = profiles.get("pres").get("data") + temp = profiles.get("temp").get("data") + sphum = profiles.get("sphum").get("data") - profiles = self.atmo_profiles # dictionary - pres = profiles.get('pres').get('data') - temp = profiles.get('temp').get('data') - sphum = profiles.get('sphum').get('data') - - dewpt = mpcalc.dewpoint_from_specific_humidity(sphum, temp, pres).to('degF') + dewpt = mpcalc.dewpoint_from_specific_humidity(sphum, temp, pres).to("degF") # Pressure vs temperature - skew.plot(pres, temp, 'r', linewidth=1.5) + skew.plot(pres, temp, "r", linewidth=1.5) # Pressure vs dew point temperature - skew.plot(pres, dewpt, 'blue', linewidth=1.5) + skew.plot(pres, dewpt, "blue", linewidth=1.5) # Compute parcel profile and plot it - parcel_profile = mpcalc.parcel_profile(pres, - temp[0], - dewpt[0]).to('degC') - skew.plot(pres, - parcel_profile, - 'orange', - linestyle='dashed', - linewidth=1.2, - ) + parcel_profile = mpcalc.parcel_profile(pres, temp[0], dewpt[0]).to("degC") + skew.plot( + pres, + parcel_profile, + "orange", + linestyle="dashed", + linewidth=1.2, + ) def _plot_wind_barbs(self, skew): - # Pressure vs wind - skew.plot_barbs(self.atmo_profiles.get('pres', {}).get('data'), - self.atmo_profiles.get('u', {}).get('data'), - self.atmo_profiles.get('v', {}).get('data'), - color='blue', - linewidth=0.2, - y_clip_radius=0, - ) + skew.plot_barbs( + self.atmo_profiles.get("pres", {}).get("data"), + self.atmo_profiles.get("u", {}).get("data"), + self.atmo_profiles.get("v", {}).get("data"), + color="blue", + linewidth=0.2, + y_clip_radius=0, + ) def _setup_diagram(self): - # Create a new figure. The dimensions here give a good aspect ratio. fig = plt.figure(figsize=(12, 12)) gs = plt.GridSpec(4, 5) @@ -449,76 +476,83 @@ def _setup_diagram(self): # Celcius VALUES for those tick marks. These put the ticks in the right # spot. - labels = labels_F.to('degC').magnitude + labels = labels_F.to("degC").magnitude # Set the MINOR tick values to the CELCIUS values. skew.ax.xaxis.set_minor_locator(FixedLocator(labels)) # Set the MINOR tick labels to the FAHRENHEIT values. skew.ax.set_xticklabels(labels_F.magnitude, minor=True) - skew.ax.tick_params(which='minor', - length=8) + skew.ax.tick_params(which="minor", length=8) # Turn off the MAJOR (celcius) tick marks, label the grid lines inside # the axes. - skew.ax.tick_params(axis='x', - labelbottom=True, - labelcolor='gray', - labelright=True, - labelrotation=45, - labeltop=True, - length=0, - pad=-25, - which='major', - ) + skew.ax.tick_params( + axis="x", + labelbottom=True, + labelcolor="gray", + labelright=True, + labelrotation=45, + labeltop=True, + length=0, + pad=-25, + which="major", + ) # Add the relevant special lines with their labels dry_adiabats = np.arange(-40, 210, 10) * units.degC - skew.plot_dry_adiabats(dry_adiabats, - colors='tan', - linestyles='solid', - linewidth=0.7, - ) - utils.label_lines(ax=skew.ax, - lines=skew.dry_adiabats, - labels=dry_adiabats.magnitude, - end='top', - offset=1, - ) + skew.plot_dry_adiabats( + dry_adiabats, + colors="tan", + linestyles="solid", + linewidth=0.7, + ) + utils.label_lines( + ax=skew.ax, + lines=skew.dry_adiabats, + labels=dry_adiabats.magnitude, + end="top", + offset=1, + ) moist_adiabats = np.arange(8, 36, 4) * units.degC moist_pr = np.arange(1001, 220, -10) * units.hPa - skew.plot_moist_adiabats(moist_adiabats, - moist_pr, - colors='green', - linestyles='solid', - linewidth=0.7, - ) - utils.label_lines(ax=skew.ax, - lines=skew.moist_adiabats, - labels=moist_adiabats.magnitude, - end='top', - ) - - mixing_lines = np.array([1, 2, 3, 5, 8, 12, 16, 20]).reshape(-1, 1) / 1000 + skew.plot_moist_adiabats( + moist_adiabats, + moist_pr, + colors="green", + linestyles="solid", + linewidth=0.7, + ) + utils.label_lines( + ax=skew.ax, + lines=skew.moist_adiabats, + labels=moist_adiabats.magnitude, + end="top", + ) + + mixing_lines = np.array([1, 2, 3, 5, 8, 12, 16, 20]).reshape(-1, 1) / 1000 mix_pr = np.arange(1001, 400, -50) * units.hPa - skew.plot_mixing_lines(w=mixing_lines, p=mix_pr, - colors='green', - linestyles=(0, (5, 10)), - linewidth=0.7, - ) - utils.label_lines(ax=skew.ax, - lines=skew.mixing_lines, - labels=mixing_lines * 1000, - ) + skew.plot_mixing_lines( + w=mixing_lines, + p=mix_pr, + colors="green", + linestyles=(0, (5, 10)), + linewidth=0.7, + ) + utils.label_lines( + ax=skew.ax, + lines=skew.mixing_lines, + labels=mixing_lines * 1000, + ) hydro_subplot = fig.add_subplot(gs[:, -1], sharey=skew.ax) hydro_subplot.set_xlim(0.0001, 10.0) hydro_subplot.set_xscale("log") hydro_subplot.yaxis.tick_right() - hydro_subplot.set_aspect(23) # completely arbitrary + hydro_subplot.set_aspect(23) # completely arbitrary - plt.grid(which='major', axis='both') + plt.grid(which="major", axis="both") plt.xlabel("hydrometeors") plt.ylabel("") @@ -527,8 +561,7 @@ def _setup_diagram(self): @property @lru_cache() def thermo_variables(self): - - ''' + """ Return an ordered dictionary of thermodynamic variables needed for the skewT. Ordered because we want to print these values in this order on the SkewT diagram. @@ -538,7 +571,7 @@ def thermo_variables(self): Variables' transforms and units are handled by default specs in much the same way as in fieldData class since these are not used by MetPy explictly. - ''' + """ # OrderedDict so that we get the thermodynamic variables printed in the # same order every time in the resulting SkewT inset. The fields @@ -554,62 +587,63 @@ def thermo_variables(self): # decimals: (optional) number of decimal places to # include when formatting output. Defaults # to 0 (integer). - thermo = OrderedDict({ - 'cape': { # Convective available potential energy - 'level': 'sfc', + thermo = OrderedDict( + { + "cape": { # Convective available potential energy + "level": "sfc", }, - 'cin': { # Convective inhibition - 'level': 'sfc', + "cin": { # Convective inhibition + "level": "sfc", }, - 'mucape': { # Most Unstable CAPE - 'level': 'mu', - 'variable': 'cape', + "mucape": { # Most Unstable CAPE + "level": "mu", + "variable": "cape", }, - 'mucin': { # CIN from MUCAPE level - 'level': 'mu', - 'variable': 'cin', + "mucin": { # CIN from MUCAPE level + "level": "mu", + "variable": "cin", }, - 'li': { # Lifted Index - 'decimals': 1, - 'level': 'sfc', + "li": { # Lifted Index + "decimals": 1, + "level": "sfc", }, - 'bli': { # Best Lifted Index - 'decimals': 1, - 'level': 'best', - 'variable': 'li', + "bli": { # Best Lifted Index + "decimals": 1, + "level": "best", + "variable": "li", }, - 'lcl': { # Lifted Condensation Level + "lcl": { # Lifted Condensation Level }, - 'lpl': { # Lifted Parcel Level + "lpl": { # Lifted Parcel Level }, - 'srh03': { # 0-3 km Storm relative helicity - 'level': 'sr03', - 'variable': 'hlcy', + "srh03": { # 0-3 km Storm relative helicity + "level": "sr03", + "variable": "hlcy", }, - 'srh01': { # 0-1 km Storm relative helicity - 'level': 'sr01', - 'variable': 'hlcy', + "srh01": { # 0-1 km Storm relative helicity + "level": "sr01", + "variable": "hlcy", }, - 'shr06': { # 0-6 km Shear - 'level': '06km', - 'variable': 'shear', + "shr06": { # 0-6 km Shear + "level": "06km", + "variable": "shear", }, - 'shr01': { # 0-1 km Shear - 'level': '01km', - 'variable': 'shear', + "shr01": { # 0-1 km Shear + "level": "01km", + "variable": "shear", }, - 'cell': { # Cell motion + "cell": { # Cell motion }, - 'pwtr': { # Precipitable water - 'decimals': 1, - 'level': 'sfc', + "pwtr": { # Precipitable water + "decimals": 1, + "level": "sfc", }, - }) + } + ) for var, items in thermo.items(): - - varname = items.get('variable', var) - lev = items.get('level', 'ua') + varname = items.get("variable", var) + lev = items.get("level", "ua") spec = self.spec.get(varname, {}).get(lev) if not spec: @@ -618,47 +652,47 @@ def thermo_variables(self): try: tmp = self.values(level=lev, name=varname, one_lev=True) - - transforms = spec.get('transform') + transforms = spec.get("transform") if transforms: tmp = self.get_transform(transforms, tmp) except errors.GribReadError: + tmp = "--" - tmp = '--' - - thermo[var]['data'] = tmp - thermo[var]['units'] = spec.get('unit') + thermo[var]["data"] = tmp + thermo[var]["units"] = spec.get("unit") return thermo def _title(self): - - ''' Creates standard annotation for a skew-T. ''' + """Creates standard annotation for a skew-T.""" atime = self.date_to_str(self.anl_dt) vtime = self.date_to_str(self.valid_dt) # Top Left - plt.title(f"{self.model_name}: {atime}\nFcst Hr: {self.fhr}", - fontsize=16, - loc='left', - position=(-4.8, 1.03), - ) + plt.title( + f"{self.model_name}: {atime}\nFcst Hr: {self.fhr}", + fontsize=16, + loc="left", + position=(-4.8, 1.03), + ) # Top Right - plt.title(f"Valid: {vtime}", - fontsize=16, - loc='right', - position=(-0.20, 1.03), - ) + plt.title( + f"Valid: {vtime}", + fontsize=16, + loc="right", + position=(-0.20, 1.03), + ) # Center site = f"{self.site_code} {self.site_num} {self.site_name}" site_loc = f"{self.site_lat}, {self.site_lon}" site_title = f"{site} at nearest grid pt over land {site_loc}" - plt.title(site_title, - fontsize=12, - loc='center', - position=(-2.5, 1.0), - ) + plt.title( + site_title, + fontsize=12, + loc="center", + position=(-2.5, 1.0), + ) diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index e3c687b..fc110d4 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -1,515 +1,494 @@ # pylint: disable=too-many-public-methods -''' +""" This module sets the specifications for certain atmospheric variables. Typically this is related to a spec that needs some level of computation, i.e. a set of colors from a color map. -''' +""" import abc from itertools import chain -from matplotlib import cm -from matplotlib import colors as mpcolors + import numpy as np import yaml +from matplotlib import cm +from matplotlib import colors as mpcolors from metpy.plots import ctables -class VarSpec(abc.ABC): - ''' +class VarSpec(abc.ABC): + """ Loads a yaml config file with spec settings. Also defines methods for declaring more complex specifications for variables based on settings within the config file. - ''' + """ @property def aod_colors(self) -> np.ndarray: + """Default color map for AOD products and chem products""" - ''' Default color map for AOD products and chem products ''' - - grays = cm.get_cmap('Greys', 2)([0]) - others = cm.get_cmap(self.vspec.get('cmap'), 15)(range(1, 15, 1)) + grays = cm.get_cmap("Greys", 2)([0]) + others = cm.get_cmap(self.vspec.get("cmap"), 15)(range(1, 15, 1)) return np.concatenate((grays, others)) def centered_diff(self, cmap=None, nlev=None): - - ''' Returns the colors specified by levels and cmap in default spec, but - with white center. ''' + """Returns the colors specified by levels and cmap in default spec, but + with white center.""" if nlev is None: - clevs = self.vspec.get('clevs') + clevs = self.vspec.get("clevs") nlev = len(clevs) + 1 if cmap is None: - cmap = self.vspec.get('cmap') + cmap = self.vspec.get("cmap") colors = cm.get_cmap(cmap, nlev)(range(nlev)) mid = nlev // 2 colors[mid] = [1, 1, 1, 1] - colors[mid-1] = [1, 1, 1, 1] + colors[mid - 1] = [1, 1, 1, 1] return colors @property def cin_colors(self) -> np.ndarray: + """Default color map for Convective Inhibition""" - ''' Default color map for Convective Inhibition ''' - - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([120, 100, 90, 85, 80, 70, 60, 50, 25, 20, 18]) - grays = cm.get_cmap('Greys', 2)([0]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [120, 100, 90, 85, 80, 70, 60, 50, 25, 20, 18] + ) + grays = cm.get_cmap("Greys", 2)([0]) return np.concatenate((ncar, grays)) @property @abc.abstractmethod def clevs(self) -> np.ndarray: - - ''' An abstract method responsible for returning the np.ndarray of contour - levels for a given field. Numpy arange supports non-integer values. ''' + """An abstract method responsible for returning the np.ndarray of contour + levels for a given field. Numpy arange supports non-integer values.""" @property @abc.abstractproperty def vspec(self): - - ''' The variable plotting specification. The level-specific subgroup - from a config file like default_specs.yml. ''' + """The variable plotting specification. The level-specific subgroup + from a config file like default_specs.yml.""" @property def ceil_colors(self) -> np.ndarray: + """Default color map for Ceiling""" - ''' Default color map for Ceiling ''' - - grays = cm.get_cmap('Greys', 2)([0]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([15, 18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120]) + grays = cm.get_cmap("Greys", 2)([0]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [15, 18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120] + ) return np.concatenate((grays, ncar, grays)) @property def cldcov_colors(self) -> np.ndarray: + """Default color map for Cloud Cover""" - ''' Default color map for Cloud Cover ''' - - grays = cm.get_cmap('Greys', 7)([0, 1, 3]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([120, 100, 90, 85, 80, 70, 60, 50, 25, 20]) + grays = cm.get_cmap("Greys", 7)([0, 1, 3]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [120, 100, 90, 85, 80, 70, 60, 50, 25, 20] + ) return np.concatenate((grays, ncar)) @property def cref_colors(self) -> np.ndarray: + """Default color map for Reflectivity""" - ''' Default color map for Reflectivity ''' - - ncolors = len(self.clevs)-1 - grays = cm.get_cmap('Greys', 5)([0]) - nws = ctables.colortables.get_colortable(self.vspec.get('cmap'))(range(ncolors)) - white = cm.get_cmap('Greys', 5)([0]) + ncolors = len(self.clevs) - 1 + grays = cm.get_cmap("Greys", 5)([0]) + nws = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(ncolors)) + white = cm.get_cmap("Greys", 5)([0]) return np.concatenate((grays, nws, white)) @property def fire_power_colors(self) -> np.ndarray: - - ''' Default color map for fire power plot. ''' + """Default color map for fire power plot.""" # The scatter plot utility won't accept anything but named colors - colors = ['white', 'lightskyblue', 'darkblue', 'green', 'darkorange', \ - 'indianred', 'firebrick'] + colors = [ + "white", + "lightskyblue", + "darkblue", + "green", + "darkorange", + "indianred", + "firebrick", + ] return colors @property def smoke_emissions_colors(self) -> np.ndarray: - - ''' Default color map for smoke emissions plot. ''' + """Default color map for smoke emissions plot.""" # The scatter plot utility won't accept anything but named colors - colors = ['white', 'rebeccapurple', 'royalblue', 'cadetblue', \ - 'yellowgreen', 'mediumaquamarine', 'lightgreen', 'yellow', \ - 'gold', 'orange', 'darkorange', 'orangered', 'red', \ - 'firebrick'] + colors = [ + "white", + "rebeccapurple", + "royalblue", + "cadetblue", + "yellowgreen", + "mediumaquamarine", + "lightgreen", + "yellow", + "gold", + "orange", + "darkorange", + "orangered", + "red", + "firebrick", + ] return colors def flru_colors(self) -> np.ndarray: + """Default color map for Ceiling""" - ''' Default color map for Ceiling ''' - - ctable = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([50, 15, 90, 120]) + ctable = cm.get_cmap(self.vspec.get("cmap"), 128)([50, 15, 90, 120]) return ctable @property def frzn_colors(self) -> np.ndarray: + """Default color map for Frozen Precip %""" - ''' Default color map for Frozen Precip % ''' - - grays = cm.get_cmap('Greys', 7)([0, 2]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([120, 90, 85, 80, 70, 60, 50, 25, 20, 15]) + grays = cm.get_cmap("Greys", 7)([0, 2]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [120, 90, 85, 80, 70, 60, 50, 25, 20, 15] + ) return np.concatenate((grays, ncar)) @property def goes_colors(self) -> np.ndarray: + """Default color map for simulated GOES IR satellite""" - ''' Default color map for simulated GOES IR satellite ''' - - grays = cm.get_cmap('Greys_r', 33)(range(33)) - ctable2 = ctables.colortables.get_colortable(self.vspec.get('cmap')) \ - (range(65, 150)) + grays = cm.get_cmap("Greys_r", 33)(range(33)) + ctable2 = ctables.colortables.get_colortable(self.vspec.get("cmap"))( + range(65, 150) + ) return np.concatenate((grays[-1:], grays, ctable2, grays[1:])) @property def graupel_colors(self) -> np.ndarray: + """Default color map for Max Vertically Integrated Graupel""" - ''' Default color map for Max Vertically Integrated Graupel ''' - - grays = cm.get_cmap('Greys', 3)([0]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - (range(20, 128, 6)) + grays = cm.get_cmap("Greys", 3)([0]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)(range(20, 128, 6)) return np.concatenate((grays, ncar)) @property def hail_colors(self) -> np.ndarray: + """Default color map for Hail diameter""" - ''' Default color map for Hail diameter ''' - - grays = cm.get_cmap('Greys', 2)([0]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([100, 15, 18, 20, 25, 60, 80, 85, 90]) + grays = cm.get_cmap("Greys", 2)([0]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [100, 15, 18, 20, 25, 60, 80, 85, 90] + ) return np.concatenate((grays, ncar)) @property def heat_flux_colors(self) -> np.ndarray: + """Default color map for Latent/Sensible Heat Flux""" - ''' Default color map for Latent/Sensible Heat Flux ''' - - grays = cm.get_cmap('Greys', 8)([6, 5, 4, 3, 2]) - ctable = ctables.colortables.get_colortable(self.vspec.get('cmap')) \ - (range(0, 33, 2)) + grays = cm.get_cmap("Greys", 8)([6, 5, 4, 3, 2]) + ctable = ctables.colortables.get_colortable(self.vspec.get("cmap"))( + range(0, 33, 2) + ) return np.concatenate((grays, ctable)) @property def heat_flux_colors_g(self) -> np.ndarray: + """Default color map for Latent/Sensible Heat Flux""" - ''' Default color map for Latent/Sensible Heat Flux ''' - - colors = cm.get_cmap(self.vspec.get('cmap'), 128) \ - (range(15, 112, 8)) + colors = cm.get_cmap(self.vspec.get("cmap"), 128)(range(15, 112, 8)) return colors @property def heat_flux_colors_l(self) -> np.ndarray: + """Default color map for Latent/Sensible Heat Flux""" - ''' Default color map for Latent/Sensible Heat Flux ''' - - colors = cm.get_cmap(self.vspec.get('cmap'), 128) \ - (range(32, 129, 6)) + colors = cm.get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) return colors @property def heat_flux_colors_s(self) -> np.ndarray: + """Default color map for Latent/Sensible Heat Flux""" - ''' Default color map for Latent/Sensible Heat Flux ''' - - colors = cm.get_cmap(self.vspec.get('cmap'), 128) \ - (range(32, 129, 6)) + colors = cm.get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) return colors @property def icprb_colors(self) -> np.ndarray: + """Default color map for Icing Probability""" - ''' Default color map for Icing Probability ''' - - grays = cm.get_cmap('Greys', 2)([0]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([25, 35, 50, 60, 70, 80, 85, 90, 100]) + grays = cm.get_cmap("Greys", 2)([0]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [25, 35, 50, 60, 70, 80, 85, 90, 100] + ) return np.concatenate((grays, ncar)) def icsev_colors(self) -> np.ndarray: + """Default color map for Icing Severity""" - ''' Default color map for Icing Severity ''' - - white = cm.get_cmap('Greys', 2)([0]) - blues = cm.get_cmap(self.vspec.get('cmap'), 9) \ - ([2, 3, 4, 6, 8]) + white = cm.get_cmap("Greys", 2)([0]) + blues = cm.get_cmap(self.vspec.get("cmap"), 9)([2, 3, 4, 6, 8]) return np.concatenate((white, blues)) @property def lcl_colors(self) -> np.ndarray: + """Default color map for Lifted Condensation Level""" - ''' Default color map for Lifted Condensation Level ''' - - ctable = ctables.colortables.get_colortable(self.vspec.get('cmap')) \ - (range(50, 180, 7)) # rainbow + ctable = ctables.colortables.get_colortable(self.vspec.get("cmap"))( + range(50, 180, 7) + ) # rainbow return ctable @property def lifted_index_colors(self) -> np.ndarray: + """Default color map for Lifted Index""" - ''' Default color map for Lifted Index ''' - - ctable = cm.get_cmap(self.vspec.get('cmap'), 128) \ - (range(4, 125, 4)) + ctable = cm.get_cmap(self.vspec.get("cmap"), 128)(range(4, 125, 4)) ctable[14] = [1, 1, 1, 1] ctable[15] = [1, 1, 1, 1] return ctable @property def mdn_colors(self) -> np.ndarray: + """Default color map for Max Downdraft""" - ''' Default color map for Max Downdraft ''' - - grays = cm.get_cmap('Greys', 2)([0]) - others = cm.get_cmap(self.vspec.get('cmap'), 18)(range(18, 1, -1)) + grays = cm.get_cmap("Greys", 2)([0]) + others = cm.get_cmap(self.vspec.get("cmap"), 18)(range(18, 1, -1)) return np.concatenate((others, grays)) @property def mean_vvel_colors(self) -> np.ndarray: + """Default color map for Mean Vertical Velocity""" - ''' Default color map for Mean Vertical Velocity ''' - - ctable = cm.get_cmap(self.vspec.get('cmap'), 128)(range(0, 114, 6)) + ctable = cm.get_cmap(self.vspec.get("cmap"), 128)(range(0, 114, 6)) ctable[9] = [1, 1, 1, 1] return ctable @property def mup_colors(self) -> np.ndarray: + """Default color map for Max Updraft""" - ''' Default color map for Max Updraft ''' - - grays = cm.get_cmap('Greys', 2)([0]) - others = cm.get_cmap(self.vspec.get('cmap'), 18)(range(1, 18, 1)) + grays = cm.get_cmap("Greys", 2)([0]) + others = cm.get_cmap(self.vspec.get("cmap"), 18)(range(1, 18, 1)) return np.concatenate((grays, others)) @property def pbl_colors(self) -> np.ndarray: + """Default color map for PBL Height""" - ''' Default color map for PBL Height ''' - - return ctables.colortables.get_colortable(self.vspec.get('cmap')) \ - (range(15, 60, 3)) + return ctables.colortables.get_colortable(self.vspec.get("cmap"))( + range(15, 60, 3) + ) @property def pcp_colors(self) -> np.ndarray: + """Default color map for Hourly Precipitation""" - ''' Default color map for Hourly Precipitation ''' - - grays = cm.get_cmap('Greys', 6)([0, 3]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([25, 50, 60, 70, 80, 85, 90, 115]) + grays = cm.get_cmap("Greys", 6)([0, 3]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [25, 50, 60, 70, 80, 85, 90, 115] + ) return np.concatenate((grays, ncar)) @property def pcp_colors_high(self) -> np.ndarray: + """High values color map for Hourly Precipitation""" - ''' High values color map for Hourly Precipitation ''' - - grays = cm.get_cmap('Greys', 2)([0]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([70, 80, 85, 90, 115]) + grays = cm.get_cmap("Greys", 2)([0]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([70, 80, 85, 90, 115]) return np.concatenate((grays, ncar)) @property def pmsl_colors(self) -> np.ndarray: + """Default color map for Surface Pressure""" - ''' Default color map for Surface Pressure ''' - - ncolors = len(self.vspec.get('clevs')) + ncolors = len(self.vspec.get("clevs")) incr = 128 // ncolors - colors = cm.get_cmap(self.vspec.get('cmap'), 128)(range(incr, 128, incr)) + colors = cm.get_cmap(self.vspec.get("cmap"), 128)(range(incr, 128, incr)) return np.asarray(colors) @property def ps_colors(self) -> np.ndarray: + """Default color map for Surface Pressure""" - ''' Default color map for Surface Pressure ''' - - grays = cm.get_cmap('Greys', 13)(range(13)) + grays = cm.get_cmap("Greys", 13)(range(13)) segments = [[16, 53], [86, 105], [110, 151, 2], [172, 202, 2]] - ncar = cm.get_cmap('gist_ncar', 200)(list(chain(*[range(*i) for i in segments]))) + ncar = cm.get_cmap("gist_ncar", 200)( + list(chain(*[range(*i) for i in segments])) + ) return np.concatenate((grays, ncar)) @property def pw_colors(self) -> np.ndarray: - - ''' Default color map for Precipitable Water ''' - - grays = cm.get_cmap('Greys', 5)([1, 3]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([120, 100, 95, 85, 80, 70, 65, 50, 25, 22, 20, 17]) - bupu = cm.get_cmap('BuPu', 15)([13, 14]) - cool = cm.get_cmap('cool', 15)([10, 9, 12, 7, 5]) + """Default color map for Precipitable Water""" + + grays = cm.get_cmap("Greys", 5)([1, 3]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [120, 100, 95, 85, 80, 70, 65, 50, 25, 22, 20, 17] + ) + bupu = cm.get_cmap("BuPu", 15)([13, 14]) + cool = cm.get_cmap("cool", 15)([10, 9, 12, 7, 5]) return np.concatenate((grays, ncar, bupu, cool)) @property def radiation_colors(self) -> np.ndarray: + """Default color map for Longwave Radiation""" - ''' Default color map for Longwave Radiation ''' - - grays = cm.get_cmap('Greys', 2)([0]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - (range(0, 126, 5)) + grays = cm.get_cmap("Greys", 2)([0]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)(range(0, 126, 5)) return np.concatenate((grays, ncar)) @property def radiation_bw_colors(self) -> np.ndarray: + """Default grayscale map for Outgoing Shortwave Radiation""" - ''' Default grayscale map for Outgoing Shortwave Radiation ''' - - return cm.get_cmap(self.vspec.get('cmap'), 128) \ - (range(30, 110)) + return cm.get_cmap(self.vspec.get("cmap"), 128)(range(30, 110)) @property def radiation_mix_colors(self) -> np.ndarray: + """Default color map for Longwave Radiation""" - ''' Default color map for Longwave Radiation ''' - - ncar = ctables.colortables.get_colortable(self.vspec.get('cmap')) \ - (range(0, 40)) - grays = cm.get_cmap('Greys', 100)(range(10, 100)) + ncar = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(0, 40)) + grays = cm.get_cmap("Greys", 100)(range(10, 100)) return np.concatenate((ncar, grays)) @property def rainbow11_colors(self) -> np.ndarray: + """Default color map for Hourly Wildfire Potential""" - ''' Default color map for Hourly Wildfire Potential ''' - - grays = cm.get_cmap('Greys', 2)([0]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120]) + grays = cm.get_cmap("Greys", 2)([0]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120] + ) return np.concatenate((grays, ncar)) @property def rainbow12_colors(self) -> np.ndarray: + """Default color map for ACPCP, ACSNOD, HLCY, RH, and SNOD""" - ''' Default color map for ACPCP, ACSNOD, HLCY, RH, and SNOD ''' - - grays = cm.get_cmap('Greys', 2)([0]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([15, 18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120]) + grays = cm.get_cmap("Greys", 2)([0]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [15, 18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120] + ) return np.concatenate((grays, ncar)) @property def rainbow12_reverse(self) -> np.ndarray: - - ''' Default color map for min helicity ''' + """Default color map for min helicity""" return np.flip(self.rainbow12_colors, 0) @property def rainbow16_colors(self) -> np.ndarray: + """Default color map for helicity""" - ''' Default color map for helicity ''' - - grays = cm.get_cmap('Greys', 5)([0, 2]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([9, 15, 18, 20, 25, 48, 57, 65, 74, 79, 87, 94, 102, 109, 120]) + grays = cm.get_cmap("Greys", 5)([0, 2]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [9, 15, 18, 20, 25, 48, 57, 65, 74, 79, 87, 94, 102, 109, 120] + ) return np.concatenate((grays, ncar)) @property def shear_colors(self) -> np.ndarray: + """Default color map for Vertical Shear""" - ''' Default color map for Vertical Shear ''' - - ctable = cm.get_cmap(self.vspec.get('cmap'), 16) \ - (range(5, 15)) + ctable = cm.get_cmap(self.vspec.get("cmap"), 16)(range(5, 15)) ctable[9] = [1, 1, 1, 1] return ctable @property def slw_colors(self) -> np.ndarray: + """Default color map for Max Vertically Integrated Graupel""" - ''' Default color map for Max Vertically Integrated Graupel ''' - - white = cm.get_cmap('Greys', 3)([0]) - purples = cm.get_cmap('nipy_spectral', 30)([3, 1]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 15) \ - ([2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]) + white = cm.get_cmap("Greys", 3)([0]) + purples = cm.get_cmap("nipy_spectral", 30)([3, 1]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 15)( + [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + ) return np.concatenate((white, purples, ncar)) @property def smoke_colors(self) -> np.ndarray: + """Default color map for smoke plots.""" - ''' Default color map for smoke plots. ''' - - white = cm.get_cmap('Greys', 2)([0]) - blues = cm.get_cmap('Blues', 6)(range(1, 5)) - green_yellow_red = cm.get_cmap('RdYlGn_r', 18)([1, 3, 5, 9, 12, 13, 14, 16, 18]) - purple = np.array([mpcolors.to_rgba('xkcd:vivid purple')]) + white = cm.get_cmap("Greys", 2)([0]) + blues = cm.get_cmap("Blues", 6)(range(1, 5)) + green_yellow_red = cm.get_cmap("RdYlGn_r", 18)([1, 3, 5, 9, 12, 13, 14, 16, 18]) + purple = np.array([mpcolors.to_rgba("xkcd:vivid purple")]) return np.concatenate((white, blues, green_yellow_red, purple)) - @property def snow_colors(self) -> np.ndarray: + """Default color map for Snow fields""" - ''' Default color map for Snow fields ''' - - grays = cm.get_cmap('Greys', 5)([0, 2]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([15, 18, 20, 25, 50, 60, 74, 81, 85, 90, 100]) + grays = cm.get_cmap("Greys", 5)([0, 2]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [15, 18, 20, 25, 50, 60, 74, 81, 85, 90, 100] + ) return np.concatenate((grays, ncar)) @property def soilm_colors(self) -> np.ndarray: + """Default color map for Soil Moisture Availability""" - ''' Default color map for Soil Moisture Availability ''' - - ncar = cm.get_cmap(self.vspec.get('cmap'), 128)(range(0, 122, 11)) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)(range(0, 122, 11)) return ncar @property def soilw_colors(self) -> np.ndarray: + """Default color map for Soil Moisture""" - ''' Default color map for Soil Moisture ''' - - grays = cm.get_cmap('Greys', 2)([1]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 110) \ - ([0, 10, 20, 25, 35, 40, 60, 73, 80, 85, 95, 105]) + grays = cm.get_cmap("Greys", 2)([1]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 110)( + [0, 10, 20, 25, 35, 40, 60, 73, 80, 85, 95, 105] + ) return np.concatenate((grays, ncar)) @property def t_colors(self) -> np.ndarray: - - ''' Default color map for Potential Temperature ''' + """Default color map for Potential Temperature""" ncolors = len(self.clevs) - return cm.get_cmap(self.vspec.get('cmap', 'jet'), ncolors)(range(ncolors)) + return cm.get_cmap(self.vspec.get("cmap", "jet"), ncolors)(range(ncolors)) @property def tsfc_colors(self) -> np.ndarray: + """Default color map for Surface Temperature""" - ''' Default color map for Surface Temperature ''' - - purples = cm.get_cmap('Purples', 16)([14, 12, 8, 6, 4, 2]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([15, 20, 25, 33, 50, 60, 70, 80, 85, 90, 115]) - grays = cm.get_cmap('Greys', 15)([2, 4, 6, 8]) + purples = cm.get_cmap("Purples", 16)([14, 12, 8, 6, 4, 2]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [15, 20, 25, 33, 50, 60, 70, 80, 85, 90, 115] + ) + grays = cm.get_cmap("Greys", 15)([2, 4, 6, 8]) return np.concatenate((purples, ncar, grays)) @property def terrain_colors(self) -> np.ndarray: + """Default color map for Terrain""" - ''' Default color map for Terrain ''' - - ctable = ctables.colortables.get_colortable(self.vspec.get('cmap')) \ - (range(54, 157, 6)) + ctable = ctables.colortables.get_colortable(self.vspec.get("cmap"))( + range(54, 157, 6) + ) return ctable @property def ua_temp_colors(self) -> np.ndarray: + """Default color map for Upper-Air Temperature""" - ''' Default color map for Upper-Air Temperature ''' - - grays = cm.get_cmap('Greys', 27)(range(17, 1, -2)) - purples = cm.get_cmap('Purples', 27)(range(17, 1, -2)) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([30, 34, 36, 40, 45, 55, 60, 65, 70, \ - 75, 80, 85, 90, 95, 100, 115]) + grays = cm.get_cmap("Greys", 27)(range(17, 1, -2)) + purples = cm.get_cmap("Purples", 27)(range(17, 1, -2)) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [30, 34, 36, 40, 45, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 115] + ) return np.concatenate((grays, purples, ncar)) @property def vis_colors(self) -> np.ndarray: - - ''' Default color map for Visibility + """Default color map for Visibility section names are based on Aviation Flight Rule visibility categories LIFR (Low Instrument Flight Rules) -- less than 1 mile @@ -518,55 +497,52 @@ def vis_colors(self) -> np.ndarray: VFR (Visual Flight Rules) -- greater than 5 miles the gray range is arbitrary compared to the official flight levels - ''' + """ - lifr = cm.get_cmap('RdPu_r', 20)(range(0, 11)) - ifr = cm.get_cmap('autumn', 30)(range(0, 30)) - mvfr = cm.get_cmap('Blues', 20)(range(10, 20)) - vfr1 = cm.get_cmap('YlGn_r', 60)(range(0, 50)) - vfr2 = cm.get_cmap('Greys', 25)(np.full(10, 9)) - hi01 = cm.get_cmap('Greys', 25)(np.full(10, 6)) - hi02 = cm.get_cmap('Greys', 25)(np.full(20, 3)) - hi03 = cm.get_cmap('Greys', 25)(np.full(1, 0)) + lifr = cm.get_cmap("RdPu_r", 20)(range(0, 11)) + ifr = cm.get_cmap("autumn", 30)(range(0, 30)) + mvfr = cm.get_cmap("Blues", 20)(range(10, 20)) + vfr1 = cm.get_cmap("YlGn_r", 60)(range(0, 50)) + vfr2 = cm.get_cmap("Greys", 25)(np.full(10, 9)) + hi01 = cm.get_cmap("Greys", 25)(np.full(10, 6)) + hi02 = cm.get_cmap("Greys", 25)(np.full(20, 3)) + hi03 = cm.get_cmap("Greys", 25)(np.full(1, 0)) return np.concatenate((lifr, ifr, mvfr, vfr1, vfr2, hi01, hi02, hi03)) @property def vvel_colors(self) -> np.ndarray: + """Default color map for Vetical Velocity""" - ''' Default color map for Vetical Velocity ''' - - ncar1 = cm.get_cmap(self.vspec.get('cmap'), 128)([15, 18, 20, 25]) - grays = cm.get_cmap('Greys', 2)([0]) - ncar2 = cm.get_cmap(self.vspec.get('cmap'), 128)([60, 70, 80, 85, 90, 100, 120]) + ncar1 = cm.get_cmap(self.vspec.get("cmap"), 128)([15, 18, 20, 25]) + grays = cm.get_cmap("Greys", 2)([0]) + ncar2 = cm.get_cmap(self.vspec.get("cmap"), 128)([60, 70, 80, 85, 90, 100, 120]) return np.concatenate((ncar1, grays, ncar2)) @property def vort_colors(self) -> np.ndarray: + """Default color map for Absolute Vorticity""" - ''' Default color map for Absolute Vorticity ''' - - grays = cm.get_cmap('Greys', 2)([0]) - ncar = cm.get_cmap(self.vspec.get('cmap'), 128) \ - ([15, 18, 20, 25, 50, 60, 70, 80, 83, 90, 100, 120]) + grays = cm.get_cmap("Greys", 2)([0]) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + [15, 18, 20, 25, 50, 60, 70, 80, 83, 90, 100, 120] + ) return np.concatenate((grays, ncar)) @property def wind_colors(self) -> np.ndarray: + """Default color map for Wind Speed""" - ''' Default color map for Wind Speed ''' - - low = cm.get_cmap(self.vspec.get('cmap'), 129)(range(129, 109, -5)) - high1 = cm.get_cmap(self.vspec.get('cmap'), 129)(range(16, 29, 3)) - high2 = cm.get_cmap(self.vspec.get('cmap'), 129)(range(48, 103, 6)) + low = cm.get_cmap(self.vspec.get("cmap"), 129)(range(129, 109, -5)) + high1 = cm.get_cmap(self.vspec.get("cmap"), 129)(range(16, 29, 3)) + high2 = cm.get_cmap(self.vspec.get("cmap"), 129)(range(48, 103, 6)) return np.concatenate((low, high1, high2)) @property def wind_colors_high(self) -> np.ndarray: + """Default color map for High Wind Speed""" - ''' Default color map for High Wind Speed ''' - - low = cm.get_cmap(self.vspec.get('cmap'), 129)(range(129, 108, -7)) - high1 = cm.get_cmap(self.vspec.get('cmap'), 129)(range(16, 29, 4)) - high2 = cm.get_cmap(self.vspec.get('cmap'), 129)(range(46, 95, 7)) + low = cm.get_cmap(self.vspec.get("cmap"), 129)(range(129, 108, -7)) + high1 = cm.get_cmap(self.vspec.get("cmap"), 129)(range(16, 29, 4)) + high2 = cm.get_cmap(self.vspec.get("cmap"), 129)(range(46, 95, 7)) return np.concatenate((low, high1, high2)) diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 113cf92..e26300c 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -1,53 +1,53 @@ # pylint: disable=invalid-name -''' +""" A set of generic utilities available to all the adb_graphics components. -''' +""" import argparse import datetime as dt import functools import glob import importlib as il -from math import atan2, degrees -from multiprocessing import Process -from string import digits, ascii_letters import os import subprocess import sys import time +from math import atan2, degrees +from multiprocessing import Process +from string import ascii_letters, digits import numpy as np import yaml -def cfgrib_spec(config, model): - if (spec := config.get(model)): +def cfgrib_spec(config, model): + if spec := config.get(model): return spec return config def create_zip(files_to_zip, zipf): + """Create a zip file. Use a locking mechanism -- write a lock file to disk.""" - ''' Create a zip file. Use a locking mechanism -- write a lock file to disk. ''' - - lock_file = f'{zipf}._lock' + lock_file = f"{zipf}._lock" retry = 2 count = 0 while True: if not os.path.exists(lock_file): # Create the lock - fd = open(lock_file, 'w') - print(f'Writing to zip file {zipf} for files like: {files_to_zip[0][-10:]}') + fd = open(lock_file, "w") + print(f"Writing to zip file {zipf} for files like: {files_to_zip[0][-10:]}") - cmd = f'zip -uj {zipf} {" ".join(files_to_zip)}' - print(f'Running command: {cmd}') + cmd = f"zip -uj {zipf} {' '.join(files_to_zip)}" + print(f"Running command: {cmd}") try: - subprocess.run(cmd, - check=True, - shell=True, - ) - except: # pylint: disable=bare-except - print(f'Error on writing zip file! {sys.exc_info()[0]}') + subprocess.run( + cmd, + check=True, + shell=True, + ) + except: # pylint: disable=bare-except + print(f"Error on writing zip file! {sys.exc_info()[0]}") count += 1 if count >= retry: raise @@ -65,9 +65,9 @@ def create_zip(files_to_zip, zipf): # Wait before trying to obtain the lock on the file time.sleep(5) -def fhr_list(args): - ''' +def fhr_list(args): + """ Given an argparse list argument, return the sequence of forecast hours to process. @@ -81,7 +81,7 @@ def fhr_list(args): argparse should provide a list of at least one item (nargs='+'). Must ensure that the list contains integers. - ''' + """ args = args if isinstance(args, list) else [args] arg_len = len(args) @@ -91,41 +91,44 @@ def fhr_list(args): return args + def from_datetime(date): - ''' Return a string like YYYYMMDDHH given a datetime object. ''' - return dt.datetime.strftime(date, '%Y%m%d%H') + """Return a string like YYYYMMDDHH given a datetime object.""" + return dt.datetime.strftime(date, "%Y%m%d%H") -def get_func(val: str): - ''' +def get_func(val: str): + """ Given an input string, val, returns the corresponding callable function. This function is borrowed from stackoverflow.com response to "Python: YAML dictionary of functions: how to load without converting to strings." - ''' + """ - if '.' in val: - module_name, fun_name = val.rsplit('.', 1) + if "." in val: + module_name, fun_name = val.rsplit(".", 1) else: - module_name = '__main__' + module_name = "__main__" fun_name = val - mod_spec = il.util.find_spec(module_name, package='adb_graphics') + mod_spec = il.util.find_spec(module_name, package="adb_graphics") if mod_spec is None: - mod_spec = il.util.find_spec('.' + module_name, package='adb_graphics') + mod_spec = il.util.find_spec("." + module_name, package="adb_graphics") try: __import__(mod_spec.name) except ImportError as exc: - print(f'Could not load {module_name} while trying to locate function in get_func') + print( + f"Could not load {module_name} while trying to locate function in get_func" + ) raise exc module = sys.modules[mod_spec.name] fun = getattr(module, fun_name) return fun + # pylint: disable=unused-argument def join_ranges(loader, node): - - ''' + """ Merge two or more different ranges into a single array for color bar clevs. e.g.: in default_specs.yml, clevs for visibility can be assigned as @@ -137,7 +140,7 @@ def join_ranges(loader, node): resolution than the rest, while keeping the colorbar from looking squished. Note that a "yaml.add_constructor" is required, as shown after the method. - ''' + """ list_ = [] for seq_node in node.value: @@ -149,13 +152,14 @@ def join_ranges(loader, node): return np.concatenate(list_, axis=0) + # SafeLoader doesn't seem compatible with our numpy contructors, using Loader here yaml.add_constructor("!join_ranges", join_ranges, Loader=yaml.Loader) + # pylint: disable=invalid-name, too-many-locals def label_line(ax, label, segment, **kwargs): - - ''' + """ Label a single line with line2D label data. Input: @@ -171,66 +175,66 @@ def label_line(ax, label, segment, **kwargs): offset index to use for the "end" of the array Any kwargs accepted by matplotlib's text box. - ''' + """ # Strip non-text-box key word arguments and set default if they don't exist - align = kwargs.pop('align', True) - end = kwargs.pop('end', 'bottom') - offset = kwargs.pop('offset', 0) + align = kwargs.pop("align", True) + end = kwargs.pop("end", "bottom") + offset = kwargs.pop("offset", 0) # Label location - if end == 'bottom': + if end == "bottom": x, y = segment[0 + offset, :] ip = 1 + offset - elif end == 'top': + elif end == "top": x, y = segment[-1 - offset, :] ip = -1 - offset if align: - #Compute the slope - dx = segment[ip, 0] - segment[ip-1, 0] - dy = segment[ip, 1] - segment[ip-1, 1] + # Compute the slope + dx = segment[ip, 0] - segment[ip - 1, 0] + dy = segment[ip, 1] - segment[ip - 1, 1] ang = degrees(atan2(dy, dx)) - #Transform to screen co-ordinates + # Transform to screen co-ordinates pt = np.array([x, y]).reshape((1, 2)) - trans_angle = ax.transData.transform_angles(np.array((ang, )), pt)[0] + trans_angle = ax.transData.transform_angles(np.array((ang,)), pt)[0] - if end == 'top': + if end == "top": trans_angle -= 180 else: trans_angle = 0 - #Set a bunch of keyword arguments - if ('horizontalalignment' not in kwargs) and ('ha' not in kwargs): - kwargs['ha'] = 'center' + # Set a bunch of keyword arguments + if ("horizontalalignment" not in kwargs) and ("ha" not in kwargs): + kwargs["ha"] = "center" - if ('verticalalignment' not in kwargs) and ('va' not in kwargs): - kwargs['va'] = 'center' + if ("verticalalignment" not in kwargs) and ("va" not in kwargs): + kwargs["va"] = "center" - if 'backgroundcolor' not in kwargs: - kwargs['backgroundcolor'] = ax.get_facecolor() + if "backgroundcolor" not in kwargs: + kwargs["backgroundcolor"] = ax.get_facecolor() - if 'clip_on' not in kwargs: - kwargs['clip_on'] = True + if "clip_on" not in kwargs: + kwargs["clip_on"] = True - if 'fontsize' not in kwargs: - kwargs['fontsize'] = 'larger' + if "fontsize" not in kwargs: + kwargs["fontsize"] = "larger" - if 'fontweight' not in kwargs: - kwargs['fontweight'] = 'bold' + if "fontweight" not in kwargs: + kwargs["fontweight"] = "bold" # Larger value (e.g., 2.0) to move box in front of other diagram elements - if 'zorder' not in kwargs: - kwargs['zorder'] = 1.50 + if "zorder" not in kwargs: + kwargs["zorder"] = 1.50 # Place the text box label on the line. ax.text(x, y, label, rotation=trans_angle, **kwargs) -def label_lines(ax, lines, labels, offset=0, **kwargs): - ''' +def label_lines(ax, lines, labels, offset=0, **kwargs): + """ Plots labels on a set of lines from SkewT. Input: @@ -245,96 +249,97 @@ def label_lines(ax, lines, labels, offset=0, **kwargs): color line color Along with any other kwargs accepted by matplotlib's text box. - ''' + """ - if 'color' not in kwargs: - kwargs['color'] = lines.get_color()[0] + if "color" not in kwargs: + kwargs["color"] = lines.get_color()[0] for i, line in enumerate(lines.get_segments()): label = int(labels[i]) label_line(ax, label, line, align=True, offset=offset, **kwargs) -def load_sites(arg): - ''' Check that the sites file exists, and return its contents. ''' +def load_sites(arg): + """Check that the sites file exists, and return its contents.""" # Check that the file exists path = path_exists(arg) - with open(path, 'r') as sites_file: + with open(path, "r") as sites_file: sites = sites_file.readlines() return sites + def uniq_wgrib2_list(inlist): - ''' Given a list of wgrib2 output fields, returns a uniq list of fields for + """Given a list of wgrib2 output fields, returns a uniq list of fields for simplifying a grib2 dataset. Uniqueness is defined by the wgrib output from field 3 (colon delimted) onward, although the original full grib record must be included in the wgrib2 command below. - ''' + """ uniq_field_set = set() uniq_list = [] for infield in inlist: - infield_info = infield.split(':') + infield_info = infield.split(":") if len(infield_info) <= 3: continue - infield_str = ':'.join(infield_info[3:]) + infield_str = ":".join(infield_info[3:]) if infield_str not in uniq_field_set: uniq_list.append(infield) uniq_field_set.add(infield_str) return uniq_list -def load_specs(arg): - ''' Check to make sure arg file exists. Return its contents. ''' +def load_specs(arg): + """Check to make sure arg file exists. Return its contents.""" spec_file = path_exists(arg) - with open(spec_file, 'r') as fn: + with open(spec_file, "r") as fn: specs = yaml.load(fn, Loader=yaml.Loader) - specs['file'] = spec_file + specs["file"] = spec_file return specs -def numeric_level(index_match=True, level=None, split=None): - ''' +def numeric_level(index_match=True, level=None, split=None): + """ Split the numeric level and unit associated with the level key. A blank string is returned for lev_val for levels that do not contain a numeric, e.g., 'sfc' or 'ua'. - ''' + """ level = level if level is not None else 0 # Gather all the numbers in the string - lev_val = ''.join([c for c in level if (c in digits or c == '.')]) + lev_val = "".join([c for c in level if (c in digits or c == ".")]) # Convert the numbers to a list, and make integers or floats if lev_val: if split is not None: lev_val = [int(lev) for lev in lev_val] else: - lev_val = [float(lev_val) if '.' in lev_val else int(lev_val)] + lev_val = [float(lev_val) if "." in lev_val else int(lev_val)] # Gather all the letters - lev_unit = ''.join([c for c in level if c in ascii_letters]) + lev_unit = "".join([c for c in level if c in ascii_letters]) if index_match: - if lev_unit == 'cm': - lev_val = [val / 100. for val in lev_val] - if lev_unit in ['mb', 'mxmb']: - lev_val = [val * 100. for val in lev_val] - if lev_unit in ['in', 'km', 'mn', 'mx', 'sr']: - lev_val = [val * 1000. for val in lev_val] + if lev_unit == "cm": + lev_val = [val / 100.0 for val in lev_val] + if lev_unit in ["mb", "mxmb"]: + lev_val = [val * 100.0 for val in lev_val] + if lev_unit in ["in", "km", "mn", "mx", "sr"]: + lev_val = [val * 1000.0 for val in lev_val] return lev_val, lev_unit -def old_enough(age, file_path): - ''' +def old_enough(age, file_path): + """ Helper function to test the age of a file. Input: @@ -345,26 +350,26 @@ def old_enough(age, file_path): Output: bool whether the file is at least age minutes old - ''' + """ file_time = dt.datetime.fromtimestamp(os.path.getctime(file_path)) max_age = dt.datetime.now() - dt.timedelta(minutes=age) return file_time < max_age -def path_exists(path: str): - ''' Checks whether a file exists, and returns the path if it does. ''' +def path_exists(path: str): + """Checks whether a file exists, and returns the path if it does.""" if not os.path.exists(path): - msg = f'{path} does not exist!' + msg = f"{path} does not exist!" raise argparse.ArgumentTypeError(msg) return path -def timer(func): - ''' Decorator function that provides an elapsed time for a method. ''' +def timer(func): + """Decorator function that provides an elapsed time for a method.""" @functools.wraps(func) def wrapper_timer(*args, **kwargs): @@ -374,17 +379,19 @@ def wrapper_timer(*args, **kwargs): elapsed_time = toc - tic print(f"{func.__name__} Elapsed time: {elapsed_time:0.4f} seconds") return value + return wrapper_timer + def to_datetime(string): - ''' Return a datetime object give a string like YYYYMMDDHH. ''' + """Return a datetime object give a string like YYYYMMDDHH.""" + + return dt.datetime.strptime(string, "%Y%m%d%H") - return dt.datetime.strptime(string, '%Y%m%d%H') @timer def zip_products(fhr, workdir, zipfiles): - - ''' Spin up a subprocess to zip all the product files into the staged zip files. + """Spin up a subprocess to zip all the product files into the staged zip files. Input: @@ -394,18 +401,19 @@ def zip_products(fhr, workdir, zipfiles): Output: None - ''' + """ for tile, zipf in zipfiles.items(): - if tile == 'skewt_csv': - file_tmpl = f'*.skewt.*_f{fhr:03d}.csv' + if tile == "skewt_csv": + file_tmpl = f"*.skewt.*_f{fhr:03d}.csv" else: - file_tmpl = f'*_{tile}_*{fhr:02d}.png' + file_tmpl = f"*_{tile}_*{fhr:02d}.png" product_files = glob.glob(os.path.join(workdir, file_tmpl)) if product_files: - zip_proc = Process(group=None, - target=create_zip, - args=(product_files, zipf), - ) + zip_proc = Process( + group=None, + target=create_zip, + args=(product_files, zipf), + ) zip_proc.start() zip_proc.join() diff --git a/conftest.py b/conftest.py index ea5a1a3..0de6ef4 100644 --- a/conftest.py +++ b/conftest.py @@ -1,37 +1,38 @@ -''' +""" Add command line options to the pytest suite. Each CLA needs to be defined in pytest_addoption and to have a pytest.fixture function defined. -''' +""" import pytest def pytest_addoption(parser): + """Define command line arguments to be parsed.""" - ''' Define command line arguments to be parsed. ''' + parser.addoption( + "--nat-file", + action="store", + help="Path to nat-file.", + ) - parser.addoption('--nat-file', - action='store', - help='Path to nat-file.', - ) + parser.addoption( + "--prs-file", + action="store", + help="Path to prs-file.", + ) - parser.addoption('--prs-file', - action='store', - help='Path to prs-file.', - ) @pytest.fixture def natfile(request): + """Interface to pass a grib file to pytest""" - ''' Interface to pass a grib file to pytest''' + return request.config.getoption("--nat-file") - return request.config.getoption('--nat-file') @pytest.fixture def prsfile(request): + """Interface to pass a grib file to pytest""" - ''' Interface to pass a grib file to pytest''' - - return request.config.getoption('--prs-file') + return request.config.getoption("--prs-file") diff --git a/create_graphics.py b/create_graphics.py index 152455c..bc47767 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -1,44 +1,44 @@ # pylint: disable=invalid-name -''' +""" Driver for creating all the SkewT diagrams needed for a specific input dataset. -''' +""" # pylint: disable=wrong-import-position, wrong-import-order import matplotlib as mpl -mpl.use('Agg') + +mpl.use("Agg") # pylint: enable=wrong-import-position, wrong-import-order import argparse import copy - import glob -from multiprocessing import Pool import os import random import string import subprocess import sys import time +from multiprocessing import Pool import yaml -from adb_graphics.datahandler import gribfile import adb_graphics.errors as errors +import adb_graphics.utils as utils +from adb_graphics.datahandler import gribfile from adb_graphics.figure_builders import parallel_maps, parallel_skewt from adb_graphics.figures import maps -import adb_graphics.utils as utils +AIRPORTS = "static/Airports_locs.txt" -AIRPORTS = 'static/Airports_locs.txt' - -COMBINED_FN = 'combined_{fhr:03d}_{uniq}.grib2' -TMP_FN = 'combined_{fhr:03d}_{uniq}.tmp.grib2' +COMBINED_FN = "combined_{fhr:03d}_{uniq}.grib2" +TMP_FN = "combined_{fhr:03d}_{uniq}.tmp.grib2" LOG_BREAK = f"{('-' * 80)}\n{('-' * 80)}" + def check_file(cla, fhr, data_root=None, file_tmpl=None, mem=None): - ''' Given the command line arguments, the forecast hour, and a potential - ensemble member, build a full path to the file and ensure it exists. ''' + """Given the command line arguments, the forecast hour, and a potential + ensemble member, build a full path to the file and ensure it exists.""" if data_root is None: data_root = cla.data_root[0] @@ -51,112 +51,124 @@ def check_file(cla, fhr, data_root=None, file_tmpl=None, mem=None): else: grib_path = grib_path.format(FCST_TIME=fhr) - print(f'Checking on file {grib_path}') - old_enough = utils.old_enough(cla.data_age, grib_path) if \ - os.path.exists(grib_path) else False + print(f"Checking on file {grib_path}") + old_enough = ( + utils.old_enough(cla.data_age, grib_path) + if os.path.exists(grib_path) + else False + ) return grib_path, old_enough -def create_skewt(cla, fhr, grib_path, workdir): - ''' Generate arguments for parallel processing of Skew T graphics, - and generate a pool of workers to complete the tasks. ''' +def create_skewt(cla, fhr, grib_path, workdir): + """Generate arguments for parallel processing of Skew T graphics, + and generate a pool of workers to complete the tasks.""" # Create the file object to load the contents gfile = gribfile.GribFile(grib_path) args = [(cla, fhr, gfile.contents, site, workdir) for site in cla.sites] - print(f'Queueing {len(args)} Skew Ts') + print(f"Queueing {len(args)} Skew Ts") with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_skewt, args) -def create_maps(cla, fhr, grib_path, workdir, grib_path2=None): - ''' Generate arguments for parallel processing of plan-view maps and - generate a pool of workers to complete the task. ''' +def create_maps(cla, fhr, grib_path, workdir, grib_path2=None): + """Generate arguments for parallel processing of plan-view maps and + generate a pool of workers to complete the task.""" model = cla.images[0] for tile in cla.tiles: args = [] for variable, levels in cla.images[1].items(): for level in levels: - # Load the spec for the current variable spec = cla.specs.get(variable, {}).get(level) if not spec: - msg = f'graphics: {variable} {level}' + msg = f"graphics: {variable} {level}" raise errors.NoGraphicsDefinitionForVariable(msg) - args.append((cla, fhr, grib_path, level, model, spec, - variable, workdir, tile, grib_path2)) + args.append( + ( + cla, + fhr, + grib_path, + level, + model, + spec, + variable, + workdir, + tile, + grib_path2, + ) + ) - print(f'Queueing {len(args)} maps') + print(f"Queueing {len(args)} maps") # parallel_maps(*args[-1]) with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_maps, args) -def gather_gribfiles(cla, fhr, filename, gribfiles): - ''' Returns the appropriate gribfiles object for the type of graphics being +def gather_gribfiles(cla, fhr, filename, gribfiles): + """Returns the appropriate gribfiles object for the type of graphics being generated -- whether it's for a single forecast time or all forecast lead - times. ''' + times.""" - filenames = {'01fcst': [], 'free_fcst': []} + filenames = {"01fcst": [], "free_fcst": []} fcst_hour = int(fhr) - first_fcst = 6 if 'global' in cla.images[0] else 1 + first_fcst = 6 if "global" in cla.images[0] else 1 if fcst_hour <= first_fcst: - filenames['01fcst'].append(filename) + filenames["01fcst"].append(filename) else: - filenames['free_fcst'].append(filename) + filenames["free_fcst"].append(filename) if gribfiles is None or not cla.all_leads: - # Create a new GribFiles object, include all hours, or just this one, # depending on command line argument flag gribfiles = gribfile.GribFiles( - coord_dims={'fcst_hr': [fhr]}, + coord_dims={"fcst_hr": [fhr]}, filenames=filenames, filetype=cla.file_type, model=cla.images[0], - ) + ) else: - # Append a single forecast hour to the existing GribFiles object. - gribfiles.coord_dims.get('fcst_hr').append(fhr) + gribfiles.coord_dims.get("fcst_hr").append(fhr) gribfiles.append(filenames) return gribfiles -def generate_tile_list(arg_list): - ''' Given the input arguments -- a list if the argument is provided, return +def generate_tile_list(arg_list): + """Given the input arguments -- a list if the argument is provided, return the list. If no arg is provided, defaults to the full domain, and if 'all' - is provided, the full domain, and all subdomains are plotted. ''' + is provided, the full domain, and all subdomains are plotted.""" if not arg_list: - return ['full'] + return ["full"] - if ',' in arg_list[0]: - arg_list = arg_list[0].split(',') + if "," in arg_list[0]: + arg_list = arg_list[0].split(",") - hrrr_ak_only = ('Anchorage', 'AKRange', 'Juneau') - rap_only = ('AK', 'AKZoom', 'conus', 'HI') - if 'all' in arg_list: - all_list = ['full'] + list(maps.TILE_DEFS.keys()) + hrrr_ak_only = ("Anchorage", "AKRange", "Juneau") + rap_only = ("AK", "AKZoom", "conus", "HI") + if "all" in arg_list: + all_list = ["full"] + list(maps.TILE_DEFS.keys()) return [tile for tile in all_list if tile not in hrrr_ak_only + rap_only] return arg_list -def load_images(arg): - ''' Check that input image file exists, and that it contains the +def load_images(arg): + """Check that input image file exists, and that it contains the requested section. Return a 2-list (required by argparse) of the file path and dictionary of images to be created. - ''' + """ # Agument is expected to be a 2-list of file name and internal # section name. @@ -168,202 +180,204 @@ def load_images(arg): image_file = utils.path_exists(image_file) # Load yaml file - with open(image_file, 'r') as fn: + with open(image_file, "r") as fn: images = yaml.load(fn, Loader=yaml.Loader)[image_set] - return [images.get('model'), images.get('variables')] + return [images.get("model"), images.get("variables")] -def parse_args(argv): - ''' Set up argparse command line arguments, and return the Namespace - containing the settings. ''' +def parse_args(argv): + """Set up argparse command line arguments, and return the Namespace + containing the settings.""" - parser = argparse.ArgumentParser(description='Script to drive the \ - creation of graphices files.') + parser = argparse.ArgumentParser( + description="Script to drive the \ + creation of graphices files." + ) # Positional argument parser.add_argument( - 'graphic_type', - choices=['maps', 'skewts', 'enspanel', 'diff'], - help='The type of graphics to create.', - ) + "graphic_type", + choices=["maps", "skewts", "enspanel", "diff"], + help="The type of graphics to create.", + ) # Short args parser.add_argument( - '-r', - dest='img_res', + "-r", + dest="img_res", default=72, required=False, - help='Resolution of output images in DPI. Recommended to stay below 1000. Default = 72', + help="Resolution of output images in DPI. Recommended to stay below 1000. Default = 72", type=int, - ) + ) parser.add_argument( - '-a', - dest='data_age', + "-a", + dest="data_age", default=3, - help='Age in minutes required for data files to be complete. Default = 3', + help="Age in minutes required for data files to be complete. Default = 3", type=int, - ) + ) parser.add_argument( - '-d', - dest='data_root', - help='Cycle-independant data directory location. Provide more than one \ + "-d", + dest="data_root", + help="Cycle-independant data directory location. Provide more than one \ data path if data input files should be combined. When providing \ multiple options, the same number of options is required for the \ - --file_tmpl flag.', - nargs='+', + --file_tmpl flag.", + nargs="+", required=True, - ) + ) parser.add_argument( - '-f', - dest='fcst_hour', - help='A list describing forecast hours. If one argument, \ + "-f", + dest="fcst_hour", + help="A list describing forecast hours. If one argument, \ one fhr will be processed. If 2 or 3 arguments, a sequence \ of forecast hours [start, stop, [increment]] will be \ processed. If more than 3 arguments, the list is processed \ - as-is.', - nargs='+', + as-is.", + nargs="+", required=True, type=int, - ) + ) parser.add_argument( - '-m', - default='Unnamed Experiment', - dest='model_name', - help='string to use in title of graphic.', + "-m", + default="Unnamed Experiment", + dest="model_name", + help="string to use in title of graphic.", type=str, - ) + ) parser.add_argument( - '-n', + "-n", default=1, - dest='nprocs', - help='Number of processes to use for parallelization.', + dest="nprocs", + help="Number of processes to use for parallelization.", type=int, - ) + ) parser.add_argument( - '-o', - dest='output_path', - help='Directory location desired for the output graphics files.', + "-o", + dest="output_path", + help="Directory location desired for the output graphics files.", required=True, - ) + ) parser.add_argument( - '-s', - dest='start_time', - help='Start time in YYYYMMDDHH format', + "-s", + dest="start_time", + help="Start time in YYYYMMDDHH format", required=True, type=utils.to_datetime, - ) + ) parser.add_argument( - '-w', - dest='wait_time', + "-w", + dest="wait_time", default=10, - help='Time in minutes to wait on data files to be available. Default = 10', + help="Time in minutes to wait on data files to be available. Default = 10", type=int, - ) + ) parser.add_argument( - '-z', - dest='zip_dir', - help='Full path to zip directory.', - ) + "-z", + dest="zip_dir", + help="Full path to zip directory.", + ) # Long args parser.add_argument( - '--all_leads', - action='store_true', - help='Use --all_leads to accumulate all forecast lead times.', - ) + "--all_leads", + action="store_true", + help="Use --all_leads to accumulate all forecast lead times.", + ) parser.add_argument( - '--file_tmpl', - default='wrfnat_hrconus_{FCST_TIME:02d}.grib2', - nargs='+', - help='File naming convention. Use FCST_TIME to indicate forecast hour. \ + "--file_tmpl", + default="wrfnat_hrconus_{FCST_TIME:02d}.grib2", + nargs="+", + help="File naming convention. Use FCST_TIME to indicate forecast hour. \ Provide more than one template when data files should be combined. \ When providing multiple options, the same number of options is required \ - for the -d flag.', \ - ) + for the -d flag.", + ) parser.add_argument( - '--file_type', - choices=('nat', 'prs'), - default='nat', - help='Type of levels contained in grib file.', - ) + "--file_type", + choices=("nat", "prs"), + default="nat", + help="Type of levels contained in grib file.", + ) # SkewT-specific args - skewt_group = parser.add_argument_group('SkewT Arguments') + skewt_group = parser.add_argument_group("SkewT Arguments") skewt_group.add_argument( - '--max_plev', - help='Maximum pressure level to plot for profiles.', + "--max_plev", + help="Maximum pressure level to plot for profiles.", type=int, - ) + ) skewt_group.add_argument( - '--sites', - help='Path to a sites file.', + "--sites", + help="Path to a sites file.", type=utils.load_sites, - ) + ) # Map-specific args - map_group = parser.add_argument_group('Map Arguments') + map_group = parser.add_argument_group("Map Arguments") map_group.add_argument( - '--images', - help='Path to YAML config file specifying which \ - variables to map and the top-level section to use.', - metavar=('[FILE,', 'SECTION]'), + "--images", + help="Path to YAML config file specifying which \ + variables to map and the top-level section to use.", + metavar=("[FILE,", "SECTION]"), nargs=2, - ) + ) map_group.add_argument( - '--obs_file_path', - help='Path to an observation file. Currently this \ + "--obs_file_path", + help="Path to an observation file. Currently this \ feature is only supported for ensemble panel plots and \ - composite reflectivity.', + composite reflectivity.", type=utils.path_exists, - ) + ) map_group.add_argument( - '--specs', - default='adb_graphics/default_specs.yml', - help='Path to the specs YAML file.', - ) + "--specs", + default="adb_graphics/default_specs.yml", + help="Path to the specs YAML file.", + ) map_group.add_argument( - '--subh_freq', + "--subh_freq", default=60, - help='Sub-hourly frequency in minutes.', - ) + help="Sub-hourly frequency in minutes.", + ) map_group.add_argument( - '--tiles', - default=['full'], - help='The domains to plot. Choose from any of those listed. Special ' \ - 'choices: full is full model output domain, and all is the full domain, ' \ - 'plus all of the sub domains. ' \ - f'Choices: {["full", "all"] + maps.FULL_TILES + list(maps.TILE_DEFS.keys())}', - nargs='+', - ) + "--tiles", + default=["full"], + help="The domains to plot. Choose from any of those listed. Special " + "choices: full is full model output domain, and all is the full domain, " + "plus all of the sub domains. " + f"Choices: {['full', 'all'] + maps.FULL_TILES + list(maps.TILE_DEFS.keys())}", + nargs="+", + ) # Ensemble panel-specific args - ens_group = parser.add_argument_group('Ensemble Panel Arguments') + ens_group = parser.add_argument_group("Ensemble Panel Arguments") ens_group.add_argument( - '--ens_size', + "--ens_size", default=10, - help='Number of ensemble members.', + help="Number of ensemble members.", type=int, - ) + ) # Diff args - diff_group = parser.add_argument_group('Difference Maps Arguments') + diff_group = parser.add_argument_group("Difference Maps Arguments") diff_group.add_argument( - '--data_root2', - help='Cycle-independant data directory location. The order of the ' \ - 'difference will be generated in order: data_root - data_root2.', - ) + "--data_root2", + help="Cycle-independant data directory location. The order of the " + "difference will be generated in order: data_root - data_root2.", + ) diff_group.add_argument( - '--file_tmpl2', - default='wrfnat_hrconus_{FCST_TIME:02d}.grib2', - help='File naming convention for second set of files used in \ - difference maps. Use FCST_TIME to indicate forecast hour.', - ) + "--file_tmpl2", + default="wrfnat_hrconus_{FCST_TIME:02d}.grib2", + help="File naming convention for second set of files used in \ + difference maps. Use FCST_TIME to indicate forecast hour.", + ) return parser.parse_args(argv) -def pre_proc_grib_files(cla, fhr): - ''' Use the command line argument object (cla) to determine the grib file +def pre_proc_grib_files(cla, fhr): + """Use the command line argument object (cla) to determine the grib file location at a given forecast hour. If multiple data input paths and file templates are provided by user, concatenate the files and remove the duplicates. Return the file path of the file to be used by the graphics data @@ -379,110 +393,116 @@ def pre_proc_grib_files(cla, fhr): old_enough bool stating whether the file is old enough as defined by user settings. Combined files here are presumed old enough by default - ''' + """ if len(cla.data_root) == 1 and len(cla.file_tmpl) == 1: # Nothing to do, return the original file location return check_file(cla, fhr) # Generate a list of files to be joined. - file_list = [os.path.join(*path).format(FCST_TIME=fhr) for path in - zip(cla.data_root, cla.file_tmpl)] + file_list = [ + os.path.join(*path).format(FCST_TIME=fhr) + for path in zip(cla.data_root, cla.file_tmpl) + ] for file_path in file_list: - if not os.path.exists(file_path) \ - or not utils.old_enough(cla.data_age, file_path): + if not os.path.exists(file_path) or not utils.old_enough( + cla.data_age, file_path + ): return file_path, False - print(f'Combining input files: ') + print(f"Combining input files: ") for fn in file_list: - print(f' {fn}') - - file_rand = ''.join([random.choice(string.ascii_letters + string.digits) \ - for _ in range(8)]) - combined_fp = os.path.join(cla.output_path, - COMBINED_FN.format(fhr=fhr, uniq=file_rand)) - tmp_fp = os.path.join(cla.output_path, - TMP_FN.format(fhr=fhr, uniq=file_rand)) - - cmd = f'cat {" ".join(file_list)} > {tmp_fp}' - output = subprocess.run(cmd, - capture_output=True, - check=True, - shell=True, - ) + print(f" {fn}") + + file_rand = "".join( + [random.choice(string.ascii_letters + string.digits) for _ in range(8)] + ) + combined_fp = os.path.join( + cla.output_path, COMBINED_FN.format(fhr=fhr, uniq=file_rand) + ) + tmp_fp = os.path.join(cla.output_path, TMP_FN.format(fhr=fhr, uniq=file_rand)) + + cmd = f"cat {' '.join(file_list)} > {tmp_fp}" + output = subprocess.run( + cmd, + capture_output=True, + check=True, + shell=True, + ) if output.returncode != 0: - msg = f'{cmd} returned exit status: {output.returncode}!' + msg = f"{cmd} returned exit status: {output.returncode}!" raise OSError(msg) # Gather all grib2 entries from combined file - cmd = f'wgrib2 {tmp_fp} -submsg 1' - output = subprocess.run(cmd, - capture_output=True, - check=True, - shell=True, - ) - wgrib2_list = output.stdout.decode("utf-8").split('\n') + cmd = f"wgrib2 {tmp_fp} -submsg 1" + output = subprocess.run( + cmd, + capture_output=True, + check=True, + shell=True, + ) + wgrib2_list = output.stdout.decode("utf-8").split("\n") # Create a unique list of grib fields. wgrib2_list = utils.uniq_wgrib2_list(wgrib2_list) # Remove duplicate grib2 entries in grib file - cmd = f'wgrib2 -i {tmp_fp} -GRIB {combined_fp}' - input_arg = '\n'.join(wgrib2_list).encode("utf-8") - - output = subprocess.run(cmd, - capture_output=True, - check=True, - input=input_arg, - shell=True, - ) + cmd = f"wgrib2 -i {tmp_fp} -GRIB {combined_fp}" + input_arg = "\n".join(wgrib2_list).encode("utf-8") + + output = subprocess.run( + cmd, + capture_output=True, + check=True, + input=input_arg, + shell=True, + ) if output.returncode != 0: - msg = f'{cmd} returned exit status: {output.returncode}' + msg = f"{cmd} returned exit status: {output.returncode}" raise OSError(msg) - os.remove(f'{tmp_fp}') + os.remove(f"{tmp_fp}") - return f'{combined_fp}', True + return f"{combined_fp}", True -def remove_accumulated_images(cla): - ''' Searches for all images that correspond with specs that have the +def remove_accumulated_images(cla): + """Searches for all images that correspond with specs that have the accumulate entry set to True and removes them from the list of images to - create. ''' + create.""" for variable, levels in cla.images[1].items(): for level in levels: spec = cla.specs.get(variable, {}).get(level) if not spec: - msg = f'graphics: {variable} {level}' + msg = f"graphics: {variable} {level}" raise errors.NoGraphicsDefinitionForVariable(msg) - accumulate = spec.get('accumulate', False) + accumulate = spec.get("accumulate", False) if accumulate: - print(f'Will not plot {variable}:{level}') + print(f"Will not plot {variable}:{level}") cla.images[1][variable].remove(level) if not cla.images[1][variable]: del cla.images[1][variable] -def remove_proc_grib_files(cla): - ''' Find all processed grib files produced by this script and remove them. - ''' +def remove_proc_grib_files(cla): + """Find all processed grib files produced by this script and remove them.""" # Prepare template with all viable forecast hours -- glob accepts * - combined_fn = COMBINED_FN.format(fhr=999, uniq=999).replace('999', '*') + combined_fn = COMBINED_FN.format(fhr=999, uniq=999).replace("999", "*") combined_fp = os.path.join(cla.output_path, combined_fn) combined_files = glob.glob(combined_fp) if combined_files: - print(f'Removing combined files: ') + print(f"Removing combined files: ") for file_path in combined_files: - print(f' {file_path}') + print(f" {file_path}") os.remove(file_path) -def stage_zip_files(tiles, zip_dir): - ''' Stage the zip files in the appropriate directory for each tile to be +def stage_zip_files(tiles, zip_dir): + """Stage the zip files in the appropriate directory for each tile to be plotted. Return the dictionary of zipfile paths. Input: @@ -495,25 +515,25 @@ def stage_zip_files(tiles, zip_dir): Returns: zipfiles dictionary of tile keys, and zip directory values. - ''' + """ zipfiles = {} for tile in tiles: tile_zip_dir = os.path.join(zip_dir, tile) os.makedirs(tile_zip_dir, exist_ok=True) - tile_zip_file = os.path.join(tile_zip_dir, 'files.zip') + tile_zip_file = os.path.join(tile_zip_dir, "files.zip") zipfiles[tile] = tile_zip_file return zipfiles + @utils.timer def graphics_driver(cla): - # pylint: disable=too-many-statements # This whole script has likely reached the point of neededing refactoring # into an object oriented design....each graphics type is it's own object # sharing a base class. - ''' + """ Function that interprets the command line arguments to locate the input grib file, create the output directory, and call the graphic-specifc function. @@ -521,15 +541,15 @@ def graphics_driver(cla): cla Namespace object containing command line arguments. - ''' + """ # pylint: disable=too-many-branches, too-many-locals # Create an empty zip file if cla.zip_dir: - tiles = cla.tiles if cla.graphic_type in ["maps", "enspanel"] else ['skewt'] - if 'skewt' in tiles: - tiles.append('skewt_csv') + tiles = cla.tiles if cla.graphic_type in ["maps", "enspanel"] else ["skewt"] + if "skewt" in tiles: + tiles.append("skewt_csv") zipfiles = stage_zip_files(tiles, cla.zip_dir) fcst_hours = copy.deepcopy(cla.fcst_hour) @@ -544,27 +564,28 @@ def graphics_driver(cla): # load all of those into gribfiles up front. # This is not an operational feature. Exit if files don't exist. - if cla.graphic_type == 'maps': - first_fcst = 6 if 'global' in cla.images[0] else 0 - fcst_inc = 6 if 'global' in cla.images[0] else 1 + if cla.graphic_type == "maps": + first_fcst = 6 if "global" in cla.images[0] else 0 + fcst_inc = 6 if "global" in cla.images[0] else 1 if len(cla.fcst_hour) == 1 and cla.all_leads: for fhr in range(first_fcst, int(cla.fcst_hour[0]), fcst_inc): grib_path, old_enough = pre_proc_grib_files(cla, fhr) if not os.path.exists(grib_path) or not old_enough: - msg = (f'File {grib_path} does not exist! Cannot accumulate', - f'data for this forecast lead time!') + msg = ( + f"File {grib_path} does not exist! Cannot accumulate", + f"data for this forecast lead time!", + ) remove_proc_grib_files(cla) - raise FileNotFoundError(' '.join(msg)) + raise FileNotFoundError(" ".join(msg)) gribfiles = gather_gribfiles(cla, fhr, grib_path, gribfiles) - # Allow this task to run concurrently with UPP by continuing to check for # new files as they become available. while fcst_hours: timer_sleep = time.time() old_enough = False for fhr in sorted(fcst_hours): - if cla.graphic_type == 'enspanel': + if cla.graphic_type == "enspanel": # Expand template to create a list of ensemble member files and # check if they exist and that they're old enough grib_paths = [] @@ -586,68 +607,78 @@ def graphics_driver(cla): else: if cla.all_leads: # Wait on the missing file for an arbitrary 90% of wait time - if time.time() - timer_end > cla.wait_time * 60 * .9: - print(f"Giving up waiting on {grib_path}. \n", - f"Removing accumulated variables from image list \n", - f"{LOG_BREAK}\n") + if time.time() - timer_end > cla.wait_time * 60 * 0.9: + print( + f"Giving up waiting on {grib_path}. \n", + f"Removing accumulated variables from image list \n", + f"{LOG_BREAK}\n", + ) remove_accumulated_images(cla) # Explicitly set -all_leads to False cla.all_leads = False else: # Break out of loop, wait for the desired period, and start # back at this forecast hour. - print(f'Waiting for {grib_path} to be available.') + print(f"Waiting for {grib_path} to be available.") break # It's safe to continue on processing the next forecast hour - print(f'Cannot find specified file(s), continuing to check on \n \ - next forecast hour.') + print( + f"Cannot find specified file(s), continuing to check on \n \ + next forecast hour." + ) continue # Create the working directory - workdir = os.path.join(cla.output_path, - f"{utils.from_datetime(cla.start_time)}{fhr:02d}") + workdir = os.path.join( + cla.output_path, f"{utils.from_datetime(cla.start_time)}{fhr:02d}" + ) os.makedirs(workdir, exist_ok=True) - print(f'{LOG_BREAK}\n', - f'Graphics will be created for input files\n', - f'Output graphics directory: {workdir} \n' - f'{LOG_BREAK}') + print( + f"{LOG_BREAK}\n", + f"Graphics will be created for input files\n", + f"Output graphics directory: {workdir} \n{LOG_BREAK}", + ) - if cla.graphic_type == 'skewts': + if cla.graphic_type == "skewts": create_skewt(cla, fhr, grib_path, workdir) - elif cla.graphic_type == 'maps': - create_maps(cla, - fhr=fhr, - grib_path=grib_path, - workdir=workdir, - ) - elif cla.graphic_type == 'diff': + elif cla.graphic_type == "maps": + create_maps( + cla, + fhr=fhr, + grib_path=grib_path, + workdir=workdir, + ) + elif cla.graphic_type == "diff": gribfiles = gather_gribfiles(cla, fhr, grib_path, gribfiles) grib_path2, _ = check_file( cla, fhr, data_root=cla.data_root2, - file_tmpl=cla.file_tmpl2,) + file_tmpl=cla.file_tmpl2, + ) gribfiles2 = gather_gribfiles(cla, fhr, grib_path2, gribfiles2) - create_maps(cla, - fhr=fhr, - grib_contents=gribfiles.contents, - grib_contents2=gribfiles2.contents, - workdir=workdir, - ) + create_maps( + cla, + fhr=fhr, + grib_contents=gribfiles.contents, + grib_contents2=gribfiles2.contents, + workdir=workdir, + ) else: gribfiles = gribfile.GribFiles( - coord_dims={'ens_mem': ens_members, 'fcst_hr': fhr_as_list}, - filenames={'free_fcst': grib_paths}, + coord_dims={"ens_mem": ens_members, "fcst_hr": fhr_as_list}, + filenames={"free_fcst": grib_paths}, filetype=cla.file_type, model=cla.images[0], - ) - create_maps(cla, - fhr=fhr, - grib_contents=gribfiles.contents, - workdir=workdir, - ) + ) + create_maps( + cla, + fhr=fhr, + grib_contents=gribfiles.contents, + workdir=workdir, + ) # Zip png files and remove the originals in a subprocess if cla.zip_dir: @@ -660,24 +691,26 @@ def graphics_driver(cla): # wait_time mins. This accounts for slower UPP processes. Default for # most CONUS-sized domains is 10 mins. if time.time() - timer_end > cla.wait_time * 60: - print(f"Exiting with forecast hours remaining: {fcst_hours}", - f"{LOG_BREAK}") + print( + f"Exiting with forecast hours remaining: {fcst_hours}", f"{LOG_BREAK}" + ) break # Wait for a bit if it's been < 2 minutes (about the length of time UPP # takes) since starting last loop if fcst_hours and time.time() - timer_sleep < 120: - print(f"Waiting for a minute for forecast hours: {fcst_hours}", - f"{LOG_BREAK}") + print( + f"Waiting for a minute for forecast hours: {fcst_hours}", f"{LOG_BREAK}" + ) time.sleep(60) remove_proc_grib_files(cla) def create_graphics(argv): - ''' + """ Function to perform a series of checks on command line arguments. - ''' + """ CLARGS = parse_args(argv) CLARGS.fcst_hour = utils.fhr_list(CLARGS.fcst_hour) @@ -689,45 +722,51 @@ def create_graphics(argv): # Ensure wgrib command is available in environment before getting too far # down this path... if len(CLARGS.data_root) > 1: - retcode = subprocess.run('which wgrib2', shell=True, check=True) + retcode = subprocess.run("which wgrib2", shell=True, check=True) if retcode.returncode != 0: - errmsg = 'Could not find wgrib2, please make sure it is loaded \ - in your environment.' + errmsg = "Could not find wgrib2, please make sure it is loaded \ + in your environment." raise OSError(errmsg) # Only need to load the default in memory if we're making maps. - if CLARGS.graphic_type in ['maps', 'enspanel', 'diff']: + if CLARGS.graphic_type in ["maps", "enspanel", "diff"]: CLARGS.specs = utils.load_specs(CLARGS.specs) CLARGS.images = load_images(CLARGS.images) CLARGS.tiles = generate_tile_list(CLARGS.tiles) # Make sure the second data root is provided when doing diffs - if CLARGS.graphic_type == 'diff': + if CLARGS.graphic_type == "diff": if not CLARGS.data_root2: errmsg = "Must specify a second data root (--data_root2) for creating difference maps" raise argparse.ArgumentError(CLARGS.data_root2, errmsg) if CLARGS.all_leads: - warning = ("Warning! Plotting differences in graphics-accumulated ", - "fields is not supported!") + warning = ( + "Warning! Plotting differences in graphics-accumulated ", + "fields is not supported!", + ) print(warning) # Make sure both required arguments (--max_plev, --sites) are provided when doing skewTs - if CLARGS.graphic_type == 'skewts': + if CLARGS.graphic_type == "skewts": if not CLARGS.max_plev: - argparse.ArgumentParser.exit(0, "Must specify maximum pressure level \ - (--max_plev) when creating skewTs") + argparse.ArgumentParser.exit( + 0, + "Must specify maximum pressure level \ + (--max_plev) when creating skewTs", + ) if not CLARGS.sites: - argparse.ArgumentParser.exit(0, "Must specify sites (--sites) when creating skewTs") + argparse.ArgumentParser.exit( + 0, "Must specify sites (--sites) when creating skewTs" + ) - print(f"Running script for {CLARGS.graphic_type} with args: ", - f"{LOG_BREAK}") + print(f"Running script for {CLARGS.graphic_type} with args: ", f"{LOG_BREAK}") for name, val in CLARGS.__dict__.items(): - if name not in ['specs', 'sites']: + if name not in ["specs", "sites"]: print(f"{name:>15s}: {val}") graphics_driver(CLARGS) -if __name__ == '__main__': +if __name__ == "__main__": create_graphics(sys.argv[1:]) diff --git a/tests/test_common.py b/tests/test_common.py index 1d7393a..e72b0a0 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,6 +1,6 @@ # pylint: disable=invalid-name -''' +""" Pytests for the common utilities included in this package. Includes: - conversions.py @@ -11,34 +11,34 @@ python -m pytest --nat-file [path/to/gribfile] --prs-file [path/to/gribfile] -''' +""" +import warnings from inspect import getfullargspec from string import ascii_letters, digits -import warnings +import numpy as np +import yaml from matplotlib import cm from matplotlib import colors as mcolors from metpy.plots import ctables -import numpy as np -import yaml import adb_graphics.conversions as conversions +import adb_graphics.datahandler.gribdata as gribdata import adb_graphics.specs as specs import adb_graphics.utils as utils -import adb_graphics.datahandler.gribdata as gribdata -def test_conversion(): - ''' Test that conversions return at numpy array for input of np.ndarray, - list, or int ''' +def test_conversion(): + """Test that conversions return at numpy array for input of np.ndarray, + list, or int""" a = np.ones([3, 2]) * 300 c = a[0, 0] # Check for the right answer assert np.array_equal(conversions.k_to_c(a), a - 273.15) - assert np.array_equal(conversions.k_to_f(a), (a - 273.15) * 9/5 + 32) + assert np.array_equal(conversions.k_to_f(a), (a - 273.15) * 9 / 5 + 32) assert np.array_equal(conversions.kgm2_to_in(a), a * 0.03937) assert np.array_equal(conversions.m_to_dm(a), a / 10) assert np.array_equal(conversions.m_to_in(a), a * 39.3701) @@ -47,10 +47,10 @@ def test_conversion(): assert np.array_equal(conversions.ms_to_kt(a), a * 1.9438) assert np.array_equal(conversions.pa_to_hpa(a), a / 100) assert np.array_equal(conversions.percent(a), a * 100) - assert np.array_equal(conversions.to_micro(a), a * 1E6) - assert np.array_equal(conversions.to_micrograms_per_m3(a), a * 1E9) + assert np.array_equal(conversions.to_micro(a), a * 1e6) + assert np.array_equal(conversions.to_micrograms_per_m3(a), a * 1e9) assert np.array_equal(conversions.vvel_scale(a), a * -10) - assert np.array_equal(conversions.vort_scale(a), a / 1E-05) + assert np.array_equal(conversions.vort_scale(a), a / 1e-05) assert np.array_equal(conversions.weasd_to_1hsnw(a), a * 10) functions = [ @@ -69,7 +69,7 @@ def test_conversion(): conversions.vvel_scale, conversions.vort_scale, conversions.weasd_to_1hsnw, - ] + ] # Check that all functions return a np.ndarray given a collection, or single float for f in functions: @@ -78,8 +78,7 @@ def test_conversion(): class MockSpecs(specs.VarSpec): - - ''' Mock class for the VarSpec abstract class ''' + """Mock class for the VarSpec abstract class""" @property def clevs(self): @@ -91,10 +90,9 @@ def vspec(self): def test_specs(): + """Test VarSpec properties.""" - ''' Test VarSpec properties. ''' - - config = 'adb_graphics/default_specs.yml' + config = "adb_graphics/default_specs.yml" varspec = MockSpecs(config) # Ensure correct return type @@ -108,88 +106,80 @@ def test_specs(): def test_utils(): + """Test that utils works appropriately.""" - ''' Test that utils works appropriately. ''' - - assert callable(utils.get_func('conversions.k_to_c')) + assert callable(utils.get_func("conversions.k_to_c")) def test_join_ranges_constructor(): + """Test that the join_ranges constructor works as expected.""" - ''' Test that the join_ranges constructor works as expected. ''' - - yaml.add_constructor('!join_ranges', utils.join_ranges, Loader=yaml.SafeLoader) - yaml_str = ''' + yaml.add_constructor("!join_ranges", utils.join_ranges, Loader=yaml.SafeLoader) + yaml_str = """ foo: !join_ranges [[0, 15, 0.1], [20, 61, 20]] foo2: !join_ranges [[0, 15, 0.1]] foo3: !join_ranges [[0, 15, 0.1], [20, 40, 10], [40, 61, 20]] - ''' + """ cfg = yaml.load(yaml_str, Loader=yaml.SafeLoader) - expected = np.concatenate((np.arange(0, 15, 0.1), - np.arange(20, 61, 20)), axis=0) + expected = np.concatenate((np.arange(0, 15, 0.1), np.arange(20, 61, 20)), axis=0) expected2 = np.arange(0, 15, 0.1) - expected3 = np.concatenate((np.arange(0, 15, 0.1), - np.arange(20, 40, 10), - np.arange(40, 61, 20)), axis=0) + expected3 = np.concatenate( + (np.arange(0, 15, 0.1), np.arange(20, 40, 10), np.arange(40, 61, 20)), axis=0 + ) - assert np.array_equal(expected, cfg['foo']) - assert np.array_equal(expected2, cfg['foo2']) - assert np.array_equal(expected3, cfg['foo3']) + assert np.array_equal(expected, cfg["foo"]) + assert np.array_equal(expected2, cfg["foo2"]) + assert np.array_equal(expected3, cfg["foo3"]) -class TestDefaultSpecs(): +class TestDefaultSpecs: + """Test contents of default_specs.yml.""" - ''' Test contents of default_specs.yml. ''' - - config = 'adb_graphics/default_specs.yml' + config = "adb_graphics/default_specs.yml" varspec = MockSpecs(config) cfg = varspec.yml @property def allowable(self): - - ''' Each entry in the dict names a function that tests a key in - default_specs.yml. ''' + """Each entry in the dict names a function that tests a key in + default_specs.yml.""" return { - 'accumulate': self.is_bool, - 'annotate': self.is_bool, - 'annotate_decimal': self.is_int, - 'clevs': self.is_a_clev, - 'cmap': self.is_a_cmap, - 'colors': self.is_a_color, - 'contours': self.is_a_contour_dict, - 'include_obs': self.is_bool, - 'hatches': self.is_a_contourf_dict, - 'labels': self.is_a_contourf_dict, - 'ncl_name': True, - 'plot_airports': self.is_bool, - 'plot_scatter': self.is_bool, - 'print_units': True, - 'split': self.is_bool, - 'ticks': self.is_number, - 'title': self.is_string, - 'transform': self.check_transform, - 'unit': self.is_string, - 'vertical_index': self.is_int, - 'vertical_level_name': self.is_string, - 'wind': self.is_wind, - } + "accumulate": self.is_bool, + "annotate": self.is_bool, + "annotate_decimal": self.is_int, + "clevs": self.is_a_clev, + "cmap": self.is_a_cmap, + "colors": self.is_a_color, + "contours": self.is_a_contour_dict, + "include_obs": self.is_bool, + "hatches": self.is_a_contourf_dict, + "labels": self.is_a_contourf_dict, + "ncl_name": True, + "plot_airports": self.is_bool, + "plot_scatter": self.is_bool, + "print_units": True, + "split": self.is_bool, + "ticks": self.is_number, + "title": self.is_string, + "transform": self.check_transform, + "unit": self.is_string, + "vertical_index": self.is_int, + "vertical_level_name": self.is_string, + "wind": self.is_wind, + } def check_kwargs(self, accepted_args, kwargs): - - ''' Ensure a dictionary entry matches the kwargs accepted by a function. - ''' + """Ensure a dictionary entry matches the kwargs accepted by a function.""" assert isinstance(kwargs, dict) for key, args in kwargs.items(): - lev = None - if '_' in key: - short_name, lev = key.split('_') + if "_" in key: + short_name, lev = key.split("_") else: short_name = key @@ -204,8 +194,7 @@ def check_kwargs(self, accepted_args, kwargs): return True def check_transform(self, entry): - - ''' + """ Check that the transform entry is either a single transformation function, a list of transformation functions, or a dictionary containing the functions list and the kwargs list like so: @@ -217,7 +206,7 @@ def check_transform(self, entry): sec_arg: value The functions listed under functions MUST be methods, not attributes! - ''' + """ kwargs = dict() @@ -228,22 +217,19 @@ def check_transform(self, entry): # If the transform entry is a dictionary, check that it has the # appropriate contents elif isinstance(entry, dict): - - funcs = entry.get('funcs') + funcs = entry.get("funcs") assert funcs is not None # Make sure funcs is a list funcs = funcs if isinstance(funcs, list) else [funcs] # Key word arguments may not be present. - kwargs = entry.get('kwargs') - + kwargs = entry.get("kwargs") transforms = [] for func in funcs: callables = self.get_callable(func) - callables = callables if isinstance(callables, list) else \ - [callables] + callables = callables if isinstance(callables, list) else [callables] transforms.extend(callables) # The argspecs bit gives us a list of all the accepted arguments @@ -251,12 +237,12 @@ def check_transform(self, entry): # when provided arguments don't appear in all_params. # arguments not in that list, we fail. if kwargs: - argspecs = [getfullargspec(func) for func in transforms if - callable(func)] + argspecs = [ + getfullargspec(func) for func in transforms if callable(func) + ] all_params = [] for argspec in argspecs: - # Make sure all functions accept key word arguments assert argspec.varkw is not None @@ -268,26 +254,21 @@ def check_transform(self, entry): for key in kwargs.keys(): if key not in all_params: - msg = f'Function key {key} is not an expicit parameter \ - in any of the transforms: {funcs}!' + msg = f"Function key {key} is not an expicit parameter \ + in any of the transforms: {funcs}!" warnings.warn(msg, UserWarning) - return True - # pylint: disable=inconsistent-return-statements def get_callable(self, func): - - - ''' Return the callable function given a function name. ''' + """Return the callable function given a function name.""" if func in dir(self.varspec): return self.varspec.__getattribute__(func) # Check datahandler.gribdata objects if a single word is provided - if len(func.split('.')) == 1: - + if len(func.split(".")) == 1: # Check all the classes in the gribdata module for attr in dir(gribdata): # pylint: disable=no-member @@ -300,19 +281,18 @@ def get_callable(self, func): if callable(utils.get_func(func)): return utils.get_func(func) - raise ValueError('{func} is not a known callable function!') + raise ValueError("{func} is not a known callable function!") @staticmethod def is_a_clev(clev): - - ''' Returns true for a clev that is a list, a range, or a callable function. ''' + """Returns true for a clev that is a list, a range, or a callable function.""" if isinstance(clev, (list, np.ndarray)): return True - if 'range' in clev.split('[')[0]: - clean = lambda x: x.strip().split('-')[-1].replace('.', '1') - items = clev.split(' ', 1)[1].strip('[').strip(']').split(',') + if "range" in clev.split("[")[0]: + clean = lambda x: x.strip().split("-")[-1].replace(".", "1") + items = clev.split(" ", 1)[1].strip("[").strip("]").split(",") nums = [clean(i).isnumeric() for i in items] return all(nums) @@ -320,19 +300,36 @@ def is_a_clev(clev): @staticmethod def is_a_cmap(cmap): - - ''' Returns true for a cmap that is a Colormap object. ''' + """Returns true for a cmap that is a Colormap object.""" return cmap in dir(cm) + list(ctables.colortables.keys()) def is_a_contour_dict(self, entry): - - ''' Set up the accepted arguments for plt.contour, and check the given - arguments. ''' - - args = ['X', 'Y', 'Z', 'levels', - 'corner_mask', 'colors', 'alpha', 'cmap', 'norm', 'vmin', - 'vmax', 'origin', 'extent', 'locator', 'extend', 'xunits', - 'yunits', 'antialiased', 'nchunk', 'linewidths', 'linestyles'] + """Set up the accepted arguments for plt.contour, and check the given + arguments.""" + + args = [ + "X", + "Y", + "Z", + "levels", + "corner_mask", + "colors", + "alpha", + "cmap", + "norm", + "vmin", + "vmax", + "origin", + "extent", + "locator", + "extend", + "xunits", + "yunits", + "antialiased", + "nchunk", + "linewidths", + "linestyles", + ] if entry is None: return True @@ -340,15 +337,33 @@ def is_a_contour_dict(self, entry): return self.check_kwargs(args, entry) def is_a_contourf_dict(self, entry): - - ''' Set up the accepted arguments for plt.contourf, and check the given - arguments. ''' - - args = ['X', 'Y', 'Z', 'levels', - 'corner_mask', 'colors', 'alpha', 'cmap', 'labels', 'norm', 'vmin', - 'vmax', 'origin', 'extent', 'locator', 'extend', 'xunits', - 'yunits', 'antialiased', 'nchunk', 'linewidths', - 'hatches'] + """Set up the accepted arguments for plt.contourf, and check the given + arguments.""" + + args = [ + "X", + "Y", + "Z", + "levels", + "corner_mask", + "colors", + "alpha", + "cmap", + "labels", + "norm", + "vmin", + "vmax", + "origin", + "extent", + "locator", + "extend", + "xunits", + "yunits", + "antialiased", + "nchunk", + "linewidths", + "hatches", + ] if entry is None: return True @@ -356,11 +371,9 @@ def is_a_contourf_dict(self, entry): return self.check_kwargs(args, entry) def is_a_color(self, color): + """Returns true if color is contained in the list of recognized colors.""" - ''' Returns true if color is contained in the list of recognized colors. ''' - - colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS, - **ctables.colortables) + colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS, **ctables.colortables) if color in colors.keys(): return True @@ -372,8 +385,7 @@ def is_a_color(self, color): @staticmethod def is_a_level(key): - - ''' + """ Returns true if the key fits one of the level descriptor formats. Allowable formats include: @@ -382,56 +394,56 @@ def is_a_level(key): [numeric][lev_type] e.g. 500mb, or 2m [stat][numeric] e.g. mn02, mx25 - ''' + """ allowed_levels = [ - 'agl', # above ground level - 'best', # Best - 'bndylay', # boundary layer cld cover - 'esbl', # ??? - 'esblmn', # ??? - 'high', # high clouds - 'int', # vertical integral - 'low', # low clouds - 'max', # maximum in column - 'maxsfc', # max surface value - 'mdn', # maximum downward - 'mid', # mid-level clouds - 'mnsfc', # min surface value - 'msl', # mean sea level - 'mu', # most unstable - 'mul', # most unstable layer - 'mup', # maximum upward - 'mu', # most unstable - 'obs', # observations - 'pw', # wrt precipitable water - 'sat', # satellite - 'sfc', # surface - 'sfclt', # surface (less than) - 'top', # nominal top of atmosphere - 'total', # total clouds - 'ua', # upper air - ] + "agl", # above ground level + "best", # Best + "bndylay", # boundary layer cld cover + "esbl", # ??? + "esblmn", # ??? + "high", # high clouds + "int", # vertical integral + "low", # low clouds + "max", # maximum in column + "maxsfc", # max surface value + "mdn", # maximum downward + "mid", # mid-level clouds + "mnsfc", # min surface value + "msl", # mean sea level + "mu", # most unstable + "mul", # most unstable layer + "mup", # maximum upward + "mu", # most unstable + "obs", # observations + "pw", # wrt precipitable water + "sat", # satellite + "sfc", # surface + "sfclt", # surface (less than) + "top", # nominal top of atmosphere + "total", # total clouds + "ua", # upper air + ] allowed_lev_type = [ - 'cm', # centimeters - 'ds', # difference - 'ft', # feet - 'km', # kilometers - 'm', # meters - 'mm', # millimeters - 'mb', # milibars - 'sr', # storm relative - ] + "cm", # centimeters + "ds", # difference + "ft", # feet + "km", # kilometers + "m", # meters + "mm", # millimeters + "mb", # milibars + "sr", # storm relative + ] allowed_stat = [ - 'in', # ??? - 'ens', # ensemble - 'm', # ??? - 'maxm', # ??? - 'mn', # minimum - 'mx', # maximum - ] + "in", # ??? + "ens", # ensemble + "m", # ??? + "maxm", # ??? + "mn", # minimum + "mx", # maximum + ] # Easy check first -- it is in the allowed_levels list if key in allowed_levels: @@ -440,11 +452,11 @@ def is_a_level(key): # Check for [numeric][lev_type] or [lev_type][numeric] pattern # Numbers come at beginning or end, only - numeric = ''.join([c for c in key if c in digits + '.']) in key + numeric = "".join([c for c in key if c in digits + "."]) in key # The level is allowed level_str = [c for c in key if c in ascii_letters] - allowed = ''.join(level_str) in allowed_lev_type + allowed_stat + allowed = "".join(level_str) in allowed_lev_type + allowed_stat # Check the other direction - level string contains one of the allowed # types. @@ -460,21 +472,18 @@ def is_a_level(key): return False def is_a_key(self, key): - - ''' Returns true if key exists as a key in the config file. ''' + """Returns true if key exists as a key in the config file.""" return self.cfg.get(key) is not None @staticmethod def is_bool(k): - - ''' Returns true if k is a boolean variable. ''' + """Returns true if k is a boolean variable.""" return isinstance(k, bool) def is_callable(self, funcs): - - ''' Returns true if func in funcs list is the name of a callable function. ''' + """Returns true if func in funcs list is the name of a callable function.""" funcs = funcs if isinstance(funcs, list) else [funcs] @@ -495,46 +504,40 @@ def is_callable(self, funcs): @staticmethod def is_dict(d): - - ''' Returns true if d is a dictionary ''' + """Returns true if d is a dictionary""" return isinstance(d, dict) @staticmethod def is_int(i): - - ''' Returns true if i is an integer. ''' + """Returns true if i is an integer.""" if isinstance(i, int): return True - return i.isnumeric() and len(i.split('.')) == 1 + return i.isnumeric() and len(i.split(".")) == 1 @staticmethod def is_number(i): - - ''' Returns true if i is a number. ''' + """Returns true if i is a number.""" if isinstance(i, (int, float)): return True - return i.isnumeric() and len(i.split('.')) <= 2 + return i.isnumeric() and len(i.split(".")) <= 2 @staticmethod def is_string(s): - - ''' Returns true if s is a string. ''' + """Returns true if s is a string.""" return isinstance(s, str) def is_wind(self, wind): - - ''' Returns true if wind is a bool or is_a_level. ''' + """Returns true if wind is a bool or is_a_level.""" return isinstance(wind, bool) or self.is_a_level(wind) def check_keys(self, d, depth=0): - - ''' Helper function that recursively checks the keys in the dictionary by calling the - function defined in allowable. ''' + """Helper function that recursively checks the keys in the dictionary by calling the + function defined in allowable.""" max_depth = 2 @@ -546,7 +549,7 @@ def check_keys(self, d, depth=0): if depth >= max_depth: return - level = depth+1 + level = depth + 1 for k, v in d.items(): # Check that the key is allowable @@ -565,9 +568,8 @@ def check_keys(self, d, depth=0): self.check_keys(v, depth=level) def test_keys(self): - - ''' Tests each of top-level variables in the config file by calling the helper function. ''' + """Tests each of top-level variables in the config file by calling the helper function.""" for short_name, spec in self.cfg.items(): - assert '_' not in short_name + assert "_" not in short_name self.check_keys(spec) diff --git a/tests/test_grib.py b/tests/test_grib.py index a9f42b6..83abe31 100644 --- a/tests/test_grib.py +++ b/tests/test_grib.py @@ -1,33 +1,32 @@ # pylint: disable=invalid-name -''' Test suite for grib datahandler. ''' +"""Test suite for grib datahandler.""" import datetime import numpy as np -from matplotlib import colors as mcolors import xarray as xr +from matplotlib import colors as mcolors import adb_graphics.datahandler.gribdata as gribdata import adb_graphics.datahandler.gribfile as gribfile DATAARRAY = xr.core.dataarray.DataArray -def test_UPPData(natfile, prsfile): - ''' Test the UPPData class methods on both types of input files. ''' +def test_UPPData(natfile, prsfile): + """Test the UPPData class methods on both types of input files.""" nat_ds = gribfile.GribFile(natfile) prs_ds = gribfile.GribFile(prsfile) class UPP(gribdata.UPPData): - - ''' Test class needed to define the values as an abstract class ''' + """Test class needed to define the values as an abstract class""" def values(self, level=None, name=None, **kwargs): return 1 - upp_nat = UPP(nat_ds.contents, fhr=2, filetype='nat', short_name='temp') - upp_prs = UPP(prs_ds.contents, fhr=2, short_name='temp') + upp_nat = UPP(nat_ds.contents, fhr=2, filetype="nat", short_name="temp") + upp_prs = UPP(prs_ds.contents, fhr=2, short_name="temp") # Ensure appropriate typing and size (where applicable) for upp in [upp_nat, upp_prs]: @@ -45,14 +44,14 @@ def values(self, level=None, name=None, **kwargs): assert isinstance(upp.vspec, dict) # Test for appropriate date formatting test_date = datetime.datetime(2020, 12, 5, 12) - assert upp.date_to_str(test_date) == '20201205 12 UTC' + assert upp.date_to_str(test_date) == "20201205 12 UTC" -def test_fieldData(prsfile): - ''' Test the fieldData class methods on a prs file''' +def test_fieldData(prsfile): + """Test the fieldData class methods on a prs file""" prs_ds = gribfile.GribFile(prsfile) - field = gribdata.fieldData(prs_ds.contents, fhr=2, level='500mb', short_name='temp') + field = gribdata.fieldData(prs_ds.contents, fhr=2, level="500mb", short_name="temp") assert isinstance(field.cmap, mcolors.Colormap) assert isinstance(field.colors, np.ndarray) @@ -64,48 +63,53 @@ def test_fieldData(prsfile): assert isinstance(field.wind(True), list) assert len(field.corners) == 4 assert len(field.wind(True)) == 2 - assert len(field.wind('850mb')) == 2 + assert len(field.wind("850mb")) == 2 for component in field.wind(True): assert isinstance(component, DATAARRAY) # Test retrieving other values - assert np.array_equal(field.values(), field.values(name='temp', level='500mb')) + assert np.array_equal(field.values(), field.values(name="temp", level="500mb")) # Return zeros by subtracting same field - diff = field.field_diff(field.values(), variable2='temp', level2='500mb') + diff = field.field_diff(field.values(), variable2="temp", level2="500mb") assert isinstance(diff, DATAARRAY) assert not np.any(diff) # Test transform - assert np.array_equal(field.get_transform('conversions.k_to_f', field.values()), \ - (field.values() - 273.15) * 9/5 +32) - - field2 = gribdata.fieldData(prs_ds.contents, fhr=2, level='ua', short_name='ceil') - transforms = field2.vspec.get('transform') - assert np.array_equal(field2.get_transform(transforms, field2.values()), \ - field2.field_diff(field2.values(), variable2='gh', level2='sfc') / 304.8) + assert np.array_equal( + field.get_transform("conversions.k_to_f", field.values()), + (field.values() - 273.15) * 9 / 5 + 32, + ) + + field2 = gribdata.fieldData(prs_ds.contents, fhr=2, level="ua", short_name="ceil") + transforms = field2.vspec.get("transform") + assert np.array_equal( + field2.get_transform(transforms, field2.values()), + field2.field_diff(field2.values(), variable2="gh", level2="sfc") / 304.8, + ) # Expected size of values assert len(np.shape((field.values()))) == 2 - assert len(np.shape((field.values(name='u')))) == 2 - assert len(np.shape((field.values(name='u', level='850mb')))) == 2 + assert len(np.shape((field.values(name="u")))) == 2 + assert len(np.shape((field.values(name="u", level="850mb")))) == 2 -def test_profileData(natfile): - ''' Test the profileData class methods on a nat file''' +def test_profileData(natfile): + """Test the profileData class methods on a nat file""" nat_ds = gribfile.GribFile(natfile) - loc = ' BNA 9999 99999 36.12 86.69 597 Nashville, TN\n' - profile = gribdata.profileData(nat_ds.contents, - fhr=2, - filetype='nat', - loc=loc, - short_name='temp', - ) - - assert isinstance(profile.get_xypoint(40., -100.), tuple) + loc = " BNA 9999 99999 36.12 86.69 597 Nashville, TN\n" + profile = gribdata.profileData( + nat_ds.contents, + fhr=2, + filetype="nat", + loc=loc, + short_name="temp", + ) + + assert isinstance(profile.get_xypoint(40.0, -100.0), tuple) assert isinstance(profile.values(), DATAARRAY) # The values should return a single number (0) or a 1D array (1) - assert len(np.shape((profile.values(level='best', name='li')))) == 0 - assert len(np.shape((profile.values(name='temp')))) == 1 + assert len(np.shape((profile.values(level="best", name="li")))) == 0 + assert len(np.shape((profile.values(name="temp")))) == 1 diff --git a/tests/test_hrrr_maps.py b/tests/test_hrrr_maps.py index 19fdd27..c07ca30 100644 --- a/tests/test_hrrr_maps.py +++ b/tests/test_hrrr_maps.py @@ -1,37 +1,74 @@ -#pylint: disable=unused-variable -''' Tests for create_graphics driver ''' +# pylint: disable=unused-variable +"""Tests for create_graphics driver""" + import os + import pytest -from create_graphics import create_graphics -from create_graphics import parse_args + +from create_graphics import create_graphics, parse_args DATA_LOC = os.environ.get("data_loc") OUTPUT_LOC = os.environ.get("data_loc") + @pytest.fixture(name="_setup") def build_maps(): - ''' Builds HRRR 12-hour accumulated maps ''' - args = ['maps', '-d', DATA_LOC, '-f', '0', '12', '1', '-o', OUTPUT_LOC,\ - '-s', '2023031500', '--file_tmpl', 'hrrr.t00z.wrfprsf{FCST_TIME:02d}.grib2', \ - '--images', './image_lists/hrrr_test.yml', 'hourly', '--all_leads', '--file_type=prs'] + """Builds HRRR 12-hour accumulated maps""" + args = [ + "maps", + "-d", + DATA_LOC, + "-f", + "0", + "12", + "1", + "-o", + OUTPUT_LOC, + "-s", + "2023031500", + "--file_tmpl", + "hrrr.t00z.wrfprsf{FCST_TIME:02d}.grib2", + "--images", + "./image_lists/hrrr_test.yml", + "hourly", + "--all_leads", + "--file_type=prs", + ] create_graphics(args) def test_parse_args(): - ''' Test parse_args for basic parsing success. - Checks if parse_args returns 'maps' in the graphic_type field. - ''' - args = ['maps', '-d', DATA_LOC, '-f', '0', '12', '1', '-o', OUTPUT_LOC,\ - '-s', '2021052315', '--file_tmpl', 'hrrr.t00z.wrfprsf{FCST_TIME:02d}.grib2', \ - '--images', './image_lists/hrrr_test.yml', 'hourly', '--all_leads', '--file_type=prs'] + """Test parse_args for basic parsing success. + Checks if parse_args returns 'maps' in the graphic_type field. + """ + args = [ + "maps", + "-d", + DATA_LOC, + "-f", + "0", + "12", + "1", + "-o", + OUTPUT_LOC, + "-s", + "2021052315", + "--file_tmpl", + "hrrr.t00z.wrfprsf{FCST_TIME:02d}.grib2", + "--images", + "./image_lists/hrrr_test.yml", + "hourly", + "--all_leads", + "--file_type=prs", + ] test_args = parse_args(args) - assert test_args.graphic_type == 'maps' + assert test_args.graphic_type == "maps" def test_folder_existence(_setup): - ''' Tests for existence of folders. - Can be extended to cover multiple folders. - ''' + """Tests for existence of folders. + Can be extended to cover multiple folders. + """ folder = "/202303150000" full_path = OUTPUT_LOC + folder file_path = os.path.isdir(full_path) @@ -39,9 +76,9 @@ def test_folder_existence(_setup): def test_file_count(_setup): - ''' Test for file count in directory. - Can be extended to cover multiple folders. - ''' + """Test for file count in directory. + Can be extended to cover multiple folders. + """ # Based on the hrrr_test.yml file, only 6 maps will be created map_count = 6 count = 0 From fdc0624f42fc9f987b553f7582dbdae5b434a6b4 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 21 Oct 2025 14:03:50 -0600 Subject: [PATCH 11/98] Executable. --- recipe/run_test.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 recipe/run_test.sh diff --git a/recipe/run_test.sh b/recipe/run_test.sh old mode 100644 new mode 100755 From f87eb135eb95865cf32a6e7530875976410d6c14 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 21 Oct 2025 17:52:55 -0600 Subject: [PATCH 12/98] Partially done formatting and linting. --- .github/workflows/hrrr_maps_tests.yml | 10 +- adb_graphics/datahandler/gribdata.py | 115 ++++--- adb_graphics/datahandler/gribfile.py | 53 +-- adb_graphics/figure_builders.py | 16 +- adb_graphics/figures/maps.py | 478 +++++++++++++------------- adb_graphics/figures/skewt.py | 86 +++-- adb_graphics/specs.py | 197 +++++------ adb_graphics/utils.py | 114 +++--- conftest.py | 4 +- create_graphics.py | 236 ++++++------- recipe/run_test.sh | 1 - tests/test_common.py | 87 ++--- tests/test_grib.py | 37 +- tests/test_hrrr_maps.py | 39 +-- 14 files changed, 722 insertions(+), 751 deletions(-) diff --git a/.github/workflows/hrrr_maps_tests.yml b/.github/workflows/hrrr_maps_tests.yml index 40f6791..a0d24ad 100644 --- a/.github/workflows/hrrr_maps_tests.yml +++ b/.github/workflows/hrrr_maps_tests.yml @@ -1,7 +1,7 @@ name: hrrr_maps_tests env: - data_loc: ${{ github.workspace }}/input_data - output_loc: ${{ github.workspace }}/output + DATA_LOC: ${{ github.workspace }}/input_data + OUTPUT_LOC: ${{ github.workspace }}/output on: push: branches: @@ -21,11 +21,11 @@ jobs: uses: actions/checkout@v3 - name: Create data and output folders run: | - mkdir -p $output_loc - mkdir -p $data_loc + mkdir -p $OUTPUT_LOC + mkdir -p $DATA_LOC - name: Fetch Grib Files run: | - wget -N -P $data_loc -i - << EOF + wget -N -P $DATA_LOC -i - << EOF https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf00.grib2 https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf01.grib2 https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf02.grib2 diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 1b0bcc1..e4a17cb 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -32,6 +32,7 @@ class UPPData(specs.VarSpec): Keyword Arguments: config: path to a user-specified configuration file model: string describing the model type + """ def __init__(self, ds, short_name, spec, **kwargs): @@ -49,8 +50,10 @@ def __init__(self, ds, short_name, spec, **kwargs): @property def anl_dt(self) -> datetime: - """Returns the initial time of the grib file as a datetime object from - the grib file.""" + """ + Returns the initial time of the grib file as a datetime object from + the grib file. + """ return datetime.fromisoformat(str(self.field.time.values).split(".")[0]) @@ -81,21 +84,24 @@ def clevs(self) -> np.ndarray: @staticmethod def date_to_str(date: datetime) -> str: - """Returns a formatted string (for graphic title) from a datetime - object""" + """ + Returns a formatted string (for graphic title) from a datetime + object + """ return date.strftime("%Y%m%d %H UTC") @property def field(self): - """Wrapper that calls get_field method for the current variable. - Returns the NioVariable object""" + """ + Wrapper that calls get_field method for the current variable. + Returns the NioVariable object + """ return self.ds.__getattr__([x for x in self.ds.data_vars][0]) def field_column_max(self, values, variable, level, **kwargs): # pylint: disable=unused-argument - """Returns the column max of the values.""" vals = self.values(name=variable, level=level, one_lev=False) @@ -105,7 +111,6 @@ def field_column_max(self, values, variable, level, **kwargs): def field_sum(self, values, variable2, level2, **kwargs): # pylint: disable=unused-argument - """Return the sum of the values.""" value2 = self.values(name=variable2, level=level2) @@ -116,7 +121,6 @@ def field_sum(self, values, variable2, level2, **kwargs): def field_diff(self, values, variable2, level2, **kwargs): # pylint: disable=unused-argument - """Subtracts the values from variable2 from self.field.""" value2 = self.values(name=variable2, level=level2) @@ -127,7 +131,6 @@ def field_diff(self, values, variable2, level2, **kwargs): def field_mean(self, values, variable, levels, global_levels, **kwargs): # pylint: disable=unused-argument - """Returns the mean of the values.""" levs = [int(x[:-2]) for x in levels] @@ -135,8 +138,10 @@ def field_mean(self, values, variable, levels, global_levels, **kwargs): return ret def _get_data_levels(self, vertical_dim): - """Return a list of vertical dimension values corresponding to the - requested vertical dimension to get the values of those dimensions""" + """ + Return a list of vertical dimension values corresponding to the + requested vertical dimension to get the values of those dimensions + """ fcst_hr = 0 if self.ds.sizes.get("fcst_hr", 0) <= 1 else int(self.fhr) @@ -154,10 +159,10 @@ def _get_field(self, spec): return ds.__getattr__([x for x in ds.data_vars][0]) def _get_level(self, field, level, spec, **kwargs): - """Returns the value of the level to for a 3D array + """ + Returns the value of the level to for a 3D array Arguments: - field dataset object for a given variable level string describing the level atmospheric level; corresponds to a key in default specs @@ -171,9 +176,9 @@ def _get_level(self, field, level, spec, **kwargs): Return: - Integer value corresponding to the array index for the atmospheric level. + """ # The index of the requested level @@ -227,7 +232,8 @@ def _get_level(self, field, level, spec, **kwargs): raise ValueError(msg) def get_transform(self, transforms, val): - """Applies a set of one or more transforms to an np.array of + """ + Applies a set of one or more transforms to an np.array of data values. Input: @@ -237,9 +243,9 @@ def get_transform(self, transforms, val): transformed Return: - val: updated values after transforms have been applied + """ transform_kwargs = {} @@ -260,7 +266,7 @@ def get_transform(self, transforms, val): val = utils.get_func(transform)(val, **transform_kwargs) return val - @lru_cache() + @lru_cache def get_xypoint(self, site_lat, site_lon) -> tuple: """ Return the X, Y grid point corresponding to the site location. No @@ -287,8 +293,10 @@ def get_xypoint(self, site_lat, site_lon) -> tuple: @property def grid_suffix(self): - """Return the suffix of the first variable with 4 sections (split on _) - in the file. This should correspond to the grid tag.""" + """ + Return the suffix of the first variable with 4 sections (split on _) + in the file. This should correspond to the grid tag. + """ for var in self.ds.keys(): vsplit = var.split("_") @@ -346,15 +354,16 @@ def numeric_level(self, index_match=True, level=None, split=None): @staticmethod def opposite(values, **kwargs): # pylint: disable=unused-argument - """Returns the opposite of input values""" return -values @property def valid_dt(self) -> datetime: - """Returns a datetime object corresponding to the forecast hour's valid - time as set in the Grib file.""" + """ + Returns a datetime object corresponding to the forecast hour's valid + time as set in the Grib file. + """ fh = timedelta(hours=int(self.fhr)) return self.anl_dt + fh @@ -366,9 +375,11 @@ def values(self, level=None, name=None, **kwargs): @staticmethod def vertical_dim(field): - """Determine the vertical dimension of the variable by looking through + """ + Determine the vertical dimension of the variable by looking through the field's dimensions for one that includes "lv". Return the first - matching instance.""" + matching instance. + """ vert_dim = [dim for dim in field.dims if ("lv" in dim or "probability" in dim)] if vert_dim: @@ -399,6 +410,7 @@ class fieldData(UPPData): config: path to a user-specified configuration file member: integer describing the ensemble member number to grab data for + """ def __init__(self, ds, level, short_name, **kwargs): @@ -406,11 +418,10 @@ def __init__(self, ds, level, short_name, **kwargs): self.level = level self.contour_kwargs = kwargs.get("contour_kwargs", {}) - self.mem = kwargs.get("member", None) + self.mem = kwargs.get("member") def aviation_flight_rules(self, values, **kwargs): # pylint: disable=unused-argument - """ Generates a field of Aviation Flight Rules from Ceil and Vis """ @@ -431,8 +442,10 @@ def aviation_flight_rules(self, values, **kwargs): @property def cmap(self): - """Returns the LinearSegmentedColormap specified by the config key - "cmap" """ + """ + Returns the LinearSegmentedColormap specified by the config key + "cmap" + """ return cm.get_cmap(self.vspec["cmap"]) @@ -476,7 +489,6 @@ def corners(self) -> list: def fire_weather_index(self, values, **kwargs): # pylint: disable=unused-argument - """ Generates a field of Fire Weather Index @@ -527,9 +539,7 @@ def _load_field(level, short_name): # Set all others vegetation types to 1 veg = np.where(veg > 0, 1, veg) - fwi = veg * ( - 2.37 * (gust_max**1.11) * (dewpt_depression**0.92) * (mois**6.95) * snowc - ) + fwi = veg * (2.37 * (gust_max**1.11) * (dewpt_depression**0.92) * (mois**6.95) * snowc) fwi = fwi / 10.0 @@ -614,7 +624,6 @@ def grid_info(self): def icing_adjust_trace(self, values, **kwargs): # pylint: disable=unused-argument,no-self-use - """Changes the value of ICSEV trace from 4.0 to 0.5, to maintain ascending order""" vals = np.where(values == 4.0, 0.5, values) @@ -644,7 +653,6 @@ def run_total(self, values, **kwargs): def supercooled_liquid_water(self, values, **kwargs): # pylint: disable=unused-argument - """ Generates a field of Supercooled Liquid Water @@ -675,9 +683,7 @@ def supercooled_liquid_water(self, values, **kwargs): pres_layer = 2 * (pres_sfc[:, :] - pres_nat_lev[n, :, :]) # layer depth pres_sigma = pres_sfc - pres_layer # pressure at next sigma level else: - pres_layer = 2 * ( - pres_sigma[:, :] - pres_nat_lev[n, :, :] - ) # layer depth + pres_layer = 2 * (pres_sigma[:, :] - pres_nat_lev[n, :, :]) # layer depth pres_sigma = pres_sigma - pres_layer # pressure at next sigma level # compute supercooled water in layer and add to previous values supercool_locs = np.where( @@ -696,22 +702,28 @@ def supercooled_liquid_water(self, values, **kwargs): @property def ticks(self) -> int: - """Returns the number of color bar tick marks from the yaml config - settings.""" + """ + Returns the number of color bar tick marks from the yaml config + settings. + """ return self.vspec.get("ticks", 10) @property def units(self) -> str: - """Returns the variable unit from the yaml config, if available. If not - specified in the yaml file, returns the value set in the Grib file.""" + """ + Returns the variable unit from the yaml config, if available. If not + specified in the yaml file, returns the value set in the Grib file. + """ return self.vspec.get("unit", self.field.units) @property def data(self): - """Sets the data property on the object for use when we need to update - the values associated with a given object -- helpful for differences.""" + """ + Sets the data property on the object for use when we need to update + the values associated with a given object -- helpful for differences. + """ if not hasattr(self, "_data"): return self.values() return self._data @@ -736,6 +748,7 @@ def values(self, level=None, name=None, **kwargs): one_lev bool flag. if True, get the single level of the variable (default: True) vertical_index the index (int) of the desired vertical level + """ level = level or self.level @@ -823,7 +836,6 @@ def vector_magnitude( **kwargs, ): # pylint: disable=unused-argument - """ Returns the vector magnitude of two component vector fields. The input fields can be either NCL names (string) or full data fields. The @@ -832,9 +844,7 @@ def vector_magnitude( if cfkeys: if cfkeys.get("level") is None: - cfkeys["level"] = utils.numeric_level( - level=self.level, index_match=False - )[0] + cfkeys["level"] = utils.numeric_level(level=self.level, index_match=False)[0] field2_spec = {"cfgrib": cfkeys} else: var, lev = field2_id.split(".") @@ -925,9 +935,7 @@ def __init__(self, ds, loc, short_name, **kwargs): # minus sign to convert the longitude to deg East, and then need to # adjust to the 0 to 360 system. self.site_lat = float(lat) - self.site_lon = -float( - lon - ) # lons are -180 but without minus sign in input file + self.site_lon = -float(lon) # lons are -180 but without minus sign in input file if self.site_lon < 0: self.site_lon = self.site_lon + 360.0 @@ -947,6 +955,7 @@ def values(self, level=None, name=None, **kwargs): split bool flag. if True, level string numbers are split into a list, e.g. used to get [0, 6000] from 06km vertical_index the index of the required level + """ # Set the defaults here since this is an instance of an abstract method @@ -993,9 +1002,7 @@ def values(self, level=None, name=None, **kwargs): profile = profile[:, x, y] return profile - def vector_magnitude( - self, field1, field2, level="ua", vertical_index=None, **kwargs - ): + def vector_magnitude(self, field1, field2, level="ua", vertical_index=None, **kwargs): """ Returns the vector magnitude of two component vector profiles. The input fields can be either NCL names (string) or full data fields. diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 13b5692..f624e0e 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -18,8 +18,10 @@ def __init__(self, filename, var_config, **kwargs): self.contents = self._load() def _load(self): - """Internal method that opens the grib file. Returns a grib message - iterator.""" + """ + Internal method that opens the grib file. Returns a grib message + iterator. + """ return xr.open_dataset( self.filename, @@ -30,13 +32,14 @@ def _load(self): class GribFiles: - """Class for loading in a set of grib files and combining them over - forecast hours.""" + """ + Class for loading in a set of grib files and combining them over + forecast hours. + """ def __init__(self, coord_dims, filenames, filetype, **kwargs): """ Arguments: - coord_dims dict containing the name of the dimension to concat (key), and a list of its values (value). Ex: {'fhr': [2, 3, 4]} @@ -47,6 +50,7 @@ def __init__(self, coord_dims, filenames, filetype, **kwargs): Keyword Arguments: model string describing the model type + """ self.model = kwargs.get("model", "") @@ -58,14 +62,18 @@ def __init__(self, coord_dims, filenames, filetype, **kwargs): self.contents = self._load() def append(self, filenames): - """Add a single new slice to existing data set. Must match coord_dims - and filetype of original dataset. Updates current contents of Object""" + """ + Add a single new slice to existing data set. Must match coord_dims + and filetype of original dataset. Updates current contents of Object + """ self.contents = self._load(filenames) def free_fcst_names(self, ds, fcst_type): - """Given an opened dataset, return a dict of original variable names - (key) and the desired name (value)""" + """ + Given an opened dataset, return a dict of original variable names + (key) and the desired name (value) + """ ret = {} @@ -104,7 +112,7 @@ def free_fcst_names(self, ds, fcst_type): ret[var] = var.replace(suffix, new_suffix) # MASSDEN is a special case when ending in "avg_1'" if var.split("_")[0] == "MASSDEN" and var.split("_")[-2] == "avg": - print(f"Special change to MASSDEN avg_1 name to avg1h_1") + print("Special change to MASSDEN avg_1 name to avg1h_1") ret[var] = var.replace("avg", "avg1h") else: # Only rename these variables at late hours @@ -128,11 +136,7 @@ def free_fcst_names(self, ds, fcst_type): # others! At 7 hours, it starts averaging since 6h. From 0-6 # h it's named with suffix avg, after its named avg1h, # avg2h, etc. - if ( - self.model == "rrfs" - and variable == "LRGHR" - and suffix == f"{suf}1h" - ): + if self.model == "rrfs" and variable == "LRGHR" and suffix == f"{suf}1h": contains_suffix.append(suf) # RRFS_A has fields that have the suffix 'acc0h' but we don't @@ -209,8 +213,10 @@ def free_fcst_names(self, ds, fcst_type): @staticmethod def _get_grid_suffix(filenames): - """Return the suffix of the first variable with 4 sections (split on _) - in the file. This should correspond to the grid tag.""" + """ + Return the suffix of the first variable with 4 sections (split on _) + in the file. This should correspond to the grid tag. + """ for files in filenames.values(): if files: @@ -250,7 +256,7 @@ def _load(self, filenames=None): renaming = self.free_fcst_names(dataset, fcst_type) if renaming and self.model not in ["hrrre", "rrfse"]: - print(f"RENAMING VARIABLES:") + print("RENAMING VARIABLES:") for old_name, new_name in renaming.items(): print(f" {old_name:>30s} -> {new_name}") dataset = dataset.rename_vars(renaming) @@ -272,10 +278,7 @@ def _load(self, filenames=None): for bad_var in bad_vars: # Check to see if the bad variable is in the current # dataset and NOT in the original dataset. - if ( - bad_var not in og_ds.variables - and dataset.get(bad_var) is not None - ): + if bad_var not in og_ds.variables and dataset.get(bad_var) is not None: print(f"Adding {bad_var} to og ds") # Duplicate the accumulated variable with the # required name @@ -293,8 +296,10 @@ def _load(self, filenames=None): @property def open_kwargs(self): - """Defines the key word arguments used by the various calls to XArray - open_mfdataset""" + """ + Defines the key word arguments used by the various calls to XArray + open_mfdataset + """ return dict( backend_kwargs=dict(format="grib2"), diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 18f80a7..75a0184 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -9,20 +9,19 @@ import matplotlib.pyplot as plt import numpy as np -import yaml -import adb_graphics.errors as errors from adb_graphics.datahandler import gribdata, gribfile from adb_graphics.figures import maps, skewt -from adb_graphics.utils import numeric_level AIRPORTS = "static/Airports_locs.txt" def add_obs_panel(ax, model_name, obs_file, proj_info, short_name, tile): # pylint: disable=too-many-arguments - """Plot observation data provided by the obs_file - path using the assigned projection.""" + """ + Plot observation data provided by the obs_file + path using the assigned projection. + """ gribobs = gribfile.GribFile(filename=obs_file) ax.axis("on") @@ -57,7 +56,6 @@ def parallel_maps( ): # pylint: disable=too-many-arguments,too-many-locals # pylint: disable=too-many-branches,too-many-statements - """ Function that creates plan-view maps, either a single panel, or multipanel for a forecast ensemble. Can be used in parallel. @@ -243,8 +241,10 @@ def parallel_skewt(cla, fhr, ds, site, workdir): def set_figure(model_name, graphic_type, tile): - """Create the figure and subplots appropriate for the model and - graphics type. Return the figure handle and list of axes.""" + """ + Create the figure and subplots appropriate for the model and + graphics type. Return the figure handle and list of axes. + """ if model_name == "HRRR-HI": inches = 12.2 diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index e0175d9..5f420e6 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -8,8 +8,10 @@ barbs, and descriptive annotation. """ +from collections.abc import Callable from copy import copy, deepcopy from math import isnan +from pathlib import Path import matplotlib.image as mpimg import matplotlib.offsetbox as mpob @@ -106,9 +108,157 @@ } +class MapFields: + """ + Class that packages all the field objects need for producing + desired map content, i.e. an object that contains all filled + contours, hatched spaces, and overlayed contours needed for a full + product. + """ + + def __init__( + self, + fhr: int, + fields_spec: dict, + grib_path: Path, + level: str, + name: str, + map_type: str | None = None, + **kwargs, + ): + self.grib_path = grib_path + self.fhr = fhr + self.fields_spec = deepcopy(fields_spec) + self.level = level + self.map_type = map_type + self.model = kwargs.get("model") + self.name = name + self.tile = kwargs.get("tile", "full") + + self.map_spec = deepcopy(self.fields_spec[self.name][self.level]) + self.set_level(self.level, self.map_spec) + # Required if map_type is "diff" + self.grib_path2 = kwargs.get("grib_path2") + + def set_level(self, level: str, spec: dict): + nlevel, _ = numeric_level(level=level, index_match=False) + level_info = any( + key + for keys in cfgrib_spec(spec["cfgrib"], self.model) + for key in ("level", "top", "bottom", "Surface") + if key in keys + ) + if nlevel and not level_info: + if spec["cfgrib"].get(self.model): + spec["cfgrib"][self.model]["level"] = nlevel + else: + spec["cfgrib"]["level"] = nlevel + # if spec["cfgrib"].get("level") is None and not spec["cfgrib"].get("stepRange") and not \ + # spec["cfgrib"].get("topLevel") and not \ + # spec["cfgrib"].get("typeOfLevel") == "surface" and not \ + # spec["cfgrib"].get("scaledValueOfFirstFixedSurface"): + + @property + def shaded(self): + cf = cfgrib_spec(self.map_spec["cfgrib"], self.model) + ds = gribfile.GribFile(self.grib_path, cf).contents + args = { + "ds": ds, + "fhr": self.fhr, + "level": self.level, + "model": self.model, + "short_name": self.name, + "spec": self.fields_spec, + "grib_path": self.grib_path, + } + field = gribdata.fieldData(**args) + + if self.map_type == "diff": + args["ds"] = gribfile.GribFile(self.grib_path2, cf).contents + args["grib_path"] = self.grib_path2 + field2 = gribdata.fieldData(**args) + field.data = field.values() - field2.values() + + return field + + @property + def contours(self): + """Return the list of contour fieldData objects.""" + + # We won't plot contours on multipanel plots, or full global + # plots. + if self.map_type == "enspanel": + return [] + + if "global" in self.model and self.tile in ["full"]: + return [] + + return self._overlay_fields("contours") + + @property + def hatches(self): + """Return the list of hatch fieldData objects.""" + + return self._overlay_fields("hatches") + + def wind_fields(self, level: str | None = None): + """Return u, v tuple of wind fields.""" + + lev = level or self.level + winds = [] + for var in ("u", "v"): + wind_spec = self.fields_spec[var][lev] + self.set_level(lev, wind_spec) + ds = gribfile.GribFile( + self.grib_path, cfgrib_spec(wind_spec["cfgrib"], self.model) + ).contents + args = { + "ds": ds, + "fhr": self.fhr, + "level": lev, + "model": self.model, + "short_name": var, + "spec": self.fields_spec, + "grib_path": self.grib_path, + } + winds.append(gribdata.fieldData(**args)) + return winds + + def _overlay_fields(self, spec_sect: str) -> list: + """ + Create fieldData objects for the specified overlay type - hatches or contours. + """ + + overlay_fields = [] + for overlay, overlay_kwargs in self.map_spec.get(spec_sect, {}).items(): + if "_" in overlay: + var, lev = overlay.split("_") + else: + var, lev = overlay, self.level + + overlay_spec = deepcopy(self.fields_spec[var][lev]) + self.set_level(lev, overlay_spec) + ds = gribfile.GribFile( + self.grib_path, cfgrib_spec(overlay_spec["cfgrib"], self.model) + ).contents + args = { + "ds": ds, + "fhr": self.fhr, + "level": lev, + "model": self.model, + "short_name": var, + "spec": self.fields_spec, + "grib_path": self.grib_path, + } + overlay_obj = gribdata.fieldData(**args) + # Set the attributes for the overlay field + overlay_obj.contour_kwargs = overlay_kwargs + overlay_fields.append(overlay_obj) + return overlay_fields + + class Map: # pylint: disable=too-many-instance-attributes - """ Class includes utilities needed to create a Basemap object, add airport locations, and draw the blank map. @@ -118,8 +268,7 @@ class Map: airport_fn full path to airport file ax figure axis - Keyword arguments: - + Keyword Arguments: map_proj dict describing the map projection to use. The only options currently are for lcc settings in _get_basemap() @@ -132,9 +281,10 @@ class Map: certain plots, default is True tile a string corresponding to a pre-defined tile in the TILE_DEFS dictionary + """ - def __init__(self, airport_fn, ax, **kwargs): + def __init__(self, airport_fn: Path, ax: plt, **kwargs): self.ax = ax self.grid_info = kwargs.get("grid_info", {}) self.model = kwargs.get("model") @@ -204,9 +354,7 @@ def draw(self): """Draw a map with political boundaries and airports only.""" self.boundaries() - if ( - self.plot_airports and "global" not in self.model - ): # airports are too dense in global + if self.plot_airports and "global" not in self.model: # airports are too dense in global self.draw_airports() def draw_airports(self): @@ -230,7 +378,7 @@ def draw_airports(self): del y def _get_basemap(self, **get_basemap_kwargs): - """Wrapper around basemap creation""" + """Wrapper around basemap creation.""" basemap_args = dict( ax=self.ax, @@ -260,8 +408,9 @@ def _get_basemap(self, **get_basemap_kwargs): def get_corners(self): """ - Gather the corners for a specific tile. Corners are supplied in the - following format: + Gather the corners for a specific tile. + + Corners are supplied in the following format: lat and lon of lower left (ll) and upper right(ur) corners: ll_lat, ur_lat, ll_lon, ur_lon @@ -284,17 +433,16 @@ def get_height(self): return TILE_DEFS[self.tile]["height"] @staticmethod - def load_airports(fn): + def load_airports(fn: Path): """Load lat, lon pairs from a text file, return a list of lists.""" - with open(fn, "r") as f: + with fn.open() as f: data = f.readlines() - return np.array([l.strip().split(",") for l in data], dtype=float) + return np.array([line.strip().split(",") for line in data], dtype=float) class DataMap: # pylint: disable=too-many-arguments - """ Class that combines the input data and the chosen map to plot both together. @@ -308,8 +456,7 @@ class DataMap: """ - # pylint: disable=unused-argument - def __init__(self, map_fields, map_, model_name=None, **kwargs): + def __init__(self, map_fields: MapFields, map_: plt, model_name: str | None = None, **kwargs): # noqa: ARG002 self.field = map_fields.shaded self.contour_fields = map_fields.contours self.hatch_fields = map_fields.hatches @@ -318,11 +465,11 @@ def __init__(self, map_fields, map_, model_name=None, **kwargs): self.model_name = model_name self.plot_scatter = map_fields.fields_spec.get("plot_scatter", False) - def wind_fields(self, level): + def wind_fields(self, level: str): return self.map_fields.wind_fields(level) @staticmethod - def add_logo(ax): + def add_logo(ax: plt): """Puts the NOAA logo at the bottom left of the matplotlib axes.""" logo = mpimg.imread("static/noaa-logo-50x50.png") @@ -338,10 +485,12 @@ def add_logo(ax): ax.add_artist(ab) - def _colorbar(self, cc, ax): - """Internal method that plots the color bar for a contourf field. - If ticks is set to zero, use a user-defined list of clevs from default_specs - If ticks is less than zero, use abs(ticks) as the step for labeling clevs""" + def _colorbar(self, cc: plt, ax: plt): + """ + Plot the colorbar for the contourf field. + If ticks is set to zero, use a user-defined list of clevs from default_specs. + If ticks is less than zero, use abs(ticks) as the step for labeling clevs. + """ if self.field.ticks > 0: ticks = np.arange( @@ -369,15 +518,15 @@ def _colorbar(self, cc, ax): # this step is done to allow proper order of icing severity levels (trace before light) if self.field.short_name == "icsev": - ticks = [ - label.rjust(30) for label in ["TRACE", "LIGHT", "MODERATE", "HEAVY", ""] - ] + ticks = [label.rjust(30) for label in ["TRACE", "LIGHT", "MODERATE", "HEAVY", ""]] cbar.ax.set_xticklabels(ticks, fontsize=12) - def draw(self, show=False): - """Main method for creating the plot. Set show=True to display the - figure from the command line.""" + def draw(self, show: bool = False): + """ + Main method for creating the plot. Set show=True to display the + figure from the command line. + """ cf = self._draw_panel() @@ -394,7 +543,7 @@ def draw(self, show=False): self.add_logo(self.map.ax) - def _draw_panel(self, wind_barbs=True): # pylint: disable=too-many-locals, too-many-branches + def _draw_panel(self, wind_barbs: bool = True): ax = self.map.ax # Draw a map and add the shaded field @@ -428,10 +577,15 @@ def _draw_panel(self, wind_barbs=True): # pylint: disable=too-many-locals, too- # Add field values at airports annotate = self.field.vspec.get("annotate", False) model_name = self.model_name - if annotate and "global" not in self.map.model: # too dense in global - if model_name not in ["RRFS NA 3km"]: # too dense in full RRFS domain - if model_name == "RAP-NCEP" and self.map.tile not in ["full"]: - self._draw_field_values(ax) + # too dense in global and rrfs NA + if ( + annotate + and "global" not in self.map.model + and model_name not in ["RRFS NA 3km"] + and model_name == "RAP-NCEP" + and self.map.tile not in ["full"] + ): + self._draw_field_values(ax) # Add scatter plot, if requested if self.plot_scatter: @@ -439,7 +593,7 @@ def _draw_panel(self, wind_barbs=True): # pylint: disable=too-many-locals, too- return cf - def _draw_contours(self, ax, not_labeled): + def _draw_contours(self, ax: plt, not_labeled: bool): """Draw the contour fields requested.""" model_name = self.model_name @@ -448,13 +602,12 @@ def _draw_contours(self, ax, not_labeled): for contour_field in self.contour_fields: levels = contour_field.contour_kwargs.pop("levels", contour_field.clevs) - if model_name in ["RAP-NCEP", "RRFS-NCEP", "RRFS NA 3km"]: - if ( - main_field == "totp" - and contour_field.short_name == "pres" - and self.map.tile == "full" - ): - levels = np.arange(650, 1051, 8) + if model_name in ["RAP-NCEP", "RRFS-NCEP", "RRFS NA 3km"] and ( + main_field == "totp" + and contour_field.short_name == "pres" + and self.map.tile == "full" + ): + levels = np.arange(650, 1051, 8) cc = self._draw_field( ax=ax, @@ -479,7 +632,7 @@ def _draw_contours(self, ax, not_labeled): {self.field.level}" ) - def _draw_scatter(self, ax): + def _draw_scatter(self, ax: plt): """Plot dots at locations on the map that meet a threshold.""" field = self.field @@ -497,9 +650,7 @@ def _draw_scatter(self, ax): value_to_color, ) else: - value_to_color = np.where( - vals > levels[i], colors[i + 1], value_to_color - ) + value_to_color = np.where(vals > levels[i], colors[i + 1], value_to_color) vtc1d = np.ravel(value_to_color) @@ -516,7 +667,7 @@ def _draw_scatter(self, ax): **field.contour_kwargs, ) - def _draw_field(self, ax, field, func, **kwargs): + def _draw_field(self, ax: plt, field: str, func: Callable, **kwargs): """ Internal implementation that calls a matplotlib function. @@ -525,12 +676,13 @@ def _draw_field(self, ax, field, func, **kwargs): field: Field to be plotted func: Matplotlib function to be called. - Keyword args: + kwargs: Can be any of the keyword args accepted by original func in matplotlib. Return: The return from the function called. + """ x, y = self._xy_mesh(field) @@ -561,7 +713,7 @@ def _draw_field(self, ax, field, func, **kwargs): print(f"CLOSE ERROR: {field.short_name} {field.level}") return ret - def _draw_field_values(self, ax): + def _draw_field_values(self, ax: plt): """Add the text value of the field at airport locations.""" annotate_decimal = self.field.vspec.get("annotate_decimal", 0) lats = self.map.airports[:, 0] @@ -579,16 +731,15 @@ def _draw_field_values(self, ax): if crnrs[1] > lat > crnrs[0] and crnrs[3] > lons[i] > crnrs[2]: xgrid, ygrid = self.field.get_xypoint(lat, lons[i]) data_value = data_values[xgrid, ygrid].values.item() - if xgrid > 0 and ygrid > 0: - if (not isnan(data_value)) and (data_value != 0.0): - ax.annotate( - f"{data_value:.{annotate_decimal}f}", - xy=(x[i], y[i]), - fontsize=10, - ) + if xgrid > 0 and ygrid > 0 and (not isnan(data_value)) and (data_value != 0.0): + ax.annotate( + f"{data_value:.{annotate_decimal}f}", + xy=(x[i], y[i]), + fontsize=10, + ) data_values.close() - def _draw_hatches(self, ax): + def _draw_hatches(self, ax: plt): """Draw the hatched regions requested.""" # Levels should be included in the settings dict here since they don't @@ -625,8 +776,10 @@ def _draw_hatches(self, ax): plt.legend(handles=handles, loc=[0.25, 0.03]) def _set_overlay_string(self): - """Creates the main title of the plot with select hatched and - contoured fields defined.""" + """ + Creates the main title of the plot with select hatched and + contoured fields defined. + """ f = self.field @@ -672,10 +825,7 @@ def _title(self): ) level, lev_unit = f.numeric_level(index_match=False) - if f.vspec.get("print_units", True): - units = f"({f.units}, shaded)" - else: - units = f"" + units = f"({f.units}, shaded)" if f.vspec.get("print_units", True) else "" # Title or Atmospheric level and unit in the high center if f.vspec.get("title"): @@ -693,11 +843,13 @@ def _title(self): fontsize=14, ) - def _wind_barbs(self, level): - """Draws the wind barbs. A decent stride can be found if you divide the + def _wind_barbs(self, level: bool | str): + """ + Draws the wind barbs. A decent stride can be found if you divide the number of grid points on the shorter side by 35. Subdomains are defined by lat,lon so the stride is set in the TILE_DEFS. For the globalCONUS - subdomains, further dividing by 2.5 works well.""" + subdomains, further dividing by 2.5 works well. + """ lev = level if not isinstance(level, bool) else self.field.level u, v = [f.data for f in self.wind_fields(lev)] @@ -707,28 +859,17 @@ def _wind_barbs(self, level): # Set the stride and size of the barbs to be plotted with a masked array. if full_tile: - if u.shape[0] < u.shape[1]: - stride = int(round(u.shape[0] / 35)) - else: - stride = int(round(u.shape[1] / 35)) + stride = round(u.shape[0] / 35) if u.shape[0] < u.shape[1] else round(u.shape[1] / 35) length = 5 else: stride = TILE_DEFS[tile]["stride"] length = TILE_DEFS[tile]["length"] if self.map.model == "globalCONUS": - stride = int(round(stride / 2.5)) + stride = round(stride / 2.5) length = 5 - if ( - self.map.model == "hrrr" - and self.model_name == "WFIP3-FULL" - and tile == "WFIP3-d02" - ): + if self.map.model == "hrrr" and self.model_name == "WFIP3-FULL" and tile == "WFIP3-d02": stride = 6 - if ( - self.map.model == "hrrr" - and self.model_name == "WFIP3-NEST" - and tile == "WFIP3-d02" - ): + if self.map.model == "hrrr" and self.model_name == "WFIP3-NEST" and tile == "WFIP3-d02": stride = 17 mask = np.ones_like(u) @@ -758,7 +899,7 @@ def _wind_barbs(self, level): sizes={"spacing": 0.25}, ) - def _xy_mesh(self, field): + def _xy_mesh(self, field: gribdata.fieldData): """Helper function to create mesh for various plot.""" lat, lon = field.latlons() @@ -775,7 +916,7 @@ class DiffMap(DataMap): and will not plot overlays and such. """ - def _colorbar(self, cc, ax): + def _colorbar(self, cc: plt, ax: plt): """Set the colorbar for a difference field.""" plt.colorbar( @@ -786,7 +927,7 @@ def _colorbar(self, cc, ax): shrink=1.0, ) - def _draw_panel(self, wind_barbs=False): + def _draw_panel(self): """Draw a map of the difference field.""" ax = self.map.ax @@ -798,7 +939,7 @@ def _draw_panel(self, wind_barbs=False): # in the linspace call in self._eq_contours. 21 seems reasonable, but is # arbitrary. colors = self.field.centered_diff(cmap="Spectral_r", nlev=21) - cf = self._draw_field( + return self._draw_field( ax=ax, colors=colors, extend="both", @@ -806,10 +947,9 @@ def _draw_panel(self, wind_barbs=False): func=self.map.m.contourf, levels=self._eq_contours(), ) - return cf def _eq_contours(self): - """Center the contours based on the data min/max""" + """Center the contours based on the data min/max.""" minval = np.amin(self.field.data) maxval = np.amax(self.field.data) @@ -834,10 +974,7 @@ def _title(self): ) level, lev_unit = f.numeric_level(index_match=False) - if f.vspec.get("print_units", True): - units = f"({f.units}, shaded)" - else: - units = f"" + units = f"({f.units}, shaded)" if f.vspec.get("print_units", True) else "" # Title or Atmospheric level and unit in the high center if f.vspec.get("title"): @@ -852,19 +989,24 @@ class MultiPanelDataMap(DataMap): """ Class that extends a DataMap for handling multiple panels. - Keyword arguments: + Keyword Arguments: last_panel flag for multipanel plots to designate last panel drawn + """ - def __init__(self, map_fields, map_, member, model_name=None, **kwargs): + def __init__( + self, map_fields: list, map_: plt, member: int, model_name: str | None = None, **kwargs + ): super().__init__(map_fields, map_, model_name=model_name) self.last_panel = kwargs.get("last_panel", False) self.member = str(member) - def draw(self, show=False): - """Main method for creating the plot. Set show=True to display the - figure from the command line.""" + def draw(self, show: bool = False): + """ + Main method for creating the plot. Set show=True to display the + figure from the command line. + """ cf = self._draw_panel(wind_barbs=False) @@ -885,7 +1027,7 @@ def draw(self, show=False): return cf def _label_member(self): - """Add the member label to the top left of the plot""" + """Add the member label to the top left of the plot.""" ax = self.map.ax ax.text( @@ -919,10 +1061,7 @@ def title(self): ) level, lev_unit = f.numeric_level(index_match=False) - if f.vspec.get("print_units", True): - units = f"({f.units}, shaded)" - else: - units = f"" + units = f"({f.units}, shaded)" if f.vspec.get("print_units", True) else "" # Title or Atmospheric level and unit in the high center if f.vspec.get("title"): @@ -951,142 +1090,3 @@ def title(self): fontsize=14, transform=ax.transAxes, ) - - -class MapFields: - """Class that packages all the field objects need for producing - desired map content, i.e. an object that contains all filled - contours, hatched spaces, and overlayed contours needed for a full - product.""" - - def __init__( - self, fhr, fields_spec, grib_path, level, name, map_type=None, **kwargs - ): - self.grib_path = grib_path - self.fhr = fhr - self.fields_spec = deepcopy(fields_spec) - self.level = level - self.map_type = map_type - self.model = kwargs.get("model") - self.name = name - self.tile = kwargs.get("tile", "full") - - self.map_spec = deepcopy(self.fields_spec[self.name][self.level]) - self.set_level(self.level, self.map_spec) - # Required if map_type is "diff" - self.grib_path2 = kwargs.get("grib_path2") - - def set_level(self, level, spec): - nlevel, _ = numeric_level(level=level, index_match=False) - level_info = any( - x - for x in cfgrib_spec(spec["cfgrib"], self.model) - for l in ("level", "top", "bottom", "Surface") - if l in x - ) - if nlevel and not level_info: - if spec["cfgrib"].get(self.model): - spec["cfgrib"][self.model]["level"] = nlevel - else: - spec["cfgrib"]["level"] = nlevel - # if spec["cfgrib"].get("level") is None and not spec["cfgrib"].get("stepRange") and not \ - # spec["cfgrib"].get("topLevel") and not \ - # spec["cfgrib"].get("typeOfLevel") == "surface" and not \ - # spec["cfgrib"].get("scaledValueOfFirstFixedSurface"): - - @property - def shaded(self): - cf = cfgrib_spec(self.map_spec["cfgrib"], self.model) - ds = gribfile.GribFile(self.grib_path, cf).contents - args = { - "ds": ds, - "fhr": self.fhr, - "level": self.level, - "model": self.model, - "short_name": self.name, - "spec": self.fields_spec, - "grib_path": self.grib_path, - } - field = gribdata.fieldData(**args) - - if self.map_type == "diff": - args["ds"] = gribfile.GribFile(self.grib_path2, cf).contents - args["grib_path"] == self.grib_path2 - field2 = gribdata.fieldData(**args) - field.data = field.values() - field2.values() - - return field - - @property - def contours(self): - """Return the list of contour fieldData objects""" - - # We won't plot contours on multipanel plots, or full global - # plots. - if self.map_type == "enspanel": - return [] - - if "global" in self.model and self.tile in ["full"]: - return [] - - return self._overlay_fields("contours") - - @property - def hatches(self): - """Return the list of hatch fieldData objects""" - - return self._overlay_fields("hatches") - - def wind_fields(self, level=None): - """Return u, v tuple of wind fields""" - - lev = level or self.level - winds = [] - for var in ("u", "v"): - wind_spec = self.fields_spec[var][lev] - self.set_level(lev, wind_spec) - ds = gribfile.GribFile( - self.grib_path, cfgrib_spec(wind_spec["cfgrib"], self.model) - ).contents - args = { - "ds": ds, - "fhr": self.fhr, - "level": lev, - "model": self.model, - "short_name": var, - "spec": self.fields_spec, - "grib_path": self.grib_path, - } - winds.append(gribdata.fieldData(**args)) - return winds - - def _overlay_fields(self, spec_sect): - """Generate a list of fieldData objects for the specified type - of overlay -- hatches or contours""" - - overlay_fields = [] - for overlay, overlay_kwargs in self.map_spec.get(spec_sect, {}).items(): - if "_" in overlay: - var, lev = overlay.split("_") - else: - var, lev = overlay, self.level - - overlay_spec = deepcopy(self.fields_spec[var][lev]) - self.set_level(lev, overlay_spec) - ds = gribfile.GribFile( - self.grib_path, cfgrib_spec(overlay_spec["cfgrib"], self.model) - ).contents - args = { - "ds": ds, - "fhr": self.fhr, - "level": lev, - "model": self.model, - "short_name": var, - "spec": self.fields_spec, - "grib_path": self.grib_path, - } - overlay_obj = gribdata.fieldData(**args) - # Set the attributes for the overlay field - overlay_obj.contour_kwargs = overlay_kwargs - overlay_fields.append(overlay_obj) - return overlay_fields diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 32474a2..87b16d1 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -5,7 +5,8 @@ """ from collections import OrderedDict -from functools import lru_cache +from functools import cached_property +from pathlib import Path import matplotlib.font_manager as fm import matplotlib.lines as mlines @@ -18,14 +19,15 @@ from metpy.plots import Hodograph, SkewT from metpy.units import units from mpl_toolkits.axes_grid1.inset_locator import inset_axes +from xarray import Dataset -import adb_graphics.datahandler.gribdata as gribdata -import adb_graphics.errors as errors -import adb_graphics.utils as utils +from adb_graphics import errors, utils +from adb_graphics.datahandler import gribdata class SkewTDiagram(gribdata.profileData): - """The class responsible for gathering all data needed from a grib file to + """ + The class responsible for gathering all data needed from a grib file to produce a Skew-T Log-P diagram. Input: @@ -43,7 +45,7 @@ class SkewTDiagram(gribdata.profileData): be included. """ - def __init__(self, ds, loc, **kwargs): + def __init__(self, ds: Dataset, loc: str, **kwargs): # Initialize on the temperature field since we need to gather # field-specific data from this object, e.g. dates, lat, lon, etc. @@ -57,7 +59,7 @@ def __init__(self, ds, loc, **kwargs): self.max_plev = kwargs.get("max_plev", 0) self.model_name = kwargs.get("model_name", "Analysis") - def _add_hydrometeors(self, hydro_subplot): + def _add_hydrometeors(self, hydro_subplot: plt): # pylint: disable=too-many-locals mixing_ratios = OrderedDict( { @@ -109,8 +111,9 @@ def _add_hydrometeors(self, hydro_subplot): lines = [ "Vert. Integrated Amt\n(Resolved, Total)\n" - + "(supercool layers shaded,\nwith filled markers)" + "(supercool layers shaded,\nwith filled markers)" ] + freezing_f = 32.0 for mixr, settings in mixing_ratios.items(): # Get the profile values @@ -130,9 +133,9 @@ def _add_hydrometeors(self, hydro_subplot): pres_sigma = pres_sigma - pres_layer # pressure at next sigma level mixr_total = mixr_total + pres_layer / gravity * profile[n] - # limit values to upper and lower values of lotting range - profile = np.where((profile > 0.0) & (profile < 1.0e-4), 1.0e-4, profile) - profile = np.where((profile > 10.0), 10.0, profile) + # limit values to upper and lower values of plotting range + profile = np.where((profile > 0.0) & (profile < 1.0e-4), 1.0e-4, profile) # noqa: PLR2004 + profile = np.where((profile > 10.0), 10.0, profile) # noqa: PLR2004 # plot line hydro_subplot.plot( @@ -146,8 +149,8 @@ def _add_hydrometeors(self, hydro_subplot): ) if mixr in ["clwmr", "rwmr"]: hydro_subplot.plot( - profile[temp.magnitude < 32.0], - pres[temp.magnitude < 32.0], + profile[temp.magnitude < freezing_f], + pres[temp.magnitude < freezing_f], settings.get("color"), fillstyle="full", linewidth=0.5, @@ -156,10 +159,10 @@ def _add_hydrometeors(self, hydro_subplot): ) layer = False for i, profile_lev in enumerate(profile): - if (profile_lev > 0.0 and temp[i].magnitude < 32.0) and not layer: + if (profile_lev > 0.0 and temp[i].magnitude < freezing_f) and not layer: layer = True p_base = pres[i].magnitude - elif (profile_lev <= 0.0 or temp[i].magnitude > 32.0) and layer: + elif (profile_lev <= 0.0 or temp[i].magnitude > freezing_f) and layer: # Shade the supercooled water depth p_top = pres[i - 1].magnitude rect = plt.Rectangle( @@ -174,8 +177,7 @@ def _add_hydrometeors(self, hydro_subplot): # compute vertically integrated amount and add legend line line = ( - f"{settings.get('label'):<7s} {mixr_total.magnitude:>10.3f} " - f"{settings.get('units')}" + f"{settings.get('label'):<7s} {mixr_total.magnitude:>10.3f} {settings.get('units')}" ) if scale != 1.0: line = ( @@ -215,7 +217,7 @@ def _add_hydrometeors(self, hydro_subplot): verticalalignment="top", ) - def _add_thermo_inset(self, skew): + def _add_thermo_inset(self, skew: plt): # Build up the text that goes in the thermo-dyniamics box lines = [] for name, items in self.thermo_variables.items(): @@ -223,15 +225,11 @@ def _add_thermo_inset(self, skew): decimals = items.get("decimals", 0) value = items["data"] if value != "--": - value = ( - int(value) - if decimals == 0 - else value.round(decimals=decimals).values - ) + value = int(value) if decimals == 0 else value.round(decimals=decimals).to_numpy() # Sure would have been nice to use a variable in the f string to # denote the format per variable. - line = f"{name.upper():<7s} {str(value):>6} {items['units']}" + line = f"{name.upper():<7s} {value!s:>6} {items['units']}" lines.append(line) contents = "\n".join(lines) @@ -248,8 +246,7 @@ def _add_thermo_inset(self, skew): verticalalignment="top", ) - @property - @lru_cache() + @cached_property def atmo_profiles(self): """ Return a dictionary of atmospheric data profiles for each variable @@ -313,8 +310,10 @@ def atmo_profiles(self): return atmo_vars def create_diagram(self): - """Calls the private methods for creating each component of the SkewT - Diagram.""" + """ + Calls the private methods for creating each component of the SkewT + Diagram. + """ skew, hydro_subplot = self._setup_diagram() self._title() @@ -326,12 +325,12 @@ def create_diagram(self): self._add_thermo_inset(skew) self._add_hydrometeors(hydro_subplot) - def create_csv(self, csv_path): + def create_csv(self, csv_path: Path | str): """Calls the private methods for writing each of the SkewT Data.""" self._write_profile(csv_path) - def _plot_hodograph(self, skew): + def _plot_hodograph(self, skew: plt): # Create an array that indicates which layer (10-3, 3-1, 0-1 km) the # wind belongs to. The array, agl, will be set to the height # corresponding to the top of the layer. The resulting array will look @@ -371,14 +370,12 @@ def _plot_hodograph(self, skew): # Local function to create a proxy line object for creating a legend on # a LineCollection returned from plot_colormapped. Using lines and # colors from outside scope. - def make_proxy(zval, idx=None, **kwargs): + def make_proxy(zval: int, idx: int | None = None, **kwargs): color = colors[idx] if idx < len(colors) else lines.cmap(zval - 1) return Line2D([0, 1], [0, 1], color=color, linewidth=line_width, **kwargs) # Make a list of proxies - proxies = [ - make_proxy(item, idx=i) for i, item in enumerate(intervals.magnitude) - ] + proxies = [make_proxy(item, idx=i) for i, item in enumerate(intervals.magnitude)] # Draw the legend ax.legend( @@ -389,11 +386,11 @@ def make_proxy(zval, idx=None, **kwargs): ) @staticmethod - def _plot_labels(skew): + def _plot_labels(skew: plt): skew.ax.set_xlabel("Temperature (F)") skew.ax.set_ylabel("Pressure (hPa)") - def _write_profile(self, csv_path): + def _write_profile(self, csv_path: str | Path): profiles = self.atmo_profiles # dictionary pres = profiles.get("pres").get("data") u = profiles.get("u").get("data") @@ -401,9 +398,7 @@ def _write_profile(self, csv_path): temp = profiles.get("temp").get("data").to("degC") sphum = profiles.get("sphum").get("data") - dewpt = np.array( - mpcalc.dewpoint_from_specific_humidity(sphum, temp, pres).to("degC") - ) + dewpt = np.array(mpcalc.dewpoint_from_specific_humidity(sphum, temp, pres).to("degC")) wspd = np.array(mpcalc.wind_speed(u, v)) wdir = np.array(mpcalc.wind_direction(u, v)) @@ -422,7 +417,7 @@ def _write_profile(self, csv_path): profile.to_csv(csv_path, index=False, float_format="%10.2f") - def _plot_profile(self, skew): + def _plot_profile(self, skew: plt): profiles = self.atmo_profiles # dictionary pres = profiles.get("pres").get("data") temp = profiles.get("temp").get("data") @@ -446,7 +441,7 @@ def _plot_profile(self, skew): linewidth=1.2, ) - def _plot_wind_barbs(self, skew): + def _plot_wind_barbs(self, skew: plt): # Pressure vs wind skew.plot_barbs( self.atmo_profiles.get("pres", {}).get("data"), @@ -472,17 +467,17 @@ def _setup_diagram(self): # display in Fahrenheit. # Fahrenheit tick labels that will display - labels_F = list(range(-20, 125, 20)) * units.degF + labels_f = list(range(-20, 125, 20)) * units.degF # Celcius VALUES for those tick marks. These put the ticks in the right # spot. - labels = labels_F.to("degC").magnitude + labels = labels_f.to("degC").magnitude # Set the MINOR tick values to the CELCIUS values. skew.ax.xaxis.set_minor_locator(FixedLocator(labels)) # Set the MINOR tick labels to the FAHRENHEIT values. - skew.ax.set_xticklabels(labels_F.magnitude, minor=True) + skew.ax.set_xticklabels(labels_f.magnitude, minor=True) skew.ax.tick_params(which="minor", length=8) # Turn off the MAJOR (celcius) tick marks, label the grid lines inside @@ -558,8 +553,7 @@ def _setup_diagram(self): return skew, hydro_subplot - @property - @lru_cache() + @cached_property def thermo_variables(self): """ Return an ordered dictionary of thermodynamic variables needed for the skewT. diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index fc110d4..71c8fd8 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -9,7 +9,6 @@ from itertools import chain import numpy as np -import yaml from matplotlib import cm from matplotlib import colors as mpcolors from metpy.plots import ctables @@ -24,15 +23,17 @@ class VarSpec(abc.ABC): @property def aod_colors(self) -> np.ndarray: - """Default color map for AOD products and chem products""" + """Default color map for AOD products and chem products.""" grays = cm.get_cmap("Greys", 2)([0]) others = cm.get_cmap(self.vspec.get("cmap"), 15)(range(1, 15, 1)) return np.concatenate((grays, others)) - def centered_diff(self, cmap=None, nlev=None): - """Returns the colors specified by levels and cmap in default spec, but - with white center.""" + def centered_diff(self, cmap: str | None = None, nlev: int | None = None): + """ + Returns the colors specified by levels and cmap in default spec, but + with white center. + """ if nlev is None: clevs = self.vspec.get("clevs") @@ -51,7 +52,7 @@ def centered_diff(self, cmap=None, nlev=None): @property def cin_colors(self) -> np.ndarray: - """Default color map for Convective Inhibition""" + """Default color map for Convective Inhibition.""" ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( [120, 100, 90, 85, 80, 70, 60, 50, 25, 20, 18] @@ -62,18 +63,22 @@ def cin_colors(self) -> np.ndarray: @property @abc.abstractmethod def clevs(self) -> np.ndarray: - """An abstract method responsible for returning the np.ndarray of contour - levels for a given field. Numpy arange supports non-integer values.""" + """ + An abstract method responsible for returning the np.ndarray of contour + levels for a given field. Numpy arange supports non-integer values. + """ @property @abc.abstractproperty def vspec(self): - """The variable plotting specification. The level-specific subgroup - from a config file like default_specs.yml.""" + """ + The variable plotting specification. The level-specific subgroup + from a config file like default_specs.yml. + """ @property def ceil_colors(self) -> np.ndarray: - """Default color map for Ceiling""" + """Default color map for Ceiling.""" grays = cm.get_cmap("Greys", 2)([0]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( @@ -83,17 +88,15 @@ def ceil_colors(self) -> np.ndarray: @property def cldcov_colors(self) -> np.ndarray: - """Default color map for Cloud Cover""" + """Default color map for Cloud Cover.""" grays = cm.get_cmap("Greys", 7)([0, 1, 3]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( - [120, 100, 90, 85, 80, 70, 60, 50, 25, 20] - ) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([120, 100, 90, 85, 80, 70, 60, 50, 25, 20]) return np.concatenate((grays, ncar)) @property def cref_colors(self) -> np.ndarray: - """Default color map for Reflectivity""" + """Default color map for Reflectivity.""" ncolors = len(self.clevs) - 1 grays = cm.get_cmap("Greys", 5)([0]) @@ -106,7 +109,7 @@ def fire_power_colors(self) -> np.ndarray: """Default color map for fire power plot.""" # The scatter plot utility won't accept anything but named colors - colors = [ + return [ "white", "lightskyblue", "darkblue", @@ -116,14 +119,12 @@ def fire_power_colors(self) -> np.ndarray: "firebrick", ] - return colors - @property def smoke_emissions_colors(self) -> np.ndarray: """Default color map for smoke emissions plot.""" # The scatter plot utility won't accept anything but named colors - colors = [ + return [ "white", "rebeccapurple", "royalblue", @@ -140,37 +141,30 @@ def smoke_emissions_colors(self) -> np.ndarray: "firebrick", ] - return colors - def flru_colors(self) -> np.ndarray: - """Default color map for Ceiling""" + """Default color map for Ceiling.""" - ctable = cm.get_cmap(self.vspec.get("cmap"), 128)([50, 15, 90, 120]) - return ctable + return cm.get_cmap(self.vspec.get("cmap"), 128)([50, 15, 90, 120]) @property def frzn_colors(self) -> np.ndarray: - """Default color map for Frozen Precip %""" + """Default color map for Frozen Precip %.""" grays = cm.get_cmap("Greys", 7)([0, 2]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( - [120, 90, 85, 80, 70, 60, 50, 25, 20, 15] - ) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([120, 90, 85, 80, 70, 60, 50, 25, 20, 15]) return np.concatenate((grays, ncar)) @property def goes_colors(self) -> np.ndarray: - """Default color map for simulated GOES IR satellite""" + """Default color map for simulated GOES IR satellite.""" grays = cm.get_cmap("Greys_r", 33)(range(33)) - ctable2 = ctables.colortables.get_colortable(self.vspec.get("cmap"))( - range(65, 150) - ) + ctable2 = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(65, 150)) return np.concatenate((grays[-1:], grays, ctable2, grays[1:])) @property def graupel_colors(self) -> np.ndarray: - """Default color map for Max Vertically Integrated Graupel""" + """Default color map for Max Vertically Integrated Graupel.""" grays = cm.get_cmap("Greys", 3)([0]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)(range(20, 128, 6)) @@ -178,57 +172,48 @@ def graupel_colors(self) -> np.ndarray: @property def hail_colors(self) -> np.ndarray: - """Default color map for Hail diameter""" + """Default color map for Hail diameter.""" grays = cm.get_cmap("Greys", 2)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( - [100, 15, 18, 20, 25, 60, 80, 85, 90] - ) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([100, 15, 18, 20, 25, 60, 80, 85, 90]) return np.concatenate((grays, ncar)) @property def heat_flux_colors(self) -> np.ndarray: - """Default color map for Latent/Sensible Heat Flux""" + """Default color map for Latent/Sensible Heat Flux.""" grays = cm.get_cmap("Greys", 8)([6, 5, 4, 3, 2]) - ctable = ctables.colortables.get_colortable(self.vspec.get("cmap"))( - range(0, 33, 2) - ) + ctable = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(0, 33, 2)) return np.concatenate((grays, ctable)) @property def heat_flux_colors_g(self) -> np.ndarray: - """Default color map for Latent/Sensible Heat Flux""" + """Default color map for Latent/Sensible Heat Flux.""" - colors = cm.get_cmap(self.vspec.get("cmap"), 128)(range(15, 112, 8)) - return colors + return cm.get_cmap(self.vspec.get("cmap"), 128)(range(15, 112, 8)) @property def heat_flux_colors_l(self) -> np.ndarray: - """Default color map for Latent/Sensible Heat Flux""" + """Default color map for Latent/Sensible Heat Flux.""" - colors = cm.get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) - return colors + return cm.get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) @property def heat_flux_colors_s(self) -> np.ndarray: - """Default color map for Latent/Sensible Heat Flux""" + """Default color map for Latent/Sensible Heat Flux.""" - colors = cm.get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) - return colors + return cm.get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) @property def icprb_colors(self) -> np.ndarray: - """Default color map for Icing Probability""" + """Default color map for Icing Probability.""" grays = cm.get_cmap("Greys", 2)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( - [25, 35, 50, 60, 70, 80, 85, 90, 100] - ) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([25, 35, 50, 60, 70, 80, 85, 90, 100]) return np.concatenate((grays, ncar)) def icsev_colors(self) -> np.ndarray: - """Default color map for Icing Severity""" + """Default color map for Icing Severity.""" white = cm.get_cmap("Greys", 2)([0]) blues = cm.get_cmap(self.vspec.get("cmap"), 9)([2, 3, 4, 6, 8]) @@ -236,16 +221,15 @@ def icsev_colors(self) -> np.ndarray: @property def lcl_colors(self) -> np.ndarray: - """Default color map for Lifted Condensation Level""" + """Default color map for Lifted Condensation Level.""" - ctable = ctables.colortables.get_colortable(self.vspec.get("cmap"))( + return ctables.colortables.get_colortable(self.vspec.get("cmap"))( range(50, 180, 7) ) # rainbow - return ctable @property def lifted_index_colors(self) -> np.ndarray: - """Default color map for Lifted Index""" + """Default color map for Lifted Index.""" ctable = cm.get_cmap(self.vspec.get("cmap"), 128)(range(4, 125, 4)) ctable[14] = [1, 1, 1, 1] @@ -254,7 +238,7 @@ def lifted_index_colors(self) -> np.ndarray: @property def mdn_colors(self) -> np.ndarray: - """Default color map for Max Downdraft""" + """Default color map for Max Downdraft.""" grays = cm.get_cmap("Greys", 2)([0]) others = cm.get_cmap(self.vspec.get("cmap"), 18)(range(18, 1, -1)) @@ -262,7 +246,7 @@ def mdn_colors(self) -> np.ndarray: @property def mean_vvel_colors(self) -> np.ndarray: - """Default color map for Mean Vertical Velocity""" + """Default color map for Mean Vertical Velocity.""" ctable = cm.get_cmap(self.vspec.get("cmap"), 128)(range(0, 114, 6)) ctable[9] = [1, 1, 1, 1] @@ -270,7 +254,7 @@ def mean_vvel_colors(self) -> np.ndarray: @property def mup_colors(self) -> np.ndarray: - """Default color map for Max Updraft""" + """Default color map for Max Updraft.""" grays = cm.get_cmap("Greys", 2)([0]) others = cm.get_cmap(self.vspec.get("cmap"), 18)(range(1, 18, 1)) @@ -278,25 +262,21 @@ def mup_colors(self) -> np.ndarray: @property def pbl_colors(self) -> np.ndarray: - """Default color map for PBL Height""" + """Default color map for PBL Height.""" - return ctables.colortables.get_colortable(self.vspec.get("cmap"))( - range(15, 60, 3) - ) + return ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(15, 60, 3)) @property def pcp_colors(self) -> np.ndarray: - """Default color map for Hourly Precipitation""" + """Default color map for Hourly Precipitation.""" grays = cm.get_cmap("Greys", 6)([0, 3]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( - [25, 50, 60, 70, 80, 85, 90, 115] - ) + ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([25, 50, 60, 70, 80, 85, 90, 115]) return np.concatenate((grays, ncar)) @property def pcp_colors_high(self) -> np.ndarray: - """High values color map for Hourly Precipitation""" + """High values color map for Hourly Precipitation.""" grays = cm.get_cmap("Greys", 2)([0]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([70, 80, 85, 90, 115]) @@ -304,7 +284,7 @@ def pcp_colors_high(self) -> np.ndarray: @property def pmsl_colors(self) -> np.ndarray: - """Default color map for Surface Pressure""" + """Default color map for Surface Pressure.""" ncolors = len(self.vspec.get("clevs")) incr = 128 // ncolors @@ -313,18 +293,16 @@ def pmsl_colors(self) -> np.ndarray: @property def ps_colors(self) -> np.ndarray: - """Default color map for Surface Pressure""" + """Default color map for Surface Pressure.""" grays = cm.get_cmap("Greys", 13)(range(13)) segments = [[16, 53], [86, 105], [110, 151, 2], [172, 202, 2]] - ncar = cm.get_cmap("gist_ncar", 200)( - list(chain(*[range(*i) for i in segments])) - ) + ncar = cm.get_cmap("gist_ncar", 200)(list(chain(*[range(*i) for i in segments]))) return np.concatenate((grays, ncar)) @property def pw_colors(self) -> np.ndarray: - """Default color map for Precipitable Water""" + """Default color map for Precipitable Water.""" grays = cm.get_cmap("Greys", 5)([1, 3]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( @@ -336,7 +314,7 @@ def pw_colors(self) -> np.ndarray: @property def radiation_colors(self) -> np.ndarray: - """Default color map for Longwave Radiation""" + """Default color map for Longwave Radiation.""" grays = cm.get_cmap("Greys", 2)([0]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)(range(0, 126, 5)) @@ -344,21 +322,21 @@ def radiation_colors(self) -> np.ndarray: @property def radiation_bw_colors(self) -> np.ndarray: - """Default grayscale map for Outgoing Shortwave Radiation""" + """Default grayscale map for Outgoing Shortwave Radiation.""" return cm.get_cmap(self.vspec.get("cmap"), 128)(range(30, 110)) @property def radiation_mix_colors(self) -> np.ndarray: - """Default color map for Longwave Radiation""" + """Default color map for Longwave Radiation.""" - ncar = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(0, 40)) + ncar = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(40)) grays = cm.get_cmap("Greys", 100)(range(10, 100)) return np.concatenate((ncar, grays)) @property def rainbow11_colors(self) -> np.ndarray: - """Default color map for Hourly Wildfire Potential""" + """Default color map for Hourly Wildfire Potential.""" grays = cm.get_cmap("Greys", 2)([0]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( @@ -368,7 +346,7 @@ def rainbow11_colors(self) -> np.ndarray: @property def rainbow12_colors(self) -> np.ndarray: - """Default color map for ACPCP, ACSNOD, HLCY, RH, and SNOD""" + """Default color map for ACPCP, ACSNOD, HLCY, RH, and SNOD.""" grays = cm.get_cmap("Greys", 2)([0]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( @@ -378,13 +356,13 @@ def rainbow12_colors(self) -> np.ndarray: @property def rainbow12_reverse(self) -> np.ndarray: - """Default color map for min helicity""" + """Default color map for min helicity.""" return np.flip(self.rainbow12_colors, 0) @property def rainbow16_colors(self) -> np.ndarray: - """Default color map for helicity""" + """Default color map for helicity.""" grays = cm.get_cmap("Greys", 5)([0, 2]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( @@ -394,7 +372,7 @@ def rainbow16_colors(self) -> np.ndarray: @property def shear_colors(self) -> np.ndarray: - """Default color map for Vertical Shear""" + """Default color map for Vertical Shear.""" ctable = cm.get_cmap(self.vspec.get("cmap"), 16)(range(5, 15)) ctable[9] = [1, 1, 1, 1] @@ -402,13 +380,11 @@ def shear_colors(self) -> np.ndarray: @property def slw_colors(self) -> np.ndarray: - """Default color map for Max Vertically Integrated Graupel""" + """Default color map for Max Vertically Integrated Graupel.""" white = cm.get_cmap("Greys", 3)([0]) purples = cm.get_cmap("nipy_spectral", 30)([3, 1]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 15)( - [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] - ) + ncar = cm.get_cmap(self.vspec.get("cmap"), 15)([2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]) return np.concatenate((white, purples, ncar)) @property @@ -423,7 +399,7 @@ def smoke_colors(self) -> np.ndarray: @property def snow_colors(self) -> np.ndarray: - """Default color map for Snow fields""" + """Default color map for Snow fields.""" grays = cm.get_cmap("Greys", 5)([0, 2]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( @@ -433,14 +409,13 @@ def snow_colors(self) -> np.ndarray: @property def soilm_colors(self) -> np.ndarray: - """Default color map for Soil Moisture Availability""" + """Default color map for Soil Moisture Availability.""" - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)(range(0, 122, 11)) - return ncar + return cm.get_cmap(self.vspec.get("cmap"), 128)(range(0, 122, 11)) @property def soilw_colors(self) -> np.ndarray: - """Default color map for Soil Moisture""" + """Default color map for Soil Moisture.""" grays = cm.get_cmap("Greys", 2)([1]) ncar = cm.get_cmap(self.vspec.get("cmap"), 110)( @@ -450,14 +425,14 @@ def soilw_colors(self) -> np.ndarray: @property def t_colors(self) -> np.ndarray: - """Default color map for Potential Temperature""" + """Default color map for Potential Temperature.""" ncolors = len(self.clevs) return cm.get_cmap(self.vspec.get("cmap", "jet"), ncolors)(range(ncolors)) @property def tsfc_colors(self) -> np.ndarray: - """Default color map for Surface Temperature""" + """Default color map for Surface Temperature.""" purples = cm.get_cmap("Purples", 16)([14, 12, 8, 6, 4, 2]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( @@ -468,16 +443,13 @@ def tsfc_colors(self) -> np.ndarray: @property def terrain_colors(self) -> np.ndarray: - """Default color map for Terrain""" + """Default color map for Terrain.""" - ctable = ctables.colortables.get_colortable(self.vspec.get("cmap"))( - range(54, 157, 6) - ) - return ctable + return ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(54, 157, 6)) @property def ua_temp_colors(self) -> np.ndarray: - """Default color map for Upper-Air Temperature""" + """Default color map for Upper-Air Temperature.""" grays = cm.get_cmap("Greys", 27)(range(17, 1, -2)) purples = cm.get_cmap("Purples", 27)(range(17, 1, -2)) @@ -488,7 +460,8 @@ def ua_temp_colors(self) -> np.ndarray: @property def vis_colors(self) -> np.ndarray: - """Default color map for Visibility + """ + Default color map for Visibility. section names are based on Aviation Flight Rule visibility categories LIFR (Low Instrument Flight Rules) -- less than 1 mile @@ -499,10 +472,10 @@ def vis_colors(self) -> np.ndarray: the gray range is arbitrary compared to the official flight levels """ - lifr = cm.get_cmap("RdPu_r", 20)(range(0, 11)) - ifr = cm.get_cmap("autumn", 30)(range(0, 30)) + lifr = cm.get_cmap("RdPu_r", 20)(range(11)) + ifr = cm.get_cmap("autumn", 30)(range(30)) mvfr = cm.get_cmap("Blues", 20)(range(10, 20)) - vfr1 = cm.get_cmap("YlGn_r", 60)(range(0, 50)) + vfr1 = cm.get_cmap("YlGn_r", 60)(range(50)) vfr2 = cm.get_cmap("Greys", 25)(np.full(10, 9)) hi01 = cm.get_cmap("Greys", 25)(np.full(10, 6)) hi02 = cm.get_cmap("Greys", 25)(np.full(20, 3)) @@ -512,7 +485,7 @@ def vis_colors(self) -> np.ndarray: @property def vvel_colors(self) -> np.ndarray: - """Default color map for Vetical Velocity""" + """Default color map for Vetical Velocity.""" ncar1 = cm.get_cmap(self.vspec.get("cmap"), 128)([15, 18, 20, 25]) grays = cm.get_cmap("Greys", 2)([0]) @@ -521,7 +494,7 @@ def vvel_colors(self) -> np.ndarray: @property def vort_colors(self) -> np.ndarray: - """Default color map for Absolute Vorticity""" + """Default color map for Absolute Vorticity.""" grays = cm.get_cmap("Greys", 2)([0]) ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( @@ -531,7 +504,7 @@ def vort_colors(self) -> np.ndarray: @property def wind_colors(self) -> np.ndarray: - """Default color map for Wind Speed""" + """Default color map for Wind Speed.""" low = cm.get_cmap(self.vspec.get("cmap"), 129)(range(129, 109, -5)) high1 = cm.get_cmap(self.vspec.get("cmap"), 129)(range(16, 29, 3)) @@ -540,7 +513,7 @@ def wind_colors(self) -> np.ndarray: @property def wind_colors_high(self) -> np.ndarray: - """Default color map for High Wind Speed""" + """Default color map for High Wind Speed.""" low = cm.get_cmap(self.vspec.get("cmap"), 129)(range(129, 108, -7)) high1 = cm.get_cmap(self.vspec.get("cmap"), 129)(range(16, 29, 4)) diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index e26300c..834ee9f 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -4,38 +4,39 @@ """ import argparse -import datetime as dt import functools import glob -import importlib as il -import os import subprocess import sys import time +from collections.abc import Callable +from datetime import datetime, timedelta +from importlib.util import find_spec from math import atan2, degrees from multiprocessing import Process +from pathlib import Path from string import ascii_letters, digits import numpy as np import yaml -def cfgrib_spec(config, model): +def cfgrib_spec(config: dict, model: str) -> dict: if spec := config.get(model): return spec return config -def create_zip(files_to_zip, zipf): +def create_zip(files_to_zip: list[str] | list[Path], zipf: Path | str): """Create a zip file. Use a locking mechanism -- write a lock file to disk.""" - lock_file = f"{zipf}._lock" + lock_file = Path(f"{zipf}._lock") retry = 2 count = 0 while True: - if not os.path.exists(lock_file): + if not lock_file.exists(): # Create the lock - fd = open(lock_file, "w") + fd = lock_file.open() print(f"Writing to zip file {zipf} for files like: {files_to_zip[0][-10:]}") cmd = f"zip -uj {zipf} {' '.join(files_to_zip)}" @@ -46,7 +47,7 @@ def create_zip(files_to_zip, zipf): check=True, shell=True, ) - except: # pylint: disable=bare-except + except: # noqa: E722 print(f"Error on writing zip file! {sys.exc_info()[0]}") count += 1 if count >= retry: @@ -54,19 +55,17 @@ def create_zip(files_to_zip, zipf): else: # Zipping was successful. Remove files that were zipped for file_to_zip in files_to_zip: - if os.path.exists(file_to_zip): - os.remove(file_to_zip) + Path(file_to_zip).unlink(missing_ok=True) finally: # Remove the lock fd.close() - if os.path.exists(lock_file): - os.remove(lock_file) + lock_file.unlink(missing_ok=True) break # Wait before trying to obtain the lock on the file time.sleep(5) -def fhr_list(args): +def fhr_list(args: list[int]) -> list[int]: """ Given an argparse list argument, return the sequence of forecast hours to process. @@ -92,13 +91,15 @@ def fhr_list(args): return args -def from_datetime(date): +def from_datetime(date: datetime) -> str: """Return a string like YYYYMMDDHH given a datetime object.""" - return dt.datetime.strftime(date, "%Y%m%d%H") + return datetime.strftime(date, "%Y%m%d%H") def get_func(val: str): """ + Gets a callable function. + Given an input string, val, returns the corresponding callable function. This function is borrowed from stackoverflow.com response to "Python: YAML dictionary of functions: how to load without converting to strings." @@ -110,24 +111,20 @@ def get_func(val: str): module_name = "__main__" fun_name = val - mod_spec = il.util.find_spec(module_name, package="adb_graphics") + mod_spec = find_spec(module_name, package="adb_graphics") if mod_spec is None: - mod_spec = il.util.find_spec("." + module_name, package="adb_graphics") + mod_spec = find_spec("." + module_name, package="adb_graphics") try: __import__(mod_spec.name) - except ImportError as exc: - print( - f"Could not load {module_name} while trying to locate function in get_func" - ) - raise exc + except ImportError: + print(f"Could not load {module_name} while trying to locate function in get_func") + raise module = sys.modules[mod_spec.name] - fun = getattr(module, fun_name) - return fun + return getattr(module, fun_name) -# pylint: disable=unused-argument -def join_ranges(loader, node): +def join_ranges(loader: str, node: str) -> np.ndarray: # noqa: ARG001 """ Merge two or more different ranges into a single array for color bar clevs. @@ -144,9 +141,7 @@ def join_ranges(loader, node): list_ = [] for seq_node in node.value: - range_args = [] - for scalar_node in seq_node.value: - range_args.append(float(scalar_node.value)) + range_args = [float(scalar_node.value) for scalar_node in seq_node.value] list_.append(np.arange(*range_args)) @@ -157,8 +152,7 @@ def join_ranges(loader, node): yaml.add_constructor("!join_ranges", join_ranges, Loader=yaml.Loader) -# pylint: disable=invalid-name, too-many-locals -def label_line(ax, label, segment, **kwargs): +def label_line(ax: list, label: list, segment: list, **kwargs): """ Label a single line with line2D label data. @@ -233,7 +227,7 @@ def label_line(ax, label, segment, **kwargs): ax.text(x, y, label, rotation=trans_angle, **kwargs) -def label_lines(ax, lines, labels, offset=0, **kwargs): +def label_lines(ax: list, lines: list, labels: list[str], offset: float = 0, **kwargs): """ Plots labels on a set of lines from SkewT. @@ -259,19 +253,19 @@ def label_lines(ax, lines, labels, offset=0, **kwargs): label_line(ax, label, line, align=True, offset=offset, **kwargs) -def load_sites(arg): +def load_sites(arg: str | Path) -> list[str]: """Check that the sites file exists, and return its contents.""" # Check that the file exists path = path_exists(arg) - with open(path, "r") as sites_file: - sites = sites_file.readlines() - return sites + with path.open() as sites_file: + return sites_file.readlines() -def uniq_wgrib2_list(inlist): - """Given a list of wgrib2 output fields, returns a uniq list of fields for +def uniq_wgrib2_list(inlist: list[str]): + """ + Given a list of wgrib2 output fields, returns a uniq list of fields for simplifying a grib2 dataset. Uniqueness is defined by the wgrib output from field 3 (colon delimted) onward, although the original full grib record must be included in the wgrib2 command below. @@ -281,7 +275,7 @@ def uniq_wgrib2_list(inlist): uniq_list = [] for infield in inlist: infield_info = infield.split(":") - if len(infield_info) <= 3: + if len(infield_info) <= 3: # noqa: PLR2004 continue infield_str = ":".join(infield_info[3:]) if infield_str not in uniq_field_set: @@ -291,12 +285,13 @@ def uniq_wgrib2_list(inlist): return uniq_list -def load_specs(arg): +def load_specs(arg: str | Path) -> dict: """Check to make sure arg file exists. Return its contents.""" - spec_file = path_exists(arg) + spec_file = Path(arg) + assert spec_file.exists() - with open(spec_file, "r") as fn: + with spec_file.open() as fn: specs = yaml.load(fn, Loader=yaml.Loader) specs["file"] = spec_file @@ -304,7 +299,7 @@ def load_specs(arg): return specs -def numeric_level(index_match=True, level=None, split=None): +def numeric_level(index_match: bool = True, level: str | None = None): """ Split the numeric level and unit associated with the level key. @@ -319,10 +314,7 @@ def numeric_level(index_match=True, level=None, split=None): # Convert the numbers to a list, and make integers or floats if lev_val: - if split is not None: - lev_val = [int(lev) for lev in lev_val] - else: - lev_val = [float(lev_val) if "." in lev_val else int(lev_val)] + lev_val = [float(lev_val) if "." in lev_val else int(lev_val)] # Gather all the letters lev_unit = "".join([c for c in level if c in ascii_letters]) @@ -338,7 +330,7 @@ def numeric_level(index_match=True, level=None, split=None): return lev_val, lev_unit -def old_enough(age, file_path): +def old_enough(age: int, file_path: Path): """ Helper function to test the age of a file. @@ -352,23 +344,24 @@ def old_enough(age, file_path): bool whether the file is at least age minutes old """ - file_time = dt.datetime.fromtimestamp(os.path.getctime(file_path)) - max_age = dt.datetime.now() - dt.timedelta(minutes=age) + file_time = datetime.fromtimestamp(file_path.stat().st_ctime) + max_age = datetime.now() - timedelta(minutes=age) return file_time < max_age -def path_exists(path: str): +def path_exists(path: Path | str): """Checks whether a file exists, and returns the path if it does.""" - if not os.path.exists(path): + ret_path = Path(path) + if not ret_path.exists(): msg = f"{path} does not exist!" raise argparse.ArgumentTypeError(msg) - return path + return ret_path -def timer(func): +def timer(func: Callable): """Decorator function that provides an elapsed time for a method.""" @functools.wraps(func) @@ -383,15 +376,16 @@ def wrapper_timer(*args, **kwargs): return wrapper_timer -def to_datetime(string): +def to_datetime(string: str): """Return a datetime object give a string like YYYYMMDDHH.""" - return dt.datetime.strptime(string, "%Y%m%d%H") + return datetime.strptime(string, "%Y%m%d%H") @timer -def zip_products(fhr, workdir, zipfiles): - """Spin up a subprocess to zip all the product files into the staged zip files. +def zip_products(fhr: int, workdir: Path, zipfiles: dict) -> None: + """ + Spin up a subprocess to zip all the product files into the staged zip files. Input: @@ -408,7 +402,7 @@ def zip_products(fhr, workdir, zipfiles): file_tmpl = f"*.skewt.*_f{fhr:03d}.csv" else: file_tmpl = f"*_{tile}_*{fhr:02d}.png" - product_files = glob.glob(os.path.join(workdir, file_tmpl)) + product_files = glob.glob(workdir / file_tmpl) if product_files: zip_proc = Process( group=None, diff --git a/conftest.py b/conftest.py index 0de6ef4..d6fdf78 100644 --- a/conftest.py +++ b/conftest.py @@ -26,13 +26,13 @@ def pytest_addoption(parser): @pytest.fixture def natfile(request): - """Interface to pass a grib file to pytest""" + """Interface to pass a grib file to pytest.""" return request.config.getoption("--nat-file") @pytest.fixture def prsfile(request): - """Interface to pass a grib file to pytest""" + """Interface to pass a grib file to pytest.""" return request.config.getoption("--prs-file") diff --git a/create_graphics.py b/create_graphics.py index bc47767..b5b6902 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -9,22 +9,21 @@ mpl.use("Agg") # pylint: enable=wrong-import-position, wrong-import-order -import argparse import copy import glob -import os import random import string import subprocess import sys import time +from argparse import ArgumentError, ArgumentParser, Namespace from multiprocessing import Pool import yaml +from libpath import Path -import adb_graphics.errors as errors -import adb_graphics.utils as utils -from adb_graphics.datahandler import gribfile +from adb_graphics import errors, utils +from adb_graphics.datahandler.gribfile import GribFile, GribFiles from adb_graphics.figure_builders import parallel_maps, parallel_skewt from adb_graphics.figures import maps @@ -36,36 +35,42 @@ LOG_BREAK = f"{('-' * 80)}\n{('-' * 80)}" -def check_file(cla, fhr, data_root=None, file_tmpl=None, mem=None): - """Given the command line arguments, the forecast hour, and a potential - ensemble member, build a full path to the file and ensure it exists.""" +def check_file( + cla: Namespace, + fhr: int, + data_root: Path | None = None, + file_tmpl: str | None = None, + mem: int | None = None, +) -> (Path, bool): + """ + Given the command line arguments, the forecast hour, and a potential + ensemble member, build a full path to the file and ensure it exists. + """ if data_root is None: data_root = cla.data_root[0] if file_tmpl is None: file_tmpl = cla.file_tmpl[0] - grib_path = os.path.join(data_root, file_tmpl) + grib_path = data_root / file_tmpl if mem is not None: grib_path = grib_path.format(FCST_TIME=fhr, mem=mem) else: grib_path = grib_path.format(FCST_TIME=fhr) print(f"Checking on file {grib_path}") - old_enough = ( - utils.old_enough(cla.data_age, grib_path) - if os.path.exists(grib_path) - else False - ) + old_enough = utils.old_enough(cla.data_age, grib_path) if grib_path.exists() else False return grib_path, old_enough -def create_skewt(cla, fhr, grib_path, workdir): - """Generate arguments for parallel processing of Skew T graphics, - and generate a pool of workers to complete the tasks.""" +def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): + """ + Generate arguments for parallel processing of Skew T graphics, + and generate a pool of workers to complete the tasks. + """ # Create the file object to load the contents - gfile = gribfile.GribFile(grib_path) + gfile = GribFile(grib_path) args = [(cla, fhr, gfile.contents, site, workdir) for site in cla.sites] @@ -74,9 +79,13 @@ def create_skewt(cla, fhr, grib_path, workdir): pool.starmap(parallel_skewt, args) -def create_maps(cla, fhr, grib_path, workdir, grib_path2=None): - """Generate arguments for parallel processing of plan-view maps and - generate a pool of workers to complete the task.""" +def create_maps( + cla: Namespace, fhr: int, grib_path: Path, workdir: Path, grib_path2: Path | None = None +): + """ + Generate arguments for parallel processing of plan-view maps and + generate a pool of workers to complete the task. + """ model = cla.images[0] for tile in cla.tiles: @@ -111,10 +120,12 @@ def create_maps(cla, fhr, grib_path, workdir, grib_path2=None): pool.starmap(parallel_maps, args) -def gather_gribfiles(cla, fhr, filename, gribfiles): - """Returns the appropriate gribfiles object for the type of graphics being +def gather_gribfiles(cla: Namespace, fhr: int, filename: str, gribfiles: None | GribFiles): + """ + Returns the appropriate gribfiles object for the type of graphics being generated -- whether it's for a single forecast time or all forecast lead - times.""" + times. + """ filenames = {"01fcst": [], "free_fcst": []} @@ -130,7 +141,7 @@ def gather_gribfiles(cla, fhr, filename, gribfiles): # Create a new GribFiles object, include all hours, or just this one, # depending on command line argument flag - gribfiles = gribfile.GribFiles( + gribfiles = GribFiles( coord_dims={"fcst_hr": [fhr]}, filenames=filenames, filetype=cla.file_type, @@ -144,10 +155,12 @@ def gather_gribfiles(cla, fhr, filename, gribfiles): return gribfiles -def generate_tile_list(arg_list): - """Given the input arguments -- a list if the argument is provided, return +def generate_tile_list(arg_list: list) -> list[str]: + """ + Given the input arguments -- a list if the argument is provided, return the list. If no arg is provided, defaults to the full domain, and if 'all' - is provided, the full domain, and all subdomains are plotted.""" + is provided, the full domain, and all subdomains are plotted. + """ if not arg_list: return ["full"] @@ -158,14 +171,15 @@ def generate_tile_list(arg_list): hrrr_ak_only = ("Anchorage", "AKRange", "Juneau") rap_only = ("AK", "AKZoom", "conus", "HI") if "all" in arg_list: - all_list = ["full"] + list(maps.TILE_DEFS.keys()) + all_list = ["full", *list(maps.TILE_DEFS.keys())] return [tile for tile in all_list if tile not in hrrr_ak_only + rap_only] return arg_list -def load_images(arg): - """Check that input image file exists, and that it contains the +def load_images(arg: Path | str): + """ + Check that input image file exists, and that it contains the requested section. Return a 2-list (required by argparse) of the file path and dictionary of images to be created. """ @@ -173,24 +187,26 @@ def load_images(arg): # Agument is expected to be a 2-list of file name and internal # section name. - image_file = arg[0] + image_file = Path(arg[0]) image_set = arg[1] # Check that the file exists - image_file = utils.path_exists(image_file) + assert image_file.exists() # Load yaml file - with open(image_file, "r") as fn: + with Path.open(image_file) as fn: images = yaml.load(fn, Loader=yaml.Loader)[image_set] return [images.get("model"), images.get("variables")] -def parse_args(argv): - """Set up argparse command line arguments, and return the Namespace - containing the settings.""" +def parse_args(argv: list) -> Namespace: + """ + Set up argparse command line arguments, and return the Namespace + containing the settings. + """ - parser = argparse.ArgumentParser( + parser = ArgumentParser( description="Script to drive the \ creation of graphices files." ) @@ -227,6 +243,7 @@ def parse_args(argv): --file_tmpl flag.", nargs="+", required=True, + type=Path, ) parser.add_argument( "-f", @@ -259,6 +276,7 @@ def parse_args(argv): dest="output_path", help="Directory location desired for the output graphics files.", required=True, + type=Path, ) parser.add_argument( "-s", @@ -347,7 +365,7 @@ def parse_args(argv): help="The domains to plot. Choose from any of those listed. Special " "choices: full is full model output domain, and all is the full domain, " "plus all of the sub domains. " - f"Choices: {['full', 'all'] + maps.FULL_TILES + list(maps.TILE_DEFS.keys())}", + f"Choices: {['full', 'all', *maps.FULL_TILES, *list(maps.TILE_DEFS.keys())]}", nargs="+", ) @@ -376,8 +394,9 @@ def parse_args(argv): return parser.parse_args(argv) -def pre_proc_grib_files(cla, fhr): - """Use the command line argument object (cla) to determine the grib file +def pre_proc_grib_files(cla: Namespace, fhr: int) -> Path: + """ + Use the command line argument object (cla) to determine the grib file location at a given forecast hour. If multiple data input paths and file templates are provided by user, concatenate the files and remove the duplicates. Return the file path of the file to be used by the graphics data @@ -401,26 +420,19 @@ def pre_proc_grib_files(cla, fhr): # Generate a list of files to be joined. file_list = [ - os.path.join(*path).format(FCST_TIME=fhr) - for path in zip(cla.data_root, cla.file_tmpl) + Path(*path).format(FCST_TIME=fhr) for path in zip(cla.data_root, cla.file_tmpl, strict=True) ] for file_path in file_list: - if not os.path.exists(file_path) or not utils.old_enough( - cla.data_age, file_path - ): + if not file_path.exists() or not utils.old_enough(cla.data_age, file_path): return file_path, False - print(f"Combining input files: ") + print("Combining input files: ") for fn in file_list: print(f" {fn}") - file_rand = "".join( - [random.choice(string.ascii_letters + string.digits) for _ in range(8)] - ) - combined_fp = os.path.join( - cla.output_path, COMBINED_FN.format(fhr=fhr, uniq=file_rand) - ) - tmp_fp = os.path.join(cla.output_path, TMP_FN.format(fhr=fhr, uniq=file_rand)) + file_rand = "".join([random.choice(string.ascii_letters + string.digits) for _ in range(8)]) + combined_fp = Path(cla.output_path, COMBINED_FN.format(fhr=fhr, uniq=file_rand)) + tmp_fp = Path(cla.output_path, TMP_FN.format(fhr=fhr, uniq=file_rand)) cmd = f"cat {' '.join(file_list)} > {tmp_fp}" output = subprocess.run( @@ -460,15 +472,17 @@ def pre_proc_grib_files(cla, fhr): if output.returncode != 0: msg = f"{cmd} returned exit status: {output.returncode}" raise OSError(msg) - os.remove(f"{tmp_fp}") + tmp_fp.unlink() - return f"{combined_fp}", True + return combined_fp, True -def remove_accumulated_images(cla): - """Searches for all images that correspond with specs that have the +def remove_accumulated_images(cla: Namespace): + """ + Searches for all images that correspond with specs that have the accumulate entry set to True and removes them from the list of images to - create.""" + create. + """ for variable, levels in cla.images[1].items(): for level in levels: @@ -485,24 +499,25 @@ def remove_accumulated_images(cla): del cla.images[1][variable] -def remove_proc_grib_files(cla): +def remove_proc_grib_files(cla: Namespace) -> None: """Find all processed grib files produced by this script and remove them.""" # Prepare template with all viable forecast hours -- glob accepts * combined_fn = COMBINED_FN.format(fhr=999, uniq=999).replace("999", "*") - combined_fp = os.path.join(cla.output_path, combined_fn) + combined_fp = cla.output_path / combined_fn combined_files = glob.glob(combined_fp) if combined_files: - print(f"Removing combined files: ") + print("Removing combined files: ") for file_path in combined_files: print(f" {file_path}") - os.remove(file_path) + Path(file_path).unlink() -def stage_zip_files(tiles, zip_dir): - """Stage the zip files in the appropriate directory for each tile to be +def stage_zip_files(tiles: list, zip_dir: Path) -> dict: + """ + Stage the zip files in the appropriate directory for each tile to be plotted. Return the dictionary of zipfile paths. Input: @@ -518,21 +533,19 @@ def stage_zip_files(tiles, zip_dir): """ zipfiles = {} for tile in tiles: - tile_zip_dir = os.path.join(zip_dir, tile) - os.makedirs(tile_zip_dir, exist_ok=True) - - tile_zip_file = os.path.join(tile_zip_dir, "files.zip") + tile_zip_dir = Path(zip_dir, tile) + tile_zip_dir.mkdir(parents=True, exist_ok=True) + tile_zip_file = tile_zip_dir / "files.zip" zipfiles[tile] = tile_zip_file return zipfiles @utils.timer -def graphics_driver(cla): - # pylint: disable=too-many-statements +def graphics_driver(cla: Namespace): + # ruff: noqa: PLR0915, PLR0912 # This whole script has likely reached the point of neededing refactoring # into an object oriented design....each graphics type is it's own object # sharing a base class. - """ Function that interprets the command line arguments to locate the input grib file, create the output directory, and call the graphic-specifc function. @@ -543,8 +556,6 @@ def graphics_driver(cla): """ - # pylint: disable=too-many-branches, too-many-locals - # Create an empty zip file if cla.zip_dir: tiles = cla.tiles if cla.graphic_type in ["maps", "enspanel"] else ["skewt"] @@ -570,10 +581,10 @@ def graphics_driver(cla): if len(cla.fcst_hour) == 1 and cla.all_leads: for fhr in range(first_fcst, int(cla.fcst_hour[0]), fcst_inc): grib_path, old_enough = pre_proc_grib_files(cla, fhr) - if not os.path.exists(grib_path) or not old_enough: + if not grib_path.exists() or not old_enough: msg = ( f"File {grib_path} does not exist! Cannot accumulate", - f"data for this forecast lead time!", + "data for this forecast lead time!", ) remove_proc_grib_files(cla) raise FileNotFoundError(" ".join(msg)) @@ -610,7 +621,7 @@ def graphics_driver(cla): if time.time() - timer_end > cla.wait_time * 60 * 0.9: print( f"Giving up waiting on {grib_path}. \n", - f"Removing accumulated variables from image list \n", + "Removing accumulated variables from image list \n", f"{LOG_BREAK}\n", ) remove_accumulated_images(cla) @@ -623,20 +634,18 @@ def graphics_driver(cla): break # It's safe to continue on processing the next forecast hour print( - f"Cannot find specified file(s), continuing to check on \n \ + "Cannot find specified file(s), continuing to check on \n \ next forecast hour." ) continue # Create the working directory - workdir = os.path.join( - cla.output_path, f"{utils.from_datetime(cla.start_time)}{fhr:02d}" - ) - os.makedirs(workdir, exist_ok=True) + workdir = Path(cla.output_path, f"{utils.from_datetime(cla.start_time)}{fhr:02d}") + workdir.mkdir(parents=True, exist_ok=True) print( f"{LOG_BREAK}\n", - f"Graphics will be created for input files\n", + "Graphics will be created for input files\n", f"Output graphics directory: {workdir} \n{LOG_BREAK}", ) @@ -667,7 +676,7 @@ def graphics_driver(cla): workdir=workdir, ) else: - gribfiles = gribfile.GribFiles( + gribfiles = GribFiles( coord_dims={"ens_mem": ens_members, "fcst_hr": fhr_as_list}, filenames={"free_fcst": grib_paths}, filetype=cla.file_type, @@ -691,56 +700,53 @@ def graphics_driver(cla): # wait_time mins. This accounts for slower UPP processes. Default for # most CONUS-sized domains is 10 mins. if time.time() - timer_end > cla.wait_time * 60: - print( - f"Exiting with forecast hours remaining: {fcst_hours}", f"{LOG_BREAK}" - ) + print(f"Exiting with forecast hours remaining: {fcst_hours}", f"{LOG_BREAK}") break # Wait for a bit if it's been < 2 minutes (about the length of time UPP # takes) since starting last loop - if fcst_hours and time.time() - timer_sleep < 120: - print( - f"Waiting for a minute for forecast hours: {fcst_hours}", f"{LOG_BREAK}" - ) + two_mins = 120 + if fcst_hours and time.time() - timer_sleep < two_mins: + print(f"Waiting for a minute for forecast hours: {fcst_hours}", f"{LOG_BREAK}") time.sleep(60) remove_proc_grib_files(cla) -def create_graphics(argv): +def create_graphics(argv: list): """ Function to perform a series of checks on command line arguments. """ - CLARGS = parse_args(argv) - CLARGS.fcst_hour = utils.fhr_list(CLARGS.fcst_hour) + clargs = parse_args(argv) + clargs.fcst_hour = utils.fhr_list(clargs.fcst_hour) # Check that the same number of entries exists in -d and --file_tmpl - if len(CLARGS.data_root) != len(CLARGS.file_tmpl): + if len(clargs.data_root) != len(clargs.file_tmpl): errmsg = "Must specify the same number of arguments for -d and --file_tmpl" - argparse.ArgumentParser.exit(0, errmsg) + ArgumentParser.exit(0, errmsg) # Ensure wgrib command is available in environment before getting too far # down this path... - if len(CLARGS.data_root) > 1: - retcode = subprocess.run("which wgrib2", shell=True, check=True) + if len(clargs.data_root) > 1: + retcode = subprocess.run("/usr/bin/which wgrib2", shell=True, check=True) if retcode.returncode != 0: errmsg = "Could not find wgrib2, please make sure it is loaded \ in your environment." raise OSError(errmsg) # Only need to load the default in memory if we're making maps. - if CLARGS.graphic_type in ["maps", "enspanel", "diff"]: - CLARGS.specs = utils.load_specs(CLARGS.specs) + if clargs.graphic_type in ["maps", "enspanel", "diff"]: + clargs.specs = utils.load_specs(clargs.specs) - CLARGS.images = load_images(CLARGS.images) - CLARGS.tiles = generate_tile_list(CLARGS.tiles) + clargs.images = load_images(clargs.images) + clargs.tiles = generate_tile_list(clargs.tiles) # Make sure the second data root is provided when doing diffs - if CLARGS.graphic_type == "diff": - if not CLARGS.data_root2: + if clargs.graphic_type == "diff": + if not clargs.data_root2: errmsg = "Must specify a second data root (--data_root2) for creating difference maps" - raise argparse.ArgumentError(CLARGS.data_root2, errmsg) - if CLARGS.all_leads: + raise ArgumentError(clargs.data_root2, errmsg) + if clargs.all_leads: warning = ( "Warning! Plotting differences in graphics-accumulated ", "fields is not supported!", @@ -748,24 +754,22 @@ def create_graphics(argv): print(warning) # Make sure both required arguments (--max_plev, --sites) are provided when doing skewTs - if CLARGS.graphic_type == "skewts": - if not CLARGS.max_plev: - argparse.ArgumentParser.exit( + if clargs.graphic_type == "skewts": + if not clargs.max_plev: + ArgumentParser.exit( 0, "Must specify maximum pressure level \ (--max_plev) when creating skewTs", ) - if not CLARGS.sites: - argparse.ArgumentParser.exit( - 0, "Must specify sites (--sites) when creating skewTs" - ) + if not clargs.sites: + ArgumentParser.exit(0, "Must specify sites (--sites) when creating skewTs") - print(f"Running script for {CLARGS.graphic_type} with args: ", f"{LOG_BREAK}") + print(f"Running script for {clargs.graphic_type} with args: ", f"{LOG_BREAK}") - for name, val in CLARGS.__dict__.items(): + for name, val in clargs.__dict__.items(): if name not in ["specs", "sites"]: print(f"{name:>15s}: {val}") - graphics_driver(CLARGS) + graphics_driver(clargs) if __name__ == "__main__": diff --git a/recipe/run_test.sh b/recipe/run_test.sh index e95e14b..9c05f13 100755 --- a/recipe/run_test.sh +++ b/recipe/run_test.sh @@ -50,7 +50,6 @@ unittest() { msg OK } -test "${CONDEV_SHELL:-}" = 1 && cd $(dirname $0)/../src || cd ../test_files if [[ -n "${1:-}" ]]; then # Run single specified code-quality tool. $1 diff --git a/tests/test_common.py b/tests/test_common.py index e72b0a0..2d90c75 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,7 +1,9 @@ # pylint: disable=invalid-name """ -Pytests for the common utilities included in this package. Includes: +Pytests for the common utilities included in this package. + +Includes: - conversions.py - specs.py @@ -23,15 +25,14 @@ from matplotlib import colors as mcolors from metpy.plots import ctables -import adb_graphics.conversions as conversions -import adb_graphics.datahandler.gribdata as gribdata -import adb_graphics.specs as specs -import adb_graphics.utils as utils +from adb_graphics import conversions, specs, utils +from adb_graphics.datahandler import gribdata def test_conversion(): - """Test that conversions return at numpy array for input of np.ndarray, - list, or int""" + """ + Conversions return numpy array for input of np.ndarray, list, or int. + """ a = np.ones([3, 2]) * 300 c = a[0, 0] @@ -78,7 +79,7 @@ def test_conversion(): class MockSpecs(specs.VarSpec): - """Mock class for the VarSpec abstract class""" + """Mock class for the VarSpec abstract class.""" @property def clevs(self): @@ -143,8 +144,10 @@ class TestDefaultSpecs: @property def allowable(self): - """Each entry in the dict names a function that tests a key in - default_specs.yml.""" + """ + Each entry in the dict names a function that tests a key in + default_specs.yml. + """ return { "accumulate": self.is_bool, @@ -188,16 +191,18 @@ def check_kwargs(self, accepted_args, kwargs): if lev: assert self.cfg.get(short_name).get(lev) is not None - for arg in args.keys(): + for arg in args: assert arg in accepted_args return True def check_transform(self, entry): """ - Check that the transform entry is either a single transformation - function, a list of transformation functions, or a dictionary containing - the functions list and the kwargs list like so: + Check structure of transform entry. + + The transform entry should be either a single transformation function, a list of + transformation functions, or a dictionary containing the functions list and the kwargs list + like so: transform: funcs: [list, of, functions] @@ -237,9 +242,7 @@ def check_transform(self, entry): # when provided arguments don't appear in all_params. # arguments not in that list, we fail. if kwargs: - argspecs = [ - getfullargspec(func) for func in transforms if callable(func) - ] + argspecs = [getfullargspec(func) for func in transforms if callable(func)] all_params = [] for argspec in argspecs: @@ -252,11 +255,11 @@ def check_transform(self, entry): parameters.extend(argtype) all_params.extend(parameters) - for key in kwargs.keys(): + for key in kwargs: if key not in all_params: msg = f"Function key {key} is not an expicit parameter \ in any of the transforms: {funcs}!" - warnings.warn(msg, UserWarning) + warnings.warn(msg, UserWarning, stacklevel=2) return True @@ -281,7 +284,8 @@ def get_callable(self, func): if callable(utils.get_func(func)): return utils.get_func(func) - raise ValueError("{func} is not a known callable function!") + msg = f"{func} is not a known callable function!" + raise ValueError(msg) @staticmethod def is_a_clev(clev): @@ -304,8 +308,10 @@ def is_a_cmap(cmap): return cmap in dir(cm) + list(ctables.colortables.keys()) def is_a_contour_dict(self, entry): - """Set up the accepted arguments for plt.contour, and check the given - arguments.""" + """ + Set up the accepted arguments for plt.contour, and check the given + arguments. + """ args = [ "X", @@ -337,8 +343,10 @@ def is_a_contour_dict(self, entry): return self.check_kwargs(args, entry) def is_a_contourf_dict(self, entry): - """Set up the accepted arguments for plt.contourf, and check the given - arguments.""" + """ + Set up the accepted arguments for plt.contourf, and check the given + arguments. + """ args = [ "X", @@ -375,13 +383,10 @@ def is_a_color(self, color): colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS, **ctables.colortables) - if color in colors.keys(): - return True - - if color in dir(self.varspec): + if color in colors: return True - return False + return color in dir(self.varspec) @staticmethod def is_a_level(key): @@ -466,10 +471,7 @@ def is_a_level(key): allowed = True break - if numeric and allowed: - return True - - return False + return numeric and allowed def is_a_key(self, key): """Returns true if key exists as a key in the config file.""" @@ -493,9 +495,7 @@ def is_callable(self, funcs): callable_ = callable_ if isinstance(callable_, list) else [callable_] for clbl in callable_: - if isinstance(clbl, np.ndarray): - callables.append(True) - elif callable(clbl): + if isinstance(clbl, np.ndarray) or callable(clbl): callables.append(True) else: callables.append(False) @@ -504,7 +504,7 @@ def is_callable(self, funcs): @staticmethod def is_dict(d): - """Returns true if d is a dictionary""" + """Returns true if d is a dictionary.""" return isinstance(d, dict) @@ -536,8 +536,10 @@ def is_wind(self, wind): return isinstance(wind, bool) or self.is_a_level(wind) def check_keys(self, d, depth=0): - """Helper function that recursively checks the keys in the dictionary by calling the - function defined in allowable.""" + """ + Helper function that recursively checks the keys in the dictionary by calling the + function defined in allowable. + """ max_depth = 2 @@ -553,7 +555,7 @@ def check_keys(self, d, depth=0): for k, v in d.items(): # Check that the key is allowable - assert (k in self.allowable.keys()) or self.is_a_level(k) + assert (k in self.allowable) or self.is_a_level(k) # Call a checker if one exists for the key, otherwise descend into # next level of dict @@ -563,9 +565,8 @@ def check_keys(self, d, depth=0): assert checker else: assert checker(v) - else: - if isinstance(v, dict): - self.check_keys(v, depth=level) + elif isinstance(v, dict): + self.check_keys(v, depth=level) def test_keys(self): """Tests each of top-level variables in the config file by calling the helper function.""" diff --git a/tests/test_grib.py b/tests/test_grib.py index 83abe31..844782e 100644 --- a/tests/test_grib.py +++ b/tests/test_grib.py @@ -1,14 +1,13 @@ -# pylint: disable=invalid-name """Test suite for grib datahandler.""" -import datetime +from datetime import datetime +from pathlib import Path import numpy as np import xarray as xr from matplotlib import colors as mcolors -import adb_graphics.datahandler.gribdata as gribdata -import adb_graphics.datahandler.gribfile as gribfile +from adb_graphics.datahandler import gribdata, gribfile DATAARRAY = xr.core.dataarray.DataArray @@ -20,19 +19,21 @@ def test_UPPData(natfile, prsfile): prs_ds = gribfile.GribFile(prsfile) class UPP(gribdata.UPPData): - """Test class needed to define the values as an abstract class""" + """Test class needed to define the values as an abstract class.""" - def values(self, level=None, name=None, **kwargs): + def values(self, level=None, name=None, **kwargs): # noqa: ARG002 return 1 upp_nat = UPP(nat_ds.contents, fhr=2, filetype="nat", short_name="temp") upp_prs = UPP(prs_ds.contents, fhr=2, short_name="temp") + cycle = datetime(2025, 10, 2, 17) + # Ensure appropriate typing and size (where applicable) for upp in [upp_nat, upp_prs]: - assert isinstance(upp.anl_dt, datetime.datetime) + assert isinstance(upp.anl_dt, datetime) assert isinstance(upp.clevs, np.ndarray) - assert isinstance(upp.date_to_str(datetime.datetime.now()), str) + assert isinstance(upp.date_to_str(cycle, str)) assert isinstance(upp.fhr, str) assert isinstance(upp.field, DATAARRAY) assert isinstance(upp.latlons(), list) @@ -40,15 +41,15 @@ def values(self, level=None, name=None, **kwargs): assert isinstance(upp.ncl_name(upp.vspec), str) assert isinstance(upp.numeric_level(), tuple) assert isinstance(upp.spec, dict) - assert isinstance(upp.valid_dt, datetime.datetime) + assert isinstance(upp.valid_dt, datetime) assert isinstance(upp.vspec, dict) # Test for appropriate date formatting - test_date = datetime.datetime(2020, 12, 5, 12) + test_date = datetime(2020, 12, 5, 12) assert upp.date_to_str(test_date) == "20201205 12 UTC" def test_fieldData(prsfile): - """Test the fieldData class methods on a prs file""" + """Test the fieldData class methods on a prs file.""" prs_ds = gribfile.GribFile(prsfile) field = gribdata.fieldData(prs_ds.contents, fhr=2, level="500mb", short_name="temp") @@ -89,13 +90,13 @@ def test_fieldData(prsfile): ) # Expected size of values - assert len(np.shape((field.values()))) == 2 - assert len(np.shape((field.values(name="u")))) == 2 - assert len(np.shape((field.values(name="u", level="850mb")))) == 2 + assert len(np.shape(field.values())) == 2 + assert len(np.shape(field.values(name="u"))) == 2 + assert len(np.shape(field.values(name="u", level="850mb"))) == 2 -def test_profileData(natfile): - """Test the profileData class methods on a nat file""" +def test_profile_data(natfile: Path): + """Test the profileData class methods on a nat file.""" nat_ds = gribfile.GribFile(natfile) loc = " BNA 9999 99999 36.12 86.69 597 Nashville, TN\n" @@ -111,5 +112,5 @@ def test_profileData(natfile): assert isinstance(profile.values(), DATAARRAY) # The values should return a single number (0) or a 1D array (1) - assert len(np.shape((profile.values(level="best", name="li")))) == 0 - assert len(np.shape((profile.values(name="temp")))) == 1 + assert len(np.shape(profile.values(level="best", name="li"))) == 0 + assert len(np.shape(profile.values(name="temp"))) == 1 diff --git a/tests/test_hrrr_maps.py b/tests/test_hrrr_maps.py index c07ca30..f69a163 100644 --- a/tests/test_hrrr_maps.py +++ b/tests/test_hrrr_maps.py @@ -1,20 +1,20 @@ -# pylint: disable=unused-variable -"""Tests for create_graphics driver""" +"""Tests for create_graphics driver.""" import os import pytest +from path import Path from create_graphics import create_graphics, parse_args -DATA_LOC = os.environ.get("data_loc") -OUTPUT_LOC = os.environ.get("data_loc") +DATA_LOC = os.environ.get("DATA_LOC") +OUTPUT_LOC = Path(os.environ.get("OUTPUT_LOC")) -@pytest.fixture(name="_setup") -def build_maps(): - """Builds HRRR 12-hour accumulated maps""" - args = [ +@pytest.fixture +def maps_args() -> list: + """Builds HRRR 12-hour accumulated maps.""" + return [ "maps", "-d", DATA_LOC, @@ -34,11 +34,11 @@ def build_maps(): "--all_leads", "--file_type=prs", ] - create_graphics(args) def test_parse_args(): - """Test parse_args for basic parsing success. + """ + Test parse_args for basic parsing success. Checks if parse_args returns 'maps' in the graphic_type field. """ args = [ @@ -65,25 +65,18 @@ def test_parse_args(): assert test_args.graphic_type == "maps" -def test_folder_existence(_setup): - """Tests for existence of folders. - Can be extended to cover multiple folders. +def test_file_count(maps_args): """ - folder = "/202303150000" - full_path = OUTPUT_LOC + folder - file_path = os.path.isdir(full_path) - assert file_path - - -def test_file_count(_setup): - """Test for file count in directory. + Test for file count in directory. Can be extended to cover multiple folders. """ # Based on the hrrr_test.yml file, only 6 maps will be created + create_graphics(maps_args) map_count = 6 count = 0 folder = "/202303150000/" - for file_name in os.listdir(OUTPUT_LOC + folder): - if os.path.isfile(OUTPUT_LOC + folder + file_name): + assert (OUTPUT_LOC / folder).isdir() + for file_name in (OUTPUT_LOC / folder).iterdir(): + if (OUTPUT_LOC / folder / file_name).is_file(): count += 1 assert count == map_count From df1da6d8c8b9a43a5554b8636d3816b44e6138f4 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 22 Oct 2025 13:20:44 -0600 Subject: [PATCH 13/98] Fully linted. --- adb_graphics/conversions.py | 69 +++---- adb_graphics/datahandler/gribdata.py | 294 ++++++++++++--------------- adb_graphics/datahandler/gribfile.py | 60 ++---- adb_graphics/errors.py | 18 +- adb_graphics/figure_builders.py | 54 +++-- adb_graphics/figures/maps.py | 16 +- adb_graphics/figures/skewt.py | 8 +- create_graphics.py | 4 +- tests/test_grib.py | 12 +- 9 files changed, 240 insertions(+), 295 deletions(-) diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index 6ce4498..0936d4d 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -5,105 +5,106 @@ converted values as output. """ +from xarray import DataArray, DataSet from xarray.ufuncs import sqrt, square -def k_to_c(field, **kwargs): - """Conversion from Kelvin to Celsius""" +def k_to_c(field: DataArray, **kwargs): + """Conversion from Kelvin to Celsius.""" return field - 273.15 -def k_to_f(field, **kwargs): - """Conversion from Kelvin to Farenheit""" +def k_to_f(field: DataArray, **kwargs): + """Conversion from Kelvin to Farenheit.""" return (field - 273.15) * 9 / 5 + 32 -def kgm2_to_in(field, **kwargs): - """Conversion from kg per m^2 to inches""" +def kgm2_to_in(field: DataArray, **kwargs): + """Conversion from kg per m^2 to inches.""" return field * 0.03937 -def magnitude(a, b, **kwargs): - """Return the magnitude of vector components""" +def magnitude(a: DataSet, b: DataSet, **kwargs): + """Return the magnitude of vector components.""" return sqrt(square(a) + square(b)) -def m_to_dm(field, **kwargs): - """Conversion from meters to decameters""" +def m_to_dm(field: DataArray, **kwargs): + """Conversion from meters to decameters.""" return field / 10.0 -def m_to_in(field, **kwargs): - """Conversion from meters to inches""" +def m_to_in(field: DataArray, **kwargs): + """Conversion from meters to inches.""" return field * 39.3701 -def m_to_kft(field, **kwargs): - """Conversion from meters to kilofeet""" +def m_to_kft(field: DataArray, **kwargs): + """Conversion from meters to kilofeet.""" return field / 304.8 -def m_to_mi(field, **kwargs): - """Conversion from meters to miles""" +def m_to_mi(field: DataArray, **kwargs): + """Conversion from meters to miles.""" return field / 1609.344 -def ms_to_kt(field, **kwargs): - """Conversion from m s-1 to knots""" +def ms_to_kt(field: DataArray, **kwargs): + """Conversion from m s-1 to knots.""" return field * 1.9438 -def pa_to_hpa(field, **kwargs): - """Conversion from Pascals to hectopascals""" +def pa_to_hpa(field: DataArray, **kwargs): + """Conversion from Pascals to hectopascals.""" return field / 100.0 -def percent(field, **kwargs): - """Conversion from values between 0 - 1 to percent""" +def percent(field: DataArray, **kwargs): + """Conversion from values between 0 - 1 to percent.""" return field * 100.0 -def to_micro(field, **kwargs): - """Convert field to micro""" +def to_micro(field: DataArray, **kwargs): + """Convert field to micro.""" return field * 1e6 -def to_micrograms_per_m3(field, **kwargs): - """Convert field to micrograms per cubic meter""" +def to_micrograms_per_m3(field: DataArray, **kwargs): + """Convert field to micrograms per cubic meter.""" return field * 1e9 -def vvel_scale(field, **kwargs): - """Scale vertical velocity for plotting""" +def vvel_scale(field: DataArray, **kwargs): + """Scale vertical velocity for plotting.""" return field * -10 -def vort_scale(field, **kwargs): - """Scale vorticity for plotting""" +def vort_scale(field: DataArray, **kwargs): + """Scale vorticity for plotting.""" return field / 1e-05 -def weasd_to_1hsnw(field, **kwargs): - """Conversion from snow water equiv to snow (10:1 ratio)""" +def weasd_to_1hsnw(field: DataArray, **kwargs): + """Conversion from snow water equiv to snow (10:1 ratio).""" return field * 10.0 -def sden_to_slr(field, **kwargs): - """Convert snow density (kg m-3) to snow-liquid ratio""" +def sden_to_slr(field: DataArray, **kwargs): + """Convert snow density (kg m-3) to snow-liquid ratio.""" return 1000.0 / field diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index e4a17cb..d652dc3 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -7,18 +7,16 @@ import abc from copy import deepcopy from datetime import datetime, timedelta -from functools import lru_cache from string import ascii_letters, digits import numpy as np -import xarray as xr from matplotlib import cm +from xarray import DataArray, DataSet +from adb_graphics import conversions, errors, specs, utils from adb_graphics.datahandler import gribfile from adb_graphics.utils import cfgrib_spec -from .. import conversions, errors, specs, utils - class UPPData(specs.VarSpec): """ @@ -29,13 +27,13 @@ class UPPData(specs.VarSpec): ds: xarray dataset from grib file short_name: name of variable corresponding to entry in specs configuration - Keyword Arguments: + kwargs: config: path to a user-specified configuration file model: string describing the model type """ - def __init__(self, ds, short_name, spec, **kwargs): + def __init__(self, ds: DataSet, short_name: str, spec: dict, **kwargs): # Parse kwargs first self.model = kwargs.get("model") self.grib_path = kwargs.get("grib_path") @@ -85,8 +83,7 @@ def clevs(self) -> np.ndarray: @staticmethod def date_to_str(date: datetime) -> str: """ - Returns a formatted string (for graphic title) from a datetime - object + Returns a formatted string (for graphic title) from a datetime object. """ return date.strftime("%Y%m%d %H UTC") @@ -94,22 +91,18 @@ def date_to_str(date: datetime) -> str: @property def field(self): """ - Wrapper that calls get_field method for the current variable. - Returns the NioVariable object + Get the first DataArray out of the DataSet. """ - return self.ds.__getattr__([x for x in self.ds.data_vars][0]) + return self.ds.__getattr__(list(self.ds.data_vars)[0]) - def field_column_max(self, values, variable, level, **kwargs): - # pylint: disable=unused-argument + def field_column_max(self, values: DataArray, variable: str, level: str, **kwargs): # noqa: ARG002 """Returns the column max of the values.""" vals = self.values(name=variable, level=level, one_lev=False) - maxvals = vals.max(axis=0) - - return maxvals + return vals.max(axis=0) - def field_sum(self, values, variable2, level2, **kwargs): + def field_sum(self, values: DataArray, variable2: str, level2: str, **kwargs): # noqa: ARG002 # pylint: disable=unused-argument """Return the sum of the values.""" @@ -119,8 +112,7 @@ def field_sum(self, values, variable2, level2, **kwargs): return sum2 - def field_diff(self, values, variable2, level2, **kwargs): - # pylint: disable=unused-argument + def field_diff(self, values: DataArray, variable2: str, level2: str, **kwargs): # noqa: ARG002 """Subtracts the values from variable2 from self.field.""" value2 = self.values(name=variable2, level=level2) @@ -129,18 +121,23 @@ def field_diff(self, values, variable2, level2, **kwargs): return diff - def field_mean(self, values, variable, levels, global_levels, **kwargs): - # pylint: disable=unused-argument - """Returns the mean of the values.""" + def field_mean( + self, + values: DataArray, + levels: list, + **kwargs, # noqa: ARG002 + ): + """Returns the mean of the values over the vertical dimension.""" levs = [int(x[:-2]) for x in levels] - ret = values.sel(isobaricInhPa=levs).mean("isobaricInhPa") - return ret + return values.sel(isobaricInhPa=levs).mean("isobaricInhPa") - def _get_data_levels(self, vertical_dim): + def _get_data_levels(self, vertical_dim: str): """ - Return a list of vertical dimension values corresponding to the - requested vertical dimension to get the values of those dimensions + Values of the vertical dimension. + + Arg: + vertical_dim the name of the vertical dimension """ fcst_hr = 0 if self.ds.sizes.get("fcst_hr", 0) <= 1 else int(self.fhr) @@ -152,28 +149,32 @@ def _get_data_levels(self, vertical_dim): ret.append(self.ds[dim].sel(**selector).values) return ret - def _get_field(self, spec): - """Given an ncl_name, return the NioVariable object.""" + def _get_field(self, spec: dict) -> DataArray: + """ + Given a cfgrib block, return the DataArray. + + Arg: + spec the specifications dictionary to use for the variable in + question + """ ds = gribfile.GribFile(self.grib_path, spec).contents - return ds.__getattr__([x for x in ds.data_vars][0]) + return ds.__getattr__(list(ds.data_vars)[0]) - def _get_level(self, field, level, spec, **kwargs): + def _get_level(self, field: DataSet, level: str, spec: dict, **kwargs): """ - Returns the value of the level to for a 3D array + Returns the value of the level to for a 3D array. - Arguments: - field dataset object for a given variable - level string describing the level atmospheric level; corresponds + Arg: + field: dataset object for a given variable + level: string describing the level atmospheric level; corresponds to a key in default specs - spec the specifications dictionary to use for the variable in + spec: the specifications dictionary to use for the variable in question - - Keyword Arguments: - split bool sometimes passed in through transforms that indicates - a level string should be split, e.g. 06km. - + kwargs: + split: bool sometimes passed in through transforms that indicates + a level string should be split, e.g. 06km. Return: Integer value corresponding to the array index for the atmospheric @@ -201,7 +202,7 @@ def _get_level(self, field, level, spec, **kwargs): # For split-level variables, like 0-6km, find the matching index by # looping through both the possible vertical level arrays. if len(data_levels) == 2 and len(requested_level) == 2: - for lev, levset in enumerate(zip(*[list(lev) for lev in data_levels])): + for lev, levset in enumerate(zip(*[list(lev) for lev in data_levels], strict=True)): if sorted(levset) == requested_level: return lev @@ -212,8 +213,7 @@ def _get_level(self, field, level, spec, **kwargs): lev = np.argwhere(dim_levels == requested_level[0]) try: if lev or lev == [0]: - lev = int(lev[0]) - return lev + return int(lev[0]) except ValueError: print(f"BAD LEVEL is {lev} for {field.name}") @@ -231,7 +231,7 @@ def _get_level(self, field, level, spec, **kwargs): ) raise ValueError(msg) - def get_transform(self, transforms, val): + def get_transform(self, transforms: dict, val: float | list | DataArray): """ Applies a set of one or more transforms to an np.array of data values. @@ -266,8 +266,7 @@ def get_transform(self, transforms, val): val = utils.get_func(transform)(val, **transform_kwargs) return val - @lru_cache - def get_xypoint(self, site_lat, site_lon) -> tuple: + def get_xypoint(self, site_lat: float, site_lon: float) -> tuple: """ Return the X, Y grid point corresponding to the site location. No interpolation is used. @@ -298,14 +297,14 @@ def grid_suffix(self): in the file. This should correspond to the grid tag. """ - for var in self.ds.keys(): + for var in self.ds: vsplit = var.split("_") if len(vsplit) == 4: return vsplit[-1] return "GRID NOT FOUND" def latlons(self): - """Returns the set of latitudes and longitudes""" + """Returns the set of latitudes and longitudes.""" coords = sorted( [c for c in list(self.ds.coords) if any(ele in c for ele in ["lat", "lon"])] @@ -318,7 +317,9 @@ def lev_descriptor(self): return self.field.level_type - def numeric_level(self, index_match=True, level=None, split=None): + def numeric_level( + self, index_match: bool = True, level: str | None = None, split: str | None = None + ): """ Split the numeric level and unit associated with the level key. @@ -352,9 +353,8 @@ def numeric_level(self, index_match=True, level=None, split=None): return lev_val, lev_unit @staticmethod - def opposite(values, **kwargs): - # pylint: disable=unused-argument - """Returns the opposite of input values""" + def opposite(values: DataArray, **kwargs): # noqa: ARG004 + """Returns the opposite of input values.""" return -values @@ -369,16 +369,16 @@ def valid_dt(self) -> datetime: return self.anl_dt + fh @abc.abstractmethod - def values(self, level=None, name=None, **kwargs): + def values(self, level: str | None = None, name: str | None = None, **kwargs): """Returns the values of a given variable.""" - ... @staticmethod - def vertical_dim(field): + def vertical_dim(field: DataSet): """ - Determine the vertical dimension of the variable by looking through - the field's dimensions for one that includes "lv". Return the first - matching instance. + Find the name of the vertical dimension. + + Looking through the field's dimensions for one that includes "lv". Return the first matching + instance. """ vert_dim = [dim for dim in field.dims if ("lv" in dim or "probability" in dim)] @@ -392,11 +392,11 @@ def vspec(self): vspec = self.spec.get(self.short_name, {}).get(self.level) if not vspec: - raise errors.NoGraphicsDefinitionForVariable(self.short_name, self.level) + raise errors.NoGraphicsDefinitionForVariableError(self.short_name, self.level) return vspec -class fieldData(UPPData): +class FieldData(UPPData): """ Class provides interface for accessing field (2D plan view) data from UPP in Grib2 format. @@ -413,17 +413,16 @@ class fieldData(UPPData): """ - def __init__(self, ds, level, short_name, **kwargs): + def __init__(self, ds: DataSet, level: str, short_name: str, **kwargs): super().__init__(ds, short_name, **kwargs) self.level = level self.contour_kwargs = kwargs.get("contour_kwargs", {}) self.mem = kwargs.get("member") - def aviation_flight_rules(self, values, **kwargs): - # pylint: disable=unused-argument + def aviation_flight_rules(self, values: DataArray, **kwargs): # noqa: ARG002 """ - Generates a field of Aviation Flight Rules from Ceil and Vis + Generates a field of Aviation Flight Rules from Ceil and Vis. """ ceil = values.to_dataarray().squeeze() @@ -438,13 +437,12 @@ def aviation_flight_rules(self, values, **kwargs): vis.close() - return xr.DataArray(flru) + return DataArray(flru) @property def cmap(self): """ - Returns the LinearSegmentedColormap specified by the config key - "cmap" + The LinearSegmentedColormap specified by the config key 'cmap'. """ return cm.get_cmap(self.vspec["cmap"]) @@ -466,14 +464,17 @@ def colors(self) -> np.ndarray: ret = self.__getattribute__(color_spec) if callable(ret): return ret() - return ret except AttributeError: return color_spec + return ret @property def corners(self) -> list: """ - Returns lat and lon of lower left (ll) and upper right(ur) corners: + + Returns lat and lon of lower left (ll) and upper right(ur) corners. + + Order: ll_lat, ur_lat, ll_lon, ur_lon """ @@ -487,17 +488,16 @@ def corners(self) -> list: return ret - def fire_weather_index(self, values, **kwargs): - # pylint: disable=unused-argument + def fire_weather_index(self, values: DataArray, **kwargs): # noqa: ARG002 """ - Generates a field of Fire Weather Index + Generates a field of Fire Weather Index. This method uses wrfprs data to find regions where weather conditions are most likely to lead to wildfires. """ - def _load_field(level, short_name): + def _load_field(level: str, short_name: str): spec = cfgrib_spec(self.spec[short_name][level]["cfgrib"], self.model) ds = gribfile.GribFile(self.grib_path, spec).contents args = { @@ -509,7 +509,7 @@ def _load_field(level, short_name): "spec": self.spec, "grib_path": self.grib_path, } - return fieldData(**args).values(do_transform=False) + return FieldData(**args).values(do_transform=False) # Gather fields from the input veg = ( @@ -622,39 +622,33 @@ def grid_info(self): return grid_info - def icing_adjust_trace(self, values, **kwargs): - # pylint: disable=unused-argument,no-self-use - """Changes the value of ICSEV trace from 4.0 to 0.5, to maintain ascending order""" - - vals = np.where(values == 4.0, 0.5, values) + @staticmethod + def icing_adjust_trace(values: DataArray, **kwargs): # noqa: ARG004 + """Changes the value of ICSEV trace from 4.0 to 0.5, to maintain ascending order.""" - return vals + return np.where(values == 4.0, 0.5, values) - def run_max(self, values, **kwargs): + @staticmethod + def run_max(values: DataArray, **kwargs): # noqa: ARG004 """Finds the max hourly value over all the forecast lead times available.""" - # pylint: disable=unused-argument,no-self-use - return values.max(dim="fcst_hr") - def run_min(self, values, **kwargs): + @staticmethod + def run_min(values: DataArray, **kwargs): # noqa: ARG004 """Finds the min hourly value over all the forecast lead times available.""" - # pylint: disable=unused-argument,no-self-use - return values.min(dim="fcst_hr") - def run_total(self, values, **kwargs): + @staticmethod + def run_total(values: DataArray, **kwargs): # noqa: ARG004 """Sums over all the forecast lead times available.""" - # pylint: disable=unused-argument,no-self-use - return values.sum(dim="fcst_hr") - def supercooled_liquid_water(self, values, **kwargs): - # pylint: disable=unused-argument + def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 """ - Generates a field of Supercooled Liquid Water + Generates a field of Supercooled Liquid Water. This method uses wrfnat data to find regions where cloud and rain moisture are in below-freezing temps. @@ -729,10 +723,10 @@ def data(self): return self._data @data.setter - def data(self, value): + def data(self, value: DataSet): self._data = value - def values(self, level=None, name=None, **kwargs): + def values(self, level: str | None = None, name: str | None = None, **kwargs): """ Returns the numpy array of values at the requested level for the variable after applying any unit conversion to the original data. @@ -741,7 +735,7 @@ def values(self, level=None, name=None, **kwargs): name the name of a field other than defined in self level the desired level of the named field - Keyword Args: + kwargs: do_transform bool flag. to call, or not, the transform specified in specs (default: True) ncl_name the NCL-assigned Grib2 name (default: '') @@ -753,89 +747,42 @@ def values(self, level=None, name=None, **kwargs): level = level or self.level vals = self.ds - - # one_lev = kwargs.get('one_lev', True) - # vertical_index = kwargs.get('vertical_index') - - # ncl_name = kwargs.get('ncl_name', '') - # ncl_name = ncl_name.format(fhr=self.fhr, grid=self.grid_suffix) + spec = self.vspec do_transform = kwargs.get("do_transform", True) - if name is None: - # Use field and spec from the current object - field = self.field - spec = self.vspec - - else: + if name is not None: # Get the spec dict and ncl_name for the given variable name spec = deepcopy(self.spec.get(name, {}).get(level, {})) if not spec and name is not None: - raise errors.NoGraphicsDefinitionForVariable(name, level) + raise errors.NoGraphicsDefinitionForVariableError(name, level) cfkeys = utils.cfgrib_spec(spec["cfgrib"], self.model) nlevel = utils.numeric_level(level=level, index_match=False)[0] level_info = any( - x - for x in utils.cfgrib_spec(spec["cfgrib"], self.model) - for l in ("level", "top", "bottom", "Surface") - if l in x + key + for keys in utils.cfgrib_spec(spec["cfgrib"], self.model) + for key in ("level", "top", "bottom", "Surface") + if key in keys ) if nlevel and not level_info: cfkeys["level"] = utils.numeric_level(level=level, index_match=False)[0] vals = self._get_field(cfkeys) - # lev = vertical_index - # vals = field - # if one_lev: - - # # Check if it's a 3D variable (lv in any dimension field) - # dim_name = self.vertical_dim(field) - - # if dim_name: # Field has a vertical dimension - - # # Use vertical_index if provided in kwargs - # lev = vertical_index if vertical_index is not None else \ - # self._get_level(field, level, spec) - - # if lev is None or dim_name is None: - # print(f'ERROR: Could not find dim_name ({dim_name}) or' \ - # f'lev {lev} for {vals}') - # raise ValueError - - # try: - # vals = vals.isel(**{dim_name: lev}) - # except: - # print(f'Error for {vals.name} : {dim_name} {lev} \ - # {level} {spec}') - # raise - - # if self.mem is not None: - # vals = vals.isel(**{'ens_mem': self.mem}) - - ## Select a single forecast hour (only if there are many) - # if not spec.get('accumulate', False): - # if 'fcst_hr' in vals.dims: - # fcst_hr = 0 if self.ds.sizes['fcst_hr'] <= 1 else int(self.fhr) - # vals = vals.sel(**{'fcst_hr': fcst_hr}) - transforms = spec.get("transform") if transforms and do_transform: vals = self.get_transform(transforms, vals) - if isinstance(vals, xr.Dataset): + if isinstance(vals, DataSet): return vals.to_dataarray().squeeze() return vals def vector_magnitude( self, - field1, - cfkeys=None, - field2_id=None, - level=None, - vertical_index=None, - **kwargs, + field1: DataSet, + cfkeys: dict | None = None, + field2_id: str | None = None, + **kwargs, # noqa: ARG002 ): - # pylint: disable=unused-argument """ Returns the vector magnitude of two component vector fields. The input fields can be either NCL names (string) or full data fields. The @@ -862,7 +809,7 @@ def vector_magnitude( "spec": self.spec, "grib_path": self.grib_path, } - field2 = fieldData(**args).ds + field2 = FieldData(**args).ds mag = conversions.magnitude( field1.to_dataarray().squeeze(), field2.to_dataarray().squeeze() @@ -872,7 +819,7 @@ def vector_magnitude( return mag - def wind(self, level) -> [np.ndarray, np.ndarray]: + def wind(self, level: str) -> [np.ndarray, np.ndarray]: """ Returns the u, v wind components as a list (length 2) of arrays. @@ -888,8 +835,8 @@ def wind(self, level) -> [np.ndarray, np.ndarray]: if not level: return False - # Create fieldData objects for u, v components - field_lambda = lambda ds, level, var: fieldData( + # Create FieldData objects for u, v components + field_lambda = lambda ds, level, var: FieldData( ds=ds, fhr=self.fhr, level=level, @@ -900,7 +847,7 @@ def wind(self, level) -> [np.ndarray, np.ndarray]: return [u, v] -class profileData(UPPData): +class ProfileData(UPPData): """ Class provides methods for getting profiles from a specific lat/lon location from a grib file. @@ -919,7 +866,7 @@ class profileData(UPPData): """ - def __init__(self, ds, loc, short_name, **kwargs): + def __init__(self, ds: DataSet, loc: str, short_name: str, **kwargs): super().__init__(ds, short_name, **kwargs) # The first 31 columns are space delimted @@ -939,7 +886,7 @@ def __init__(self, ds, loc, short_name, **kwargs): if self.site_lon < 0: self.site_lon = self.site_lon + 360.0 - def values(self, level=None, name=None, **kwargs): + def values(self, level: str | None = None, name: str | None = None, **kwargs): """ Returns the numpy array of values at the object's x, y location for the requested variable. Transforms are performed in the child class. @@ -949,7 +896,7 @@ def values(self, level=None, name=None, **kwargs): level the level of the alternate field to use, default='ua' for upper air - Keyword Args: + kwargs: ncl_name the NCL name of the variable to be retrieved one_lev bool flag. if True, get the single level of the variable split bool flag. if True, level string numbers are split @@ -980,7 +927,7 @@ def values(self, level=None, name=None, **kwargs): ncl_name = ncl_name.format(fhr=self.fhr, grid=self.grid_suffix) if not ncl_name: - raise errors.NoGraphicsDefinitionForVariable( + raise errors.NoGraphicsDefinitionForVariableError( name, "ua", ) @@ -990,8 +937,10 @@ def values(self, level=None, name=None, **kwargs): profile = field[::] lev = 0 + # 2D if len(profile.shape) == 2: profile = profile[x, y] + # 3D elif len(profile.shape) == 3: if one_lev: lev = vertical_index @@ -1002,13 +951,20 @@ def values(self, level=None, name=None, **kwargs): profile = profile[:, x, y] return profile - def vector_magnitude(self, field1, field2, level="ua", vertical_index=None, **kwargs): + def vector_magnitude( + self, + field1: DataSet, + field2: DataSet, + level: str = "ua", + vertical_index: int | None = None, + **kwargs, + ) -> DataArray: """ - Returns the vector magnitude of two component vector profiles. The - input fields can be either NCL names (string) or full data fields. + The vector magnitude of two component vector profiles. + + The input fields can be either NCL names (string) or full data fields. - If no layer or level is provided, the default 'ua' will be used in - self.values. + If no layer or level is provided, the default 'ua' will be used in self.values. """ if isinstance(field1, str): diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index f624e0e..f138f2c 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -4,13 +4,15 @@ Classes that load grib files. """ +from pathlib import Path + import xarray as xr class GribFile: """Wrappers and helper functions for interfacing with cfgrib.""" - def __init__(self, filename, var_config, **kwargs): + def __init__(self, filename: Path | str, var_config: dict): # pylint: disable=unused-argument self.filename = filename @@ -37,9 +39,10 @@ class GribFiles: forecast hours. """ - def __init__(self, coord_dims, filenames, filetype, **kwargs): + def __init__(self, coord_dims: dict[str, list[int]], filenames: dict, filetype: str, **kwargs): """ - Arguments: + Initialize GribFiles object. + coord_dims dict containing the name of the dimension to concat (key), and a list of its values (value). Ex: {'fhr': [2, 3, 4]} @@ -48,9 +51,8 @@ def __init__(self, coord_dims, filenames, filetype, **kwargs): hours after that ('free_fcst'). filetype key to use for dict when setting variable_names - Keyword Arguments: + kwargs: model string describing the model type - """ self.model = kwargs.get("model", "") @@ -61,18 +63,21 @@ def __init__(self, coord_dims, filenames, filetype, **kwargs): self.grid_suffix = self._get_grid_suffix(filenames) self.contents = self._load() - def append(self, filenames): + def append(self, filenames: list[str]): """ - Add a single new slice to existing data set. Must match coord_dims - and filetype of original dataset. Updates current contents of Object + Add a single new slice to existing data set. + Must match coord_dims and filetype of the original dataset. Updates current contents + property. """ self.contents = self._load(filenames) - def free_fcst_names(self, ds, fcst_type): + def free_fcst_names(self, ds: xr.Dataset, fcst_type: str): # noqa: PLR0915,PLR0912 """ + Variable names to rename. + Given an opened dataset, return a dict of original variable names - (key) and the desired name (value) + (key) and the desired name (value). """ ret = {} @@ -182,7 +187,7 @@ def free_fcst_names(self, ds, fcst_type): "WEASD_P8_L1_GST0_acc3h", "FROZR_P8_L1_GST0_acc3h", ] - if self.model == "rap" and fhr != 3 and var in bad_3h_vars: + if self.model == "rap" and fhr != 3 and var in bad_3h_vars: # noqa: PLR2004 print(f"dropping {var}") ds.drop(var) continue @@ -195,7 +200,7 @@ def free_fcst_names(self, ds, fcst_type): "APCP_P8_L1_GLC0_acc12h", "APCP_P8_L1_GST0_acc12h", ] - if fhr != 12 and var in bad_12h_vars: + if fhr != 12 and var in bad_12h_vars: # noqa: PLR2004 print(f"dropping {var}") ds.drop(var) continue @@ -211,30 +216,7 @@ def free_fcst_names(self, ds, fcst_type): return ret - @staticmethod - def _get_grid_suffix(filenames): - """ - Return the suffix of the first variable with 4 sections (split on _) - in the file. This should correspond to the grid tag. - """ - - for files in filenames.values(): - if files: - gfile = xr.open_dataset( - files[0], - cache=False, - engine="pynio", - lock=False, - backend_kwargs=dict(format="grib2"), - ) - for var in gfile.keys(): - vsplit = var.split("_") - if len(vsplit) == 4: - gfile.close() - return vsplit[-1] - return "GRID NOT FOUND" - - def _load(self, filenames=None): + def _load(self, filenames: list[str] | None = None): """Load the set of files into a single XArray structure.""" all_leads = [] if filenames is None else [self.contents] @@ -285,20 +267,18 @@ def _load(self, filenames=None): og_ds[bad_var] = og_ds.get(f"{bad_var}1h") all_leads.append(dataset) - ret = xr.combine_nested( + return xr.combine_nested( all_leads, compat="override", concat_dim=list(self.coord_dims.keys())[0], coords="minimal", data_vars="all", ) - return ret @property def open_kwargs(self): """ - Defines the key word arguments used by the various calls to XArray - open_mfdataset + Defines the kwargs used by calls to XArray open_mfdataset. """ return dict( diff --git a/adb_graphics/errors.py b/adb_graphics/errors.py index e90f9be..69687a5 100644 --- a/adb_graphics/errors.py +++ b/adb_graphics/errors.py @@ -1,18 +1,14 @@ """Errors specific to the ADB Graphics package.""" -class Error(Exception): - """Base class for handling errors""" +class FieldNotUniqueError(Exception): + """Exception raised when multiple Grib fields are found with input parameters.""" -class FieldNotUnique(Error): - """Exception raised when multiple Grib fields are found with input parameters""" - - -class GribReadError(Error): +class GribReadError(Exception): """Exception raised when there is an error reading the grib file.""" - def __init__(self, name, message="was not found"): + def __init__(self, name: str, message: str = "was not found"): self.name = name self.message = message @@ -22,13 +18,13 @@ def __str__(self): return f'"{self.name}" {self.message}' -class NoGraphicsDefinitionForVariable(Error): +class NoGraphicsDefinitionForVariableError(Exception): """Exception raised when there is no configuration for the variable.""" -class LevelNotFound(Error): +class LevelNotFoundError(Exception): """Exception raised when there is no configuration for the variable.""" -class OutsideDomain(Error): +class OutsideDomainError(Exception): """Exception raised when there is no configuration for the variable.""" diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 75a0184..e6ced48 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -5,10 +5,12 @@ """ import gc -import os +from argparse import Namespace +from pathlib import Path import matplotlib.pyplot as plt import numpy as np +from xarray import Dataset from adb_graphics.datahandler import gribdata, gribfile from adb_graphics.figures import maps, skewt @@ -16,8 +18,9 @@ AIRPORTS = "static/Airports_locs.txt" -def add_obs_panel(ax, model_name, obs_file, proj_info, short_name, tile): - # pylint: disable=too-many-arguments +def add_obs_panel( + ax: plt, model_name: str, obs_file: Path, proj_info: dict, short_name: str, tile: str +): """ Plot observation data provided by the obs_file path using the assigned projection. @@ -25,7 +28,7 @@ def add_obs_panel(ax, model_name, obs_file, proj_info, short_name, tile): gribobs = gribfile.GribFile(filename=obs_file) ax.axis("on") - field = gribdata.fieldData( + field = gribdata.FieldData( ds=gribobs.contents, fhr=0, level="obs", @@ -52,7 +55,16 @@ def add_obs_panel(ax, model_name, obs_file, proj_info, short_name, tile): def parallel_maps( - cla, fhr, grib_path, level, model, spec, variable, workdir, tile="full", dp2=None + cla: Namespace, + fhr: int, + grib_path: Path, + level: str, + model: str, + spec: dict, + variable: str, + workdir: Path, + tile: str = "full", + dp2: Path | None = None, ): # pylint: disable=too-many-arguments,too-many-locals # pylint: disable=too-many-branches,too-many-statements @@ -104,9 +116,11 @@ def parallel_maps( continue # Shenanigans to match ensemble member to panel index - mem = 0 if index == 4 else index - mem = mem if mem < 4 else index - 1 - mem = mem if mem < 8 else index - 2 + center_left = 4 + lower_left = 8 + mem = 0 if index == center_left else index + mem = mem if mem < center_left else index - 1 + mem = mem if mem < lower_left else index - 2 # Create an object that holds all the fields for this map map_fields = maps.MapFields( @@ -145,7 +159,7 @@ def parallel_maps( if index == 0: dm.title() dm.add_logo(current_ax) - elif index == 8: + elif index == lower_left: if spec.get("include_obs", False) and cla.obs_file_path: # Add observation panel to lower left. Currently only # supported for composite reflectivity. @@ -153,7 +167,7 @@ def parallel_maps( ax=axes[8], model_name=cla.model_name, obs_file=cla.obs_file_path, - proj_info=field.grid_info(), + proj_info=map_fields.shaded.grid_info(), short_name=variable, tile=tile, ) @@ -165,7 +179,7 @@ def parallel_maps( # Build the output path png_file = f"{variable}_{tile}_{level}_f{fhr:03d}.png" png_file = png_file.replace("__", "_") - png_path = os.path.join(workdir, png_file) + png_path = workdir / png_file print("*" * 120) print(f"Creating image file: {png_path}") @@ -192,9 +206,11 @@ def parallel_maps( gc.collect() -def parallel_skewt(cla, fhr, ds, site, workdir): +def parallel_skewt(cla: Namespace, fhr: int, ds: Dataset, site: str, workdir: Path): """ - Function that creates a single SkewT plot. Can be used in parallel. + Function that creates a single SkewT plot. + + Can be used in parallel. Input: cla command line arguments Namespace object @@ -214,8 +230,7 @@ def parallel_skewt(cla, fhr, ds, site, workdir): ) skew.create_diagram() outfile = f"{skew.site_code}_{skew.site_num}_skewt_f{fhr:03d}.png" - png_path = os.path.join(workdir, outfile) - + png_path = workdir / outfile print("*" * 80) print(f"Creating image file: {png_path}") print("*" * 80) @@ -231,7 +246,7 @@ def parallel_skewt(cla, fhr, ds, site, workdir): start_time = cla.start_time.strftime("%Y%m%d%H") csvfile = f"{skew.site_code}.{skew.site_num}.skewt.{start_time}_f{fhr:03d}.csv" - csv_path = os.path.join(workdir, csvfile) + csv_path = workdir / csvfile print("*" * 80) print(f"Creating csv file: {csv_path}") print("*" * 80) @@ -240,16 +255,13 @@ def parallel_skewt(cla, fhr, ds, site, workdir): plt.close() -def set_figure(model_name, graphic_type, tile): +def set_figure(model_name: str, graphic_type: str, tile: str): """ Create the figure and subplots appropriate for the model and graphics type. Return the figure handle and list of axes. """ - if model_name == "HRRR-HI": - inches = 12.2 - else: - inches = 10 + inches = 12.2 if model_name == "HRRR-HI" else 10 # Settings for a default single map x_aspect = 1 diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 5f420e6..53bfe71 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -171,19 +171,19 @@ def shaded(self): "spec": self.fields_spec, "grib_path": self.grib_path, } - field = gribdata.fieldData(**args) + field = gribdata.FieldData(**args) if self.map_type == "diff": args["ds"] = gribfile.GribFile(self.grib_path2, cf).contents args["grib_path"] = self.grib_path2 - field2 = gribdata.fieldData(**args) + field2 = gribdata.FieldData(**args) field.data = field.values() - field2.values() return field @property def contours(self): - """Return the list of contour fieldData objects.""" + """Return the list of contour FieldData objects.""" # We won't plot contours on multipanel plots, or full global # plots. @@ -197,7 +197,7 @@ def contours(self): @property def hatches(self): - """Return the list of hatch fieldData objects.""" + """Return the list of hatch FieldData objects.""" return self._overlay_fields("hatches") @@ -221,12 +221,12 @@ def wind_fields(self, level: str | None = None): "spec": self.fields_spec, "grib_path": self.grib_path, } - winds.append(gribdata.fieldData(**args)) + winds.append(gribdata.FieldData(**args)) return winds def _overlay_fields(self, spec_sect: str) -> list: """ - Create fieldData objects for the specified overlay type - hatches or contours. + Create FieldData objects for the specified overlay type - hatches or contours. """ overlay_fields = [] @@ -250,7 +250,7 @@ def _overlay_fields(self, spec_sect: str) -> list: "spec": self.fields_spec, "grib_path": self.grib_path, } - overlay_obj = gribdata.fieldData(**args) + overlay_obj = gribdata.FieldData(**args) # Set the attributes for the overlay field overlay_obj.contour_kwargs = overlay_kwargs overlay_fields.append(overlay_obj) @@ -899,7 +899,7 @@ def _wind_barbs(self, level: bool | str): sizes={"spacing": 0.25}, ) - def _xy_mesh(self, field: gribdata.fieldData): + def _xy_mesh(self, field: gribdata.FieldData): """Helper function to create mesh for various plot.""" lat, lon = field.latlons() diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 87b16d1..e1265ea 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -25,7 +25,7 @@ from adb_graphics.datahandler import gribdata -class SkewTDiagram(gribdata.profileData): +class SkewTDiagram(gribdata.ProfileData): """ The class responsible for gathering all data needed from a grib file to produce a Skew-T Log-P diagram. @@ -41,7 +41,7 @@ class SkewTDiagram(gribdata.profileData): max_plev maximum pressure level to plot in mb model_name model name to use for plotting - Additional keyword arguments for the gribdata.profileData base class should also + Additional keyword arguments for the gribdata.ProfileData base class should also be included. """ @@ -563,7 +563,7 @@ def thermo_variables(self): includes the value of the metric. Variables' transforms and units are handled by default specs in much the - same way as in fieldData class since these are not used by MetPy + same way as in FieldData class since these are not used by MetPy explictly. """ @@ -641,7 +641,7 @@ def thermo_variables(self): spec = self.spec.get(varname, {}).get(lev) if not spec: - raise errors.NoGraphicsDefinitionForVariable(varname, lev) + raise errors.NoGraphicsDefinitionForVariableError(varname, lev) try: tmp = self.values(level=lev, name=varname, one_lev=True) diff --git a/create_graphics.py b/create_graphics.py index b5b6902..b9a04e0 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -97,7 +97,7 @@ def create_maps( if not spec: msg = f"graphics: {variable} {level}" - raise errors.NoGraphicsDefinitionForVariable(msg) + raise errors.NoGraphicsDefinitionForVariableError(msg) args.append( ( @@ -489,7 +489,7 @@ def remove_accumulated_images(cla: Namespace): spec = cla.specs.get(variable, {}).get(level) if not spec: msg = f"graphics: {variable} {level}" - raise errors.NoGraphicsDefinitionForVariable(msg) + raise errors.NoGraphicsDefinitionForVariableError(msg) accumulate = spec.get("accumulate", False) if accumulate: diff --git a/tests/test_grib.py b/tests/test_grib.py index 844782e..3d3f30d 100644 --- a/tests/test_grib.py +++ b/tests/test_grib.py @@ -48,11 +48,11 @@ def values(self, level=None, name=None, **kwargs): # noqa: ARG002 assert upp.date_to_str(test_date) == "20201205 12 UTC" -def test_fieldData(prsfile): - """Test the fieldData class methods on a prs file.""" +def test_FieldData(prsfile): + """Test the FieldData class methods on a prs file.""" prs_ds = gribfile.GribFile(prsfile) - field = gribdata.fieldData(prs_ds.contents, fhr=2, level="500mb", short_name="temp") + field = gribdata.FieldData(prs_ds.contents, fhr=2, level="500mb", short_name="temp") assert isinstance(field.cmap, mcolors.Colormap) assert isinstance(field.colors, np.ndarray) @@ -82,7 +82,7 @@ def test_fieldData(prsfile): (field.values() - 273.15) * 9 / 5 + 32, ) - field2 = gribdata.fieldData(prs_ds.contents, fhr=2, level="ua", short_name="ceil") + field2 = gribdata.FieldData(prs_ds.contents, fhr=2, level="ua", short_name="ceil") transforms = field2.vspec.get("transform") assert np.array_equal( field2.get_transform(transforms, field2.values()), @@ -96,11 +96,11 @@ def test_fieldData(prsfile): def test_profile_data(natfile: Path): - """Test the profileData class methods on a nat file.""" + """Test the ProfileData class methods on a nat file.""" nat_ds = gribfile.GribFile(natfile) loc = " BNA 9999 99999 36.12 86.69 597 Nashville, TN\n" - profile = gribdata.profileData( + profile = gribdata.ProfileData( nat_ds.contents, fhr=2, filetype="nat", From 48617df9d69fc782305df2fdd4fb26aee5a5ea3a Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 22 Oct 2025 14:55:47 -0600 Subject: [PATCH 14/98] WIP with mypy. --- adb_graphics/datahandler/gribfile.py | 4 +-- adb_graphics/figure_builders.py | 26 ++++++++------ adb_graphics/figures/maps.py | 2 +- adb_graphics/utils.py | 4 ++- create_graphics.py | 54 ++++++++++++++-------------- tests/test_hrrr_maps.py | 23 ++++++------ 6 files changed, 60 insertions(+), 53 deletions(-) diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index f138f2c..084e481 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -63,7 +63,7 @@ def __init__(self, coord_dims: dict[str, list[int]], filenames: dict, filetype: self.grid_suffix = self._get_grid_suffix(filenames) self.contents = self._load() - def append(self, filenames: list[str]): + def append(self, filenames: dict[str, list[Path]]): """ Add a single new slice to existing data set. Must match coord_dims and filetype of the original dataset. Updates current contents @@ -216,7 +216,7 @@ def free_fcst_names(self, ds: xr.Dataset, fcst_type: str): # noqa: PLR0915,PLR0 return ret - def _load(self, filenames: list[str] | None = None): + def _load(self, filenames: list[Path] | None = None): """Load the set of files into a single XArray structure.""" all_leads = [] if filenames is None else [self.contents] diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index e6ced48..4920f55 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -10,32 +10,37 @@ import matplotlib.pyplot as plt import numpy as np +from matplotlib import axes from xarray import Dataset -from adb_graphics.datahandler import gribdata, gribfile from adb_graphics.figures import maps, skewt -AIRPORTS = "static/Airports_locs.txt" +AIRPORTS = Path("static/Airports_locs.txt") def add_obs_panel( - ax: plt, model_name: str, obs_file: Path, proj_info: dict, short_name: str, tile: str + ax: axes.Axes, + model_name: str, + obs_file: Path, + proj_info: dict, + spec: dict, + short_name: str, + tile: str, ): """ Plot observation data provided by the obs_file path using the assigned projection. """ - gribobs = gribfile.GribFile(filename=obs_file) ax.axis("on") - field = gribdata.FieldData( - ds=gribobs.contents, + map_fields = maps.MapFields( fhr=0, + fields_spec=spec, + grib_path=obs_file, level="obs", model="obs", - short_name=short_name, + name=short_name, ) - map_fields = maps.MapFields(main_field=field) m = maps.Map( airport_fn=AIRPORTS, ax=ax, @@ -169,6 +174,7 @@ def parallel_maps( obs_file=cla.obs_file_path, proj_info=map_fields.shaded.grid_info(), short_name=variable, + spec=cla.specs, tile=tile, ) else: @@ -264,8 +270,8 @@ def set_figure(model_name: str, graphic_type: str, tile: str): inches = 12.2 if model_name == "HRRR-HI" else 10 # Settings for a default single map - x_aspect = 1 - y_aspect = 1 + x_aspect = 1.0 + y_aspect = 1.0 nrows = 1 ncols = 1 diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 53bfe71..28b6986 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -995,7 +995,7 @@ class MultiPanelDataMap(DataMap): """ def __init__( - self, map_fields: list, map_: plt, member: int, model_name: str | None = None, **kwargs + self, map_fields: MapFields, map_: plt, member: str, model_name: str | None = None, **kwargs ): super().__init__(map_fields, map_, model_name=model_name) diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 834ee9f..9dff31d 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -330,7 +330,7 @@ def numeric_level(index_match: bool = True, level: str | None = None): return lev_val, lev_unit -def old_enough(age: int, file_path: Path): +def old_enough(age: int, file_path: Path | str): """ Helper function to test the age of a file. @@ -344,6 +344,8 @@ def old_enough(age: int, file_path: Path): bool whether the file is at least age minutes old """ + file_path = Path(file_path) if isinstance(file_path, str) else file_path + file_time = datetime.fromtimestamp(file_path.stat().st_ctime) max_age = datetime.now() - timedelta(minutes=age) diff --git a/create_graphics.py b/create_graphics.py index b9a04e0..11cce05 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -18,9 +18,9 @@ import time from argparse import ArgumentError, ArgumentParser, Namespace from multiprocessing import Pool +from pathlib import Path import yaml -from libpath import Path from adb_graphics import errors, utils from adb_graphics.datahandler.gribfile import GribFile, GribFiles @@ -41,7 +41,7 @@ def check_file( data_root: Path | None = None, file_tmpl: str | None = None, mem: int | None = None, -) -> (Path, bool): +) -> tuple[Path, bool]: """ Given the command line arguments, the forecast hour, and a potential ensemble member, build a full path to the file and ensure it exists. @@ -54,9 +54,10 @@ def check_file( grib_path = data_root / file_tmpl if mem is not None: - grib_path = grib_path.format(FCST_TIME=fhr, mem=mem) + grib_path = str(grib_path).format(FCST_TIME=fhr, mem=mem) else: - grib_path = grib_path.format(FCST_TIME=fhr) + grib_path = str(grib_path).format(FCST_TIME=fhr) + grib_path = Path(grib_path) print(f"Checking on file {grib_path}") old_enough = utils.old_enough(cla.data_age, grib_path) if grib_path.exists() else False @@ -70,7 +71,7 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): """ # Create the file object to load the contents - gfile = GribFile(grib_path) + gfile = GribFile(grib_path, var_config={}) args = [(cla, fhr, gfile.contents, site, workdir) for site in cla.sites] @@ -120,38 +121,36 @@ def create_maps( pool.starmap(parallel_maps, args) -def gather_gribfiles(cla: Namespace, fhr: int, filename: str, gribfiles: None | GribFiles): +def gather_gribfiles(cla: Namespace, fhr: int, grib_path: Path, gribfiles: None | GribFiles): """ Returns the appropriate gribfiles object for the type of graphics being generated -- whether it's for a single forecast time or all forecast lead times. """ - filenames = {"01fcst": [], "free_fcst": []} + filenames: dict[str, list[Path]] = {"01fcst": [], "free_fcst": []} fcst_hour = int(fhr) first_fcst = 6 if "global" in cla.images[0] else 1 if fcst_hour <= first_fcst: - filenames["01fcst"].append(filename) + filenames["01fcst"].append(grib_path) else: - filenames["free_fcst"].append(filename) + filenames["free_fcst"].append(grib_path) if gribfiles is None or not cla.all_leads: # Create a new GribFiles object, include all hours, or just this one, # depending on command line argument flag - gribfiles = GribFiles( + return GribFiles( coord_dims={"fcst_hr": [fhr]}, filenames=filenames, filetype=cla.file_type, model=cla.images[0], ) - else: - # Append a single forecast hour to the existing GribFiles object. - gribfiles.coord_dims.get("fcst_hr").append(fhr) - gribfiles.append(filenames) - + # Append a single forecast hour to the existing GribFiles object. + gribfiles.coord_dims.get("fcst_hr", []).append(fhr) + gribfiles.append(filenames) return gribfiles @@ -177,7 +176,7 @@ def generate_tile_list(arg_list: list) -> list[str]: return arg_list -def load_images(arg: Path | str): +def load_images(arg: list[Path | str]): """ Check that input image file exists, and that it contains the requested section. Return a 2-list (required by argparse) of the @@ -394,7 +393,7 @@ def parse_args(argv: list) -> Namespace: return parser.parse_args(argv) -def pre_proc_grib_files(cla: Namespace, fhr: int) -> Path: +def pre_proc_grib_files(cla: Namespace, fhr: int) -> tuple[Path, bool]: """ Use the command line argument object (cla) to determine the grib file location at a given forecast hour. If multiple data input paths and file @@ -420,11 +419,12 @@ def pre_proc_grib_files(cla: Namespace, fhr: int) -> Path: # Generate a list of files to be joined. file_list = [ - Path(*path).format(FCST_TIME=fhr) for path in zip(cla.data_root, cla.file_tmpl, strict=True) + "/".join(path).format(FCST_TIME=fhr) + for path in zip(cla.data_root, cla.file_tmpl, strict=True) ] for file_path in file_list: - if not file_path.exists() or not utils.old_enough(cla.data_age, file_path): - return file_path, False + if not Path(file_path).exists() or not utils.old_enough(cla.data_age, file_path): + return Path(file_path), False print("Combining input files: ") for fn in file_list: @@ -671,8 +671,8 @@ def graphics_driver(cla: Namespace): create_maps( cla, fhr=fhr, - grib_contents=gribfiles.contents, - grib_contents2=gribfiles2.contents, + grib_path=grib_path, + grib_path2=grib_path2, workdir=workdir, ) else: @@ -685,7 +685,7 @@ def graphics_driver(cla: Namespace): create_maps( cla, fhr=fhr, - grib_contents=gribfiles.contents, + grib_path=grib_path, workdir=workdir, ) @@ -723,7 +723,7 @@ def create_graphics(argv: list): # Check that the same number of entries exists in -d and --file_tmpl if len(clargs.data_root) != len(clargs.file_tmpl): errmsg = "Must specify the same number of arguments for -d and --file_tmpl" - ArgumentParser.exit(0, errmsg) + clargs.exit(errmsg, 0) # Ensure wgrib command is available in environment before getting too far # down this path... @@ -756,13 +756,13 @@ def create_graphics(argv: list): # Make sure both required arguments (--max_plev, --sites) are provided when doing skewTs if clargs.graphic_type == "skewts": if not clargs.max_plev: - ArgumentParser.exit( - 0, + clargs.exit( "Must specify maximum pressure level \ (--max_plev) when creating skewTs", + 0, ) if not clargs.sites: - ArgumentParser.exit(0, "Must specify sites (--sites) when creating skewTs") + clargs.exit("Must specify sites (--sites) when creating skewTs", 0) print(f"Running script for {clargs.graphic_type} with args: ", f"{LOG_BREAK}") diff --git a/tests/test_hrrr_maps.py b/tests/test_hrrr_maps.py index f69a163..4a3b459 100644 --- a/tests/test_hrrr_maps.py +++ b/tests/test_hrrr_maps.py @@ -2,17 +2,15 @@ import os -import pytest -from path import Path +from pytest import fixture from create_graphics import create_graphics, parse_args DATA_LOC = os.environ.get("DATA_LOC") -OUTPUT_LOC = Path(os.environ.get("OUTPUT_LOC")) -@pytest.fixture -def maps_args() -> list: +@fixture +def maps_args(tmp_path) -> list: """Builds HRRR 12-hour accumulated maps.""" return [ "maps", @@ -23,7 +21,7 @@ def maps_args() -> list: "12", "1", "-o", - OUTPUT_LOC, + tmp_path / "output", "-s", "2023031500", "--file_tmpl", @@ -36,7 +34,7 @@ def maps_args() -> list: ] -def test_parse_args(): +def test_parse_args(tmp_path): """ Test parse_args for basic parsing success. Checks if parse_args returns 'maps' in the graphic_type field. @@ -50,7 +48,7 @@ def test_parse_args(): "12", "1", "-o", - OUTPUT_LOC, + tmp_path / "output", "-s", "2021052315", "--file_tmpl", @@ -65,7 +63,7 @@ def test_parse_args(): assert test_args.graphic_type == "maps" -def test_file_count(maps_args): +def test_file_count(maps_args, tmp_path): """ Test for file count in directory. Can be extended to cover multiple folders. @@ -75,8 +73,9 @@ def test_file_count(maps_args): map_count = 6 count = 0 folder = "/202303150000/" - assert (OUTPUT_LOC / folder).isdir() - for file_name in (OUTPUT_LOC / folder).iterdir(): - if (OUTPUT_LOC / folder / file_name).is_file(): + output = tmp_path / "output" / folder + assert output.isdir() + for file_name in output.iterdir(): + if (output / file_name).is_file(): count += 1 assert count == map_count From ffc321a4306571e897f1e3e5f25ec8e241779094 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 23 Oct 2025 12:29:29 -0600 Subject: [PATCH 15/98] WIP with mypy --- adb_graphics/conversions.py | 35 ++-- adb_graphics/datahandler/gribdata.py | 2 +- adb_graphics/figures/maps.py | 70 ++++--- adb_graphics/figures/skewt.py | 299 ++++++++++++++------------- adb_graphics/utils.py | 5 +- tests/test_common.py | 32 +-- 6 files changed, 224 insertions(+), 219 deletions(-) diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index 0936d4d..482696b 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -5,23 +5,24 @@ converted values as output. """ -from xarray import DataArray, DataSet +from numpy import ndarray +from xarray import DataSet from xarray.ufuncs import sqrt, square -def k_to_c(field: DataArray, **kwargs): +def k_to_c(field: ndarray, **kwargs): """Conversion from Kelvin to Celsius.""" return field - 273.15 -def k_to_f(field: DataArray, **kwargs): +def k_to_f(field: ndarray, **kwargs): """Conversion from Kelvin to Farenheit.""" return (field - 273.15) * 9 / 5 + 32 -def kgm2_to_in(field: DataArray, **kwargs): +def kgm2_to_in(field: ndarray, **kwargs): """Conversion from kg per m^2 to inches.""" return field * 0.03937 @@ -33,78 +34,78 @@ def magnitude(a: DataSet, b: DataSet, **kwargs): return sqrt(square(a) + square(b)) -def m_to_dm(field: DataArray, **kwargs): +def m_to_dm(field: ndarray, **kwargs): """Conversion from meters to decameters.""" return field / 10.0 -def m_to_in(field: DataArray, **kwargs): +def m_to_in(field: ndarray, **kwargs): """Conversion from meters to inches.""" return field * 39.3701 -def m_to_kft(field: DataArray, **kwargs): +def m_to_kft(field: ndarray, **kwargs): """Conversion from meters to kilofeet.""" return field / 304.8 -def m_to_mi(field: DataArray, **kwargs): +def m_to_mi(field: ndarray, **kwargs): """Conversion from meters to miles.""" return field / 1609.344 -def ms_to_kt(field: DataArray, **kwargs): +def ms_to_kt(field: ndarray, **kwargs): """Conversion from m s-1 to knots.""" return field * 1.9438 -def pa_to_hpa(field: DataArray, **kwargs): +def pa_to_hpa(field: ndarray, **kwargs): """Conversion from Pascals to hectopascals.""" return field / 100.0 -def percent(field: DataArray, **kwargs): +def percent(field: ndarray, **kwargs): """Conversion from values between 0 - 1 to percent.""" return field * 100.0 -def to_micro(field: DataArray, **kwargs): +def to_micro(field: ndarray, **kwargs): """Convert field to micro.""" return field * 1e6 -def to_micrograms_per_m3(field: DataArray, **kwargs): +def to_micrograms_per_m3(field: ndarray, **kwargs): """Convert field to micrograms per cubic meter.""" return field * 1e9 -def vvel_scale(field: DataArray, **kwargs): +def vvel_scale(field: ndarray, **kwargs): """Scale vertical velocity for plotting.""" return field * -10 -def vort_scale(field: DataArray, **kwargs): +def vort_scale(field: ndarray, **kwargs): """Scale vorticity for plotting.""" return field / 1e-05 -def weasd_to_1hsnw(field: DataArray, **kwargs): +def weasd_to_1hsnw(field: ndarray, **kwargs): """Conversion from snow water equiv to snow (10:1 ratio).""" return field * 10.0 -def sden_to_slr(field: DataArray, **kwargs): +def sden_to_slr(field: ndarray, **kwargs): """Convert snow density (kg m-3) to snow-liquid ratio.""" return 1000.0 / field diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index d652dc3..62cf3bf 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -726,7 +726,7 @@ def data(self): def data(self, value: DataSet): self._data = value - def values(self, level: str | None = None, name: str | None = None, **kwargs): + def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: """ Returns the numpy array of values at the requested level for the variable after applying any unit conversion to the original data. diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 28b6986..ae2c0ec 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -18,6 +18,8 @@ import matplotlib.patches as mpatches import matplotlib.pyplot as plt import numpy as np +from matplotlib.axes import Axes +from matplotlib.contour import QuadContourSet from mpl_toolkits.basemap import Basemap, shiftgrid from adb_graphics.datahandler import gribdata, gribfile @@ -37,8 +39,7 @@ # TILE_DEFS is a dict of dicts with predefined tiles specifying the corners of the grid # to be plotted, and the stride and length of the wind barbs. # Order for corners: [lower left lat, upper right lat, lower left lon, upper right lon] - -TILE_DEFS = { +TILE_DEFS: dict = { "NC": {"corners": [36, 51, -109, -85], "stride": 10, "length": 4}, "NE": {"corners": [36, 48, -91, -62], "stride": 10, "length": 4}, "NW": {"corners": [35, 52, -126, -102], "stride": 10, "length": 4}, @@ -131,14 +132,18 @@ def __init__( self.fields_spec = deepcopy(fields_spec) self.level = level self.map_type = map_type - self.model = kwargs.get("model") + self.model = kwargs.get("model", "") self.name = name self.tile = kwargs.get("tile", "full") self.map_spec = deepcopy(self.fields_spec[self.name][self.level]) self.set_level(self.level, self.map_spec) # Required if map_type is "diff" - self.grib_path2 = kwargs.get("grib_path2") + if map_type == "diff": + self.grib_path2: Path | str = kwargs.get("grib_path2", "") + if not self.grib_path2: + msg = "Diff map requires a second grib path. Provide grib_path2 argument!" + raise ValueError(msg) def set_level(self, level: str, spec: dict): nlevel, _ = numeric_level(level=level, index_match=False) @@ -284,10 +289,10 @@ class Map: """ - def __init__(self, airport_fn: Path, ax: plt, **kwargs): + def __init__(self, airport_fn: Path, ax: Axes, **kwargs): self.ax = ax self.grid_info = kwargs.get("grid_info", {}) - self.model = kwargs.get("model") + self.model = kwargs.get("model", "") self.plot_airports = kwargs.get("plot_airports", True) self.tile = kwargs.get("tile", "full") self.airports = self.load_airports(airport_fn) @@ -456,7 +461,7 @@ class DataMap: """ - def __init__(self, map_fields: MapFields, map_: plt, model_name: str | None = None, **kwargs): # noqa: ARG002 + def __init__(self, map_fields: MapFields, map_: Map, model_name: str | None = None, **kwargs): # noqa: ARG002 self.field = map_fields.shaded self.contour_fields = map_fields.contours self.hatch_fields = map_fields.hatches @@ -469,7 +474,7 @@ def wind_fields(self, level: str): return self.map_fields.wind_fields(level) @staticmethod - def add_logo(ax: plt): + def add_logo(ax: Axes): """Puts the NOAA logo at the bottom left of the matplotlib axes.""" logo = mpimg.imread("static/noaa-logo-50x50.png") @@ -485,7 +490,7 @@ def add_logo(ax: plt): ax.add_artist(ab) - def _colorbar(self, cc: plt, ax: plt): + def _colorbar(self, cc: QuadContourSet, ax: Axes): """ Plot the colorbar for the contourf field. If ticks is set to zero, use a user-defined list of clevs from default_specs. @@ -514,13 +519,13 @@ def _colorbar(self, cc: plt, ax: plt): ) if self.field.short_name == "flru": - ticks = [label.rjust(30) for label in ["VFR", "MVFR", "IFR", "LIFR", ""]] + tick_labels = [label.rjust(30) for label in ["VFR", "MVFR", "IFR", "LIFR", ""]] # this step is done to allow proper order of icing severity levels (trace before light) if self.field.short_name == "icsev": - ticks = [label.rjust(30) for label in ["TRACE", "LIGHT", "MODERATE", "HEAVY", ""]] + tick_labels = [label.rjust(30) for label in ["TRACE", "LIGHT", "MODERATE", "HEAVY", ""]] - cbar.ax.set_xticklabels(ticks, fontsize=12) + cbar.ax.set_xticklabels(tick_labels, fontsize=12) def draw(self, show: bool = False): """ @@ -593,7 +598,7 @@ def _draw_panel(self, wind_barbs: bool = True): return cf - def _draw_contours(self, ax: plt, not_labeled: bool): + def _draw_contours(self, ax: Axes, not_labeled: list[str]): """Draw the contour fields requested.""" model_name = self.model_name @@ -632,7 +637,7 @@ def _draw_contours(self, ax: plt, not_labeled: bool): {self.field.level}" ) - def _draw_scatter(self, ax: plt): + def _draw_scatter(self, ax: Axes): """Plot dots at locations on the map that meet a threshold.""" field = self.field @@ -667,7 +672,7 @@ def _draw_scatter(self, ax: plt): **field.contour_kwargs, ) - def _draw_field(self, ax: plt, field: str, func: Callable, **kwargs): + def _draw_field(self, ax: Axes, field: gribdata.FieldData, func: Callable, **kwargs): """ Internal implementation that calls a matplotlib function. @@ -713,7 +718,7 @@ def _draw_field(self, ax: plt, field: str, func: Callable, **kwargs): print(f"CLOSE ERROR: {field.short_name} {field.level}") return ret - def _draw_field_values(self, ax: plt): + def _draw_field_values(self, ax: Axes): """Add the text value of the field at airport locations.""" annotate_decimal = self.field.vspec.get("annotate_decimal", 0) lats = self.map.airports[:, 0] @@ -739,7 +744,7 @@ def _draw_field_values(self, ax: plt): ) data_values.close() - def _draw_hatches(self, ax: plt): + def _draw_hatches(self, ax: Axes): """Draw the hatched regions requested.""" # Levels should be included in the settings dict here since they don't @@ -775,7 +780,7 @@ def _draw_hatches(self, ax: plt): if self.field.short_name == "ptyp": plt.legend(handles=handles, loc=[0.25, 0.03]) - def _set_overlay_string(self): + def _set_overlay_string(self) -> str: """ Creates the main title of the plot with select hatched and contoured fields defined. @@ -791,23 +796,23 @@ def _set_overlay_string(self): cf = self.hatch_fields[0] not_labeled.extend([h.short_name for h in self.hatch_fields]) if not any(list(set(cf.short_name).intersection(["pres"]))): - title = cf.vspec.get("title", cf.field.long_name) - contoured.append(f"{title} ({cf.units}, hatched)") + user_title = cf.vspec.get("title", cf.field.long_name) + contoured.append(f"{user_title} ({cf.units}, hatched)") # Add descriptor string for the important contoured fields if self.contour_fields: for cf in self.contour_fields: if cf.short_name not in not_labeled: - title = cf.vspec.get("title", cf.field.long_name) - title = title.replace("Geopotential", "Geop.") - contoured.append(f"{title}") + user_title = cf.vspec.get("title", cf.field.long_name) + user_title = user_title.replace("Geopotential", "Geop.") + contoured.append(f"{user_title}") contoured_units.append(f"{cf.units}") - contoured = "\n".join(contoured) # Make 'contoured' a string with linefeeds + title = "\n".join(contoured) # Make 'contoured' a multioline string if contoured_units: - contoured = f"{contoured} ({', '.join(contoured_units)}, contoured)" + title = f"{title} ({', '.join(contoured_units)}, contoured)" - return contoured + return title def _title(self): """Draw the title for a map.""" @@ -885,6 +890,8 @@ def _wind_barbs(self, level: bool | str): u, x = shiftgrid(180.0, u, x, start=False) v, savex = shiftgrid(180.0, v, savex, start=False) y, x = np.meshgrid(y, x, sparse=False, indexing="ij") + mu: np.ma.MaskedArray + mv: np.ma.MaskedArray mu, mv = [np.ma.masked_array(c, mask=mask) for c in [u, v]] self.map.m.barbs( @@ -916,7 +923,7 @@ class DiffMap(DataMap): and will not plot overlays and such. """ - def _colorbar(self, cc: plt, ax: plt): + def _colorbar(self, cc: QuadContourSet, ax: Axes): """Set the colorbar for a difference field.""" plt.colorbar( @@ -927,7 +934,7 @@ def _colorbar(self, cc: plt, ax: plt): shrink=1.0, ) - def _draw_panel(self): + def _draw_panel(self, wind_barbs: bool = False): """Draw a map of the difference field.""" ax = self.map.ax @@ -935,6 +942,9 @@ def _draw_panel(self): # Draw a map and add the shaded field self.map.draw() + if wind_barbs: + print("Wind barbs are not drawn for diff plots") + # The number of levels (nlev) here, should be the same number as is used # in the linspace call in self._eq_contours. 21 seems reasonable, but is # arbitrary. @@ -995,7 +1005,7 @@ class MultiPanelDataMap(DataMap): """ def __init__( - self, map_fields: MapFields, map_: plt, member: str, model_name: str | None = None, **kwargs + self, map_fields: MapFields, map_: Map, member: str, model_name: str | None = None, **kwargs ): super().__init__(map_fields, map_, model_name=model_name) @@ -1015,7 +1025,7 @@ def draw(self, show: bool = False): # Finish with the colorbar on the last panel only # Plot it on the full figure scale. if self.last_panel: - cax = plt.axes([0.0, 0.0, 1.0, 0.2]) + cax = plt.axes((0.0, 0.0, 1.0, 0.2)) self._colorbar(ax=cax, cc=cf) cax.axis("off") diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index e1265ea..18788b7 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -4,9 +4,9 @@ Log-P diagram using MetPy. """ -from collections import OrderedDict from functools import cached_property from pathlib import Path +from typing import TYPE_CHECKING, TypedDict import matplotlib.font_manager as fm import matplotlib.lines as mlines @@ -14,6 +14,7 @@ import metpy.calc as mpcalc import numpy as np import pandas as pd +from matplotlib.axes import Axes from matplotlib.lines import Line2D from matplotlib.ticker import FixedLocator from metpy.plots import Hodograph, SkewT @@ -24,6 +25,17 @@ from adb_graphics import errors, utils from adb_graphics.datahandler import gribdata +if TYPE_CHECKING: + from pint import UnitRegistry + + +class HydroPlotSettings(TypedDict): + color: str + label: str + marker: str + scale: float + units: str + class SkewTDiagram(gribdata.ProfileData): """ @@ -59,47 +71,46 @@ def __init__(self, ds: Dataset, loc: str, **kwargs): self.max_plev = kwargs.get("max_plev", 0) self.model_name = kwargs.get("model_name", "Analysis") - def _add_hydrometeors(self, hydro_subplot: plt): + def _add_hydrometeors(self, hydro_subplot: Axes): # pylint: disable=too-many-locals - mixing_ratios = OrderedDict( - { - "clwmr": { - "color": "blue", - "label": "CWAT", - "marker": "s", - "scale": 1.0, - "units": "g/m2", - }, - "icmr": { - "color": "red", - "label": "CICE", - "marker": "^", - "scale": 10.0, - "units": "g/m2", - }, - "rwmr": { - "color": "cyan", - "label": "RAIN", - "marker": "o", - "scale": 1.0, - "units": "g/m2", - }, - "snmr": { - "color": "purple", - "label": "SNOW", - "marker": "*", - "scale": 1.0, - "units": "g/m2", - }, - "grle": { - "color": "orange", - "label": "GRPL", - "marker": "D", - "scale": 1.0, - "units": "g/m2", - }, - } - ) + + mixing_ratios: dict[str, HydroPlotSettings] = { + "clwmr": { + "color": "blue", + "label": "CWAT", + "marker": "s", + "scale": 1.0, + "units": "g/m2", + }, + "icmr": { + "color": "red", + "label": "CICE", + "marker": "^", + "scale": 10.0, + "units": "g/m2", + }, + "rwmr": { + "color": "cyan", + "label": "RAIN", + "marker": "o", + "scale": 1.0, + "units": "g/m2", + }, + "snmr": { + "color": "purple", + "label": "SNOW", + "marker": "*", + "scale": 1.0, + "units": "g/m2", + }, + "grle": { + "color": "orange", + "label": "GRPL", + "marker": "D", + "scale": 1.0, + "units": "g/m2", + }, + } profiles = self.atmo_profiles # dictionary pres = profiles.get("pres").get("data") @@ -119,11 +130,11 @@ def _add_hydrometeors(self, hydro_subplot: plt): # Get the profile values scale = settings.get("scale") try: - profile = np.asarray(self.values(name=mixr)) * 1000.0 * scale + profile = self.values(name=mixr) * 1000.0 * scale except errors.GribReadError: print(f"missing {mixr} for hydrometeor plot, skipping that field.") continue - mixr_total = 0.0 + mixr_total: units = 0.0 for n in range(nlevs): if n == 0: pres_layer = 2 * (pres_sfc - pres[n]) # layer depth @@ -141,7 +152,7 @@ def _add_hydrometeors(self, hydro_subplot: plt): hydro_subplot.plot( profile, pres, - settings.get("color"), + settings.get("color", ""), fillstyle="none", linewidth=0.5, marker=settings.get("marker"), @@ -151,7 +162,7 @@ def _add_hydrometeors(self, hydro_subplot: plt): hydro_subplot.plot( profile[temp.magnitude < freezing_f], pres[temp.magnitude < freezing_f], - settings.get("color"), + settings.get("color", ""), fillstyle="full", linewidth=0.5, marker=settings.get("marker"), @@ -217,7 +228,7 @@ def _add_hydrometeors(self, hydro_subplot: plt): verticalalignment="top", ) - def _add_thermo_inset(self, skew: plt): + def _add_thermo_inset(self, skew: SkewT): # Build up the text that goes in the thermo-dyniamics box lines = [] for name, items in self.thermo_variables.items(): @@ -257,39 +268,37 @@ def atmo_profiles(self): differs from the requirements of other graphics units/transforms. """ - # OrderedDict because we need to get pressure profile first. Entries in + # We need to get pressure profile first. Entries in # the dict are as follows: # # Variable short name: consistent with default_specs.yml # transform: units string to pass to MetPy's to() function # units: the end unit of the field (after transform, # if applicable). - atmo_vars = OrderedDict( - { - "pres": { - "transform": "hectoPa", - "units": units.Pa, - }, - "gh": { - "units": units.gpm, - }, - "sphum": { - "units": units.dimensionless, - }, - "temp": { - "transform": "degF", - "units": units.degK, - }, - "u": { - "transform": "knots", - "units": units.meter_per_second, - }, - "v": { - "transform": "knots", - "units": units.meter_per_second, - }, - } - ) + atmo_vars = { + "pres": { + "transform": "hectoPa", + "units": units.Pa, + }, + "gh": { + "units": units.gpm, + }, + "sphum": { + "units": units.dimensionless, + }, + "temp": { + "transform": "degF", + "units": units.degK, + }, + "u": { + "transform": "knots", + "units": units.meter_per_second, + }, + "v": { + "transform": "knots", + "units": units.meter_per_second, + }, + } top = None for var, items in atmo_vars.items(): @@ -330,7 +339,7 @@ def create_csv(self, csv_path: Path | str): self._write_profile(csv_path) - def _plot_hodograph(self, skew: plt): + def _plot_hodograph(self, skew: SkewT): # Create an array that indicates which layer (10-3, 3-1, 0-1 km) the # wind belongs to. The array, agl, will be set to the height # corresponding to the top of the layer. The resulting array will look @@ -341,7 +350,8 @@ def _plot_hodograph(self, skew: plt): # Where the values above 10 km are unchanged, and there are three levels # in each of the 3 layers of interest. # - agl = np.copy(self.atmo_profiles.get("gh", {}).get("data")).to("km") + data_copy: units = np.copy(self.atmo_profiles.get("gh", {}).get("data")) + agl = data_copy.to("km") # Retrieve the wind data profiles u_wind = self.atmo_profiles.get("u", {}).get("data") @@ -353,7 +363,7 @@ def _plot_hodograph(self, skew: plt): h = Hodograph(ax, component_range=80.0) h.add_grid(increment=20, linewidth=0.5) - intervals = [0, 1, 3, 10] * agl.units + intervals: UnitRegistry = np.array([0, 1, 3, 10]) * agl.units colors = ["xkcd:salmon", "xkcd:aquamarine", "xkcd:navy blue"] line_width = 1.5 @@ -370,12 +380,12 @@ def _plot_hodograph(self, skew: plt): # Local function to create a proxy line object for creating a legend on # a LineCollection returned from plot_colormapped. Using lines and # colors from outside scope. - def make_proxy(zval: int, idx: int | None = None, **kwargs): + def make_proxy(zval: int, idx: int): color = colors[idx] if idx < len(colors) else lines.cmap(zval - 1) - return Line2D([0, 1], [0, 1], color=color, linewidth=line_width, **kwargs) + return Line2D([0, 1], [0, 1], color=color, linewidth=line_width) # Make a list of proxies - proxies = [make_proxy(item, idx=i) for i, item in enumerate(intervals.magnitude)] + proxies = [make_proxy(item, i) for i, item in enumerate(np.asarray(intervals.magnitude))] # Draw the legend ax.legend( @@ -386,7 +396,7 @@ def make_proxy(zval: int, idx: int | None = None, **kwargs): ) @staticmethod - def _plot_labels(skew: plt): + def _plot_labels(skew: SkewT): skew.ax.set_xlabel("Temperature (F)") skew.ax.set_ylabel("Pressure (hPa)") @@ -417,7 +427,7 @@ def _write_profile(self, csv_path: str | Path): profile.to_csv(csv_path, index=False, float_format="%10.2f") - def _plot_profile(self, skew: plt): + def _plot_profile(self, skew: SkewT): profiles = self.atmo_profiles # dictionary pres = profiles.get("pres").get("data") temp = profiles.get("temp").get("data") @@ -441,7 +451,7 @@ def _plot_profile(self, skew: plt): linewidth=1.2, ) - def _plot_wind_barbs(self, skew: plt): + def _plot_wind_barbs(self, skew: SkewT): # Pressure vs wind skew.plot_barbs( self.atmo_profiles.get("pres", {}).get("data"), @@ -556,20 +566,17 @@ def _setup_diagram(self): @cached_property def thermo_variables(self): """ - Return an ordered dictionary of thermodynamic variables needed for the skewT. - Ordered because we want to print these values in this order on the SkewT - diagram. - The return dictionary contains a 'data' entry for each variable that - includes the value of the metric. - - Variables' transforms and units are handled by default specs in much the - same way as in FieldData class since these are not used by MetPy - explictly. + Return a dictionary of thermodynamic variables needed for the skewT. + Ensure it's ordered because we want to print these values in this order on the SkewT + diagram. The return dictionary contains a 'data' entry for each variable that includes the + value of the metric. + + Variables' transforms and units are handled by default specs in much the same way as in + FieldData class since these are not used by MetPy explictly. """ - # OrderedDict so that we get the thermodynamic variables printed in the - # same order every time in the resulting SkewT inset. The fields - # include: + # We want the thermodynamic variables printed in the same order every time in the resulting + # SkewT inset. The fields include: # # Variable short name: can be consistent with default_specs.yml. # If not, must provide level and variable @@ -581,59 +588,57 @@ def thermo_variables(self): # decimals: (optional) number of decimal places to # include when formatting output. Defaults # to 0 (integer). - thermo = OrderedDict( - { - "cape": { # Convective available potential energy - "level": "sfc", - }, - "cin": { # Convective inhibition - "level": "sfc", - }, - "mucape": { # Most Unstable CAPE - "level": "mu", - "variable": "cape", - }, - "mucin": { # CIN from MUCAPE level - "level": "mu", - "variable": "cin", - }, - "li": { # Lifted Index - "decimals": 1, - "level": "sfc", - }, - "bli": { # Best Lifted Index - "decimals": 1, - "level": "best", - "variable": "li", - }, - "lcl": { # Lifted Condensation Level - }, - "lpl": { # Lifted Parcel Level - }, - "srh03": { # 0-3 km Storm relative helicity - "level": "sr03", - "variable": "hlcy", - }, - "srh01": { # 0-1 km Storm relative helicity - "level": "sr01", - "variable": "hlcy", - }, - "shr06": { # 0-6 km Shear - "level": "06km", - "variable": "shear", - }, - "shr01": { # 0-1 km Shear - "level": "01km", - "variable": "shear", - }, - "cell": { # Cell motion - }, - "pwtr": { # Precipitable water - "decimals": 1, - "level": "sfc", - }, - } - ) + thermo: dict = { + "cape": { # Convective available potential energy + "level": "sfc", + }, + "cin": { # Convective inhibition + "level": "sfc", + }, + "mucape": { # Most Unstable CAPE + "level": "mu", + "variable": "cape", + }, + "mucin": { # CIN from MUCAPE level + "level": "mu", + "variable": "cin", + }, + "li": { # Lifted Index + "decimals": 1, + "level": "sfc", + }, + "bli": { # Best Lifted Index + "decimals": 1, + "level": "best", + "variable": "li", + }, + "lcl": { # Lifted Condensation Level + }, + "lpl": { # Lifted Parcel Level + }, + "srh03": { # 0-3 km Storm relative helicity + "level": "sr03", + "variable": "hlcy", + }, + "srh01": { # 0-1 km Storm relative helicity + "level": "sr01", + "variable": "hlcy", + }, + "shr06": { # 0-6 km Shear + "level": "06km", + "variable": "shear", + }, + "shr01": { # 0-1 km Shear + "level": "01km", + "variable": "shear", + }, + "cell": { # Cell motion + }, + "pwtr": { # Precipitable water + "decimals": 1, + "level": "sfc", + }, + } for var, items in thermo.items(): varname = items.get("variable", var) diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 9dff31d..54fc97b 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -16,6 +16,7 @@ from multiprocessing import Process from pathlib import Path from string import ascii_letters, digits +from typing import Any import numpy as np import yaml @@ -124,7 +125,7 @@ def get_func(val: str): return getattr(module, fun_name) -def join_ranges(loader: str, node: str) -> np.ndarray: # noqa: ARG001 +def join_ranges(loader: yaml.SafeLoader, node: yaml.Node) -> np.ndarray: # noqa: ARG001 """ Merge two or more different ranges into a single array for color bar clevs. @@ -227,7 +228,7 @@ def label_line(ax: list, label: list, segment: list, **kwargs): ax.text(x, y, label, rotation=trans_angle, **kwargs) -def label_lines(ax: list, lines: list, labels: list[str], offset: float = 0, **kwargs): +def label_lines(ax: list, lines: Any, labels: np.ndarray, offset: float = 0, **kwargs): """ Plots labels on a set of lines from SkewT. diff --git a/tests/test_common.py b/tests/test_common.py index 2d90c75..3842179 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -17,6 +17,7 @@ import warnings from inspect import getfullargspec +from pathlib import Path from string import ascii_letters, digits import numpy as np @@ -81,6 +82,9 @@ def test_conversion(): class MockSpecs(specs.VarSpec): """Mock class for the VarSpec abstract class.""" + with Path("adb_graphics/default_specs.yml").open() as c: + cfg = yaml.safe_load(c) + @property def clevs(self): return np.asarray(range(15)) @@ -90,22 +94,6 @@ def vspec(self): return {} -def test_specs(): - """Test VarSpec properties.""" - - config = "adb_graphics/default_specs.yml" - varspec = MockSpecs(config) - - # Ensure correct return type - assert isinstance(varspec.t_colors, np.ndarray) - assert isinstance(varspec.ps_colors, np.ndarray) - assert isinstance(varspec.yml, dict) - - # Ensure the appropriate number of colors is returned - assert np.shape(varspec.t_colors) == (len(varspec.clevs), 4) - assert np.shape(varspec.ps_colors) == (105, 4) - - def test_utils(): """Test that utils works appropriately.""" @@ -138,9 +126,9 @@ class TestDefaultSpecs: """Test contents of default_specs.yml.""" config = "adb_graphics/default_specs.yml" - varspec = MockSpecs(config) - - cfg = varspec.yml + varspec = MockSpecs() + with Path("adb_graphics/default_specs.yml").open() as c: + cfg = yaml.safe_load(c) @property def allowable(self): @@ -229,7 +217,7 @@ def check_transform(self, entry): funcs = funcs if isinstance(funcs, list) else [funcs] # Key word arguments may not be present. - kwargs = entry.get("kwargs") + kwargs = entry.get("kwargs", {}) transforms = [] for func in funcs: @@ -244,12 +232,12 @@ def check_transform(self, entry): if kwargs: argspecs = [getfullargspec(func) for func in transforms if callable(func)] - all_params = [] + all_params: list = [] for argspec in argspecs: # Make sure all functions accept key word arguments assert argspec.varkw is not None - parameters = [] + parameters: list = [] for argtype in [argspec.args, argspec.varargs, argspec.varkw]: if argtype is not None: parameters.extend(argtype) From 717686a2f6b9803c596771fed1e77a451a077f2e Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 23 Oct 2025 15:35:43 -0600 Subject: [PATCH 16/98] mypy done! --- adb_graphics/conversions.py | 6 +- adb_graphics/datahandler/gribdata.py | 138 +++++++++++---------------- adb_graphics/datahandler/gribfile.py | 93 +++++++++--------- adb_graphics/figures/skewt.py | 10 +- adb_graphics/specs.py | 19 ++-- adb_graphics/utils.py | 33 ++++--- tests/test_grib.py | 15 ++- 7 files changed, 152 insertions(+), 162 deletions(-) diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index 482696b..1e5400d 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -6,7 +6,7 @@ """ from numpy import ndarray -from xarray import DataSet +from xarray import DataArray from xarray.ufuncs import sqrt, square @@ -28,10 +28,10 @@ def kgm2_to_in(field: ndarray, **kwargs): return field * 0.03937 -def magnitude(a: DataSet, b: DataSet, **kwargs): +def magnitude(a: DataArray, b: DataArray, **kwargs) -> DataArray: """Return the magnitude of vector components.""" - return sqrt(square(a) + square(b)) + return DataArray(sqrt(square(a) + square(b))) def m_to_dm(field: ndarray, **kwargs): diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 62cf3bf..34228cd 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -7,11 +7,13 @@ import abc from copy import deepcopy from datetime import datetime, timedelta +from pathlib import Path from string import ascii_letters, digits +from typing import Any import numpy as np from matplotlib import cm -from xarray import DataArray, DataSet +from xarray import DataArray, Dataset from adb_graphics import conversions, errors, specs, utils from adb_graphics.datahandler import gribfile @@ -33,10 +35,10 @@ class UPPData(specs.VarSpec): """ - def __init__(self, ds: DataSet, short_name: str, spec: dict, **kwargs): + def __init__(self, ds: Dataset, short_name: str, spec: dict, **kwargs): # Parse kwargs first - self.model = kwargs.get("model") - self.grib_path = kwargs.get("grib_path") + self.model = kwargs.get("model", "") + self.grib_path = Path(kwargs.get("grib_path", "")) self.spec = spec self.short_name = short_name @@ -89,12 +91,12 @@ def date_to_str(date: datetime) -> str: return date.strftime("%Y%m%d %H UTC") @property - def field(self): + def field(self) -> DataArray: """ - Get the first DataArray out of the DataSet. + Get the first DataArray out of the Dataset. """ - - return self.ds.__getattr__(list(self.ds.data_vars)[0]) + first_variable_name = list(self.ds.data_vars)[0] + return DataArray(self.ds[first_variable_name]) def field_column_max(self, values: DataArray, variable: str, level: str, **kwargs): # noqa: ARG002 """Returns the column max of the values.""" @@ -139,15 +141,8 @@ def _get_data_levels(self, vertical_dim: str): Arg: vertical_dim the name of the vertical dimension """ - - fcst_hr = 0 if self.ds.sizes.get("fcst_hr", 0) <= 1 else int(self.fhr) - - ret = [] - for dim in [var for var in self.ds.variables if vertical_dim in var]: - # Get the current forecast hour slice, if it's in the dataset - selector = {"fcst_hr": fcst_hr} if "fcst_hr" in self.ds[dim].dims else {} - ret.append(self.ds[dim].sel(**selector).values) - return ret + dim = [str(coord) for coord in self.ds.coords if vertical_dim in str(coord)][0] + return self.ds.coords[dim].values def _get_field(self, spec: dict) -> DataArray: """ @@ -159,9 +154,10 @@ def _get_field(self, spec: dict) -> DataArray: """ ds = gribfile.GribFile(self.grib_path, spec).contents - return ds.__getattr__(list(ds.data_vars)[0]) + first_variable_name = list(ds.data_vars)[0] + return DataArray(ds[first_variable_name]) - def _get_level(self, field: DataSet, level: str, spec: dict, **kwargs): + def _get_level(self, field: DataArray, level: str, spec: dict, **kwargs) -> int: """ Returns the value of the level to for a 3D array. @@ -185,7 +181,7 @@ def _get_level(self, field: DataSet, level: str, spec: dict, **kwargs): # The index of the requested level lev = spec.get("vertical_index") if lev is not None: - return lev + return int(lev) vertical_dim = self.vertical_dim(field) @@ -231,7 +227,7 @@ def _get_level(self, field: DataSet, level: str, spec: dict, **kwargs): ) raise ValueError(msg) - def get_transform(self, transforms: dict, val: float | list | DataArray): + def get_transform(self, transforms: dict | str, val: DataArray) -> DataArray: """ Applies a set of one or more transforms to an np.array of data values. @@ -248,12 +244,12 @@ def get_transform(self, transforms: dict, val: float | list | DataArray): """ - transform_kwargs = {} + transform_kwargs: dict = {} if isinstance(transforms, dict): transform_list = transforms.get("funcs") if not isinstance(transform_list, list): transform_list = [transform_list] - transform_kwargs = transforms.get("kwargs") + transform_kwargs = transforms.get("kwargs", {}) elif isinstance(transforms, str): transform_list = [transforms] else: @@ -290,24 +286,11 @@ def get_xypoint(self, site_lat: float, site_lon: float) -> tuple: return (x, y) - @property - def grid_suffix(self): - """ - Return the suffix of the first variable with 4 sections (split on _) - in the file. This should correspond to the grid tag. - """ - - for var in self.ds: - vsplit = var.split("_") - if len(vsplit) == 4: - return vsplit[-1] - return "GRID NOT FOUND" - def latlons(self): """Returns the set of latitudes and longitudes.""" coords = sorted( - [c for c in list(self.ds.coords) if any(ele in c for ele in ["lat", "lon"])] + [str(c) for c in list(self.ds.coords) if any(ele in str(c) for ele in ["lat", "lon"])] ) return [self.ds.coords[c].values for c in coords] @@ -319,7 +302,7 @@ def lev_descriptor(self): def numeric_level( self, index_match: bool = True, level: str | None = None, split: str | None = None - ): + ) -> tuple[list[float | int], str]: """ Split the numeric level and unit associated with the level key. @@ -330,14 +313,15 @@ def numeric_level( level = level if level else self.level # Gather all the numbers in the string - lev_val = "".join([c for c in level if (c in digits or c == ".")]) + numbers = "".join([c for c in level if (c in digits or c == ".")]) # Convert the numbers to a list, and make integers or floats - if lev_val: + lev_val: list[float | int] + if numbers: if split is not None: - lev_val = [int(lev) for lev in lev_val] + lev_val = [int(lev) for lev in numbers] else: - lev_val = [float(lev_val) if "." in lev_val else int(lev_val)] + lev_val = [float(numbers) if "." in numbers else int(numbers)] # Gather all the letters lev_unit = "".join([c for c in level if c in ascii_letters]) @@ -369,11 +353,11 @@ def valid_dt(self) -> datetime: return self.anl_dt + fh @abc.abstractmethod - def values(self, level: str | None = None, name: str | None = None, **kwargs): + def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: """Returns the values of a given variable.""" @staticmethod - def vertical_dim(field: DataSet): + def vertical_dim(field: DataArray) -> str: """ Find the name of the vertical dimension. @@ -381,9 +365,10 @@ def vertical_dim(field: DataSet): instance. """ - vert_dim = [dim for dim in field.dims if ("lv" in dim or "probability" in dim)] - if vert_dim: - return vert_dim[0] + for dims in list(field.dims): + dim = str(dims) + if "lv" in dim or "probability" in dim: + return dim return "" @property @@ -413,7 +398,7 @@ class FieldData(UPPData): """ - def __init__(self, ds: DataSet, level: str, short_name: str, **kwargs): + def __init__(self, ds: Dataset, level: str, short_name: str, **kwargs): super().__init__(ds, short_name, **kwargs) self.level = level @@ -448,7 +433,7 @@ def cmap(self): return cm.get_cmap(self.vspec["cmap"]) @property - def colors(self) -> np.ndarray: + def colors(self) -> Any: """ Returns a list of colors, specified by the config key "colors". @@ -570,7 +555,7 @@ def grid_info(self): Lo2="lon_2", ) - grid_info = {} + grid_info: dict[str, str | float | int | list] = {} var_info = self.field grid_def = var_info.attrs["GRIB_gridDefinitionDescription"].lower() if "lambert" in grid_def: @@ -701,7 +686,7 @@ def ticks(self) -> int: settings. """ - return self.vspec.get("ticks", 10) + return int(self.vspec.get("ticks", 10)) @property def units(self) -> str: @@ -710,10 +695,10 @@ def units(self) -> str: specified in the yaml file, returns the value set in the Grib file. """ - return self.vspec.get("unit", self.field.units) + return str(self.vspec.get("unit", self.field.units)) @property - def data(self): + def data(self) -> DataArray: """ Sets the data property on the object for use when we need to update the values associated with a given object -- helpful for differences. @@ -723,7 +708,7 @@ def data(self): return self._data @data.setter - def data(self, value: DataSet): + def data(self, value: DataArray): self._data = value def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: @@ -746,7 +731,7 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> """ level = level or self.level - vals = self.ds + vals: DataArray = self.ds.to_dataarray().sqeeze() spec = self.vspec do_transform = kwargs.get("do_transform", True) @@ -770,15 +755,13 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> transforms = spec.get("transform") if transforms and do_transform: - vals = self.get_transform(transforms, vals) + vals = self.get_transform(transforms, self.field) - if isinstance(vals, DataSet): - return vals.to_dataarray().squeeze() return vals def vector_magnitude( self, - field1: DataSet, + field1: Dataset, cfkeys: dict | None = None, field2_id: str | None = None, **kwargs, # noqa: ARG002 @@ -793,11 +776,14 @@ def vector_magnitude( if cfkeys.get("level") is None: cfkeys["level"] = utils.numeric_level(level=self.level, index_match=False)[0] field2_spec = {"cfgrib": cfkeys} - else: + elif field2_id: var, lev = field2_id.split(".") field2_spec = self.spec for key in (var, lev): field2_spec = field2_spec[key] + else: + msg = "Must supply a field2_id if cfkeys is not explicitly provided." + raise ValueError(msg) ds = gribfile.GribFile(self.grib_path, field2_spec["cfgrib"]).contents args = { @@ -819,7 +805,7 @@ def vector_magnitude( return mag - def wind(self, level: str) -> [np.ndarray, np.ndarray]: + def wind(self, level: bool | str) -> list[DataArray]: """ Returns the u, v wind components as a list (length 2) of arrays. @@ -833,7 +819,7 @@ def wind(self, level: str) -> [np.ndarray, np.ndarray]: # Just in case wind gets called with level=False if not level: - return False + return [] # Create FieldData objects for u, v components field_lambda = lambda ds, level, var: FieldData( @@ -866,7 +852,7 @@ class ProfileData(UPPData): """ - def __init__(self, ds: DataSet, loc: str, short_name: str, **kwargs): + def __init__(self, ds: Dataset, loc: str, short_name: str, **kwargs): super().__init__(ds, short_name, **kwargs) # The first 31 columns are space delimted @@ -886,7 +872,7 @@ def __init__(self, ds: DataSet, loc: str, short_name: str, **kwargs): if self.site_lon < 0: self.site_lon = self.site_lon + 360.0 - def values(self, level: str | None = None, name: str | None = None, **kwargs): + def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: """ Returns the numpy array of values at the object's x, y location for the requested variable. Transforms are performed in the child class. @@ -919,21 +905,8 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs): # Retrive the location for the profile x, y = self.get_xypoint(self.site_lat, self.site_lon) - # Retrieve the default_specs section for the specified level - var_spec = self.spec.get(name, {}).get(level, {}) - - # Set the NCL name from the specs section, unless otherwise specified - ncl_name = kwargs.get("ncl_name") or self.ncl_name(var_spec) - ncl_name = ncl_name.format(fhr=self.fhr, grid=self.grid_suffix) - - if not ncl_name: - raise errors.NoGraphicsDefinitionForVariableError( - name, - "ua", - ) - # Get the full 2- or 3-D field - field = self.ds[ncl_name] + field = self.field profile = field[::] lev = 0 @@ -943,9 +916,10 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs): # 3D elif len(profile.shape) == 3: if one_lev: - lev = vertical_index if vertical_index is None: - lev = self._get_level(field, level, var_spec, split=split) + lev = self._get_level(field, level, {}, split=split) + else: + lev = int(vertical_index) profile = profile[lev, x, y] else: profile = profile[:, x, y] @@ -953,8 +927,8 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs): def vector_magnitude( self, - field1: DataSet, - field2: DataSet, + field1: DataArray, + field2: DataArray, level: str = "ua", vertical_index: int | None = None, **kwargs, diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 084e481..e8d22d0 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -39,7 +39,13 @@ class GribFiles: forecast hours. """ - def __init__(self, coord_dims: dict[str, list[int]], filenames: dict, filetype: str, **kwargs): + def __init__( + self, + coord_dims: dict[str, list[int]], + filenames: dict[str, list[Path]], + filetype: str, + **kwargs, + ): """ Initialize GribFiles object. @@ -60,7 +66,6 @@ def __init__(self, coord_dims: dict[str, list[int]], filenames: dict, filetype: self.filenames = filenames self.filetype = filetype self.coord_dims = coord_dims - self.grid_suffix = self._get_grid_suffix(filenames) self.contents = self._load() def append(self, filenames: dict[str, list[Path]]): @@ -85,7 +90,8 @@ def free_fcst_names(self, ds: xr.Dataset, fcst_type: str): # noqa: PLR0915,PLR0 fhr = self.coord_dims["fcst_hr"][-1] special_suffixes = ["max", "min", "acc", "avg"] - for var in ds.variables: + for ds_var in ds.variables: + var = str(ds_var) suffix = var.split("_")[-1] # Keeping lists of misbehaving "accumulated" variables here because @@ -216,7 +222,7 @@ def free_fcst_names(self, ds: xr.Dataset, fcst_type: str): # noqa: PLR0915,PLR0 return ret - def _load(self, filenames: list[Path] | None = None): + def _load(self, filenames: dict[str, list[Path]] | None = None): """Load the set of files into a single XArray structure.""" all_leads = [] if filenames is None else [self.contents] @@ -226,46 +232,45 @@ def _load(self, filenames: list[Path] | None = None): # the rest of the forecast hours. Rename those accumulated variables if # needed. for fcst_type in ["01fcst", "free_fcst"]: - if filenames.get(fcst_type): - for filename in filenames.get(fcst_type): - print(f"Loading grib2 file: {fcst_type}, {filename}") - - # Rename variables to match free forecast variables - dataset = xr.open_mfdataset( - filenames[fcst_type], - **self.open_kwargs, - ) - - renaming = self.free_fcst_names(dataset, fcst_type) - if renaming and self.model not in ["hrrre", "rrfse"]: - print("RENAMING VARIABLES:") - for old_name, new_name in renaming.items(): - print(f" {old_name:>30s} -> {new_name}") - dataset = dataset.rename_vars(renaming) - - if len(all_leads) == 1: - # Check that specific variables exist in the xarray that is - # already loaded (presumably 0hr), and add them if they - # don't. This implementation is relying on pointers to - # update "in place" - og_ds = all_leads[0] - bad_vars = [ - "APCP_P8_L1_{grid}_acc", - "ACPCP_P8_L1_{grid}_acc", - "FROZR_P8_L1_{grid}_acc", - "NCPCP_P8_L1_{grid}_acc", - "WEASD_P8_L1_{grid}_acc", - ] - bad_vars = [v.format(grid=self.grid_suffix) for v in bad_vars] - for bad_var in bad_vars: - # Check to see if the bad variable is in the current - # dataset and NOT in the original dataset. - if bad_var not in og_ds.variables and dataset.get(bad_var) is not None: - print(f"Adding {bad_var} to og ds") - # Duplicate the accumulated variable with the - # required name - og_ds[bad_var] = og_ds.get(f"{bad_var}1h") - all_leads.append(dataset) + for filename in filenames.get(fcst_type, {}): + print(f"Loading grib2 file: {fcst_type}, {filename}") + + # Rename variables to match free forecast variables + dataset = xr.open_mfdataset( + filenames[fcst_type], + **self.open_kwargs, + ) + + # renaming = self.free_fcst_names(dataset, fcst_type) + # if renaming and self.model not in ["hrrre", "rrfse"]: + # print("RENAMING VARIABLES:") + # for old_name, new_name in renaming.items(): + # print(f" {old_name:>30s} -> {new_name}") + # dataset = dataset.rename_vars(renaming) + + # if len(all_leads) == 1: + # # Check that specific variables exist in the xarray that is + # # already loaded (presumably 0hr), and add them if they + # # don't. This implementation is relying on pointers to + # # update "in place" + # og_ds = all_leads[0] + # bad_vars = [ + # "APCP_P8_L1_{grid}_acc", + # "ACPCP_P8_L1_{grid}_acc", + # "FROZR_P8_L1_{grid}_acc", + # "NCPCP_P8_L1_{grid}_acc", + # "WEASD_P8_L1_{grid}_acc", + # ] + # bad_vars = [v.format(grid=self.grid_suffix) for v in bad_vars] + # for bad_var in bad_vars: + # # Check to see if the bad variable is in the current + # # dataset and NOT in the original dataset. + # if bad_var not in og_ds.variables and dataset.get(bad_var) is not None: + # print(f"Adding {bad_var} to og ds") + # # Duplicate the accumulated variable with the + # # required name + # og_ds[bad_var] = og_ds.get(f"{bad_var}1h") + all_leads.append(dataset) return xr.combine_nested( all_leads, diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 18788b7..bab7efb 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -20,7 +20,7 @@ from metpy.plots import Hodograph, SkewT from metpy.units import units from mpl_toolkits.axes_grid1.inset_locator import inset_axes -from xarray import Dataset +from xarray import DataArray, Dataset from adb_graphics import errors, utils from adb_graphics.datahandler import gribdata @@ -128,7 +128,7 @@ def _add_hydrometeors(self, hydro_subplot: Axes): for mixr, settings in mixing_ratios.items(): # Get the profile values - scale = settings.get("scale") + scale = settings.get("scale", 1.0) try: profile = self.values(name=mixr) * 1000.0 * scale except errors.GribReadError: @@ -145,8 +145,8 @@ def _add_hydrometeors(self, hydro_subplot: Axes): mixr_total = mixr_total + pres_layer / gravity * profile[n] # limit values to upper and lower values of plotting range - profile = np.where((profile > 0.0) & (profile < 1.0e-4), 1.0e-4, profile) # noqa: PLR2004 - profile = np.where((profile > 10.0), 10.0, profile) # noqa: PLR2004 + profile = DataArray.where((profile > 0.0) & (profile < 1.0e-4), 1.0e-4, profile) # noqa: PLR2004 + profile = DataArray.where((profile > 10.0), 10.0, profile) # noqa: PLR2004 # plot line hydro_subplot.plot( @@ -656,7 +656,7 @@ def thermo_variables(self): tmp = self.get_transform(transforms, tmp) except errors.GribReadError: - tmp = "--" + tmp = DataArray([]) thermo[var]["data"] = tmp thermo[var]["units"] = spec.get("unit") diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index 71c8fd8..3643306 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -105,7 +105,7 @@ def cref_colors(self) -> np.ndarray: return np.concatenate((grays, nws, white)) @property - def fire_power_colors(self) -> np.ndarray: + def fire_power_colors(self) -> list[str]: """Default color map for fire power plot.""" # The scatter plot utility won't accept anything but named colors @@ -120,7 +120,7 @@ def fire_power_colors(self) -> np.ndarray: ] @property - def smoke_emissions_colors(self) -> np.ndarray: + def smoke_emissions_colors(self) -> list[str]: """Default color map for smoke emissions plot.""" # The scatter plot utility won't accept anything but named colors @@ -223,9 +223,10 @@ def icsev_colors(self) -> np.ndarray: def lcl_colors(self) -> np.ndarray: """Default color map for Lifted Condensation Level.""" - return ctables.colortables.get_colortable(self.vspec.get("cmap"))( - range(50, 180, 7) - ) # rainbow + # rainbow + return np.ndarray( + ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(50, 180, 7)) + ) @property def lifted_index_colors(self) -> np.ndarray: @@ -264,7 +265,9 @@ def mup_colors(self) -> np.ndarray: def pbl_colors(self) -> np.ndarray: """Default color map for PBL Height.""" - return ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(15, 60, 3)) + return np.ndarray( + ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(15, 60, 3)) + ) @property def pcp_colors(self) -> np.ndarray: @@ -445,7 +448,9 @@ def tsfc_colors(self) -> np.ndarray: def terrain_colors(self) -> np.ndarray: """Default color map for Terrain.""" - return ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(54, 157, 6)) + return np.ndarray( + ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(54, 157, 6)) + ) @property def ua_temp_colors(self) -> np.ndarray: diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 54fc97b..bd2ec2f 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -20,15 +20,17 @@ import numpy as np import yaml +from matplotlib.axes import Axes def cfgrib_spec(config: dict, model: str) -> dict: - if spec := config.get(model): + spec: dict = config.get(model, {}) + if spec and isinstance(spec, dict): return spec return config -def create_zip(files_to_zip: list[str] | list[Path], zipf: Path | str): +def create_zip(files_to_zip: list[str], zipf: Path | str): """Create a zip file. Use a locking mechanism -- write a lock file to disk.""" lock_file = Path(f"{zipf}._lock") @@ -115,6 +117,9 @@ def get_func(val: str): mod_spec = find_spec(module_name, package="adb_graphics") if mod_spec is None: mod_spec = find_spec("." + module_name, package="adb_graphics") + if mod_spec is None: + msg = "Could not find {module_name} in current environment." + raise ValueError(msg) try: __import__(mod_spec.name) @@ -125,7 +130,7 @@ def get_func(val: str): return getattr(module, fun_name) -def join_ranges(loader: yaml.SafeLoader, node: yaml.Node) -> np.ndarray: # noqa: ARG001 +def join_ranges(loader: yaml.SafeLoader, node: yaml.Node) -> Any: # noqa: ARG001 """ Merge two or more different ranges into a single array for color bar clevs. @@ -150,10 +155,10 @@ def join_ranges(loader: yaml.SafeLoader, node: yaml.Node) -> np.ndarray: # noqa # SafeLoader doesn't seem compatible with our numpy contructors, using Loader here -yaml.add_constructor("!join_ranges", join_ranges, Loader=yaml.Loader) +yaml.add_constructor("!join_ranges", join_ranges, Loader=yaml.SafeLoader) -def label_line(ax: list, label: list, segment: list, **kwargs): +def label_line(ax: Axes, label: str, segment: np.ndarray, **kwargs): """ Label a single line with line2D label data. @@ -228,7 +233,7 @@ def label_line(ax: list, label: list, segment: list, **kwargs): ax.text(x, y, label, rotation=trans_angle, **kwargs) -def label_lines(ax: list, lines: Any, labels: np.ndarray, offset: float = 0, **kwargs): +def label_lines(ax: Axes, lines: Any, labels: np.ndarray, offset: float = 0, **kwargs): """ Plots labels on a set of lines from SkewT. @@ -251,7 +256,7 @@ def label_lines(ax: list, lines: Any, labels: np.ndarray, offset: float = 0, **k for i, line in enumerate(lines.get_segments()): label = int(labels[i]) - label_line(ax, label, line, align=True, offset=offset, **kwargs) + label_line(ax, str(label), line, align=True, offset=offset, **kwargs) def load_sites(arg: str | Path) -> list[str]: @@ -261,7 +266,8 @@ def load_sites(arg: str | Path) -> list[str]: path = path_exists(arg) with path.open() as sites_file: - return sites_file.readlines() + sites: list[str] = sites_file.readlines() + return sites def uniq_wgrib2_list(inlist: list[str]): @@ -292,6 +298,7 @@ def load_specs(arg: str | Path) -> dict: spec_file = Path(arg) assert spec_file.exists() + specs: dict with spec_file.open() as fn: specs = yaml.load(fn, Loader=yaml.Loader) @@ -308,14 +315,14 @@ def numeric_level(index_match: bool = True, level: str | None = None): numeric, e.g., 'sfc' or 'ua'. """ - level = level if level is not None else 0 + level = level if level is not None else "" # Gather all the numbers in the string - lev_val = "".join([c for c in level if (c in digits or c == ".")]) + numbers = "".join([c for c in level if (c in digits or c == ".")]) # Convert the numbers to a list, and make integers or floats - if lev_val: - lev_val = [float(lev_val) if "." in lev_val else int(lev_val)] + if numbers: + lev_val = [float(numbers) if "." in numbers else int(numbers)] # Gather all the letters lev_unit = "".join([c for c in level if c in ascii_letters]) @@ -405,7 +412,7 @@ def zip_products(fhr: int, workdir: Path, zipfiles: dict) -> None: file_tmpl = f"*.skewt.*_f{fhr:03d}.csv" else: file_tmpl = f"*_{tile}_*{fhr:02d}.png" - product_files = glob.glob(workdir / file_tmpl) + product_files = glob.glob(str(workdir / file_tmpl)) if product_files: zip_proc = Process( group=None, diff --git a/tests/test_grib.py b/tests/test_grib.py index 3d3f30d..c5b4f50 100644 --- a/tests/test_grib.py +++ b/tests/test_grib.py @@ -15,8 +15,8 @@ def test_UPPData(natfile, prsfile): """Test the UPPData class methods on both types of input files.""" - nat_ds = gribfile.GribFile(natfile) - prs_ds = gribfile.GribFile(prsfile) + nat_ds = gribfile.GribFile(natfile, var_config={}) + prs_ds = gribfile.GribFile(prsfile, var_config={}) class UPP(gribdata.UPPData): """Test class needed to define the values as an abstract class.""" @@ -24,8 +24,8 @@ class UPP(gribdata.UPPData): def values(self, level=None, name=None, **kwargs): # noqa: ARG002 return 1 - upp_nat = UPP(nat_ds.contents, fhr=2, filetype="nat", short_name="temp") - upp_prs = UPP(prs_ds.contents, fhr=2, short_name="temp") + upp_nat = UPP(nat_ds.contents, fhr=2, filetype="nat", short_name="temp", spec={}) + upp_prs = UPP(prs_ds.contents, fhr=2, short_name="temp", spec={}) cycle = datetime(2025, 10, 2, 17) @@ -33,12 +33,11 @@ def values(self, level=None, name=None, **kwargs): # noqa: ARG002 for upp in [upp_nat, upp_prs]: assert isinstance(upp.anl_dt, datetime) assert isinstance(upp.clevs, np.ndarray) - assert isinstance(upp.date_to_str(cycle, str)) + assert isinstance(upp.date_to_str(cycle), str) assert isinstance(upp.fhr, str) assert isinstance(upp.field, DATAARRAY) assert isinstance(upp.latlons(), list) assert isinstance(upp.lev_descriptor, str) - assert isinstance(upp.ncl_name(upp.vspec), str) assert isinstance(upp.numeric_level(), tuple) assert isinstance(upp.spec, dict) assert isinstance(upp.valid_dt, datetime) @@ -51,7 +50,7 @@ def values(self, level=None, name=None, **kwargs): # noqa: ARG002 def test_FieldData(prsfile): """Test the FieldData class methods on a prs file.""" - prs_ds = gribfile.GribFile(prsfile) + prs_ds = gribfile.GribFile(prsfile, var_config={}) field = gribdata.FieldData(prs_ds.contents, fhr=2, level="500mb", short_name="temp") assert isinstance(field.cmap, mcolors.Colormap) @@ -98,7 +97,7 @@ def test_FieldData(prsfile): def test_profile_data(natfile: Path): """Test the ProfileData class methods on a nat file.""" - nat_ds = gribfile.GribFile(natfile) + nat_ds = gribfile.GribFile(natfile, var_config={}) loc = " BNA 9999 99999 36.12 86.69 597 Nashville, TN\n" profile = gribdata.ProfileData( nat_ds.contents, From 85fe1f963f424ddfb62f512603bd8fa7b75694f8 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 24 Oct 2025 10:09:23 -0600 Subject: [PATCH 17/98] Passing all tests that still exist. --- Makefile | 9 ++- adb_graphics/default_specs.yml | 4 +- adb_graphics/utils.py | 5 +- image_lists/hrrr_subset.yml | 2 +- pyproject.toml | 95 +++++++++++++++++++++++++++ recipe/run_test.sh | 62 ------------------ tests/test_common.py | 24 +++++-- tests/test_grib.py | 115 --------------------------------- tests/test_hrrr_maps.py | 8 +-- 9 files changed, 125 insertions(+), 199 deletions(-) create mode 100644 pyproject.toml delete mode 100755 recipe/run_test.sh delete mode 100644 tests/test_grib.py diff --git a/Makefile b/Makefile index ec1342c..144e1b1 100644 --- a/Makefile +++ b/Makefile @@ -7,14 +7,13 @@ format: @./format lint: - recipe/run_test.sh lint + ruff check . -test: - recipe/run_test.sh +test: lint typecheck unittest typecheck: - recipe/run_test.sh typecheck + mypy --install-types --non-interactive . unittest: - recipe/run_test.sh unittest + pytest --cov -k "not hrrr" -n 4 . diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 6daa68e..d47cc59 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -12,7 +12,7 @@ # clevs: specifies the contour levels by one of the following methods # - a list with no quotes # - a numpy.arange specified as "!!python/object/apply:numpy.arange [list]" -# with quotes. Specify "list" as with numpy.anumpy.arange() like this -- +# with quotes. Specify "list" as with numpy.arange() like this -- # [[start, ]stop[, increment]]]. start and increment are options. # - the name of a function to be called that will return a list. # @@ -977,7 +977,7 @@ firewx: # Fire Weather Index ticks: 0 title: Hourly Wildfire Potential unit: "%" -firewx_transform: +firewxtransform: sfc: <<: *firewx cfgrib: diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index bd2ec2f..a031d57 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -130,7 +130,7 @@ def get_func(val: str): return getattr(module, fun_name) -def join_ranges(loader: yaml.SafeLoader, node: yaml.Node) -> Any: # noqa: ARG001 +def join_ranges(loader: yaml.Loader, node: yaml.Node) -> Any: # noqa: ARG001 """ Merge two or more different ranges into a single array for color bar clevs. @@ -154,8 +154,7 @@ def join_ranges(loader: yaml.SafeLoader, node: yaml.Node) -> Any: # noqa: ARG00 return np.concatenate(list_, axis=0) -# SafeLoader doesn't seem compatible with our numpy contructors, using Loader here -yaml.add_constructor("!join_ranges", join_ranges, Loader=yaml.SafeLoader) +yaml.add_constructor("!join_ranges", join_ranges, Loader=yaml.Loader) def label_line(ax: Axes, label: str, segment: np.ndarray, **kwargs): diff --git a/image_lists/hrrr_subset.yml b/image_lists/hrrr_subset.yml index 1f9fad5..840c476 100644 --- a/image_lists/hrrr_subset.yml +++ b/image_lists/hrrr_subset.yml @@ -41,7 +41,7 @@ hourly: - 2m echotop: - sfc - firewx_transform: + firewxtransform: - sfc flru: - sfc diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b0d31d0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,95 @@ +[tool.coverage.report] +exclude_also = ["if TYPE_CHECKING:"] +fail_under = 100 +show_missing = true +skip_covered = true +omit = ["tests/*"] + + +[tool.mypy] +check_untyped_defs = true +follow_untyped_imports = true # needed for mpl_toolkits.basemap +pretty = true +warn_return_any = true + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["ALL"] +ignore = [ + "ANN002", # missing-type-args + "ANN003", # missing-type-kwargs + "ANN201", # missing-return-type-class-method + "ANN202", # missing-return-type-private-function + "ANN204", # missing-return-type-special-method + "ANN205", # missing-return-type-static-method + "ANN206", # missing-return-type-class-method + "ANN401", # any-type + "B010", # set-attr-with-constant + "C408", # unnecessary-collection-call + "C901", # complex-structure + "COM812", # missing-trailing-comma + "D100", # undocumented-public-module + "D101", # undocumented-public-class + "D102", # undocumented-public-method + "D103", # undocumented-public-function + "D104", # undocumented-public-package + "D105", # undocumented-magic-method + "D107", # undocumented-public-init + "D200", # unnecessary-multiline-docstring + "D202", # blank-line-after-function + "D203", # incorrect-blank-line-before-class + "D205", # missing-blank-line-after-summary + "D212", # multi-line-summary-first-line + "D400", # missing-trailing-period + "D401", # non-imperative-mood + "D404", # docstring-starts-with-this + "DTZ001", # call-datetime-without-tzinfo + "DTZ005", # call-datetime-now-without-tzinfo + "DTZ006", # call-datetime-fromtimestamp + "DTZ007", # call-datetime-strptime-without-zone + "E731", # lambda-assignment + "ERA001", # commented-out-code + "FBT001", # boolean-type-hint-positional-argument + "FBT002", # boolean-default-value-positional-argument + "FBT003", # boolean-positional-value-in-call + "FLY002", # static-join-to-f-string + "N813", # camelcase-imported-as-lowercase + "PLR0913", # too-many-arguments + "PT019", # pytest-fixture-param-without-value + "PTH207", # glob + "RUF015", # unnecessary-iterable-allocation-for-first-element + "S101", # assert + "S103", # bad-file-permissions + "S311", # suspicious-non-cryptographic-random-usage + "S506", # unsafe-yaml-load + "S602", # subprocess-popen-with-shell-equals-true + "S701", # jinja2-autoescape-false + "T201", # print + "TC006", # runtime-cast-value + "UP031", # printf-string-formatting + "UP032", # f-string +] + +[tool.ruff.lint.per-file-ignores] +"conftest.py" = [ + "ANN001", # missing-type-function-argument + "N802", # invalid-function-name + "PLR2004", # magic-value-comparison + "PT013", # pytest-incorrect-pytest-import + "SLF001", # private-member-access +] +"adb_graphics/conversions.py" = [ + "ARG001", # unused-function-argument +] +"adb_graphics/datahandler/gribdata.py" = [ + "PLR2004", # magic-value-comparison +] +"tests/*" = [ + "ANN001", # missing-type-function-argument + "N802", # invalid-function-name + "PLR2004", # magic-value-comparison + "PT013", # pytest-incorrect-pytest-import + "SLF001", # private-member-access +] diff --git a/recipe/run_test.sh b/recipe/run_test.sh deleted file mode 100755 index 9c05f13..0000000 --- a/recipe/run_test.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -eu - -# This script is called 1. By conda-build, to test code at package-build time, and 2. By "make test" -# to test code in a live development shell. Its name is dictated by conda-build (see URL below for -# details), but its contents derive from https://github.com/maddenp/condev/tree/main/demo. If run -# with no arguments, it executes all test types: linting, typechecking, unit testing, and basic CLI- -# tool verification. It can also be run with "lint", "typecheck", "unittest", or "cli" arguments to -# run only a single test type. (The "make lint", "make typecheck", and "make unittest" targets in -# the root-directory Makefile run the first test types individually; CLI tools can be hand-tested.) - -# https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#test-section - -cli() { - msg Testing CLI - ( - set -eux - uw --version - ) - msg OK -} - -lint() { - msg Running linter - ( - set -eux - ruff check . - ) - msg OK -} - -msg() { - echo "=> $@" -} - -typecheck() { - msg Running typechecker - ( - set -eux - mypy --install-types --non-interactive . - ) - msg OK -} - -unittest() { - msg Running unit tests - ( - set -eux - pytest --cov=uwtools -n 4 . - ) - msg OK -} - -if [[ -n "${1:-}" ]]; then - # Run single specified code-quality tool. - $1 -else - # Run all code-quality tools. - lint - typecheck - unittest - cli -fi diff --git a/tests/test_common.py b/tests/test_common.py index 3842179..61f39e5 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -29,6 +29,8 @@ from adb_graphics import conversions, specs, utils from adb_graphics.datahandler import gribdata +yaml.add_constructor("!join_ranges", utils.join_ranges, Loader=yaml.Loader) + def test_conversion(): """ @@ -83,7 +85,7 @@ class MockSpecs(specs.VarSpec): """Mock class for the VarSpec abstract class.""" with Path("adb_graphics/default_specs.yml").open() as c: - cfg = yaml.safe_load(c) + cfg = yaml.load(c, Loader=yaml.Loader) @property def clevs(self): @@ -103,13 +105,12 @@ def test_utils(): def test_join_ranges_constructor(): """Test that the join_ranges constructor works as expected.""" - yaml.add_constructor("!join_ranges", utils.join_ranges, Loader=yaml.SafeLoader) yaml_str = """ foo: !join_ranges [[0, 15, 0.1], [20, 61, 20]] foo2: !join_ranges [[0, 15, 0.1]] foo3: !join_ranges [[0, 15, 0.1], [20, 40, 10], [40, 61, 20]] """ - cfg = yaml.load(yaml_str, Loader=yaml.SafeLoader) + cfg = yaml.load(yaml_str, Loader=yaml.Loader) expected = np.concatenate((np.arange(0, 15, 0.1), np.arange(20, 61, 20)), axis=0) expected2 = np.arange(0, 15, 0.1) @@ -128,7 +129,7 @@ class TestDefaultSpecs: config = "adb_graphics/default_specs.yml" varspec = MockSpecs() with Path("adb_graphics/default_specs.yml").open() as c: - cfg = yaml.safe_load(c) + cfg = yaml.load(c, Loader=yaml.Loader) @property def allowable(self): @@ -143,11 +144,13 @@ def allowable(self): "annotate_decimal": self.is_int, "clevs": self.is_a_clev, "cmap": self.is_a_cmap, + "cfgrib": self.is_dict, "colors": self.is_a_color, "contours": self.is_a_contour_dict, "include_obs": self.is_bool, "hatches": self.is_a_contourf_dict, "labels": self.is_a_contourf_dict, + "level": self.is_number, "ncl_name": True, "plot_airports": self.is_bool, "plot_scatter": self.is_bool, @@ -230,7 +233,7 @@ def check_transform(self, entry): # when provided arguments don't appear in all_params. # arguments not in that list, we fail. if kwargs: - argspecs = [getfullargspec(func) for func in transforms if callable(func)] + argspecs = [getfullargspec(fx) for fx in transforms if callable(fx)] all_params: list = [] for argspec in argspecs: @@ -245,8 +248,10 @@ def check_transform(self, entry): for key in kwargs: if key not in all_params: - msg = f"Function key {key} is not an expicit parameter \ - in any of the transforms: {funcs}!" + msg = ( + f"Function key {key} is not an explicit parameter" + f"in any of the transforms: {funcs}!" + ) warnings.warn(msg, UserWarning, stacklevel=2) return True @@ -267,6 +272,8 @@ def get_callable(self, func): if func in dir(gribdata.__getattribute__(attr)): method = gribdata.__getattribute__(attr).__dict__.get(func) if method is not None: + if isinstance(method, staticmethod): + return method.__func__ return method if callable(utils.get_func(func)): @@ -376,6 +383,9 @@ def is_a_color(self, color): return color in dir(self.varspec) + def is_a_dict(self, cfgrib): + return isinstance(cfgrib, dict) + @staticmethod def is_a_level(key): """ diff --git a/tests/test_grib.py b/tests/test_grib.py deleted file mode 100644 index c5b4f50..0000000 --- a/tests/test_grib.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Test suite for grib datahandler.""" - -from datetime import datetime -from pathlib import Path - -import numpy as np -import xarray as xr -from matplotlib import colors as mcolors - -from adb_graphics.datahandler import gribdata, gribfile - -DATAARRAY = xr.core.dataarray.DataArray - - -def test_UPPData(natfile, prsfile): - """Test the UPPData class methods on both types of input files.""" - - nat_ds = gribfile.GribFile(natfile, var_config={}) - prs_ds = gribfile.GribFile(prsfile, var_config={}) - - class UPP(gribdata.UPPData): - """Test class needed to define the values as an abstract class.""" - - def values(self, level=None, name=None, **kwargs): # noqa: ARG002 - return 1 - - upp_nat = UPP(nat_ds.contents, fhr=2, filetype="nat", short_name="temp", spec={}) - upp_prs = UPP(prs_ds.contents, fhr=2, short_name="temp", spec={}) - - cycle = datetime(2025, 10, 2, 17) - - # Ensure appropriate typing and size (where applicable) - for upp in [upp_nat, upp_prs]: - assert isinstance(upp.anl_dt, datetime) - assert isinstance(upp.clevs, np.ndarray) - assert isinstance(upp.date_to_str(cycle), str) - assert isinstance(upp.fhr, str) - assert isinstance(upp.field, DATAARRAY) - assert isinstance(upp.latlons(), list) - assert isinstance(upp.lev_descriptor, str) - assert isinstance(upp.numeric_level(), tuple) - assert isinstance(upp.spec, dict) - assert isinstance(upp.valid_dt, datetime) - assert isinstance(upp.vspec, dict) - # Test for appropriate date formatting - test_date = datetime(2020, 12, 5, 12) - assert upp.date_to_str(test_date) == "20201205 12 UTC" - - -def test_FieldData(prsfile): - """Test the FieldData class methods on a prs file.""" - - prs_ds = gribfile.GribFile(prsfile, var_config={}) - field = gribdata.FieldData(prs_ds.contents, fhr=2, level="500mb", short_name="temp") - - assert isinstance(field.cmap, mcolors.Colormap) - assert isinstance(field.colors, np.ndarray) - assert isinstance(field.corners, list) - assert isinstance(field.ticks, int) - assert isinstance(field.units, str) - assert isinstance(field.values(), DATAARRAY) - assert isinstance(field.aviation_flight_rules(field.values()), DATAARRAY) - assert isinstance(field.wind(True), list) - assert len(field.corners) == 4 - assert len(field.wind(True)) == 2 - assert len(field.wind("850mb")) == 2 - for component in field.wind(True): - assert isinstance(component, DATAARRAY) - - # Test retrieving other values - assert np.array_equal(field.values(), field.values(name="temp", level="500mb")) - - # Return zeros by subtracting same field - diff = field.field_diff(field.values(), variable2="temp", level2="500mb") - assert isinstance(diff, DATAARRAY) - assert not np.any(diff) - - # Test transform - assert np.array_equal( - field.get_transform("conversions.k_to_f", field.values()), - (field.values() - 273.15) * 9 / 5 + 32, - ) - - field2 = gribdata.FieldData(prs_ds.contents, fhr=2, level="ua", short_name="ceil") - transforms = field2.vspec.get("transform") - assert np.array_equal( - field2.get_transform(transforms, field2.values()), - field2.field_diff(field2.values(), variable2="gh", level2="sfc") / 304.8, - ) - - # Expected size of values - assert len(np.shape(field.values())) == 2 - assert len(np.shape(field.values(name="u"))) == 2 - assert len(np.shape(field.values(name="u", level="850mb"))) == 2 - - -def test_profile_data(natfile: Path): - """Test the ProfileData class methods on a nat file.""" - - nat_ds = gribfile.GribFile(natfile, var_config={}) - loc = " BNA 9999 99999 36.12 86.69 597 Nashville, TN\n" - profile = gribdata.ProfileData( - nat_ds.contents, - fhr=2, - filetype="nat", - loc=loc, - short_name="temp", - ) - - assert isinstance(profile.get_xypoint(40.0, -100.0), tuple) - assert isinstance(profile.values(), DATAARRAY) - - # The values should return a single number (0) or a 1D array (1) - assert len(np.shape(profile.values(level="best", name="li"))) == 0 - assert len(np.shape(profile.values(name="temp"))) == 1 diff --git a/tests/test_hrrr_maps.py b/tests/test_hrrr_maps.py index 4a3b459..f1f1c4c 100644 --- a/tests/test_hrrr_maps.py +++ b/tests/test_hrrr_maps.py @@ -21,7 +21,7 @@ def maps_args(tmp_path) -> list: "12", "1", "-o", - tmp_path / "output", + str(tmp_path / "output"), "-s", "2023031500", "--file_tmpl", @@ -34,7 +34,7 @@ def maps_args(tmp_path) -> list: ] -def test_parse_args(tmp_path): +def test_hrrr_parse_args(tmp_path): """ Test parse_args for basic parsing success. Checks if parse_args returns 'maps' in the graphic_type field. @@ -48,7 +48,7 @@ def test_parse_args(tmp_path): "12", "1", "-o", - tmp_path / "output", + str(tmp_path / "output"), "-s", "2021052315", "--file_tmpl", @@ -63,7 +63,7 @@ def test_parse_args(tmp_path): assert test_args.graphic_type == "maps" -def test_file_count(maps_args, tmp_path): +def test_hrrr_file_count(maps_args, tmp_path): """ Test for file count in directory. Can be extended to cover multiple folders. From fbc8ba10b33d63e0371ef94a0207204140f091ea Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 24 Oct 2025 10:39:44 -0600 Subject: [PATCH 18/98] Conversions fully tested. --- adb_graphics/conversions.py | 9 ++-- tests/test_conversions.py | 83 +++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 tests/test_conversions.py diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index 1e5400d..9e01196 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -75,6 +75,11 @@ def percent(field: ndarray, **kwargs): return field * 100.0 +def sden_to_slr(field: ndarray, **kwargs): + """Convert snow density (kg m-3) to snow-liquid ratio.""" + + return 1000.0 / field + def to_micro(field: ndarray, **kwargs): """Convert field to micro.""" @@ -105,7 +110,3 @@ def weasd_to_1hsnw(field: ndarray, **kwargs): return field * 10.0 -def sden_to_slr(field: ndarray, **kwargs): - """Convert snow density (kg m-3) to snow-liquid ratio.""" - - return 1000.0 / field diff --git a/tests/test_conversions.py b/tests/test_conversions.py new file mode 100644 index 0000000..73dec4d --- /dev/null +++ b/tests/test_conversions.py @@ -0,0 +1,83 @@ +import numpy as np +from pytest import fixture +from xarray import DataArray + +from adb_graphics import conversions + + +@fixture +def array(): + return np.ones([3, 2]) * 300 + + +def test_k_to_c(array): + assert np.array_equal(conversions.k_to_c(array), array - 273.15) + + +def test_k_to_f(array): + assert np.array_equal(conversions.k_to_f(array), (array - 273.15) * 9 / 5 + 32) + + +def test_kgm2_to_in(array): + assert np.array_equal(conversions.kgm2_to_in(array), array * 0.03937) + + +def test_magnitude(): + ones = DataArray(np.ones([3, 2])) + field1 = ones * 3 + field2 = ones * 4 + out = conversions.magnitude(field1, field2) + assert np.array_equal(out, ones * 5) + + +def test_m_to_dm(array): + assert np.array_equal(conversions.m_to_dm(array), array / 10.0) + assert conversions.m_to_dm(array).dtype == np.float64 + + +def test_m_to_in(array): + assert np.array_equal(conversions.m_to_in(array), array * 39.3701) + + +def test_m_to_kft(array): + assert np.array_equal(conversions.m_to_kft(array), array / 304.8) + + +def test_m_to_mi(array): + assert np.array_equal(conversions.m_to_mi(array), array / 1609.344) + + +def test_ms_to_kt(array): + assert np.array_equal(conversions.ms_to_kt(array), array * 1.9438) + + +def test_pa_to_hpa(array): + assert np.array_equal(conversions.pa_to_hpa(array), array / 100) + + +def test_percent(array): + assert np.array_equal(conversions.percent(array), array * 100) + + +def test_sden_to_slr(array): + assert np.array_equal(conversions.sden_to_slr(array), 1000.0 / array) + + +def test_to_micro(array): + assert np.array_equal(conversions.to_micro(array), array * 1e6) + + +def test_to_micrograms_per_m3(array): + assert np.array_equal(conversions.to_micrograms_per_m3(array), array * 1e9) + + +def test_vvel_scale(array): + assert np.array_equal(conversions.vvel_scale(array), array * -10) + + +def test_vort_scale(array): + assert np.array_equal(conversions.vort_scale(array), array / 1e-05) + + +def test_weasd_to_1hsnw(array): + assert np.array_equal(conversions.weasd_to_1hsnw(array), array * 10) From ef756c14f86e4a96e2f0a53dd076afc579faa5e7 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 24 Oct 2025 11:35:41 -0600 Subject: [PATCH 19/98] specs.py tested. --- adb_graphics/conversions.py | 3 +- adb_graphics/specs.py | 248 ++++++++++++++++++------------------ tests/test_common.py | 51 +------- tests/test_specs.py | 103 +++++++++++++++ 4 files changed, 226 insertions(+), 179 deletions(-) create mode 100644 tests/test_specs.py diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index 9e01196..a5d562b 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -80,6 +80,7 @@ def sden_to_slr(field: ndarray, **kwargs): return 1000.0 / field + def to_micro(field: ndarray, **kwargs): """Convert field to micro.""" @@ -108,5 +109,3 @@ def weasd_to_1hsnw(field: ndarray, **kwargs): """Conversion from snow water equiv to snow (10:1 ratio).""" return field * 10.0 - - diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index 3643306..56b08b1 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -9,8 +9,8 @@ from itertools import chain import numpy as np -from matplotlib import cm from matplotlib import colors as mpcolors +from matplotlib.pyplot import get_cmap from metpy.plots import ctables @@ -25,8 +25,8 @@ class VarSpec(abc.ABC): def aod_colors(self) -> np.ndarray: """Default color map for AOD products and chem products.""" - grays = cm.get_cmap("Greys", 2)([0]) - others = cm.get_cmap(self.vspec.get("cmap"), 15)(range(1, 15, 1)) + grays = get_cmap("Greys", 2)([0]) + others = get_cmap(self.vspec.get("cmap"), 15)(range(1, 15, 1)) return np.concatenate((grays, others)) def centered_diff(self, cmap: str | None = None, nlev: int | None = None): @@ -36,13 +36,13 @@ def centered_diff(self, cmap: str | None = None, nlev: int | None = None): """ if nlev is None: - clevs = self.vspec.get("clevs") + clevs = self.vspec.get("clevs", self.clevs) nlev = len(clevs) + 1 if cmap is None: cmap = self.vspec.get("cmap") - colors = cm.get_cmap(cmap, nlev)(range(nlev)) + colors = get_cmap(cmap, nlev)(range(nlev)) mid = nlev // 2 colors[mid] = [1, 1, 1, 1] @@ -54,10 +54,8 @@ def centered_diff(self, cmap: str | None = None, nlev: int | None = None): def cin_colors(self) -> np.ndarray: """Default color map for Convective Inhibition.""" - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( - [120, 100, 90, 85, 80, 70, 60, 50, 25, 20, 18] - ) - grays = cm.get_cmap("Greys", 2)([0]) + ncar = get_cmap(self.vspec.get("cmap"), 128)([120, 100, 90, 85, 80, 70, 60, 50, 25, 20, 18]) + grays = get_cmap("Greys", 2)([0]) return np.concatenate((ncar, grays)) @property @@ -80,8 +78,8 @@ def vspec(self): def ceil_colors(self) -> np.ndarray: """Default color map for Ceiling.""" - grays = cm.get_cmap("Greys", 2)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + grays = get_cmap("Greys", 2)([0]) + ncar = get_cmap(self.vspec.get("cmap"), 128)( [15, 18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120] ) return np.concatenate((grays, ncar, grays)) @@ -90,8 +88,8 @@ def ceil_colors(self) -> np.ndarray: def cldcov_colors(self) -> np.ndarray: """Default color map for Cloud Cover.""" - grays = cm.get_cmap("Greys", 7)([0, 1, 3]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([120, 100, 90, 85, 80, 70, 60, 50, 25, 20]) + grays = get_cmap("Greys", 7)([0, 1, 3]) + ncar = get_cmap(self.vspec.get("cmap"), 128)([120, 100, 90, 85, 80, 70, 60, 50, 25, 20]) return np.concatenate((grays, ncar)) @property @@ -99,9 +97,9 @@ def cref_colors(self) -> np.ndarray: """Default color map for Reflectivity.""" ncolors = len(self.clevs) - 1 - grays = cm.get_cmap("Greys", 5)([0]) + grays = get_cmap("Greys", 5)([0]) nws = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(ncolors)) - white = cm.get_cmap("Greys", 5)([0]) + white = get_cmap("Greys", 5)([0]) return np.concatenate((grays, nws, white)) @property @@ -120,45 +118,24 @@ def fire_power_colors(self) -> list[str]: ] @property - def smoke_emissions_colors(self) -> list[str]: - """Default color map for smoke emissions plot.""" - - # The scatter plot utility won't accept anything but named colors - return [ - "white", - "rebeccapurple", - "royalblue", - "cadetblue", - "yellowgreen", - "mediumaquamarine", - "lightgreen", - "yellow", - "gold", - "orange", - "darkorange", - "orangered", - "red", - "firebrick", - ] - def flru_colors(self) -> np.ndarray: """Default color map for Ceiling.""" - return cm.get_cmap(self.vspec.get("cmap"), 128)([50, 15, 90, 120]) + return get_cmap(self.vspec.get("cmap"), 128)([50, 15, 90, 120]) @property def frzn_colors(self) -> np.ndarray: """Default color map for Frozen Precip %.""" - grays = cm.get_cmap("Greys", 7)([0, 2]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([120, 90, 85, 80, 70, 60, 50, 25, 20, 15]) + grays = get_cmap("Greys", 7)([0, 2]) + ncar = get_cmap(self.vspec.get("cmap"), 128)([120, 90, 85, 80, 70, 60, 50, 25, 20, 15]) return np.concatenate((grays, ncar)) @property def goes_colors(self) -> np.ndarray: """Default color map for simulated GOES IR satellite.""" - grays = cm.get_cmap("Greys_r", 33)(range(33)) + grays = get_cmap("Greys_r", 33)(range(33)) ctable2 = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(65, 150)) return np.concatenate((grays[-1:], grays, ctable2, grays[1:])) @@ -166,23 +143,23 @@ def goes_colors(self) -> np.ndarray: def graupel_colors(self) -> np.ndarray: """Default color map for Max Vertically Integrated Graupel.""" - grays = cm.get_cmap("Greys", 3)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)(range(20, 128, 6)) + grays = get_cmap("Greys", 3)([0]) + ncar = get_cmap(self.vspec.get("cmap"), 128)(range(20, 128, 6)) return np.concatenate((grays, ncar)) @property def hail_colors(self) -> np.ndarray: """Default color map for Hail diameter.""" - grays = cm.get_cmap("Greys", 2)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([100, 15, 18, 20, 25, 60, 80, 85, 90]) + grays = get_cmap("Greys", 2)([0]) + ncar = get_cmap(self.vspec.get("cmap"), 128)([100, 15, 18, 20, 25, 60, 80, 85, 90]) return np.concatenate((grays, ncar)) @property def heat_flux_colors(self) -> np.ndarray: """Default color map for Latent/Sensible Heat Flux.""" - grays = cm.get_cmap("Greys", 8)([6, 5, 4, 3, 2]) + grays = get_cmap("Greys", 8)([6, 5, 4, 3, 2]) ctable = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(0, 33, 2)) return np.concatenate((grays, ctable)) @@ -190,33 +167,34 @@ def heat_flux_colors(self) -> np.ndarray: def heat_flux_colors_g(self) -> np.ndarray: """Default color map for Latent/Sensible Heat Flux.""" - return cm.get_cmap(self.vspec.get("cmap"), 128)(range(15, 112, 8)) + return get_cmap(self.vspec.get("cmap"), 128)(range(15, 112, 8)) @property def heat_flux_colors_l(self) -> np.ndarray: """Default color map for Latent/Sensible Heat Flux.""" - return cm.get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) + return get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) @property def heat_flux_colors_s(self) -> np.ndarray: """Default color map for Latent/Sensible Heat Flux.""" - return cm.get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) + return get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) @property def icprb_colors(self) -> np.ndarray: """Default color map for Icing Probability.""" - grays = cm.get_cmap("Greys", 2)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([25, 35, 50, 60, 70, 80, 85, 90, 100]) + grays = get_cmap("Greys", 2)([0]) + ncar = get_cmap(self.vspec.get("cmap"), 128)([25, 35, 50, 60, 70, 80, 85, 90, 100]) return np.concatenate((grays, ncar)) + @property def icsev_colors(self) -> np.ndarray: """Default color map for Icing Severity.""" - white = cm.get_cmap("Greys", 2)([0]) - blues = cm.get_cmap(self.vspec.get("cmap"), 9)([2, 3, 4, 6, 8]) + white = get_cmap("Greys", 2)([0]) + blues = get_cmap(self.vspec.get("cmap"), 9)([2, 3, 4, 6, 8]) return np.concatenate((white, blues)) @property @@ -224,7 +202,7 @@ def lcl_colors(self) -> np.ndarray: """Default color map for Lifted Condensation Level.""" # rainbow - return np.ndarray( + return np.asarray( ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(50, 180, 7)) ) @@ -232,7 +210,7 @@ def lcl_colors(self) -> np.ndarray: def lifted_index_colors(self) -> np.ndarray: """Default color map for Lifted Index.""" - ctable = cm.get_cmap(self.vspec.get("cmap"), 128)(range(4, 125, 4)) + ctable = get_cmap(self.vspec.get("cmap"), 128)(range(4, 125, 4)) ctable[14] = [1, 1, 1, 1] ctable[15] = [1, 1, 1, 1] return ctable @@ -241,15 +219,15 @@ def lifted_index_colors(self) -> np.ndarray: def mdn_colors(self) -> np.ndarray: """Default color map for Max Downdraft.""" - grays = cm.get_cmap("Greys", 2)([0]) - others = cm.get_cmap(self.vspec.get("cmap"), 18)(range(18, 1, -1)) + grays = get_cmap("Greys", 2)([0]) + others = get_cmap(self.vspec.get("cmap"), 18)(range(18, 1, -1)) return np.concatenate((others, grays)) @property def mean_vvel_colors(self) -> np.ndarray: """Default color map for Mean Vertical Velocity.""" - ctable = cm.get_cmap(self.vspec.get("cmap"), 128)(range(0, 114, 6)) + ctable = get_cmap(self.vspec.get("cmap"), 128)(range(0, 114, 6)) ctable[9] = [1, 1, 1, 1] return ctable @@ -257,15 +235,15 @@ def mean_vvel_colors(self) -> np.ndarray: def mup_colors(self) -> np.ndarray: """Default color map for Max Updraft.""" - grays = cm.get_cmap("Greys", 2)([0]) - others = cm.get_cmap(self.vspec.get("cmap"), 18)(range(1, 18, 1)) + grays = get_cmap("Greys", 2)([0]) + others = get_cmap(self.vspec.get("cmap"), 18)(range(1, 18, 1)) return np.concatenate((grays, others)) @property def pbl_colors(self) -> np.ndarray: """Default color map for PBL Height.""" - return np.ndarray( + return np.asarray( ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(15, 60, 3)) ) @@ -273,86 +251,84 @@ def pbl_colors(self) -> np.ndarray: def pcp_colors(self) -> np.ndarray: """Default color map for Hourly Precipitation.""" - grays = cm.get_cmap("Greys", 6)([0, 3]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([25, 50, 60, 70, 80, 85, 90, 115]) + grays = get_cmap("Greys", 6)([0, 3]) + ncar = get_cmap(self.vspec.get("cmap"), 128)([25, 50, 60, 70, 80, 85, 90, 115]) return np.concatenate((grays, ncar)) @property def pcp_colors_high(self) -> np.ndarray: """High values color map for Hourly Precipitation.""" - grays = cm.get_cmap("Greys", 2)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)([70, 80, 85, 90, 115]) + grays = get_cmap("Greys", 2)([0]) + ncar = get_cmap(self.vspec.get("cmap"), 128)([70, 80, 85, 90, 115]) return np.concatenate((grays, ncar)) @property def pmsl_colors(self) -> np.ndarray: """Default color map for Surface Pressure.""" - ncolors = len(self.vspec.get("clevs")) + ncolors = len(self.vspec.get("clevs", self.clevs)) incr = 128 // ncolors - colors = cm.get_cmap(self.vspec.get("cmap"), 128)(range(incr, 128, incr)) + colors = get_cmap(self.vspec.get("cmap"), 128)(range(incr, 128, incr)) return np.asarray(colors) @property def ps_colors(self) -> np.ndarray: """Default color map for Surface Pressure.""" - grays = cm.get_cmap("Greys", 13)(range(13)) + grays = get_cmap("Greys", 13)(range(13)) segments = [[16, 53], [86, 105], [110, 151, 2], [172, 202, 2]] - ncar = cm.get_cmap("gist_ncar", 200)(list(chain(*[range(*i) for i in segments]))) + ncar = get_cmap("gist_ncar", 200)(list(chain(*[range(*i) for i in segments]))) return np.concatenate((grays, ncar)) @property def pw_colors(self) -> np.ndarray: """Default color map for Precipitable Water.""" - grays = cm.get_cmap("Greys", 5)([1, 3]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + grays = get_cmap("Greys", 5)([1, 3]) + ncar = get_cmap(self.vspec.get("cmap"), 128)( [120, 100, 95, 85, 80, 70, 65, 50, 25, 22, 20, 17] ) - bupu = cm.get_cmap("BuPu", 15)([13, 14]) - cool = cm.get_cmap("cool", 15)([10, 9, 12, 7, 5]) + bupu = get_cmap("BuPu", 15)([13, 14]) + cool = get_cmap("cool", 15)([10, 9, 12, 7, 5]) return np.concatenate((grays, ncar, bupu, cool)) @property def radiation_colors(self) -> np.ndarray: """Default color map for Longwave Radiation.""" - grays = cm.get_cmap("Greys", 2)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)(range(0, 126, 5)) + grays = get_cmap("Greys", 2)([0]) + ncar = get_cmap(self.vspec.get("cmap"), 128)(range(0, 126, 5)) return np.concatenate((grays, ncar)) @property def radiation_bw_colors(self) -> np.ndarray: """Default grayscale map for Outgoing Shortwave Radiation.""" - return cm.get_cmap(self.vspec.get("cmap"), 128)(range(30, 110)) + return get_cmap(self.vspec.get("cmap"), 128)(range(30, 110)) @property def radiation_mix_colors(self) -> np.ndarray: """Default color map for Longwave Radiation.""" ncar = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(40)) - grays = cm.get_cmap("Greys", 100)(range(10, 100)) + grays = get_cmap("Greys", 100)(range(10, 100)) return np.concatenate((ncar, grays)) @property def rainbow11_colors(self) -> np.ndarray: """Default color map for Hourly Wildfire Potential.""" - grays = cm.get_cmap("Greys", 2)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( - [18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120] - ) + grays = get_cmap("Greys", 2)([0]) + ncar = get_cmap(self.vspec.get("cmap"), 128)([18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120]) return np.concatenate((grays, ncar)) @property def rainbow12_colors(self) -> np.ndarray: """Default color map for ACPCP, ACSNOD, HLCY, RH, and SNOD.""" - grays = cm.get_cmap("Greys", 2)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + grays = get_cmap("Greys", 2)([0]) + ncar = get_cmap(self.vspec.get("cmap"), 128)( [15, 18, 20, 25, 50, 60, 70, 80, 85, 90, 100, 120] ) return np.concatenate((grays, ncar)) @@ -367,8 +343,8 @@ def rainbow12_reverse(self) -> np.ndarray: def rainbow16_colors(self) -> np.ndarray: """Default color map for helicity.""" - grays = cm.get_cmap("Greys", 5)([0, 2]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + grays = get_cmap("Greys", 5)([0, 2]) + ncar = get_cmap(self.vspec.get("cmap"), 128)( [9, 15, 18, 20, 25, 48, 57, 65, 74, 79, 87, 94, 102, 109, 120] ) return np.concatenate((grays, ncar)) @@ -377,7 +353,7 @@ def rainbow16_colors(self) -> np.ndarray: def shear_colors(self) -> np.ndarray: """Default color map for Vertical Shear.""" - ctable = cm.get_cmap(self.vspec.get("cmap"), 16)(range(5, 15)) + ctable = get_cmap(self.vspec.get("cmap"), 16)(range(5, 15)) ctable[9] = [1, 1, 1, 1] return ctable @@ -385,43 +361,63 @@ def shear_colors(self) -> np.ndarray: def slw_colors(self) -> np.ndarray: """Default color map for Max Vertically Integrated Graupel.""" - white = cm.get_cmap("Greys", 3)([0]) - purples = cm.get_cmap("nipy_spectral", 30)([3, 1]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 15)([2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]) + white = get_cmap("Greys", 3)([0]) + purples = get_cmap("nipy_spectral", 30)([3, 1]) + ncar = get_cmap(self.vspec.get("cmap"), 15)([2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]) return np.concatenate((white, purples, ncar)) @property def smoke_colors(self) -> np.ndarray: """Default color map for smoke plots.""" - white = cm.get_cmap("Greys", 2)([0]) - blues = cm.get_cmap("Blues", 6)(range(1, 5)) - green_yellow_red = cm.get_cmap("RdYlGn_r", 18)([1, 3, 5, 9, 12, 13, 14, 16, 18]) + white = get_cmap("Greys", 2)([0]) + blues = get_cmap("Blues", 6)(range(1, 5)) + green_yellow_red = get_cmap("RdYlGn_r", 18)([1, 3, 5, 9, 12, 13, 14, 16, 18]) purple = np.array([mpcolors.to_rgba("xkcd:vivid purple")]) return np.concatenate((white, blues, green_yellow_red, purple)) + @property + def smoke_emissions_colors(self) -> list[str]: + """Default color map for smoke emissions plot.""" + + # The scatter plot utility won't accept anything but named colors + return [ + "white", + "rebeccapurple", + "royalblue", + "cadetblue", + "yellowgreen", + "mediumaquamarine", + "lightgreen", + "yellow", + "gold", + "orange", + "darkorange", + "orangered", + "red", + "firebrick", + ] + @property def snow_colors(self) -> np.ndarray: """Default color map for Snow fields.""" - grays = cm.get_cmap("Greys", 5)([0, 2]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( - [15, 18, 20, 25, 50, 60, 74, 81, 85, 90, 100] - ) + grays = get_cmap("Greys", 5)([0, 2]) + ncar = get_cmap(self.vspec.get("cmap"), 128)([15, 18, 20, 25, 50, 60, 74, 81, 85, 90, 100]) return np.concatenate((grays, ncar)) @property def soilm_colors(self) -> np.ndarray: """Default color map for Soil Moisture Availability.""" - return cm.get_cmap(self.vspec.get("cmap"), 128)(range(0, 122, 11)) + return get_cmap(self.vspec.get("cmap"), 128)(range(0, 122, 11)) @property def soilw_colors(self) -> np.ndarray: """Default color map for Soil Moisture.""" - grays = cm.get_cmap("Greys", 2)([1]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 110)( + grays = get_cmap("Greys", 2)([1]) + ncar = get_cmap(self.vspec.get("cmap"), 110)( [0, 10, 20, 25, 35, 40, 60, 73, 80, 85, 95, 105] ) return np.concatenate((grays, ncar)) @@ -431,24 +427,22 @@ def t_colors(self) -> np.ndarray: """Default color map for Potential Temperature.""" ncolors = len(self.clevs) - return cm.get_cmap(self.vspec.get("cmap", "jet"), ncolors)(range(ncolors)) + return get_cmap(self.vspec.get("cmap", "jet"), ncolors)(range(ncolors)) @property def tsfc_colors(self) -> np.ndarray: """Default color map for Surface Temperature.""" - purples = cm.get_cmap("Purples", 16)([14, 12, 8, 6, 4, 2]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( - [15, 20, 25, 33, 50, 60, 70, 80, 85, 90, 115] - ) - grays = cm.get_cmap("Greys", 15)([2, 4, 6, 8]) + purples = get_cmap("Purples", 16)([14, 12, 8, 6, 4, 2]) + ncar = get_cmap(self.vspec.get("cmap"), 128)([15, 20, 25, 33, 50, 60, 70, 80, 85, 90, 115]) + grays = get_cmap("Greys", 15)([2, 4, 6, 8]) return np.concatenate((purples, ncar, grays)) @property def terrain_colors(self) -> np.ndarray: """Default color map for Terrain.""" - return np.ndarray( + return np.asarray( ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(54, 157, 6)) ) @@ -456,9 +450,9 @@ def terrain_colors(self) -> np.ndarray: def ua_temp_colors(self) -> np.ndarray: """Default color map for Upper-Air Temperature.""" - grays = cm.get_cmap("Greys", 27)(range(17, 1, -2)) - purples = cm.get_cmap("Purples", 27)(range(17, 1, -2)) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + grays = get_cmap("Greys", 27)(range(17, 1, -2)) + purples = get_cmap("Purples", 27)(range(17, 1, -2)) + ncar = get_cmap(self.vspec.get("cmap"), 128)( [30, 34, 36, 40, 45, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100, 115] ) return np.concatenate((grays, purples, ncar)) @@ -477,14 +471,14 @@ def vis_colors(self) -> np.ndarray: the gray range is arbitrary compared to the official flight levels """ - lifr = cm.get_cmap("RdPu_r", 20)(range(11)) - ifr = cm.get_cmap("autumn", 30)(range(30)) - mvfr = cm.get_cmap("Blues", 20)(range(10, 20)) - vfr1 = cm.get_cmap("YlGn_r", 60)(range(50)) - vfr2 = cm.get_cmap("Greys", 25)(np.full(10, 9)) - hi01 = cm.get_cmap("Greys", 25)(np.full(10, 6)) - hi02 = cm.get_cmap("Greys", 25)(np.full(20, 3)) - hi03 = cm.get_cmap("Greys", 25)(np.full(1, 0)) + lifr = get_cmap("RdPu_r", 20)(range(11)) + ifr = get_cmap("autumn", 30)(range(30)) + mvfr = get_cmap("Blues", 20)(range(10, 20)) + vfr1 = get_cmap("YlGn_r", 60)(range(50)) + vfr2 = get_cmap("Greys", 25)(np.full(10, 9)) + hi01 = get_cmap("Greys", 25)(np.full(10, 6)) + hi02 = get_cmap("Greys", 25)(np.full(20, 3)) + hi03 = get_cmap("Greys", 25)(np.full(1, 0)) return np.concatenate((lifr, ifr, mvfr, vfr1, vfr2, hi01, hi02, hi03)) @@ -492,17 +486,17 @@ def vis_colors(self) -> np.ndarray: def vvel_colors(self) -> np.ndarray: """Default color map for Vetical Velocity.""" - ncar1 = cm.get_cmap(self.vspec.get("cmap"), 128)([15, 18, 20, 25]) - grays = cm.get_cmap("Greys", 2)([0]) - ncar2 = cm.get_cmap(self.vspec.get("cmap"), 128)([60, 70, 80, 85, 90, 100, 120]) + ncar1 = get_cmap(self.vspec.get("cmap"), 128)([15, 18, 20, 25]) + grays = get_cmap("Greys", 2)([0]) + ncar2 = get_cmap(self.vspec.get("cmap"), 128)([60, 70, 80, 85, 90, 100, 120]) return np.concatenate((ncar1, grays, ncar2)) @property def vort_colors(self) -> np.ndarray: """Default color map for Absolute Vorticity.""" - grays = cm.get_cmap("Greys", 2)([0]) - ncar = cm.get_cmap(self.vspec.get("cmap"), 128)( + grays = get_cmap("Greys", 2)([0]) + ncar = get_cmap(self.vspec.get("cmap"), 128)( [15, 18, 20, 25, 50, 60, 70, 80, 83, 90, 100, 120] ) return np.concatenate((grays, ncar)) @@ -511,16 +505,16 @@ def vort_colors(self) -> np.ndarray: def wind_colors(self) -> np.ndarray: """Default color map for Wind Speed.""" - low = cm.get_cmap(self.vspec.get("cmap"), 129)(range(129, 109, -5)) - high1 = cm.get_cmap(self.vspec.get("cmap"), 129)(range(16, 29, 3)) - high2 = cm.get_cmap(self.vspec.get("cmap"), 129)(range(48, 103, 6)) + low = get_cmap(self.vspec.get("cmap"), 129)(range(129, 109, -5)) + high1 = get_cmap(self.vspec.get("cmap"), 129)(range(16, 29, 3)) + high2 = get_cmap(self.vspec.get("cmap"), 129)(range(48, 103, 6)) return np.concatenate((low, high1, high2)) @property def wind_colors_high(self) -> np.ndarray: """Default color map for High Wind Speed.""" - low = cm.get_cmap(self.vspec.get("cmap"), 129)(range(129, 108, -7)) - high1 = cm.get_cmap(self.vspec.get("cmap"), 129)(range(16, 29, 4)) - high2 = cm.get_cmap(self.vspec.get("cmap"), 129)(range(46, 95, 7)) + low = get_cmap(self.vspec.get("cmap"), 129)(range(129, 108, -7)) + high1 = get_cmap(self.vspec.get("cmap"), 129)(range(16, 29, 4)) + high2 = get_cmap(self.vspec.get("cmap"), 129)(range(46, 95, 7)) return np.concatenate((low, high1, high2)) diff --git a/tests/test_common.py b/tests/test_common.py index 61f39e5..4569846 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -26,61 +26,12 @@ from matplotlib import colors as mcolors from metpy.plots import ctables -from adb_graphics import conversions, specs, utils +from adb_graphics import specs, utils from adb_graphics.datahandler import gribdata yaml.add_constructor("!join_ranges", utils.join_ranges, Loader=yaml.Loader) -def test_conversion(): - """ - Conversions return numpy array for input of np.ndarray, list, or int. - """ - - a = np.ones([3, 2]) * 300 - c = a[0, 0] - - # Check for the right answer - assert np.array_equal(conversions.k_to_c(a), a - 273.15) - assert np.array_equal(conversions.k_to_f(a), (a - 273.15) * 9 / 5 + 32) - assert np.array_equal(conversions.kgm2_to_in(a), a * 0.03937) - assert np.array_equal(conversions.m_to_dm(a), a / 10) - assert np.array_equal(conversions.m_to_in(a), a * 39.3701) - assert np.array_equal(conversions.m_to_kft(a), a / 304.8) - assert np.array_equal(conversions.m_to_mi(a), a / 1609.344) - assert np.array_equal(conversions.ms_to_kt(a), a * 1.9438) - assert np.array_equal(conversions.pa_to_hpa(a), a / 100) - assert np.array_equal(conversions.percent(a), a * 100) - assert np.array_equal(conversions.to_micro(a), a * 1e6) - assert np.array_equal(conversions.to_micrograms_per_m3(a), a * 1e9) - assert np.array_equal(conversions.vvel_scale(a), a * -10) - assert np.array_equal(conversions.vort_scale(a), a / 1e-05) - assert np.array_equal(conversions.weasd_to_1hsnw(a), a * 10) - - functions = [ - conversions.k_to_c, - conversions.k_to_f, - conversions.kgm2_to_in, - conversions.m_to_dm, - conversions.m_to_in, - conversions.m_to_kft, - conversions.m_to_mi, - conversions.ms_to_kt, - conversions.pa_to_hpa, - conversions.percent, - conversions.to_micro, - conversions.to_micrograms_per_m3, - conversions.vvel_scale, - conversions.vort_scale, - conversions.weasd_to_1hsnw, - ] - - # Check that all functions return a np.ndarray given a collection, or single float - for f in functions: - for collection in [a, c]: - assert isinstance(f(collection), type(collection)) - - class MockSpecs(specs.VarSpec): """Mock class for the VarSpec abstract class.""" diff --git a/tests/test_specs.py b/tests/test_specs.py new file mode 100644 index 0000000..865ecbb --- /dev/null +++ b/tests/test_specs.py @@ -0,0 +1,103 @@ +from pathlib import Path + +import numpy as np +import yaml +from pytest import fixture, mark + +from adb_graphics import specs, utils + +yaml.add_constructor("!join_ranges", utils.join_ranges, Loader=yaml.Loader) + + +class Spec(specs.VarSpec): + """ + Concrete class for the VarSpec abstract class. + """ + + with Path("adb_graphics/default_specs.yml").open() as c: + cfg = yaml.load(c, Loader=yaml.Loader) + + @property + def clevs(self): + return np.asarray(range(15)) + + @property + def vspec(self): + return {"cmap": "rainbow"} + + +@fixture +def spec(): + return Spec() + + +def test_aod_colors(spec): + colors = spec.aod_colors + assert len(colors) == 15 + + +@mark.parametrize(("levels", "expected"), [(3, 3), (4, 4), (None, 16)]) +def test_centered_diff(levels, expected, spec): + colors = spec.centered_diff(nlev=levels) + assert len(colors) == expected + + +@mark.parametrize( + ("func", "expected"), + [ + ("aod_colors", 15), + ("cin_colors", 12), + ("ceil_colors", 14), + ("cldcov_colors", 13), + ("cref_colors", 16), + ("fire_power_colors", 7), + ("flru_colors", 4), + ("frzn_colors", 12), + ("goes_colors", 151), + ("graupel_colors", 19), + ("hail_colors", 10), + ("heat_flux_colors", 22), + ("heat_flux_colors_g", 13), + ("heat_flux_colors_l", 17), + ("heat_flux_colors_s", 17), + ("icprb_colors", 10), + ("icsev_colors", 6), + ("lcl_colors", 19), + ("lifted_index_colors", 31), + ("mdn_colors", 18), + ("mean_vvel_colors", 19), + ("mup_colors", 18), + ("pbl_colors", 15), + ("pcp_colors", 10), + ("pcp_colors_high", 6), + ("pmsl_colors", 15), + ("ps_colors", 105), + ("pw_colors", 21), + ("radiation_colors", 27), + ("radiation_bw_colors", 80), + ("radiation_mix_colors", 130), + ("rainbow11_colors", 12), + ("rainbow12_colors", 13), + ("rainbow12_reverse", 13), + ("rainbow16_colors", 17), + ("shear_colors", 10), + ("slw_colors", 16), + ("smoke_colors", 15), + ("smoke_emissions_colors", 14), + ("snow_colors", 13), + ("soilm_colors", 12), + ("soilw_colors", 13), + ("t_colors", 15), + ("tsfc_colors", 21), + ("terrain_colors", 18), + ("ua_temp_colors", 32), + ("vis_colors", 142), + ("vvel_colors", 12), + ("vort_colors", 13), + ("wind_colors", 19), + ("wind_colors_high", 14), + ], +) +def test_colors(expected, func, spec): + colors = spec.__getattribute__(func) + assert len(colors) == expected From 5b52c345181408a2c41e1600e0dd5ffc93e82220 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 24 Oct 2025 13:32:59 -0600 Subject: [PATCH 20/98] Simplified and unittested gribfile.py --- adb_graphics/datahandler/gribfile.py | 241 ++------------------------- conftest.py | 7 +- create_graphics.py | 82 +++------ pyproject.toml | 2 +- tests/datahandler/__init__.py | 0 tests/datahandler/test_gribfile.py | 37 ++++ 6 files changed, 74 insertions(+), 295 deletions(-) create mode 100644 tests/datahandler/__init__.py create mode 100644 tests/datahandler/test_gribfile.py diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index e8d22d0..cc74670 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -19,7 +19,7 @@ def __init__(self, filename: Path | str, var_config: dict): self.var_config = var_config self.contents = self._load() - def _load(self): + def _load(self) -> xr.Dataset: """ Internal method that opens the grib file. Returns a grib message iterator. @@ -41,9 +41,8 @@ class GribFiles: def __init__( self, - coord_dims: dict[str, list[int]], - filenames: dict[str, list[Path]], - filetype: str, + filenames: list[Path], + var_config: dict, **kwargs, ): """ @@ -62,237 +61,17 @@ def __init__( """ self.model = kwargs.get("model", "") - self.filenames = filenames - self.filetype = filetype - self.coord_dims = coord_dims + self.var_config = var_config self.contents = self._load() - def append(self, filenames: dict[str, list[Path]]): - """ - Add a single new slice to existing data set. - Must match coord_dims and filetype of the original dataset. Updates current contents - property. - """ - - self.contents = self._load(filenames) - - def free_fcst_names(self, ds: xr.Dataset, fcst_type: str): # noqa: PLR0915,PLR0912 - """ - Variable names to rename. - - Given an opened dataset, return a dict of original variable names - (key) and the desired name (value). - """ - - ret = {} - - fhr = self.coord_dims["fcst_hr"][-1] - - special_suffixes = ["max", "min", "acc", "avg"] - for ds_var in ds.variables: - var = str(ds_var) - suffix = var.split("_")[-1] - - # Keeping lists of misbehaving "accumulated" variables here because - # there doesn't seem to be another way to know.... - - if fcst_type == "01fcst": - # Don't rename these variables at early hours - odd_variables = [ - "ASNOW", - "FROZR", - "FRZR", - "LRGHR", - ] - if self.model == "rrfs": - odd_variables.append("WEASD") - if self.model != "rrfs": - odd_variables.extend( - [ - "CDLYR", - "TCDC", - ] - ) - needs_renaming = var.split("_")[0] not in odd_variables - if suffix in special_suffixes and needs_renaming: - if "global" not in self.model or self.model == "global_mpas": - new_suffix = f"{suffix}1h" - else: - new_suffix = f"{suffix}6h" - ret[var] = var.replace(suffix, new_suffix) - # MASSDEN is a special case when ending in "avg_1'" - if var.split("_")[0] == "MASSDEN" and var.split("_")[-2] == "avg": - print("Special change to MASSDEN avg_1 name to avg1h_1") - ret[var] = var.replace("avg", "avg1h") - else: - # Only rename these variables at late hours - odd_variables = [ - "APCP", - "CDLYR", - "FROZR", - "FRZR", - "LRGHR", - "TCDC", - "TSNOWP", - "WEASD", - ] - if self.model == "rrfs": - odd_variables.remove("WEASD") - variable = var.split("_")[0] - needs_renaming = variable in odd_variables - contains_suffix = [] - for suf in special_suffixes: - # The LRGHR variable behaves differently in RRFS than in all - # others! At 7 hours, it starts averaging since 6h. From 0-6 - # h it's named with suffix avg, after its named avg1h, - # avg2h, etc. - if self.model == "rrfs" and variable == "LRGHR" and suffix == f"{suf}1h": - contains_suffix.append(suf) - - # RRFS_A has fields that have the suffix 'acc0h' but we don't - # want those. Drop them if they come up. - bad_0h_vars = [ - "APCP_P8_L1_GLL0_acc0h", - "FROZR_P8_L1_GLC0_acc0h", - "FRZR_P8_L1_GLC0_acc0h", - "CDLYR_P8_L200_GLC0_avg0h", - "TCDC_P8_L200_GLC0_avg0h", - "APCP_P8_L1_GLC0_acc0h", - "APCP_P8_L1_GST0_acc0h", - ] - if fhr != 0 and var in bad_0h_vars: - print(f"dropping {var}") - ds.drop(var) - continue - # mpas_global has fields that have the suffix 'acc1h' but we don't - # want those since the output is 6h. Drop them if they come up. - bad_1h_vars = [ - "APCP_P8_L1_GLL0_acc1h", - "FROZR_P8_L1_GLL0_acc1h", - "FRZR_P8_L1_GLL0_acc1h", - "CDLYR_P8_L200_GLL0_avg1h", - "TCDC_P8_L200_GLL0_avg1h", - "APCP_P8_L1_GLL0_acc1h", - "APCP_P8_L1_GST0_acc1h", - "WEASD_P8_L1_GLL0_acc1h", - ] - if self.model == "global_mpas" and fhr != 0 and var in bad_1h_vars: - print(f"dropping {var}") - ds.drop(var) - continue - # For the RAP CONUS and AK domains, the APCP, WEASD, and FROZR - # variables all have 3h accumulation fields in addition to - # the 1h accumulation fields. This causes problems with the - # renaming, so just drop those fields from the dataset. - bad_3h_vars = [ - "APCP_P8_L1_GLC0_acc3h", - "WEASD_P8_L1_GLC0_acc3h", - "FROZR_P8_L1_GLC0_acc3h", - "APCP_P8_L1_GST0_acc3h", - "WEASD_P8_L1_GST0_acc3h", - "FROZR_P8_L1_GST0_acc3h", - ] - if self.model == "rap" and fhr != 3 and var in bad_3h_vars: # noqa: PLR2004 - print(f"dropping {var}") - ds.drop(var) - continue - - # Some global models will start producing 12h accumulations at - # lead times past 246h. These cause problems with the renaming, - # so we can drop those fields. - bad_12h_vars = [ - "APCP_P8_L1_GLL0_acc12h", - "APCP_P8_L1_GLC0_acc12h", - "APCP_P8_L1_GST0_acc12h", - ] - if fhr != 12 and var in bad_12h_vars: # noqa: PLR2004 - print(f"dropping {var}") - ds.drop(var) - continue - - # All the variables that need to be renamed. In most cases, - # exclude the "1h" ("6h" for global) accumulated variables - accum_freq = 6 if "global" in self.model else 1 - if suf in suffix and suffix != f"{suf}{accum_freq}h": - contains_suffix.append(suf) - - if contains_suffix and needs_renaming: - ret[var] = var.replace(suffix, contains_suffix[0]) - - return ret - - def _load(self, filenames: dict[str, list[Path]] | None = None): + def _load(self, filenames: list[Path] | None = None): """Load the set of files into a single XArray structure.""" - - all_leads = [] if filenames is None else [self.contents] filenames = self.filenames if filenames is None else filenames - - # 0h and 1h accumulated forecast variables are named differently than - # the rest of the forecast hours. Rename those accumulated variables if - # needed. - for fcst_type in ["01fcst", "free_fcst"]: - for filename in filenames.get(fcst_type, {}): - print(f"Loading grib2 file: {fcst_type}, {filename}") - - # Rename variables to match free forecast variables - dataset = xr.open_mfdataset( - filenames[fcst_type], - **self.open_kwargs, - ) - - # renaming = self.free_fcst_names(dataset, fcst_type) - # if renaming and self.model not in ["hrrre", "rrfse"]: - # print("RENAMING VARIABLES:") - # for old_name, new_name in renaming.items(): - # print(f" {old_name:>30s} -> {new_name}") - # dataset = dataset.rename_vars(renaming) - - # if len(all_leads) == 1: - # # Check that specific variables exist in the xarray that is - # # already loaded (presumably 0hr), and add them if they - # # don't. This implementation is relying on pointers to - # # update "in place" - # og_ds = all_leads[0] - # bad_vars = [ - # "APCP_P8_L1_{grid}_acc", - # "ACPCP_P8_L1_{grid}_acc", - # "FROZR_P8_L1_{grid}_acc", - # "NCPCP_P8_L1_{grid}_acc", - # "WEASD_P8_L1_{grid}_acc", - # ] - # bad_vars = [v.format(grid=self.grid_suffix) for v in bad_vars] - # for bad_var in bad_vars: - # # Check to see if the bad variable is in the current - # # dataset and NOT in the original dataset. - # if bad_var not in og_ds.variables and dataset.get(bad_var) is not None: - # print(f"Adding {bad_var} to og ds") - # # Duplicate the accumulated variable with the - # # required name - # og_ds[bad_var] = og_ds.get(f"{bad_var}1h") - all_leads.append(dataset) - - return xr.combine_nested( - all_leads, - compat="override", - concat_dim=list(self.coord_dims.keys())[0], - coords="minimal", - data_vars="all", - ) - - @property - def open_kwargs(self): - """ - Defines the kwargs used by calls to XArray open_mfdataset. - """ - - return dict( - backend_kwargs=dict(format="grib2"), - cache=False, + return xr.open_mfdataset( + filenames, + engine="cfgrib", + concat_dim="time", combine="nested", - compat="override", - concat_dim=list(self.coord_dims.keys())[0], - coords="minimal", - engine="pynio", - lock=False, + backend_kwargs=({"filter_by_keys": self.var_config}), ) diff --git a/conftest.py b/conftest.py index d6fdf78..e9e035d 100644 --- a/conftest.py +++ b/conftest.py @@ -5,6 +5,8 @@ function defined. """ +from pathlib import Path + import pytest @@ -32,7 +34,6 @@ def natfile(request): @pytest.fixture -def prsfile(request): +def prsfile(): """Interface to pass a grib file to pytest.""" - - return request.config.getoption("--prs-file") + return Path("tests", "data", "wrfprs_hrconus_07.grib2") diff --git a/create_graphics.py b/create_graphics.py index 11cce05..04bd134 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -23,7 +23,6 @@ import yaml from adb_graphics import errors, utils -from adb_graphics.datahandler.gribfile import GribFile, GribFiles from adb_graphics.figure_builders import parallel_maps, parallel_skewt from adb_graphics.figures import maps @@ -70,10 +69,7 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): and generate a pool of workers to complete the tasks. """ - # Create the file object to load the contents - gfile = GribFile(grib_path, var_config={}) - - args = [(cla, fhr, gfile.contents, site, workdir) for site in cla.sites] + args = [(cla, fhr, grib_path, site, workdir) for site in cla.sites] print(f"Queueing {len(args)} Skew Ts") with Pool(processes=cla.nprocs) as pool: @@ -81,7 +77,12 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): def create_maps( - cla: Namespace, fhr: int, grib_path: Path, workdir: Path, grib_path2: Path | None = None + cla: Namespace, + fhr: int, + grib_paths: list[Path], + workdir: Path, + grib_path2: Path | None = None, + **kwargs, ): """ Generate arguments for parallel processing of plan-view maps and @@ -104,7 +105,7 @@ def create_maps( ( cla, fhr, - grib_path, + grib_paths, level, model, spec, @@ -112,6 +113,7 @@ def create_maps( workdir, tile, grib_path2, + kwargs, ) ) @@ -121,39 +123,6 @@ def create_maps( pool.starmap(parallel_maps, args) -def gather_gribfiles(cla: Namespace, fhr: int, grib_path: Path, gribfiles: None | GribFiles): - """ - Returns the appropriate gribfiles object for the type of graphics being - generated -- whether it's for a single forecast time or all forecast lead - times. - """ - - filenames: dict[str, list[Path]] = {"01fcst": [], "free_fcst": []} - - fcst_hour = int(fhr) - - first_fcst = 6 if "global" in cla.images[0] else 1 - if fcst_hour <= first_fcst: - filenames["01fcst"].append(grib_path) - else: - filenames["free_fcst"].append(grib_path) - - if gribfiles is None or not cla.all_leads: - # Create a new GribFiles object, include all hours, or just this one, - # depending on command line argument flag - - return GribFiles( - coord_dims={"fcst_hr": [fhr]}, - filenames=filenames, - filetype=cla.file_type, - model=cla.images[0], - ) - # Append a single forecast hour to the existing GribFiles object. - gribfiles.coord_dims.get("fcst_hr", []).append(fhr) - gribfiles.append(filenames) - return gribfiles - - def generate_tile_list(arg_list: list) -> list[str]: """ Given the input arguments -- a list if the argument is provided, return @@ -568,9 +537,7 @@ def graphics_driver(cla: Namespace): # Initialize a timer used for killing the program timer_end = time.time() - gribfiles = None - gribfiles2 = None - + grib_paths = [] # When accummulating variables for preparing a single lead time, # load all of those into gribfiles up front. # This is not an operational feature. Exit if files don't exist. @@ -588,7 +555,8 @@ def graphics_driver(cla: Namespace): ) remove_proc_grib_files(cla) raise FileNotFoundError(" ".join(msg)) - gribfiles = gather_gribfiles(cla, fhr, grib_path, gribfiles) + if old_enough: + grib_paths.append(grib_path) # Allow this task to run concurrently with UPP by continuing to check for # new files as they become available. @@ -599,13 +567,14 @@ def graphics_driver(cla: Namespace): if cla.graphic_type == "enspanel": # Expand template to create a list of ensemble member files and # check if they exist and that they're old enough - grib_paths = [] + ens_paths = [] ens_members = list(range(cla.ens_size)) for mem in ens_members: mem_path, mem_old_enough = check_file(cla, fhr, mem=mem) if mem_old_enough: - grib_paths.append(mem_path) - old_enough = len(grib_paths) == cla.ens_size + ens_paths.append(mem_path) + old_enough = len(ens_paths) == cla.ens_size + grib_paths = ens_paths else: # Only checks existence/age of base file for diffs grib_path, old_enough = pre_proc_grib_files(cla, fhr) @@ -613,6 +582,7 @@ def graphics_driver(cla: Namespace): # UPP is most likely done writing if it hasn't written in data_age # mins (default is 3 to address most CONUS-sized domains) if old_enough: + grib_paths.append(grib_path) fcst_hours.remove(fhr) fhr_as_list = [fhr] else: @@ -655,38 +625,30 @@ def graphics_driver(cla: Namespace): create_maps( cla, fhr=fhr, - grib_path=grib_path, + grib_paths=grib_paths, workdir=workdir, ) elif cla.graphic_type == "diff": - gribfiles = gather_gribfiles(cla, fhr, grib_path, gribfiles) grib_path2, _ = check_file( cla, fhr, data_root=cla.data_root2, file_tmpl=cla.file_tmpl2, ) - gribfiles2 = gather_gribfiles(cla, fhr, grib_path2, gribfiles2) - create_maps( cla, fhr=fhr, - grib_path=grib_path, + grib_paths=[grib_path], grib_path2=grib_path2, workdir=workdir, ) - else: - gribfiles = GribFiles( - coord_dims={"ens_mem": ens_members, "fcst_hr": fhr_as_list}, - filenames={"free_fcst": grib_paths}, - filetype=cla.file_type, - model=cla.images[0], - ) + else: # enspanel create_maps( cla, fhr=fhr, - grib_path=grib_path, + grib_paths=grib_paths, workdir=workdir, + coord_dims={"ens_mem": ens_members, "fcst_hr": fhr_as_list}, ) # Zip png files and remove the originals in a subprocess diff --git a/pyproject.toml b/pyproject.toml index b0d31d0..5678bd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ exclude_also = ["if TYPE_CHECKING:"] fail_under = 100 show_missing = true skip_covered = true -omit = ["tests/*"] +omit = ["conftest.py", "tests/*"] [tool.mypy] diff --git a/tests/datahandler/__init__.py b/tests/datahandler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/datahandler/test_gribfile.py b/tests/datahandler/test_gribfile.py new file mode 100644 index 0000000..842baca --- /dev/null +++ b/tests/datahandler/test_gribfile.py @@ -0,0 +1,37 @@ +from pathlib import Path + +from xarray import Dataset + +from adb_graphics.datahandler import gribfile + + +def test_gribfile(prsfile): + gf = gribfile.GribFile( + filename=Path(prsfile), + var_config={ + "shortName": "sp", + "typeOfLevel": "surface", + }, + ) + assert isinstance(gf.contents, Dataset) + assert len(gf.contents.data_vars) == 1 + assert len(gf.contents.data_vars["sp"].shape) == 2 + + +def test_gribfiles(): + paths = [ + "/Users/cholt/work/pygraf_cfgrib/sample_data/rrfs_a/2025101312/rrfs.t12z.prslev.3km.f016.conus.grib2", + "/Users/cholt/work/pygraf_cfgrib/sample_data/rrfs_a/rrfs.t12z.prslev.3km.f016.conus.grib2", + ] + gribfiles = [Path(f) for f in paths] + gf = gribfile.GribFiles( + filenames=gribfiles, + var_config={ + "shortName": "sp", + "typeOfLevel": "surface", + }, + model="rrfs", + ) + assert isinstance(gf.contents, Dataset) + assert len(gf.contents.data_vars) == 1 + assert len(gf.contents.data_vars["sp"].shape) == 3 From c5b4b30c7ce47f19077295092ff5b73041fc7677 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 24 Oct 2025 16:39:14 -0600 Subject: [PATCH 21/98] WIP testing gribdata --- adb_graphics/datahandler/gribdata.py | 51 +++++-------- adb_graphics/default_specs.yml | 104 ++++++++++++------------- adb_graphics/utils.py | 13 +++- conftest.py | 6 ++ tests/datahandler/test_gribdata.py | 109 +++++++++++++++++++++++++++ tests/test_common.py | 19 ++--- tests/test_specs.py | 6 +- 7 files changed, 204 insertions(+), 104 deletions(-) create mode 100644 tests/datahandler/test_gribdata.py diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 34228cd..a62a609 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -13,6 +13,7 @@ import numpy as np from matplotlib import cm +from pandas import to_datetime from xarray import DataArray, Dataset from adb_graphics import conversions, errors, specs, utils @@ -54,8 +55,8 @@ def anl_dt(self) -> datetime: Returns the initial time of the grib file as a datetime object from the grib file. """ - - return datetime.fromisoformat(str(self.field.time.values).split(".")[0]) + ret: datetime = to_datetime(self.field.time.values) + return ret @property def clevs(self) -> np.ndarray: @@ -63,24 +64,10 @@ def clevs(self) -> np.ndarray: Uses the information contained in the yaml config file to determine the set of levels to be contoured. Returns the list of levels. - The yaml file "clevs" key may contain a list, a range, or a call to a - function. The logic to parse those options is included here. + The yaml file "clevs" key may contain a list or a range. """ - clev = np.asarray(self.vspec.get("clevs", [])) - - # Is clevs a list? - if isinstance(clev, (list, np.ndarray)): - return np.asarray(clev) - - # Is clev a call to another function? - try: - return utils.get_func(clev)() - except ImportError: - print( - f"Check yaml file definition of CLEVS for {self.short_name}. ", - "Must be a list, range, or function call!", - ) + return np.asarray(self.vspec.get("clevs", [])) @staticmethod def date_to_str(date: datetime) -> str: @@ -98,21 +85,10 @@ def field(self) -> DataArray: first_variable_name = list(self.ds.data_vars)[0] return DataArray(self.ds[first_variable_name]) - def field_column_max(self, values: DataArray, variable: str, level: str, **kwargs): # noqa: ARG002 + def field_column_max(self, **kwargs): # noqa: ARG002 """Returns the column max of the values.""" - vals = self.values(name=variable, level=level, one_lev=False) - return vals.max(axis=0) - - def field_sum(self, values: DataArray, variable2: str, level2: str, **kwargs): # noqa: ARG002 - # pylint: disable=unused-argument - """Return the sum of the values.""" - - value2 = self.values(name=variable2, level=level2) - sum2 = values + value2 - value2.close() - - return sum2 + return self.values().max(axis=0) def field_diff(self, values: DataArray, variable2: str, level2: str, **kwargs): # noqa: ARG002 """Subtracts the values from variable2 from self.field.""" @@ -134,6 +110,15 @@ def field_mean( levs = [int(x[:-2]) for x in levels] return values.sel(isobaricInhPa=levs).mean("isobaricInhPa") + def field_sum(self, values: DataArray, variable2: str, level2: str, **kwargs): # noqa: ARG002 + """Return the sum of the values.""" + + value2 = self.values(name=variable2, level=level2) + sum2 = values + value2 + value2.close() + + return sum2 + def _get_data_levels(self, vertical_dim: str): """ Values of the vertical dimension. @@ -142,7 +127,7 @@ def _get_data_levels(self, vertical_dim: str): vertical_dim the name of the vertical dimension """ dim = [str(coord) for coord in self.ds.coords if vertical_dim in str(coord)][0] - return self.ds.coords[dim].values + return self.ds.coords[dim].to_numpy() def _get_field(self, spec: dict) -> DataArray: """ @@ -292,7 +277,7 @@ def latlons(self): coords = sorted( [str(c) for c in list(self.ds.coords) if any(ele in str(c) for ele in ["lat", "lon"])] ) - return [self.ds.coords[c].values for c in coords] + return [self.ds.coords[c].to_numpy() for c in coords] @property def lev_descriptor(self): diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index d47cc59..ee5e4a0 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -11,7 +11,7 @@ # # clevs: specifies the contour levels by one of the following methods # - a list with no quotes -# - a numpy.arange specified as "!!python/object/apply:numpy.arange [list]" +# - a numpy.arange specified as "!arange [list]" # with quotes. Specify "list" as with numpy.arange() like this -- # [[start, ]stop[, increment]]]. start and increment are options. # - the name of a function to be called that will return a list. @@ -116,7 +116,7 @@ shortName: rare typeOfLevel: heightAboveGround level: 1000 - clevs: !!python/object/apply:numpy.arange [5, 76, 5] + clevs: !arange [5, 76, 5] cmap: NWSReflectivity colors: cref_colors ncl_name: REFD_P0_L103_{grid} @@ -765,7 +765,7 @@ cref: # Composite reflectivity crefmax: # Comp reflectivity (max over forecast) sfc: accumulate: True - clevs: !!python/object/apply:numpy.arange [5, 76, 5] + clevs: !arange [5, 76, 5] cmap: NWSReflectivity colors: cref_colors ncl_name: REFC_P0_L200_{grid} @@ -786,7 +786,7 @@ ctop: # Cloud top height shortName: gh typeOfLevel: cloudTop level: 0 - clevs: !!python/object/apply:numpy.arange [0, 61, 5] + clevs: !arange [0, 61, 5] cmap: gist_ncar colors: ceil_colors ncl_name: HGT_P0_L3_{grid} @@ -800,7 +800,7 @@ dewp: # Dew point temperaeure shortName: 2d level: 2 typeOfLevel: heightAboveGround - clevs: !!python/object/apply:numpy.arange [-50, 141, 10] + clevs: !arange [-50, 141, 10] cmap: gist_ncar colors: tsfc_colors ncl_name: DPT_P0_L103_{grid} @@ -814,7 +814,7 @@ dlwrf: # Downward Longwave Radiation Flux cfgrib: shortName: sdlwrf typeOfLevel: surface - clevs: !!python/object/apply:numpy.arange [200, 501, 12] + clevs: !arange [200, 501, 12] cmap: gist_ncar colors: radiation_colors ncl_name: DLWRF_P0_L1_{grid} @@ -822,7 +822,7 @@ dlwrf: # Downward Longwave Radiation Flux title: Downward LW Radiation Flux, Surface unit: W/m$^{2}$ top: # Nominal top of atmosphere - clevs: !!python/object/apply:numpy.arange [80, 341, 2] + clevs: !arange [80, 341, 2] cmap: ir_rgbv_r colors: radiation_mix_colors ncl_name: DLWRF_P0_L8_{grid} @@ -835,7 +835,7 @@ dlwrfavg: # Downward Longwave Radiation Flux Average cfgrib: shortName: avg_sdlwrf typeOfLevel: surface - clevs: !!python/object/apply:numpy.arange [200, 501, 12] + clevs: !arange [200, 501, 12] cmap: gist_ncar colors: radiation_colors ncl_name: DLWRF_P0_L1_{grid}_avg6h @@ -855,7 +855,7 @@ dswrf: # Downward Shortwave Radiation Flux title: Downward SW Radiation Flux, Surface top: # Nominal top of atmosphere <<: *radiation_flux - clevs: !!python/object/apply:numpy.arange [50, 851, 10] + clevs: !arange [50, 851, 10] cmap: Greys_r colors: radiation_bw_colors ncl_name: DSWRF_P0_L8_{grid} @@ -915,7 +915,7 @@ echotop: # Echo Top parameterCategory: 16 typeOfLevel: atmosphereSingleLayer level: 0 - clevs: !!python/object/apply:numpy.arange [4, 50, 3] + clevs: !arange [4, 50, 3] cmap: NWSReflectivity colors: cref_colors ncl_name: @@ -1014,7 +1014,7 @@ G113bt: # GOES-W Brightness temperature, water vapor (Ch 3) parameterNumber: 242 typeOfLevel: atmosphere level: 0 - clevs: !!python/object/apply:numpy.arange [-80, 41, 1] + clevs: !arange [-80, 41, 1] cmap: WVCIMSS_r colors: goes_colors ncl_name: SBT113_P0_L8_{grid} @@ -1057,7 +1057,7 @@ gh: # Geopotential height cfgrib: shortName: gh typeOfLevel: isobaricInhPa - clevs: !!python/object/apply:numpy.arange [6, 4680, 6] + clevs: !arange [6, 4680, 6] cmap: rainbow colors: terrain_colors ncl_name: @@ -1074,27 +1074,27 @@ gh: # Geopotential height <<: *ua_gh 500mb: <<: *ua_gh - clevs: !!python/object/apply:numpy.arange [504, 601, 6] + clevs: !arange [504, 601, 6] ticks: 6 700mb: <<: *ua_gh - clevs: !!python/object/apply:numpy.arange [201, 373, 3] + clevs: !arange [201, 373, 3] 850mb: <<: *ua_gh - clevs: !!python/object/apply:numpy.arange [3, 600, 3] + clevs: !arange [3, 600, 3] 925mb: <<: *ua_gh - clevs: !!python/object/apply:numpy.arange [3, 600, 3] + clevs: !arange [3, 600, 3] 1000mb: <<: *ua_gh - clevs: !!python/object/apply:numpy.arange [500, 600, 10] + clevs: !arange [500, 600, 10] sfc: <<: *ua_gh cfgrib: shortName: orog typeOfLevel: surface level: 0 - clevs: !!python/object/apply:numpy.arange [0, 5000, 250] + clevs: !arange [0, 5000, 250] cmap: gist_earth ncl_name: HGT_P0_L1_{grid} ticks: 0 @@ -1131,7 +1131,7 @@ gust: shortName: gust typeOfLevel: surface level: 0 - clevs: !!python/object/apply:numpy.arange [5, 95, 5] + clevs: !arange [5, 95, 5] cmap: gist_ncar colors: wind_colors ncl_name: GUST_P0_L1_{grid} @@ -1186,7 +1186,7 @@ hailcast: # Max 1h Hail diameter title: Max 1h Hail Diameter at Sfc from HAILCAST hlcy: # Helicity in16: &hlcy # Hourly updraft helicity over 1-6 km layer - clevs: !!python/object/apply:numpy.arange [25, 301, 25] + clevs: !arange [25, 301, 25] cmap: gist_ncar colors: rainbow12_colors ncl_name: UPHL_P0_2L103_{grid} @@ -1200,7 +1200,7 @@ hlcy: # Helicity parameterNumber: 15 topLevel: 5000 bottomLevel: 2000 - clevs: !!python/object/apply:numpy.arange [25, 301, 25] + clevs: !arange [25, 301, 25] ncl_name: UPHL_P0_2L103_{grid} split: True title: 2-5km Updraft Helicity @@ -1210,7 +1210,7 @@ hlcy: # Helicity typeOfLevel: heightAboveGroundLayer topLevel: 2000 bottomLevel: 0 - clevs: !!python/object/apply:numpy.arange [-300, -24, 25] + clevs: !arange [-300, -24, 25] cmap: gist_ncar colors: rainbow12_reverse ncl_name: VAR_0_7_200_P8_2L103_{grid}_min1h @@ -1378,7 +1378,7 @@ icprb: # Icing Probability # levels chosen are arbitrary based on initial plot samples # additional levels may be requested in the future. 500ft: &icprb - clevs: !!python/object/apply:numpy.arange [5, 86, 10] + clevs: !arange [5, 86, 10] cmap: gist_ncar colors: icprb_colors ncl_name: ICPRB_P0_L102_{grid} @@ -1440,7 +1440,7 @@ lcl: # Lifted condensation level shortName: gh typeOfLevel: adiabaticCondensation level: 0 - clevs: !!python/object/apply:numpy.arange [0, 5000, 250] + clevs: !arange [0, 5000, 250] cmap: rainbow colors: lcl_colors contours: @@ -1482,7 +1482,7 @@ li: # Lifted Index shortName: 4lftx typeOfLevel: pressureFromGroundLayer topLevel: 18000 - clevs: !!python/object/apply:numpy.arange [-15, 16] + clevs: !arange [-15, 16] cmap: Spectral colors: lifted_index_colors ncl_name: 4LFTX_P0_2L108_{grid} @@ -1538,7 +1538,7 @@ ltng: # Lightning shortName: ltng typeOfLevel: atmosphere level: 0 - clevs: !!python/object/apply:numpy.arange [5, 91, 5] + clevs: !arange [5, 91, 5] cmap: jet colors: graupel_colors ncl_name: LTNG_P8_L10_{grid}_max1h @@ -1634,7 +1634,7 @@ pres: shortName: sp level: 0 typeOfLevel: surface - clevs: !!python/object/apply:numpy.arange [650, 1051, 4] + clevs: !arange [650, 1051, 4] cmap: gist_ncar colors: ps_colors ncl_name: PRES_P0_L1_{grid} @@ -1652,7 +1652,7 @@ pres: shortName: mslet typeOfLevel: meanSea level: 0 - clevs: !!python/object/apply:numpy.arange [976, 1051, 4] + clevs: !arange [976, 1051, 4] cmap: Spectral_r colors: pmsl_colors ncl_name: @@ -1685,7 +1685,7 @@ presmin: shortName: mslet typeOfLevel: meanSea level: 0 - clevs: !!python/object/apply:numpy.arange [860, 1001, 10] + clevs: !arange [860, 1001, 10] cmap: NWSReflectivity colors: cref_colors ncl_name: @@ -1715,7 +1715,7 @@ ptmp: # Potential temperature shortName: pt typeOfLevel: surface level: 0 - clevs: !!python/object/apply:numpy.arange [210, 350, 5] + clevs: !arange [210, 350, 5] cmap: jet colors: t_colors ncl_name: POT_P0_L103_{grid} @@ -1788,7 +1788,7 @@ pwtr: # Precipitable water cfgrib: shortName: pwat typeOfLevel: atmosphereSingleLayer - clevs: !!python/object/apply:numpy.arange [4, 81, 4] + clevs: !arange [4, 81, 4] cmap: gist_ncar colors: pw_colors ncl_name: PWAT_P0_L200_{grid} @@ -2044,7 +2044,7 @@ sipd: # Supercooled Large Droplet Icing # levels chosen are arbitrary based on initial plot samples # additional levels may be requested in the future. 500ft: &sipd - clevs: !!python/object/apply:numpy.arange [5, 86, 10] + clevs: !arange [5, 86, 10] cmap: gist_ncar colors: icprb_colors ncl_name: SIPD_P0_L102_{grid} @@ -2125,7 +2125,7 @@ soilt: # Soil Temperature typeOfLevel: depthBelowLandLayer scaledValueOfFirstFixedSurface: 0 scaledValueOfSecondFixedSurface: 0 - clevs: !!python/object/apply:numpy.arange [235, 331, 5] + clevs: !arange [235, 331, 5] cmap: gist_ncar colors: tsfc_colors ncl_name: TSOIL_P0_2L106_{grid} @@ -2345,7 +2345,7 @@ temp: # Temperature shortName: 2t level: 2 typeOfLevel: heightAboveGround - clevs: !!python/object/apply:numpy.arange [-32, 33, 2] + clevs: !arange [-32, 33, 2] cmap: Spectral_r colors: centered_diff ncl_name: TMP_P0_L103_{grid} # 2m Temp @@ -2364,7 +2364,7 @@ temp: # Temperature shortName: 2t level: 2 typeOfLevel: heightAboveGround - clevs: !!python/object/apply:numpy.arange [-50, 141, 10] + clevs: !arange [-50, 141, 10] cmap: gist_ncar colors: tsfc_colors ncl_name: TMP_P0_L103_{grid} @@ -2377,7 +2377,7 @@ temp: # Temperature cfgrib: shortName: t typeOfLevel: isobaricInhPa - clevs: !!python/object/apply:numpy.arange [-40, 40, 2.5] + clevs: !arange [-40, 40, 2.5] cmap: jet colors: ua_temp_colors contours: @@ -2465,10 +2465,10 @@ totp: # Hourly total precipitation pres_msl: colors: red linewidths: 0.4 - levels: !!python/object/apply:numpy.arange [650, 1051, 2] + levels: !arange [650, 1051, 2] thick_500mb: colors: blue - levels: !!python/object/apply:numpy.arange [402, 601, 6] + levels: !arange [402, 601, 6] linewidths: 0.4 linestyles: dashed ncl_name: APCP_P8_L1_{grid}_acc1h @@ -2485,10 +2485,10 @@ totp6h: # 6-hourly total precipitation pres_msl: colors: red linewidths: 0.4 - levels: !!python/object/apply:numpy.arange [650, 1051, 2] + levels: !arange [650, 1051, 2] thick_500mb: colors: blue - levels: !!python/object/apply:numpy.arange [402, 601, 6] + levels: !arange [402, 601, 6] linewidths: 0.4 linestyles: dashed ncl_name: APCP_P8_L1_{grid}_acc6h @@ -2611,7 +2611,7 @@ ulwrf: # Upward Longwave Radiation Flux shortName: sulwrf typeOfLevel: surface stepType: instant - clevs: !!python/object/apply:numpy.arange [350, 601, 10] + clevs: !arange [350, 601, 10] cmap: gist_ncar colors: radiation_colors ncl_name: ULWRF_P0_L1_{grid} @@ -2629,7 +2629,7 @@ ulwrf: # Upward Longwave Radiation Flux typeOfLevel: nominalTop level: 0 stepType: avg - clevs: !!python/object/apply:numpy.arange [80, 341, 2] + clevs: !arange [80, 341, 2] cmap: ir_rgbv_r colors: radiation_mix_colors ncl_name: ULWRF_P0_L8_{grid} @@ -2643,7 +2643,7 @@ ulwrfavg: # Upward Longwave Radiation Flux shortName: sulwrf typeOfLevel: surface stepType: avg - clevs: !!python/object/apply:numpy.arange [350, 601, 10] + clevs: !arange [350, 601, 10] cmap: gist_ncar colors: radiation_colors ncl_name: ULWRF_P0_L1_{grid}_avg6h @@ -2655,7 +2655,7 @@ ulwrfavg: # Upward Longwave Radiation Flux parameterName: Upward long-wave radiation flux typeOfLevel: nominalTop stepType: avg - clevs: !!python/object/apply:numpy.arange [80, 341, 2] + clevs: !arange [80, 341, 2] cmap: ir_rgbv_r colors: radiation_mix_colors ncl_name: ULWRF_P0_L8_{grid}_avg6h @@ -2682,7 +2682,7 @@ uswrf: # Upward Shortwave Radiation Flux parameterCategory: 4 typeOfLevel: nominalTop level: 0 - clevs: !!python/object/apply:numpy.arange [50, 851, 10] + clevs: !arange [50, 851, 10] cmap: Greys_r colors: radiation_bw_colors ncl_name: USWRF_P0_L8_{grid} @@ -2702,7 +2702,7 @@ uswrfavg: # Upward Shortwave Radiation Flux Average title: Upward SW Radiation Flux 6h Avg, Surface top: # Nominal top of atmosphere <<: *radiation_flux - clevs: !!python/object/apply:numpy.arange [50, 851, 10] + clevs: !arange [50, 851, 10] cmap: Greys_r colors: radiation_bw_colors ncl_name: USWRF_P8_L8_{grid}_avg6h @@ -2783,7 +2783,7 @@ vig: # Vertically-integrated graupel parameterNumber: 74 typeOfLevel: atmosphereSingleLayer level: 0 - clevs: !!python/object/apply:numpy.arange [5, 91, 5] + clevs: !arange [5, 91, 5] cmap: jet colors: graupel_colors ncl_name: TCOLG_P8_L200_{grid}_max1h @@ -2819,7 +2819,7 @@ vort: # Absolute vorticity cfgrib: shortName: absv typeOfLevel: isobaricInhPa - clevs: !!python/object/apply:numpy.arange [6, 29, 2] + clevs: !arange [6, 29, 2] cmap: gist_ncar colors: vort_colors contours: @@ -2838,7 +2838,7 @@ vvel: # Vertical velocity rrfs: shortName: wz typeOfLevel: isobaricInhPa - clevs: !!python/object/apply:numpy.arange [-17, 34, 5] + clevs: !arange [-17, 34, 5] cmap: gist_ncar colors: vvel_colors contours: @@ -2866,7 +2866,7 @@ vvort: # Vertical vorticity shortName: max_vo typeOfLevel: heightAboveGroundLayer level: 1000 - clevs: !!python/object/apply:numpy.arange [0.0025, 0.0301, 0.0025] + clevs: !arange [0.0025, 0.0301, 0.0025] cmap: gist_ncar colors: vort_colors ncl_name: RELV_P8_2L103_{grid}_max1h @@ -2911,7 +2911,7 @@ windmax: shortName: max_10si typeOfLevel: heightAboveGround level: 10 - clevs: !!python/object/apply:numpy.arange [5, 95, 5] + clevs: !arange [5, 95, 5] cmap: gist_ncar colors: wind_colors ncl_name: WIND_P8_L103_{grid}_max1h @@ -2925,7 +2925,7 @@ wspeed: # Wind Speed shortName: 10u level: 10 typeOfLevel: heightAboveGround - clevs: !!python/object/apply:numpy.arange [5, 95, 5] + clevs: !arange [5, 95, 5] cmap: gist_ncar colors: wind_colors ncl_name: UGRD_P0_L103_{grid} diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index a031d57..99f5908 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -21,6 +21,8 @@ import numpy as np import yaml from matplotlib.axes import Axes +from uwtools.api.config import YAMLConfig +from uwtools.config.support import uw_yaml_loader def cfgrib_spec(config: dict, model: str) -> dict: @@ -130,7 +132,7 @@ def get_func(val: str): return getattr(module, fun_name) -def join_ranges(loader: yaml.Loader, node: yaml.Node) -> Any: # noqa: ARG001 +def join_ranges(loader: yaml.SafeLoader, node: yaml.Node) -> Any: # noqa: ARG001 """ Merge two or more different ranges into a single array for color bar clevs. @@ -154,7 +156,14 @@ def join_ranges(loader: yaml.Loader, node: yaml.Node) -> Any: # noqa: ARG001 return np.concatenate(list_, axis=0) -yaml.add_constructor("!join_ranges", join_ranges, Loader=yaml.Loader) +def arange_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> Any: # noqa: ARG001 + return np.arange(*[float(n.value) for n in node.value]) + + +def load_yaml(config: Path | str): + yaml.add_constructor("!join_ranges", join_ranges, Loader=uw_yaml_loader()) + yaml.add_constructor("!arange", arange_constructor, Loader=uw_yaml_loader()) + return YAMLConfig(config) def label_line(ax: Axes, label: str, segment: np.ndarray, **kwargs): diff --git a/conftest.py b/conftest.py index e9e035d..1593fc9 100644 --- a/conftest.py +++ b/conftest.py @@ -37,3 +37,9 @@ def natfile(request): def prsfile(): """Interface to pass a grib file to pytest.""" return Path("tests", "data", "wrfprs_hrconus_07.grib2") + + +@pytest.fixture +def spec_file(): + """Interface to pass a grib file to pytest.""" + return Path("adb_graphics", "default_specs.yml") diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py new file mode 100644 index 0000000..03bfe77 --- /dev/null +++ b/tests/datahandler/test_gribdata.py @@ -0,0 +1,109 @@ +from datetime import datetime + +import numpy as np +from pytest import fixture +from xarray import DataArray + +from adb_graphics import utils +from adb_graphics.datahandler import gribdata, gribfile + + +class ConcreteUPPData(gribdata.UPPData): + def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: # noqa: ARG002 + return self.ds.to_dataarray().squeeze() + + +@fixture +def hrrr_data(prsfile): + return gribfile.GribFile( + prsfile, + var_config={ + "shortName": "t", + "typeOfLevel": "surface", + }, + ).contents + + +@fixture +def spec(spec_file): + return utils.load_yaml(spec_file) + + +@fixture +def uppdata_obj(hrrr_data, spec): + return ConcreteUPPData( + ds=hrrr_data, + short_name="temp", + spec=spec, + fhr=15, + ) + + +def test_uppdata_anl_dt(uppdata_obj): + dt = uppdata_obj.anl_dt + assert dt == datetime(2020, 10, 9, 8) + + +def test_uppdata_clevs_array(uppdata_obj): + assert np.array_equal(uppdata_obj.clevs, np.arange(-40, 40, 2.5)) + + +def test_uppdata_clevs_list(uppdata_obj): + uppdata_obj.spec["temp"]["ua"]["clevs"] = [1, 2, 3] + assert np.array_equal(uppdata_obj.clevs, np.asarray([1, 2, 3])) + + +def test_uppdata_date_to_str(uppdata_obj): + assert uppdata_obj.date_to_str(uppdata_obj.anl_dt) == "20201009 08 UTC" + + +def test_uppdata_field(uppdata_obj): + assert np.array_equal(uppdata_obj.field, uppdata_obj.ds.t) + + +def test_uppdata_field_column_max(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + var_config={ + "shortName": "t", + "typeOfLevel": "isobaricInhPa", + }, + ).contents + uppdata_obj = ConcreteUPPData( + ds=ds, + short_name="temp", + spec=spec, + fhr=15, + ) + assert np.array_equal(uppdata_obj.field_column_max(), ds.t.max(axis=0)) + assert uppdata_obj.field_column_max().shape == (1059, 1799) + + +def test_uppdata_field_diff(uppdata_obj): + summed_field = uppdata_obj.field_diff(values=uppdata_obj.field, variable2="temp", level2="sfc") + assert np.array_equal(summed_field, uppdata_obj.ds.t * 0) + + +def test_uppdata_field_mean(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + var_config={ + "shortName": "t", + "typeOfLevel": "isobaricInhPa", + }, + ).contents + uppdata_obj = ConcreteUPPData( + ds=ds, + short_name="temp", + spec=spec, + fhr=15, + ) + levels = ["500mb", "800mb"] + mean = uppdata_obj.field_mean(values=uppdata_obj.field, levels=levels) + assert np.array_equal(mean, ds.t.sel(isobaricInhPa=[500, 800]).mean("isobaricInhPa")) + assert mean.shape == (1059, 1799) + + +def test_uppdata_field_sum(uppdata_obj): + summed_field = uppdata_obj.field_sum(values=uppdata_obj.field, variable2="temp", level2="sfc") + assert np.array_equal(summed_field, uppdata_obj.ds.t * 2) diff --git a/tests/test_common.py b/tests/test_common.py index 4569846..eef2b21 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -17,11 +17,9 @@ import warnings from inspect import getfullargspec -from pathlib import Path from string import ascii_letters, digits import numpy as np -import yaml from matplotlib import cm from matplotlib import colors as mcolors from metpy.plots import ctables @@ -29,14 +27,11 @@ from adb_graphics import specs, utils from adb_graphics.datahandler import gribdata -yaml.add_constructor("!join_ranges", utils.join_ranges, Loader=yaml.Loader) - class MockSpecs(specs.VarSpec): """Mock class for the VarSpec abstract class.""" - with Path("adb_graphics/default_specs.yml").open() as c: - cfg = yaml.load(c, Loader=yaml.Loader) + cfg = utils.load_yaml("adb_graphics/default_specs.yml") @property def clevs(self): @@ -53,15 +48,16 @@ def test_utils(): assert callable(utils.get_func("conversions.k_to_c")) -def test_join_ranges_constructor(): +def test_join_ranges_constructor(tmp_path): """Test that the join_ranges constructor works as expected.""" - yaml_str = """ + cfg_file = tmp_path / "cfg.yaml" + cfg_file.write_text(""" foo: !join_ranges [[0, 15, 0.1], [20, 61, 20]] foo2: !join_ranges [[0, 15, 0.1]] foo3: !join_ranges [[0, 15, 0.1], [20, 40, 10], [40, 61, 20]] - """ - cfg = yaml.load(yaml_str, Loader=yaml.Loader) + """) + cfg = utils.load_yaml(cfg_file) expected = np.concatenate((np.arange(0, 15, 0.1), np.arange(20, 61, 20)), axis=0) expected2 = np.arange(0, 15, 0.1) @@ -79,8 +75,7 @@ class TestDefaultSpecs: config = "adb_graphics/default_specs.yml" varspec = MockSpecs() - with Path("adb_graphics/default_specs.yml").open() as c: - cfg = yaml.load(c, Loader=yaml.Loader) + cfg = utils.load_yaml("adb_graphics/default_specs.yml") @property def allowable(self): diff --git a/tests/test_specs.py b/tests/test_specs.py index 865ecbb..e9c5c50 100644 --- a/tests/test_specs.py +++ b/tests/test_specs.py @@ -1,21 +1,17 @@ from pathlib import Path import numpy as np -import yaml from pytest import fixture, mark from adb_graphics import specs, utils -yaml.add_constructor("!join_ranges", utils.join_ranges, Loader=yaml.Loader) - class Spec(specs.VarSpec): """ Concrete class for the VarSpec abstract class. """ - with Path("adb_graphics/default_specs.yml").open() as c: - cfg = yaml.load(c, Loader=yaml.Loader) + cfg = utils.load_yaml(Path("adb_graphics/default_specs.yml")) @property def clevs(self): From da861e4e29307b52b98a0c0f2de4fc4d94e9c237 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Sat, 25 Oct 2025 15:40:05 -0600 Subject: [PATCH 22/98] WIP testing gribdata --- adb_graphics/datahandler/gribdata.py | 97 +++------------------- tests/datahandler/test_gribdata.py | 116 +++++++++++++++++++-------- 2 files changed, 91 insertions(+), 122 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index a62a609..3528231 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -142,77 +142,7 @@ def _get_field(self, spec: dict) -> DataArray: first_variable_name = list(ds.data_vars)[0] return DataArray(ds[first_variable_name]) - def _get_level(self, field: DataArray, level: str, spec: dict, **kwargs) -> int: - """ - Returns the value of the level to for a 3D array. - - Arg: - field: dataset object for a given variable - level: string describing the level atmospheric level; corresponds - to a key in default specs - spec: the specifications dictionary to use for the variable in - question - - kwargs: - split: bool sometimes passed in through transforms that indicates - a level string should be split, e.g. 06km. - - Return: - Integer value corresponding to the array index for the atmospheric - level. - - """ - - # The index of the requested level - lev = spec.get("vertical_index") - if lev is not None: - return int(lev) - - vertical_dim = self.vertical_dim(field) - - # numeric_level returns a list of length 1 (e.g. [500] for 500 mb) or of - # length 2 when split=True and it's like 0-6 km, so returns [0, 6000] - requested_level, _ = self.numeric_level( - level=level, - split=kwargs.get("split", spec.get("split")), - ) - - # data_levels contains a list of vertical dimension values - data_levels = self._get_data_levels(vertical_dim) - - # For split-level variables, like 0-6km, find the matching index by - # looping through both the possible vertical level arrays. - if len(data_levels) == 2 and len(requested_level) == 2: - for lev, levset in enumerate(zip(*[list(lev) for lev in data_levels], strict=True)): - if sorted(levset) == requested_level: - return lev - - # For single-level variables, like 500mb, use the argwhere function to - # return the matching index - if len(requested_level) == 1: - for dim_levels in data_levels: - lev = np.argwhere(dim_levels == requested_level[0]) - try: - if lev or lev == [0]: - return int(lev[0]) - except ValueError: - print(f"BAD LEVEL is {lev} for {field.name}") - - print( - f"Could not find a level for {field.name} at requested \ - level = {requested_level} for variable levels = {data_levels}. Index \ - was {lev}." - ) - - # If neither of those cases worked out appropriately, raise an error. - msg = ( - f"Length of requested_level ({len(requested_level)}) or " - f"data_levels ({len(data_levels)}) bad!" - f" {level} {field.name}" - ) - raise ValueError(msg) - - def get_transform(self, transforms: dict | str, val: DataArray) -> DataArray: + def get_transform(self, transforms: dict | list | str, val: DataArray) -> DataArray: """ Applies a set of one or more transforms to an np.array of data values. @@ -258,15 +188,18 @@ def get_xypoint(self, site_lat: float, site_lon: float) -> tuple: lons = lons + adjust max_x, max_y = np.shape(lats) + msg = f"site location is outside your domain! {site_lat} {site_lon}" + if not lats.min() < site_lat < lats.max() or not lons.min() < site_lon < lons.max(): + print(msg) + return (-1, -1) + # Numpy magic to grab the X, Y grid point nearest the profile site - # pylint: disable=unbalanced-tuple-unpacking x, y = np.unravel_index( (np.abs(lats - site_lat) + np.abs(lons - site_lon)).argmin(), lats.shape ) - # pylint: enable=unbalanced-tuple-unpacking if x <= 0 or y <= 0 or x >= max_x or y >= max_y: - print(f"site location is outside your domain! {site_lat} {site_lon}") + print(msg) return (-1, -1) return (x, y) @@ -857,7 +790,7 @@ def __init__(self, ds: Dataset, loc: str, short_name: str, **kwargs): if self.site_lon < 0: self.site_lon = self.site_lon + 360.0 - def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: + def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: # noqa: ARG002 """ Returns the numpy array of values at the object's x, y location for the requested variable. Transforms are performed in the child class. @@ -883,10 +816,6 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> if not name: name = self.short_name - one_lev = kwargs.get("one_lev", False) - vertical_index = kwargs.get("vertical_index") - split = kwargs.get("split") - # Retrive the location for the profile x, y = self.get_xypoint(self.site_lat, self.site_lon) @@ -894,20 +823,12 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> field = self.field profile = field[::] - lev = 0 # 2D if len(profile.shape) == 2: profile = profile[x, y] # 3D elif len(profile.shape) == 3: - if one_lev: - if vertical_index is None: - lev = self._get_level(field, level, {}, split=split) - else: - lev = int(vertical_index) - profile = profile[lev, x, y] - else: - profile = profile[:, x, y] + profile = profile[:, x, y] return profile def vector_magnitude( diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index 03bfe77..d3ef247 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -1,8 +1,8 @@ from datetime import datetime import numpy as np -from pytest import fixture -from xarray import DataArray +from pytest import fixture, mark +from xarray import DataArray, ones_like, zeros_like from adb_graphics import utils from adb_graphics.datahandler import gribdata, gribfile @@ -30,12 +30,30 @@ def spec(spec_file): @fixture -def uppdata_obj(hrrr_data, spec): +def uppdata_obj(hrrr_data, prsfile, spec): return ConcreteUPPData( ds=hrrr_data, short_name="temp", spec=spec, fhr=15, + grib_path=prsfile, + ) + + +@fixture +def uppdata_multilev_obj(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + var_config={ + "shortName": "t", + "typeOfLevel": "isobaricInhPa", + }, + ).contents + return ConcreteUPPData( + ds=ds, + short_name="temp", + spec=spec, + fhr=15, ) @@ -61,22 +79,11 @@ def test_uppdata_field(uppdata_obj): assert np.array_equal(uppdata_obj.field, uppdata_obj.ds.t) -def test_uppdata_field_column_max(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - var_config={ - "shortName": "t", - "typeOfLevel": "isobaricInhPa", - }, - ).contents - uppdata_obj = ConcreteUPPData( - ds=ds, - short_name="temp", - spec=spec, - fhr=15, +def test_uppdata_field_column_max(uppdata_multilev_obj): + assert np.array_equal( + uppdata_multilev_obj.field_column_max(), uppdata_multilev_obj.ds.t.max(axis=0) ) - assert np.array_equal(uppdata_obj.field_column_max(), ds.t.max(axis=0)) - assert uppdata_obj.field_column_max().shape == (1059, 1799) + assert uppdata_multilev_obj.field_column_max().shape == (1059, 1799) def test_uppdata_field_diff(uppdata_obj): @@ -84,26 +91,67 @@ def test_uppdata_field_diff(uppdata_obj): assert np.array_equal(summed_field, uppdata_obj.ds.t * 0) -def test_uppdata_field_mean(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - var_config={ - "shortName": "t", - "typeOfLevel": "isobaricInhPa", - }, - ).contents - uppdata_obj = ConcreteUPPData( - ds=ds, - short_name="temp", - spec=spec, - fhr=15, - ) +def test_uppdata_field_mean(uppdata_multilev_obj): levels = ["500mb", "800mb"] - mean = uppdata_obj.field_mean(values=uppdata_obj.field, levels=levels) - assert np.array_equal(mean, ds.t.sel(isobaricInhPa=[500, 800]).mean("isobaricInhPa")) + mean = uppdata_multilev_obj.field_mean(values=uppdata_multilev_obj.field, levels=levels) + assert np.array_equal( + mean, uppdata_multilev_obj.ds.t.sel(isobaricInhPa=[500, 800]).mean("isobaricInhPa") + ) assert mean.shape == (1059, 1799) def test_uppdata_field_sum(uppdata_obj): summed_field = uppdata_obj.field_sum(values=uppdata_obj.field, variable2="temp", level2="sfc") assert np.array_equal(summed_field, uppdata_obj.ds.t * 2) + + +def test_uppdata__get_data_levels(uppdata_multilev_obj): + assert np.array_equal( + uppdata_multilev_obj._get_data_levels("isobaricInhPa"), + uppdata_multilev_obj.ds.coords["isobaricInhPa"].to_numpy(), + ) + + +def test_uppdata__get_field(prsfile, uppdata_obj): + spec = {"shortName": "t", "typeOfLevel": "isobaricInhPa", "level": 500} + field = uppdata_obj._get_field(spec=spec) + ds = gribfile.GribFile( + prsfile, + var_config=spec, + ).contents + assert np.array_equal(field, ds.t) + + +@mark.parametrize( + "transforms", + [ + "conversions.percent", + ["conversions.percent", "opposite"], + {"funcs": "field_diff", "kwargs": {"variable2": "temp", "level2": "sfc"}}, + ], +) +def test_uppdata_get_transform(transforms, uppdata_obj): + val = ones_like(uppdata_obj.ds.t) if not isinstance(transforms, dict) else uppdata_obj.ds.t + field = uppdata_obj.get_transform(transforms, val) + expected = 0 + match transforms: + case dict(): + expected = zeros_like(uppdata_obj.ds.t) + case list(): + expected = val * -100.0 + case str(): + expected = val * 100.0 + assert np.array_equal(field, expected) + + +@mark.parametrize( + ("lat", "lon", "expected"), + [(40.019, 360 - 105.2747, (595, 679)), (25.7617, 360 - 80.1918, (109, 1487))], +) +def test_uppdata_get_xypoint(expected, lat, lon, uppdata_obj): + assert uppdata_obj.get_xypoint(lat, lon) == expected + + +@mark.parametrize(("lat", "lon"), [(88.0, 270.0), (40, 180), (10, 330), (30, 345)]) +def test_uppdata_get_xypoint_outside(lat, lon, uppdata_obj): + assert uppdata_obj.get_xypoint(lat, lon) == (-1, -1) From 145ba3e2ea1344e8a111144eee091ff51383601c Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Sun, 26 Oct 2025 17:53:54 -0600 Subject: [PATCH 23/98] WIP finished uppdata class. --- .gitattributes | 1 + adb_graphics/datahandler/gribdata.py | 67 +--------------------------- tests/datahandler/test_gribdata.py | 46 ++++++++++++++++++- 3 files changed, 48 insertions(+), 66 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3614343 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/data/wrf* filter=lfs diff=lfs merge=lfs -text diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 3528231..25be02e 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -8,7 +8,6 @@ from copy import deepcopy from datetime import datetime, timedelta from pathlib import Path -from string import ascii_letters, digits from typing import Any import numpy as np @@ -186,7 +185,6 @@ def get_xypoint(self, site_lat: float, site_lon: float) -> tuple: lats, lons = self.latlons() adjust = 360 if np.any(lons < 0) else 0 lons = lons + adjust - max_x, max_y = np.shape(lats) msg = f"site location is outside your domain! {site_lat} {site_lon}" if not lats.min() < site_lat < lats.max() or not lons.min() < site_lon < lons.max(): @@ -198,13 +196,9 @@ def get_xypoint(self, site_lat: float, site_lon: float) -> tuple: (np.abs(lats - site_lat) + np.abs(lons - site_lon)).argmin(), lats.shape ) - if x <= 0 or y <= 0 or x >= max_x or y >= max_y: - print(msg) - return (-1, -1) - return (x, y) - def latlons(self): + def latlons(self) -> list[np.ndarray]: """Returns the set of latitudes and longitudes.""" coords = sorted( @@ -212,50 +206,8 @@ def latlons(self): ) return [self.ds.coords[c].to_numpy() for c in coords] - @property - def lev_descriptor(self): - """Returns the descriptor for the variable's level type.""" - - return self.field.level_type - - def numeric_level( - self, index_match: bool = True, level: str | None = None, split: str | None = None - ) -> tuple[list[float | int], str]: - """ - Split the numeric level and unit associated with the level key. - - A blank string is returned for lev_val for levels that do not contain a - numeric, e.g., 'sfc' or 'ua'. - """ - - level = level if level else self.level - - # Gather all the numbers in the string - numbers = "".join([c for c in level if (c in digits or c == ".")]) - - # Convert the numbers to a list, and make integers or floats - lev_val: list[float | int] - if numbers: - if split is not None: - lev_val = [int(lev) for lev in numbers] - else: - lev_val = [float(numbers) if "." in numbers else int(numbers)] - - # Gather all the letters - lev_unit = "".join([c for c in level if c in ascii_letters]) - - if index_match: - if lev_unit == "cm": - lev_val = [val / 100.0 for val in lev_val] - if lev_unit in ["mb", "mxmb"]: - lev_val = [val * 100.0 for val in lev_val] - if lev_unit in ["in", "km", "mn", "mx", "sr"]: - lev_val = [val * 1000.0 for val in lev_val] - - return lev_val, lev_unit - @staticmethod - def opposite(values: DataArray, **kwargs): # noqa: ARG004 + def opposite(values: DataArray, **kwargs) -> DataArray: # noqa: ARG004 """Returns the opposite of input values.""" return -values @@ -274,21 +226,6 @@ def valid_dt(self) -> datetime: def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: """Returns the values of a given variable.""" - @staticmethod - def vertical_dim(field: DataArray) -> str: - """ - Find the name of the vertical dimension. - - Looking through the field's dimensions for one that includes "lv". Return the first matching - instance. - """ - - for dims in list(field.dims): - dim = str(dims) - if "lv" in dim or "probability" in dim: - return dim - return "" - @property def vspec(self): """Return the graphics specification for a given level.""" diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index d3ef247..6b7ed52 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -19,7 +19,7 @@ def hrrr_data(prsfile): prsfile, var_config={ "shortName": "t", - "typeOfLevel": "surface", + "typeOfLevel": "isobaricInhPa", }, ).contents @@ -155,3 +155,47 @@ def test_uppdata_get_xypoint(expected, lat, lon, uppdata_obj): @mark.parametrize(("lat", "lon"), [(88.0, 270.0), (40, 180), (10, 330), (30, 345)]) def test_uppdata_get_xypoint_outside(lat, lon, uppdata_obj): assert uppdata_obj.get_xypoint(lat, lon) == (-1, -1) + + +def test_uppdata_latlons(uppdata_obj): + lats = uppdata_obj.ds.coords["latitude"].to_numpy() + lons = uppdata_obj.ds.coords["longitude"].to_numpy() + assert [ + np.array_equal(act, exp) + for act, exp in zip(uppdata_obj.latlons(), [lats, lons], strict=True) + ] + + +@mark.parametrize("factor", [1, -1, 0, -20.0, 6543.0]) +def test_uppdata_opposite(factor, uppdata_obj): + ds = ones_like(uppdata_obj.field) * factor + assert np.array_equal(uppdata_obj.opposite(ds), -ds) + + +def test_uppdata_valid_dt(uppdata_obj): + assert uppdata_obj.valid_dt == datetime(2020, 10, 9, 23) + + +def test_uppdata_vspec(uppdata_obj): + expected = { + "cfgrib": {"shortName": "t", "typeOfLevel": "isobaricInhPa"}, + "clevs": np.arange(-40, 40, 2.5), + "cmap": "jet", + "colors": "ua_temp_colors", + "contours": { + "pres_sfc": {"levels": [0, 500], "colors": "k", "linewidths": 0.6}, + "gh": {"colors": "grey"}, + }, + "hatches": {"pres_sfc": {"hatches": ["", "..."], "levels": [0, 500]}}, + "ncl_name": {"prs": "TMP_P0_L100_{grid}", "nat": "TMP_P0_L105_{grid}"}, + "ticks": 5, + "transform": "conversions.k_to_c", + "unit": "C", + "wind": True, + } + vspec = uppdata_obj.vspec + # Can't test the array items with ==, so check them separately and then remove. + assert np.array_equal(vspec["clevs"], expected["clevs"]) + vspec.pop("clevs") + expected.pop("clevs") + assert uppdata_obj.vspec == expected From 3daa75fc3230033b36211197550526185caa2272 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 27 Oct 2025 17:05:12 -0600 Subject: [PATCH 24/98] WIP testings gribdata --- adb_graphics/datahandler/gribdata.py | 144 +++++-------- adb_graphics/datahandler/gribfile.py | 12 +- adb_graphics/default_specs.yml | 39 ++-- adb_graphics/figures/maps.py | 32 +-- adb_graphics/utils.py | 29 ++- conftest.py | 6 +- tests/data/wrfnat_hrconus_07.grib2 | 3 - tests/data/wrfnat_hrconus_16.grib2 | 3 + tests/data/wrfprs_hrconus_07.grib2 | 3 - tests/data/wrfprs_hrconus_16.grib2 | 3 + tests/datahandler/test_gribdata.py | 293 ++++++++++++++++++++++++++- tests/datahandler/test_gribfile.py | 4 +- tests/test_common.py | 2 + 13 files changed, 401 insertions(+), 172 deletions(-) delete mode 100644 tests/data/wrfnat_hrconus_07.grib2 create mode 100644 tests/data/wrfnat_hrconus_16.grib2 delete mode 100644 tests/data/wrfprs_hrconus_07.grib2 create mode 100644 tests/data/wrfprs_hrconus_16.grib2 diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 25be02e..d40de0c 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -8,10 +8,9 @@ from copy import deepcopy from datetime import datetime, timedelta from pathlib import Path -from typing import Any import numpy as np -from matplotlib import cm +from matplotlib.pyplot import get_cmap from pandas import to_datetime from xarray import DataArray, Dataset @@ -204,7 +203,11 @@ def latlons(self) -> list[np.ndarray]: coords = sorted( [str(c) for c in list(self.ds.coords) if any(ele in str(c) for ele in ["lat", "lon"])] ) - return [self.ds.coords[c].to_numpy() for c in coords] + lat = self.ds.coords[coords[0]].to_numpy() + if len(lat.shape) == 1 and lat[-1] < lat[0]: + lat = lat[::-1] + lon = self.ds.coords[coords[-1]].to_numpy() + return [lat, lon] @staticmethod def opposite(values: DataArray, **kwargs) -> DataArray: # noqa: ARG004 @@ -265,7 +268,7 @@ def aviation_flight_rules(self, values: DataArray, **kwargs): # noqa: ARG002 Generates a field of Aviation Flight Rules from Ceil and Vis. """ - ceil = values.to_dataarray().squeeze() + ceil = values vis = self.values(name="vis", level="sfc") flru = np.where((ceil > 1.0) & (ceil < 3.0), 1.01, 0.0) @@ -285,28 +288,26 @@ def cmap(self): The LinearSegmentedColormap specified by the config key 'cmap'. """ - return cm.get_cmap(self.vspec["cmap"]) + return get_cmap(self.vspec["cmap"]) @property - def colors(self) -> Any: + def colors(self) -> np.ndarray: """ - Returns a list of colors, specified by the config key "colors". - - The yaml file "colors" key may contain a list or a function to be - called. + Returns an array of colors, specified by the config key "colors". """ - color_spec = self.vspec.get("colors") - - if isinstance(color_spec, (list, np.ndarray)): - return np.asarray(color_spec) + color_spec = self.vspec.get("colors", "") + if not color_spec: + msg = f"No colors definition found for {self.short_name} at {self.level}" + raise errors.NoGraphicsDefinitionForVariableError(msg) try: ret = self.__getattribute__(color_spec) - if callable(ret): - return ret() - except AttributeError: - return color_spec - return ret + except AttributeError as e: + msg = f"There is no color definition named {color_spec}" + raise AttributeError(msg) from e + if callable(ret): + return np.asarray(ret()) + return np.asarray(ret) @property def corners(self) -> list: @@ -319,14 +320,23 @@ def corners(self) -> list: """ lat, lon = self.latlons() - if self.model in ["global", "hfip", "obs"]: - ret = [lat[-1], lat[0], lon[0], lon[-1]] - elif self.model == "global_mpas": - ret = [lat[0], lat[-1], lon[0], lon[-1]] - else: - ret = [lat[0, 0], lat[-1, -1], lon[0, 0], lon[-1, -1]] + if len(lat.shape) == 2: + return [lat[0, 0], lat[-1, -1], lon[0, 0], lon[-1, -1]] + return [lat[0], lat[-1], lon[0], lon[-1]] - return ret + @property + def data(self) -> DataArray: + """ + Sets the data property on the object for use when we need to update + the values associated with a given object -- helpful for differences. + """ + if not hasattr(self, "_data"): + return self.values() + return self._data + + @data.setter + def data(self, value: DataArray): + self._data = value def fire_weather_index(self, values: DataArray, **kwargs): # noqa: ARG002 """ @@ -352,9 +362,7 @@ def _load_field(level: str, short_name: str): return FieldData(**args).values(do_transform=False) # Gather fields from the input - veg = ( - values.to_dataarray().squeeze() - ) # Chose this value as the main one in the default_specs + veg = np.asarray(values) temp = _load_field(level="2m", short_name="temp") dewpt = _load_field(level="2m", short_name="dewp") weasd = _load_field(level="sfc", short_name="weasd") @@ -391,7 +399,7 @@ def _load_field(level: str, short_name: str): return fwi - def grid_info(self): + def grid_info(self) -> dict: """Returns a dict that includes the grid info for the full grid.""" # Keys are grib names, values are Basemap argument names @@ -472,19 +480,19 @@ def icing_adjust_trace(values: DataArray, **kwargs): # noqa: ARG004 def run_max(values: DataArray, **kwargs): # noqa: ARG004 """Finds the max hourly value over all the forecast lead times available.""" - return values.max(dim="fcst_hr") + return values.max(dim="step") @staticmethod def run_min(values: DataArray, **kwargs): # noqa: ARG004 """Finds the min hourly value over all the forecast lead times available.""" - return values.min(dim="fcst_hr") + return values.min(dim="step") @staticmethod def run_total(values: DataArray, **kwargs): # noqa: ARG004 """Sums over all the forecast lead times available.""" - return values.sum(dim="fcst_hr") + return values.sum(dim="step") def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 """ @@ -505,8 +513,8 @@ def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 pres_sfc = self.values(name="pres", level="sfc") * 100.0 # convert back to Pa pres_nat_lev = self.values(name="pres", level="ua", one_lev=False) temp = self.values(name="temp", level="ua", one_lev=False) - cloud_mixing_ratio = self.values(name="clwmr", level="ua", one_lev=False) - rain_mixing_ratio = self.values(name="rwmr", level="ua", one_lev=False) + cloud_mixing_ratio = self.values(name="clwmr", level="uanat", one_lev=False) + rain_mixing_ratio = self.values(name="rwmr", level="uanat", one_lev=False) gravity = 9.81 slw = pres_sfc * 0.0 # start with array of zero values @@ -534,7 +542,6 @@ def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 rain_mixing_ratio.close() return slw - @property def ticks(self) -> int: """ Returns the number of color bar tick marks from the yaml config @@ -552,20 +559,6 @@ def units(self) -> str: return str(self.vspec.get("unit", self.field.units)) - @property - def data(self) -> DataArray: - """ - Sets the data property on the object for use when we need to update - the values associated with a given object -- helpful for differences. - """ - if not hasattr(self, "_data"): - return self.values() - return self._data - - @data.setter - def data(self, value: DataArray): - self._data = value - def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: """ Returns the numpy array of values at the requested level for the @@ -586,7 +579,7 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> """ level = level or self.level - vals: DataArray = self.ds.to_dataarray().sqeeze() + vals: DataArray = self.ds.to_dataarray().squeeze() spec = self.vspec do_transform = kwargs.get("do_transform", True) @@ -596,21 +589,12 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> spec = deepcopy(self.spec.get(name, {}).get(level, {})) if not spec and name is not None: raise errors.NoGraphicsDefinitionForVariableError(name, level) - cfkeys = utils.cfgrib_spec(spec["cfgrib"], self.model) - nlevel = utils.numeric_level(level=level, index_match=False)[0] - level_info = any( - key - for keys in utils.cfgrib_spec(spec["cfgrib"], self.model) - for key in ("level", "top", "bottom", "Surface") - if key in keys - ) - if nlevel and not level_info: - cfkeys["level"] = utils.numeric_level(level=level, index_match=False)[0] - vals = self._get_field(cfkeys) + utils.set_level(level=level, model=self.model, spec=spec) + vals = self._get_field(spec["cfgrib"].get(self.model, spec["cfgrib"])) transforms = spec.get("transform") if transforms and do_transform: - vals = self.get_transform(transforms, self.field) + vals = self.get_transform(transforms, vals) return vals @@ -629,13 +613,15 @@ def vector_magnitude( if cfkeys: if cfkeys.get("level") is None: - cfkeys["level"] = utils.numeric_level(level=self.level, index_match=False)[0] + cfkeys["level"] = utils.numeric_level(level=self.level)[0] field2_spec = {"cfgrib": cfkeys} + utils.set_level(level=self.level, model=self.model, spec=field2_spec) elif field2_id: - var, lev = field2_id.split(".") + var, lev = field2_id.split("_") field2_spec = self.spec for key in (var, lev): field2_spec = field2_spec[key] + utils.set_level(level=lev, model=self.model, spec=field2_spec) else: msg = "Must supply a field2_id if cfkeys is not explicitly provided." raise ValueError(msg) @@ -651,7 +637,6 @@ def vector_magnitude( "grib_path": self.grib_path, } field2 = FieldData(**args).ds - mag = conversions.magnitude( field1.to_dataarray().squeeze(), field2.to_dataarray().squeeze() ) @@ -660,33 +645,6 @@ def vector_magnitude( return mag - def wind(self, level: bool | str) -> list[DataArray]: - """ - Returns the u, v wind components as a list (length 2) of arrays. - - Input: - level bool or level key. If True, use same level as self, - if a string level key is provided, use wind at that - level. - """ - - level = self.level if level and isinstance(level, bool) else level - - # Just in case wind gets called with level=False - if not level: - return [] - - # Create FieldData objects for u, v components - field_lambda = lambda ds, level, var: FieldData( - ds=ds, - fhr=self.fhr, - level=level, - short_name=var, - ).field - u, v = [field_lambda(self.ds, level, var) for var in ["u", "v"]] - - return [u, v] - class ProfileData(UPPData): """ diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index cc74670..88f752e 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -12,11 +12,11 @@ class GribFile: """Wrappers and helper functions for interfacing with cfgrib.""" - def __init__(self, filename: Path | str, var_config: dict): + def __init__(self, filename: Path | str, cfgrib_config: dict): # pylint: disable=unused-argument self.filename = filename - self.var_config = var_config + self.cfgrib_config = cfgrib_config self.contents = self._load() def _load(self) -> xr.Dataset: @@ -29,7 +29,7 @@ def _load(self) -> xr.Dataset: self.filename, engine="cfgrib", lock=False, - backend_kwargs=({"filter_by_keys": self.var_config}), + backend_kwargs=({"filter_by_keys": self.cfgrib_config}), ) @@ -42,7 +42,7 @@ class GribFiles: def __init__( self, filenames: list[Path], - var_config: dict, + cfgrib_config: dict, **kwargs, ): """ @@ -62,7 +62,7 @@ def __init__( self.model = kwargs.get("model", "") self.filenames = filenames - self.var_config = var_config + self.cfgrib_config = cfgrib_config self.contents = self._load() def _load(self, filenames: list[Path] | None = None): @@ -73,5 +73,5 @@ def _load(self, filenames: list[Path] | None = None): engine="cfgrib", concat_dim="time", combine="nested", - backend_kwargs=({"filter_by_keys": self.var_config}), + backend_kwargs=({"filter_by_keys": self.cfgrib_config}), ) diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index ee5e4a0..4fb1d58 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -636,14 +636,16 @@ cloudcover: ncl_name: TCDC_P0_L{level_type}_{grid} title: Total Cloud Cover clwmr: # Cloud water Mixing Ratio - ua: + uanat: cfgrib: - prs: - shortName: clwmr - typeOfLevel: isobaricInhPa - ncl_name: - nat: CLWMR_P0_L105_{grid} - prs: CLWMR_P0_L100_{grid} + shortName: clwmr + typeOfLevel: hybrid + ncl_name: CLWMR_P0_L105_{grid} + uaprs: + cfgrib: + shortName: clwmr + typeOfLevel: isobaricInhPa + ncl_name: CLWMR_P0_L100_{grid} coarsedust: # Coarse dust int: &coldust clevs: [1, 4, 7, 11, 15, 20, 25, 30, 40, 50, 75, 150, 250, 500] @@ -1674,8 +1676,8 @@ pres: wind: 10m ua: cfgrib: - shortName: - typeOfLevel: + shortName: pres + typeOfLevel: hybrid ncl_name: PRES_P0_L105_{grid} presmin: msl: @@ -1948,14 +1950,16 @@ rvil: # Radar-derived Vertically Integrated Liquid title: VIL (Reflectivity Derived) unit: kg/m$^{2}$ rwmr: # Rain Mixing Ratio - ua: + uanat: cfgrib: - prs: - shortName: rwmr - typeOfLevel: isobaricInhPa - ncl_name: - nat: RWMR_P0_L105_{grid} - prs: RWMR_P0_L100_{grid} + shortName: rwmr + typeOfLevel: hybrid + ncl_name: RWMR_P0_L105_{grid} + uaprs: + cfgrib: + shortName: rwmr + typeOfLevel: isobaricInhPa + ncl_name: RWMR_P0_L100_{grid} seasalt: # Fine dust, global chem sfc: clevs: [0.05, 0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 30, 40, 50, 100] @@ -2444,6 +2448,9 @@ temp: # Temperature wind: False ua: <<: *ua_temp + cfgrib: + shortName: t + typeOfLevel: hybrid thick: 500mb: <<: *ua_gh diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index ae2c0ec..44fa7d2 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -23,7 +23,7 @@ from mpl_toolkits.basemap import Basemap, shiftgrid from adb_graphics.datahandler import gribdata, gribfile -from adb_graphics.utils import cfgrib_spec, numeric_level +from adb_graphics.utils import cfgrib_spec, set_level # FULL_TILES is a list of strings that includes the labels GSL attaches to some of # the wgrib2 cutouts used for larger domains like RAP, RRFS NA, and global. @@ -137,7 +137,7 @@ def __init__( self.tile = kwargs.get("tile", "full") self.map_spec = deepcopy(self.fields_spec[self.name][self.level]) - self.set_level(self.level, self.map_spec) + set_level(self.level, self.model, self.map_spec) # Required if map_type is "diff" if map_type == "diff": self.grib_path2: Path | str = kwargs.get("grib_path2", "") @@ -145,24 +145,6 @@ def __init__( msg = "Diff map requires a second grib path. Provide grib_path2 argument!" raise ValueError(msg) - def set_level(self, level: str, spec: dict): - nlevel, _ = numeric_level(level=level, index_match=False) - level_info = any( - key - for keys in cfgrib_spec(spec["cfgrib"], self.model) - for key in ("level", "top", "bottom", "Surface") - if key in keys - ) - if nlevel and not level_info: - if spec["cfgrib"].get(self.model): - spec["cfgrib"][self.model]["level"] = nlevel - else: - spec["cfgrib"]["level"] = nlevel - # if spec["cfgrib"].get("level") is None and not spec["cfgrib"].get("stepRange") and not \ - # spec["cfgrib"].get("topLevel") and not \ - # spec["cfgrib"].get("typeOfLevel") == "surface" and not \ - # spec["cfgrib"].get("scaledValueOfFirstFixedSurface"): - @property def shaded(self): cf = cfgrib_spec(self.map_spec["cfgrib"], self.model) @@ -213,7 +195,7 @@ def wind_fields(self, level: str | None = None): winds = [] for var in ("u", "v"): wind_spec = self.fields_spec[var][lev] - self.set_level(lev, wind_spec) + set_level(lev, self.model, wind_spec) ds = gribfile.GribFile( self.grib_path, cfgrib_spec(wind_spec["cfgrib"], self.model) ).contents @@ -242,7 +224,7 @@ def _overlay_fields(self, spec_sect: str) -> list: var, lev = overlay, self.level overlay_spec = deepcopy(self.fields_spec[var][lev]) - self.set_level(lev, overlay_spec) + set_level(lev, self.model, overlay_spec) ds = gribfile.GribFile( self.grib_path, cfgrib_spec(overlay_spec["cfgrib"], self.model) ).contents @@ -829,7 +811,7 @@ def _title(self): loc="left", ) - level, lev_unit = f.numeric_level(index_match=False) + level, lev_unit = f.numeric_level() units = f"({f.units}, shaded)" if f.vspec.get("print_units", True) else "" # Title or Atmospheric level and unit in the high center @@ -983,7 +965,7 @@ def _title(self): loc="left", ) - level, lev_unit = f.numeric_level(index_match=False) + level, lev_unit = f.numeric_level() units = f"({f.units}, shaded)" if f.vspec.get("print_units", True) else "" # Title or Atmospheric level and unit in the high center @@ -1070,7 +1052,7 @@ def title(self): transform=ax.transAxes, ) - level, lev_unit = f.numeric_level(index_match=False) + level, lev_unit = f.numeric_level() units = f"({f.units}, shaded)" if f.vspec.get("print_units", True) else "" # Title or Atmospheric level and unit in the high center diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 99f5908..8bd960e 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -315,7 +315,7 @@ def load_specs(arg: str | Path) -> dict: return specs -def numeric_level(index_match: bool = True, level: str | None = None): +def numeric_level(level: str | None = None) -> tuple[float | int | str, str]: """ Split the numeric level and unit associated with the level key. @@ -330,19 +330,13 @@ def numeric_level(index_match: bool = True, level: str | None = None): # Convert the numbers to a list, and make integers or floats if numbers: - lev_val = [float(numbers) if "." in numbers else int(numbers)] + lev_val = float(numbers) if "." in numbers else int(numbers) + else: + return "", "" # Gather all the letters lev_unit = "".join([c for c in level if c in ascii_letters]) - if index_match: - if lev_unit == "cm": - lev_val = [val / 100.0 for val in lev_val] - if lev_unit in ["mb", "mxmb"]: - lev_val = [val * 100.0 for val in lev_val] - if lev_unit in ["in", "km", "mn", "mx", "sr"]: - lev_val = [val * 1000.0 for val in lev_val] - return lev_val, lev_unit @@ -400,6 +394,21 @@ def to_datetime(string: str): return datetime.strptime(string, "%Y%m%d%H") +def set_level(level: str, model: str, spec: dict): + nlevel, _ = numeric_level(level=level) + level_info = any( + key + for keys in cfgrib_spec(spec["cfgrib"], model) + for key in ("level", "top", "bottom", "Surface") + if key in keys + ) + if nlevel and not level_info: + if spec["cfgrib"].get(model): + spec["cfgrib"][model]["level"] = nlevel + else: + spec["cfgrib"]["level"] = nlevel + + @timer def zip_products(fhr: int, workdir: Path, zipfiles: dict) -> None: """ diff --git a/conftest.py b/conftest.py index 1593fc9..db8c2b8 100644 --- a/conftest.py +++ b/conftest.py @@ -27,16 +27,16 @@ def pytest_addoption(parser): @pytest.fixture -def natfile(request): +def natfile(): """Interface to pass a grib file to pytest.""" - return request.config.getoption("--nat-file") + return Path("tests", "data", "wrfnat_hrconus_16.grib2") @pytest.fixture def prsfile(): """Interface to pass a grib file to pytest.""" - return Path("tests", "data", "wrfprs_hrconus_07.grib2") + return Path("tests", "data", "wrfprs_hrconus_16.grib2") @pytest.fixture diff --git a/tests/data/wrfnat_hrconus_07.grib2 b/tests/data/wrfnat_hrconus_07.grib2 deleted file mode 100644 index 4bacf7b..0000000 --- a/tests/data/wrfnat_hrconus_07.grib2 +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:faea9adc1df68d3e848435ba9f693f1fbfb2e31a0991a5e8c23b119741ed7faf -size 740036834 diff --git a/tests/data/wrfnat_hrconus_16.grib2 b/tests/data/wrfnat_hrconus_16.grib2 new file mode 100644 index 0000000..cde057b --- /dev/null +++ b/tests/data/wrfnat_hrconus_16.grib2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6818f89f3b43f486b5a972b294bc66c7117594217945bb1cc9a32696768bff49 +size 769413571 diff --git a/tests/data/wrfprs_hrconus_07.grib2 b/tests/data/wrfprs_hrconus_07.grib2 deleted file mode 100644 index efb98a1..0000000 --- a/tests/data/wrfprs_hrconus_07.grib2 +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:76ae4d355ff35a8e3ad3693d1a0b392a1e7025a8fddbe195c1389ff73657e3fd -size 460393025 diff --git a/tests/data/wrfprs_hrconus_16.grib2 b/tests/data/wrfprs_hrconus_16.grib2 new file mode 100644 index 0000000..f99f0c4 --- /dev/null +++ b/tests/data/wrfprs_hrconus_16.grib2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d2a7b6a201ed89058f4c3856db00cefaa2ea4179ddd6b7b69f7f9225fd58675 +size 434714591 diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index 6b7ed52..11f246c 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -1,10 +1,11 @@ from datetime import datetime import numpy as np -from pytest import fixture, mark +from matplotlib.pyplot import get_cmap +from pytest import fixture, mark, raises from xarray import DataArray, ones_like, zeros_like -from adb_graphics import utils +from adb_graphics import errors, utils from adb_graphics.datahandler import gribdata, gribfile @@ -17,9 +18,9 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> def hrrr_data(prsfile): return gribfile.GribFile( prsfile, - var_config={ + cfgrib_config={ "shortName": "t", - "typeOfLevel": "isobaricInhPa", + "typeOfLevel": "surface", }, ).contents @@ -29,6 +30,18 @@ def spec(spec_file): return utils.load_yaml(spec_file) +@fixture +def fielddata_obj(hrrr_data, prsfile, spec): + return gribdata.FieldData( + ds=hrrr_data, + fhr=15, + grib_path=prsfile, + level="sfc", + short_name="temp", + spec=spec, + ) + + @fixture def uppdata_obj(hrrr_data, prsfile, spec): return ConcreteUPPData( @@ -44,7 +57,7 @@ def uppdata_obj(hrrr_data, prsfile, spec): def uppdata_multilev_obj(prsfile, spec): ds = gribfile.GribFile( prsfile, - var_config={ + cfgrib_config={ "shortName": "t", "typeOfLevel": "isobaricInhPa", }, @@ -59,7 +72,7 @@ def uppdata_multilev_obj(prsfile, spec): def test_uppdata_anl_dt(uppdata_obj): dt = uppdata_obj.anl_dt - assert dt == datetime(2020, 10, 9, 8) + assert dt == datetime(2025, 10, 6, 0) def test_uppdata_clevs_array(uppdata_obj): @@ -72,7 +85,7 @@ def test_uppdata_clevs_list(uppdata_obj): def test_uppdata_date_to_str(uppdata_obj): - assert uppdata_obj.date_to_str(uppdata_obj.anl_dt) == "20201009 08 UTC" + assert uppdata_obj.date_to_str(uppdata_obj.anl_dt) == "20251006 00 UTC" def test_uppdata_field(uppdata_obj): @@ -117,7 +130,7 @@ def test_uppdata__get_field(prsfile, uppdata_obj): field = uppdata_obj._get_field(spec=spec) ds = gribfile.GribFile( prsfile, - var_config=spec, + cfgrib_config=spec, ).contents assert np.array_equal(field, ds.t) @@ -173,12 +186,12 @@ def test_uppdata_opposite(factor, uppdata_obj): def test_uppdata_valid_dt(uppdata_obj): - assert uppdata_obj.valid_dt == datetime(2020, 10, 9, 23) + assert uppdata_obj.valid_dt == datetime(2025, 10, 6, 15) def test_uppdata_vspec(uppdata_obj): expected = { - "cfgrib": {"shortName": "t", "typeOfLevel": "isobaricInhPa"}, + "cfgrib": {"shortName": "t", "typeOfLevel": "hybrid"}, "clevs": np.arange(-40, 40, 2.5), "cmap": "jet", "colors": "ua_temp_colors", @@ -195,7 +208,265 @@ def test_uppdata_vspec(uppdata_obj): } vspec = uppdata_obj.vspec # Can't test the array items with ==, so check them separately and then remove. - assert np.array_equal(vspec["clevs"], expected["clevs"]) + assert np.array_equal(vspec["clevs"], np.asarray(expected["clevs"])) vspec.pop("clevs") expected.pop("clevs") assert uppdata_obj.vspec == expected + + +def test_fielddata_aviation_flight_rules(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + cfgrib_config={ + "shortName": "gh", + "typeOfLevel": "cloudCeiling", + }, + ) + fd = gribdata.FieldData( + ds=ds.contents, + fhr=15, + grib_path=prsfile, + level="sfc", + short_name="flru", + spec=spec, + ) + + flru = fd.aviation_flight_rules(fd.field) + assert flru.max() == 3.01 + assert flru.min() == 0.0 + + +def test_fielddata_cmap(fielddata_obj): + assert fielddata_obj.cmap == get_cmap("gist_ncar") + + +@mark.parametrize("color_def", ["aod_colors", "shear_colors", "vvel_colors"]) +def test_fielddata_colors(color_def, fielddata_obj): + fielddata_obj.vspec["colors"] = color_def + assert fielddata_obj.colors.min() == 0.0 + assert fielddata_obj.colors.max() == 1.0 + + +def test_fielddata_colors_undefined(fielddata_obj): + del fielddata_obj.vspec["colors"] + with raises(errors.NoGraphicsDefinitionForVariableError): + fielddata_obj.colors # noqa: B018 + + +def test_fielddata_colors_bad(fielddata_obj): + fielddata_obj.vspec["colors"] = "foo" + with raises(AttributeError) as e: + fielddata_obj.colors # noqa: B018 + assert "There is no color definition named foo" in str(e.value) + + +def test_fielddata_corners(fielddata_obj): + assert fielddata_obj.corners == [ + 21.13812299999999, + 47.84219502248864, + 237.28047200000003, + 299.08280722816215, + ] + + +def test_fielddata_corners_single_dim(fielddata_obj): + # Remove one dimension for the purposes of the test + fielddata_obj.ds.coords["latitude"] = fielddata_obj.ds.coords["latitude"][:, 0] + assert fielddata_obj.corners == [ + 21.13812299999999, + 47.83862349881542, + 237.28047200000003, + 225.90452026573686, + ] + + +def test_fielddata_fire_weather_index(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + cfgrib_config={ + "shortName": "vgtyp", + "typeOfLevel": "surface", + }, + ) + fd = gribdata.FieldData( + ds=ds.contents, + fhr=15, + grib_path=prsfile, + level="sfc", + short_name="firewxtransform", + spec=spec, + ) + + firewx = fd.fire_weather_index(fd.field) + assert firewx.max() <= 100 + assert firewx.min() == 0 + + +def test_fielddata_grid_info_lambert(fielddata_obj): + grid_info = fielddata_obj.grid_info() + assert grid_info == { + "corners": [21.13812299999999, 47.84219502248864, 237.28047200000003, 299.08280722816215], + "lat_0": 39.0, + "lat_1": 38.5, + "lat_2": 38.5, + "lon_0": 262.5, + "projection": "lcc", + } + + +def test_fielddata_icing_adjust_trace(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + cfgrib_config={ + "shortName": "gh", + "typeOfLevel": "cloudCeiling", + }, + ) + fd = gribdata.FieldData( + ds=ds.contents, + fhr=15, + grib_path=prsfile, + level="sfc", + short_name="flru", + spec=spec, + ) + field = ones_like(fd.field) * 4 + icing_adjust_trace = fd.icing_adjust_trace(field) + assert np.array_equal(icing_adjust_trace, ones_like(field) * 0.5) + + +def test_fielddata_supercooled_liquid_water(natfile, spec): + ds = gribfile.GribFile( + natfile, + cfgrib_config={ + "shortName": "t", + "typeOfLevel": "surface", + }, + ) + fd = gribdata.FieldData( + ds=ds.contents, + fhr=15, + grib_path=natfile, + level="sfc", + short_name="slw", + spec=spec, + ) + slw = fd.supercooled_liquid_water() + assert not np.array_equal(slw, ds.contents.t) + + +def test_fielddata_ticks_default(fielddata_obj): + assert fielddata_obj.ticks() == 10 + + +def test_fielddata_ticks_in_vspec(fielddata_obj): + ticks = 22 + fielddata_obj.vspec["ticks"] = ticks + assert fielddata_obj.ticks() == ticks + + +def test_fielddata_units_default(fielddata_obj): + assert fielddata_obj.units == "F" + + +def test_fielddata_units_in_vspec(fielddata_obj): + units = "foo" + fielddata_obj.vspec["unit"] = units + assert fielddata_obj.units == units + + +@mark.parametrize( + ("var", "lev"), [("pres", "sfc"), ("1ref", "1000m"), ("acsnw", "sfc"), ("rh", "500mb")] +) +def test_fielddata_values_args_no_transform(fielddata_obj, lev, var): + fielddata_obj.vspec["transform"] = None + fielddata_obj.model = "hrrr" + assert not np.array_equal(fielddata_obj.values(level=lev, name=var), fielddata_obj.ds.t) + + +def test_fielddata_values_args_transform(fielddata_obj): + fielddata_obj.vspec["transform"] = "opposite" + fielddata_obj.model = "hrrr" + assert np.array_equal(fielddata_obj.values(level="sfc", name="temp"), -fielddata_obj.ds.t) + + +def test_fielddata_values_no_args_no_transform(fielddata_obj): + field = ones_like(fielddata_obj.ds) + fielddata_obj.ds = field + fielddata_obj.vspec["transform"] = None + assert np.array_equal(fielddata_obj.values(), field.t) + + +def test_fielddata_values_no_args_transform(fielddata_obj): + field = ones_like(fielddata_obj.ds) + fielddata_obj.ds = field + fielddata_obj.vspec["transform"] = "opposite" + assert np.array_equal(fielddata_obj.values(), -field.t) + + +def test_fielddata_vector_magnitude_cfkeys(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + cfgrib_config={ + "shortName": "u", + "typeOfLevel": "isobaricInhPa", + "level": 250, + }, + ) + fd = gribdata.FieldData( + ds=ds.contents, + fhr=15, + grib_path=prsfile, + level="250mb", + short_name="u", + spec=spec, + ) + vm = fd.vector_magnitude( + field1=fd.ds, cfkeys={"shortName": "v", "typeOfLevel": "isobaricInhPa"} + ) + assert not np.array_equal(vm, ds.contents.u) + + +def test_fielddata_vector_magnitude_field2_id(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + cfgrib_config={ + "shortName": "u", + "typeOfLevel": "isobaricInhPa", + "level": 250, + }, + ) + fd = gribdata.FieldData( + ds=ds.contents, + fhr=15, + grib_path=prsfile, + level="250mb", + short_name="u", + spec=spec, + ) + vm = fd.vector_magnitude(field1=fd.ds, field2_id="v_250mb") + assert not np.array_equal(vm, ds.contents.u) + + +def test_fielddata_vector_magnitude_options_equal(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + cfgrib_config={ + "shortName": "u", + "typeOfLevel": "isobaricInhPa", + "level": 250, + }, + ) + fd = gribdata.FieldData( + ds=ds.contents, + fhr=15, + grib_path=prsfile, + level="250mb", + short_name="u", + spec=spec, + ) + vm_cfkeys = fd.vector_magnitude( + field1=fd.ds, cfkeys={"shortName": "v", "typeOfLevel": "isobaricInhPa"} + ) + vm_field2 = fd.vector_magnitude(field1=fd.ds, field2_id="v_250mb") + assert np.array_equal(vm_cfkeys, vm_field2) diff --git a/tests/datahandler/test_gribfile.py b/tests/datahandler/test_gribfile.py index 842baca..eaa2401 100644 --- a/tests/datahandler/test_gribfile.py +++ b/tests/datahandler/test_gribfile.py @@ -8,7 +8,7 @@ def test_gribfile(prsfile): gf = gribfile.GribFile( filename=Path(prsfile), - var_config={ + cfgrib_config={ "shortName": "sp", "typeOfLevel": "surface", }, @@ -26,7 +26,7 @@ def test_gribfiles(): gribfiles = [Path(f) for f in paths] gf = gribfile.GribFiles( filenames=gribfiles, - var_config={ + cfgrib_config={ "shortName": "sp", "typeOfLevel": "surface", }, diff --git a/tests/test_common.py b/tests/test_common.py index eef2b21..f5b5cb2 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -372,6 +372,8 @@ def is_a_level(key): "top", # nominal top of atmosphere "total", # total clouds "ua", # upper air + "uanat", # upper air native file + "uaprs", # upper air prs file ] allowed_lev_type = [ From 6f75a5cc4076fc163eda3221dada2971a0c2ba58 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 28 Oct 2025 10:43:42 -0600 Subject: [PATCH 25/98] gribdata tested! --- adb_graphics/datahandler/gribdata.py | 169 +++++++++------------ adb_graphics/errors.py | 12 +- tests/datahandler/test_gribdata.py | 212 ++++++++++++++++++--------- 3 files changed, 218 insertions(+), 175 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index d40de0c..aa9aa02 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -229,6 +229,56 @@ def valid_dt(self) -> datetime: def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: """Returns the values of a given variable.""" + def vector_magnitude( + self, + field1: Dataset, + cfkeys: dict | None = None, + field2_id: str | None = None, + **kwargs, # noqa: ARG002 + ): + """ + Returns the vector magnitude of two component vector fields. + + The second field can be specified by either a dict of cfkeys or a default_specs identifier + in the form _. + + """ + + if cfkeys: + if cfkeys.get("level") is None: + typeoflevel = cfkeys["typeOfLevel"] + cfkeys["level"] = field1.coords[typeoflevel].to_numpy().item() + field2_spec = {"cfgrib": cfkeys} + utils.set_level(level=self.level, model=self.model, spec=field2_spec) + elif field2_id: + var, lev = field2_id.split("_") + field2_spec = self.spec + for key in (var, lev): + field2_spec = field2_spec[key] + utils.set_level(level=lev, model=self.model, spec=field2_spec) + else: + msg = "Must supply a field2_id if cfkeys is not explicitly provided." + raise errors.ArgumentError(msg) + + ds = gribfile.GribFile(self.grib_path, field2_spec["cfgrib"]).contents + args = { + "ds": ds, + "fhr": self.fhr, + "level": self.level, + "model": self.model, + "short_name": self.short_name, + "spec": self.spec, + "grib_path": self.grib_path, + } + field2 = FieldData(**args).ds + mag = conversions.magnitude( + field1.to_dataarray().squeeze(), field2.to_dataarray().squeeze() + ) + field1.close() + field2.close() + + return mag + @property def vspec(self): """Return the graphics specification for a given level.""" @@ -305,8 +355,6 @@ def colors(self) -> np.ndarray: except AttributeError as e: msg = f"There is no color definition named {color_spec}" raise AttributeError(msg) from e - if callable(ret): - return np.asarray(ret()) return np.asarray(ret) @property @@ -462,11 +510,11 @@ def grid_info(self) -> dict: grid_info[bm_arg] = val del val - if self.model == "hrrrhi": - grid_info["lat_0"] = 20.44 - grid_info["lon_0"] = 202.54 - grid_info["width"] = 2000000 - grid_info["height"] = 2000000 + # if self.model == "hrrrhi": + # grid_info["lat_0"] = 20.44 + # grid_info["lon_0"] = 202.54 + # grid_info["width"] = 2000000 + # grid_info["height"] = 2000000 return grid_info @@ -480,19 +528,19 @@ def icing_adjust_trace(values: DataArray, **kwargs): # noqa: ARG004 def run_max(values: DataArray, **kwargs): # noqa: ARG004 """Finds the max hourly value over all the forecast lead times available.""" - return values.max(dim="step") + return values.max(dim="step") # pragma: no cover @staticmethod def run_min(values: DataArray, **kwargs): # noqa: ARG004 """Finds the min hourly value over all the forecast lead times available.""" - return values.min(dim="step") + return values.min(dim="step") # pragma: no cover @staticmethod def run_total(values: DataArray, **kwargs): # noqa: ARG004 """Sums over all the forecast lead times available.""" - return values.sum(dim="step") + return values.sum(dim="step") # pragma: no cover def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 """ @@ -587,7 +635,7 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> if name is not None: # Get the spec dict and ncl_name for the given variable name spec = deepcopy(self.spec.get(name, {}).get(level, {})) - if not spec and name is not None: + if not spec: raise errors.NoGraphicsDefinitionForVariableError(name, level) utils.set_level(level=level, model=self.model, spec=spec) vals = self._get_field(spec["cfgrib"].get(self.model, spec["cfgrib"])) @@ -598,53 +646,6 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> return vals - def vector_magnitude( - self, - field1: Dataset, - cfkeys: dict | None = None, - field2_id: str | None = None, - **kwargs, # noqa: ARG002 - ): - """ - Returns the vector magnitude of two component vector fields. The - input fields can be either NCL names (string) or full data fields. The - first layer of a variable is returned if none is provided. - """ - - if cfkeys: - if cfkeys.get("level") is None: - cfkeys["level"] = utils.numeric_level(level=self.level)[0] - field2_spec = {"cfgrib": cfkeys} - utils.set_level(level=self.level, model=self.model, spec=field2_spec) - elif field2_id: - var, lev = field2_id.split("_") - field2_spec = self.spec - for key in (var, lev): - field2_spec = field2_spec[key] - utils.set_level(level=lev, model=self.model, spec=field2_spec) - else: - msg = "Must supply a field2_id if cfkeys is not explicitly provided." - raise ValueError(msg) - - ds = gribfile.GribFile(self.grib_path, field2_spec["cfgrib"]).contents - args = { - "ds": ds, - "fhr": self.fhr, - "level": self.level, - "model": self.model, - "short_name": self.short_name, - "spec": self.spec, - "grib_path": self.grib_path, - } - field2 = FieldData(**args).ds - mag = conversions.magnitude( - field1.to_dataarray().squeeze(), field2.to_dataarray().squeeze() - ) - field1.close() - field2.close() - - return mag - class ProfileData(UPPData): """ @@ -688,7 +689,7 @@ def __init__(self, ds: Dataset, loc: str, short_name: str, **kwargs): def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: # noqa: ARG002 """ Returns the numpy array of values at the object's x, y location for the - requested variable. Transforms are performed in the child class. + requested variable. Optional Input: name the short name of a field other than defined in self @@ -708,16 +709,18 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> # level refers to the level key in the specs file. level = level if level is not None else "ua" - if not name: - name = self.short_name - + if name is not None: + # Get the spec dict and ncl_name for the given variable name + spec = deepcopy(self.spec.get(name, {}).get(level, {})) + if not spec: + raise errors.NoGraphicsDefinitionForVariableError(name, level) + utils.set_level(level=level, model=self.model, spec=spec) + profile = self._get_field(spec["cfgrib"].get(self.model, spec["cfgrib"])) + else: + profile = self.field[::] # Retrive the location for the profile x, y = self.get_xypoint(self.site_lat, self.site_lon) - # Get the full 2- or 3-D field - field = self.field - - profile = field[::] # 2D if len(profile.shape) == 2: profile = profile[x, y] @@ -725,37 +728,3 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> elif len(profile.shape) == 3: profile = profile[:, x, y] return profile - - def vector_magnitude( - self, - field1: DataArray, - field2: DataArray, - level: str = "ua", - vertical_index: int | None = None, - **kwargs, - ) -> DataArray: - """ - The vector magnitude of two component vector profiles. - - The input fields can be either NCL names (string) or full data fields. - - If no layer or level is provided, the default 'ua' will be used in self.values. - """ - - if isinstance(field1, str): - field1 = self.values( - level=level, - ncl_name=field1, - vertical_index=vertical_index, - **kwargs, - ) - - if isinstance(field2, str): - field2 = self.values( - level=level, - ncl_name=field2, - vertical_index=vertical_index, - **kwargs, - ) - - return conversions.magnitude(field1, field2) diff --git a/adb_graphics/errors.py b/adb_graphics/errors.py index 69687a5..a498655 100644 --- a/adb_graphics/errors.py +++ b/adb_graphics/errors.py @@ -1,6 +1,10 @@ """Errors specific to the ADB Graphics package.""" +class ArgumentError(ValueError): + """The right arguments are not provided.""" + + class FieldNotUniqueError(Exception): """Exception raised when multiple Grib fields are found with input parameters.""" @@ -20,11 +24,3 @@ def __str__(self): class NoGraphicsDefinitionForVariableError(Exception): """Exception raised when there is no configuration for the variable.""" - - -class LevelNotFoundError(Exception): - """Exception raised when there is no configuration for the variable.""" - - -class OutsideDomainError(Exception): - """Exception raised when there is no configuration for the variable.""" diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index 11f246c..65951b3 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -25,11 +25,6 @@ def hrrr_data(prsfile): ).contents -@fixture -def spec(spec_file): - return utils.load_yaml(spec_file) - - @fixture def fielddata_obj(hrrr_data, prsfile, spec): return gribdata.FieldData( @@ -42,6 +37,30 @@ def fielddata_obj(hrrr_data, prsfile, spec): ) +@fixture +def profiledata_obj(natfile, spec): + ds = gribfile.GribFile( + natfile, + cfgrib_config={ + "shortName": "t", + "typeOfLevel": "hybrid", + }, + ) + return gribdata.ProfileData( + ds=ds.contents, + fhr=15, + grib_path=natfile, + loc=" DNR 23062 72469 39.77 104.88 1611 Denver, CO", + short_name="temp", + spec=spec, + ) + + +@fixture +def spec(spec_file): + return utils.load_yaml(spec_file) + + @fixture def uppdata_obj(hrrr_data, prsfile, spec): return ConcreteUPPData( @@ -179,6 +198,19 @@ def test_uppdata_latlons(uppdata_obj): ] +def test_uppdata_latlons_lats_flipped(uppdata_obj): + # Test a 1D latitude option (like in Global, etc.) + ds = uppdata_obj.ds.sel(y=500) + lats = ds.coords["latitude"].to_numpy() + ds.coords["latitude"] = (("x"), lats[::-1]) + lons = ds.coords["longitude"].to_numpy() + uppdata_obj.ds = ds + assert [ + np.array_equal(act, exp) + for act, exp in zip(uppdata_obj.latlons(), [lats, lons], strict=True) + ] + + @mark.parametrize("factor", [1, -1, 0, -20.0, 6543.0]) def test_uppdata_opposite(factor, uppdata_obj): ds = ones_like(uppdata_obj.field) * factor @@ -189,6 +221,79 @@ def test_uppdata_valid_dt(uppdata_obj): assert uppdata_obj.valid_dt == datetime(2025, 10, 6, 15) +def test_uppdata_vector_magnitude_cfkeys(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + cfgrib_config={ + "shortName": "u", + "typeOfLevel": "isobaricInhPa", + "level": 250, + }, + ) + fd = ConcreteUPPData( + ds=ds.contents, + fhr=15, + grib_path=prsfile, + level="250mb", + short_name="u", + spec=spec, + ) + vm = fd.vector_magnitude( + field1=fd.ds, cfkeys={"shortName": "v", "typeOfLevel": "isobaricInhPa"} + ) + assert not np.array_equal(vm, ds.contents.u) + + +def test_uppdata_vector_magnitude_field2_id(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + cfgrib_config={ + "shortName": "u", + "typeOfLevel": "isobaricInhPa", + "level": 250, + }, + ) + fd = ConcreteUPPData( + ds=ds.contents, + fhr=15, + grib_path=prsfile, + level="250mb", + short_name="u", + spec=spec, + ) + vm = fd.vector_magnitude(field1=fd.ds, field2_id="v_250mb") + assert not np.array_equal(vm, ds.contents.u) + + +def test_uppdata_vector_magnitude_no_field2_args(uppdata_obj): + with raises(errors.ArgumentError): + uppdata_obj.vector_magnitude(uppdata_obj.ds) + + +def test_uppdata_vector_magnitude_options_equal(prsfile, spec): + ds = gribfile.GribFile( + prsfile, + cfgrib_config={ + "shortName": "u", + "typeOfLevel": "isobaricInhPa", + "level": 250, + }, + ) + fd = ConcreteUPPData( + ds=ds.contents, + fhr=15, + grib_path=prsfile, + level="250mb", + short_name="u", + spec=spec, + ) + vm_cfkeys = fd.vector_magnitude( + field1=fd.ds, cfkeys={"shortName": "v", "typeOfLevel": "isobaricInhPa"} + ) + vm_field2 = fd.vector_magnitude(field1=fd.ds, field2_id="v_250mb") + assert np.array_equal(vm_cfkeys, vm_field2) + + def test_uppdata_vspec(uppdata_obj): expected = { "cfgrib": {"shortName": "t", "typeOfLevel": "hybrid"}, @@ -214,6 +319,12 @@ def test_uppdata_vspec(uppdata_obj): assert uppdata_obj.vspec == expected +def test_uppdata_vspec_bad(uppdata_obj): + uppdata_obj.short_name = "foo" + with raises(errors.NoGraphicsDefinitionForVariableError): + uppdata_obj.vspec # noqa: B018 + + def test_fielddata_aviation_flight_rules(prsfile, spec): ds = gribfile.GribFile( prsfile, @@ -280,6 +391,13 @@ def test_fielddata_corners_single_dim(fielddata_obj): ] +def test_fielddata_data_getter_and_setter(fielddata_obj): + assert np.array_equal(fielddata_obj.data, fielddata_obj.values()) + new_data = ones_like(fielddata_obj.ds.t) + fielddata_obj.data = new_data + assert np.array_equal(fielddata_obj.data, new_data) + + def test_fielddata_fire_weather_index(prsfile, spec): ds = gribfile.GribFile( prsfile, @@ -404,69 +522,29 @@ def test_fielddata_values_no_args_transform(fielddata_obj): assert np.array_equal(fielddata_obj.values(), -field.t) -def test_fielddata_vector_magnitude_cfkeys(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "u", - "typeOfLevel": "isobaricInhPa", - "level": 250, - }, - ) - fd = gribdata.FieldData( - ds=ds.contents, - fhr=15, - grib_path=prsfile, - level="250mb", - short_name="u", - spec=spec, - ) - vm = fd.vector_magnitude( - field1=fd.ds, cfkeys={"shortName": "v", "typeOfLevel": "isobaricInhPa"} - ) - assert not np.array_equal(vm, ds.contents.u) +def test_fielddata_values_bad_name_level(fielddata_obj): + with raises(errors.NoGraphicsDefinitionForVariableError): + fielddata_obj.values(level="foo", name="temp") + with raises(errors.NoGraphicsDefinitionForVariableError): + fielddata_obj.values(level="sfc", name="foo") + with raises(errors.NoGraphicsDefinitionForVariableError): + fielddata_obj.values(level="bar", name="foo") -def test_fielddata_vector_magnitude_field2_id(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "u", - "typeOfLevel": "isobaricInhPa", - "level": 250, - }, - ) - fd = gribdata.FieldData( - ds=ds.contents, - fhr=15, - grib_path=prsfile, - level="250mb", - short_name="u", - spec=spec, - ) - vm = fd.vector_magnitude(field1=fd.ds, field2_id="v_250mb") - assert not np.array_equal(vm, ds.contents.u) +def test_profiledata_values(profiledata_obj): + assert profiledata_obj.values().shape == (50,) -def test_fielddata_vector_magnitude_options_equal(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "u", - "typeOfLevel": "isobaricInhPa", - "level": 250, - }, - ) - fd = gribdata.FieldData( - ds=ds.contents, - fhr=15, - grib_path=prsfile, - level="250mb", - short_name="u", - spec=spec, - ) - vm_cfkeys = fd.vector_magnitude( - field1=fd.ds, cfkeys={"shortName": "v", "typeOfLevel": "isobaricInhPa"} - ) - vm_field2 = fd.vector_magnitude(field1=fd.ds, field2_id="v_250mb") - assert np.array_equal(vm_cfkeys, vm_field2) +def test_profiledata_values_bad_name_level(profiledata_obj): + with raises(errors.NoGraphicsDefinitionForVariableError): + profiledata_obj.values(level="foo", name="temp") + with raises(errors.NoGraphicsDefinitionForVariableError): + profiledata_obj.values(level="sfc", name="foo") + with raises(errors.NoGraphicsDefinitionForVariableError): + profiledata_obj.values(level="bar", name="foo") + + +def test_profiledata_values_one_level(profiledata_obj): + value = profiledata_obj.values(name="hlcy", level="sr01") + assert value.shape == () # A single number + assert value == 47.7 From d3645547c4674eef8f1927918655217f1cc7eb2f Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 29 Oct 2025 20:18:49 -0600 Subject: [PATCH 26/98] Figure_builders tested. --- Makefile | 3 ++ adb_graphics/datahandler/gribdata.py | 40 +++------------ adb_graphics/default_specs.yml | 76 ++++++++++++++------------- adb_graphics/figure_builders.py | 77 +++++++++++++++------------- adb_graphics/figures/maps.py | 9 ++-- adb_graphics/figures/skewt.py | 45 ++++++++-------- adb_graphics/utils.py | 1 + create_graphics.py | 5 +- tests/datahandler/test_gribdata.py | 60 ++-------------------- 9 files changed, 124 insertions(+), 192 deletions(-) diff --git a/Makefile b/Makefile index 144e1b1..6dbdc64 100644 --- a/Makefile +++ b/Makefile @@ -17,3 +17,6 @@ typecheck: unittest: pytest --cov -k "not hrrr" -n 4 . +memtest: + pytest --memray -k"not hrrr" . + diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index aa9aa02..10ed52e 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -231,9 +231,8 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> def vector_magnitude( self, - field1: Dataset, - cfkeys: dict | None = None, - field2_id: str | None = None, + field1: DataArray, + field2_id: str, **kwargs, # noqa: ARG002 ): """ @@ -243,37 +242,9 @@ def vector_magnitude( in the form _. """ - - if cfkeys: - if cfkeys.get("level") is None: - typeoflevel = cfkeys["typeOfLevel"] - cfkeys["level"] = field1.coords[typeoflevel].to_numpy().item() - field2_spec = {"cfgrib": cfkeys} - utils.set_level(level=self.level, model=self.model, spec=field2_spec) - elif field2_id: - var, lev = field2_id.split("_") - field2_spec = self.spec - for key in (var, lev): - field2_spec = field2_spec[key] - utils.set_level(level=lev, model=self.model, spec=field2_spec) - else: - msg = "Must supply a field2_id if cfkeys is not explicitly provided." - raise errors.ArgumentError(msg) - - ds = gribfile.GribFile(self.grib_path, field2_spec["cfgrib"]).contents - args = { - "ds": ds, - "fhr": self.fhr, - "level": self.level, - "model": self.model, - "short_name": self.short_name, - "spec": self.spec, - "grib_path": self.grib_path, - } - field2 = FieldData(**args).ds - mag = conversions.magnitude( - field1.to_dataarray().squeeze(), field2.to_dataarray().squeeze() - ) + var, lev = field2_id.split("_") if "_" in field2_id else (field2_id, self.level) + field2 = self.values(level=lev, name=var) + mag = conversions.magnitude(field1, field2) field1.close() field2.close() @@ -590,6 +561,7 @@ def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 rain_mixing_ratio.close() return slw + @property def ticks(self) -> int: """ Returns the number of color bar tick marks from the yaml config diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 4fb1d58..b98a47e 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -460,16 +460,19 @@ cell: # Storm cell motion cfgrib: shortName: ustm typeOfLevel: heightAboveGroundLayer - level: 6000 + bottomLevel: 6000 ncl_name: USTM_P0_2L103_{grid} transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - cfkeys: - shortName: vstm - typeOfLevel: heightAboveGroundLayer - level: 6000 + field2_id: cellv unit: kt +cellv: + ua: + cfgrib: + shortName: vstm + typeOfLevel: heightAboveGroundLayer + bottomLevel: 6000 ceil: # Ceiling ua: &ceil cfgrib: @@ -1104,6 +1107,9 @@ gh: # Geopotential height unit: gpm ua: <<: *ua_gh + cfgrib: + shortName: gh + typeOfLevel: hybrid ghtfl: # Ground Heat Flux sfc: cfgrib: @@ -1786,16 +1792,18 @@ ptyp: # Hourly total precipitation transform: conversions.kgm2_to_in unit: in pwtr: # Precipitable water - sfc: + sfc: &pwtr cfgrib: shortName: pwat typeOfLevel: atmosphereSingleLayer + level: 0 clevs: !arange [4, 81, 4] cmap: gist_ncar colors: pw_colors ncl_name: PWAT_P0_L200_{grid} ticks: 4 unit: mm + ref: # Maximum reflectivity for past hour at 1 km AGL m10: <<: *refl @@ -1986,15 +1994,7 @@ shear: transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - cfkeys: - shortName: vvcsh - typeOfLevel: heightAboveGroundLayer - topLevel: 0 - level: 0 - bottomLevel: 1000 - one_lev: True - split: True - level: 01km + field2_id: vshear_01km ticks: 0 title: 0–1 km Bulk Shear unit: kt @@ -2011,16 +2011,23 @@ shear: transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - cfkeys: - shortName: vvcsh - typeOfLevel: heightAboveGroundLayer - level: 0 - topLevel: 0 - bottomLevel: 6000 - one_lev: True - split: True - level: 06km + field2_id: vshear_06km title: 0–6 km Bulk Shear +vshear: + 01km: + cfgrib: + shortName: vvcsh + typeOfLevel: heightAboveGroundLayer + topLevel: 0 + level: 0 + bottomLevel: 1000 + 06km: + cfgrib: + shortName: vvcsh + typeOfLevel: heightAboveGroundLayer + level: 0 + topLevel: 0 + bottomLevel: 6000 shtfl: # Sensible Heat Net Flux sfc: &shtflsfc cfgrib: @@ -2316,7 +2323,7 @@ sphum: # Specific humidity ua: cfgrib: shortName: q - typeOfLevel: isobaricInhPa + typeOfLevel: hybrid ncl_name: SPFH_P0_L105_{grid} ssrun: # Storm Surface Runoff sfc: &precip @@ -2611,6 +2618,9 @@ u: transform: conversions.ms_to_kt ua: <<: *ua_uwind + cfgrib: + shortName: u + typeOfLevel: hybrid ulwrf: # Upward Longwave Radiation Flux sfc: <<: *radiation_flux @@ -2768,6 +2778,9 @@ v: transform: conversions.ms_to_kt ua: <<: *ua_vwind + cfgrib: + shortName: v + typeOfLevel: hybrid vbdsf: # Incoming Direct Radiation sfc: <<: *incoming_radiation @@ -2941,10 +2954,7 @@ wspeed: # Wind Speed transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - cfkeys: - shortName: 10v - level: 10 - typeOfLevel: heightAboveGround + field2_id: v_10m unit: kt wind: True 5mb: &ua_wspeed_high @@ -2964,9 +2974,7 @@ wspeed: # Wind Speed transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - cfkeys: - shortName: v - typeOfLevel: isobaricInhPa + field2_id: v unit: kt wind: True 10mb: @@ -3005,9 +3013,7 @@ wspeed: # Wind Speed transform: funcs: [vector_magnitude, conversions.ms_to_kt] kwargs: - cfkeys: - shortName: v - typeOfLevel: isobaricInhPa + field2_id: v max: # Hourly Maximum 10m Wind <<: *ua_wspeed cfgrib: diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 4920f55..9e7e785 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -11,9 +11,11 @@ import matplotlib.pyplot as plt import numpy as np from matplotlib import axes -from xarray import Dataset -from adb_graphics.figures import maps, skewt +from adb_graphics.datahandler import gribfile +from adb_graphics.figures import skewt +from adb_graphics.figures.maps import DataMap, DiffMap, Map, MapFields, MultiPanelDataMap +from adb_graphics.utils import cfgrib_spec AIRPORTS = Path("static/Airports_locs.txt") @@ -33,7 +35,7 @@ def add_obs_panel( """ ax.axis("on") - map_fields = maps.MapFields( + map_fields = MapFields( fhr=0, fields_spec=spec, grib_path=obs_file, @@ -41,14 +43,14 @@ def add_obs_panel( model="obs", name=short_name, ) - m = maps.Map( + m = Map( airport_fn=AIRPORTS, ax=ax, grid_info=proj_info, model="obs", tile=tile, ) - dm = maps.MultiPanelDataMap( + dm = MultiPanelDataMap( map_fields=map_fields, map_=m, member="obs", @@ -56,23 +58,19 @@ def add_obs_panel( ) # Draw the map - dm.draw(show=True) + return dm.draw() -def parallel_maps( +def parallel_maps( # noqa: PLR0912 cla: Namespace, fhr: int, grib_path: Path, level: str, - model: str, - spec: dict, variable: str, workdir: Path, tile: str = "full", dp2: Path | None = None, ): - # pylint: disable=too-many-arguments,too-many-locals - # pylint: disable=too-many-branches,too-many-statements """ Function that creates plan-view maps, either a single panel, or multipanel for a forecast ensemble. Can be used in parallel. @@ -83,9 +81,6 @@ def parallel_maps( grib_path path to grib file level the vertical level of the variable to be plotted corresponding to a key in the specs file - model model name: rap, hrrr, hrrre, rrfs, rtma - spec the dictionary of specifications for the given variable - and level variable the name of the variable section in the specs file workdir output directory tile @@ -96,39 +91,49 @@ def parallel_maps( """ fig, axes = set_figure(cla.model_name, cla.graphic_type, tile) - + spec = cla.specs[variable][level] # set last_panel to send into DataMap for colorbar control last_panel = False # Declare the type of object depending on graphic type map_classes = { - "enspanel": maps.MultiPanelDataMap, - "diff": maps.DiffMap, + "enspanel": MultiPanelDataMap, + "diff": DiffMap, } - map_class = map_classes.get(cla.graphic_type, maps.DataMap) + map_class = map_classes.get(cla.graphic_type, DataMap) + top_left = 0 + center_left = 4 + lower_left = 8 for index, current_ax in enumerate(axes): - if current_ax is axes[-1] or index == cla.ens_size: + if current_ax is axes[-1]: last_panel = True mem = None if cla.graphic_type == "enspanel": # Don't put data in the top left or bottom left panels. - if index in (0, 8): + if index in (top_left, lower_left): current_ax.axis("off") - # If we have less than 10 members, skip the remaining panels. - if index > cla.ens_size: - continue + ## If we have less than 10 members, skip the remaining panels. + # if index > cla.ens_size: + # continue # Shenanigans to match ensemble member to panel index - center_left = 4 - lower_left = 8 - mem = 0 if index == center_left else index - mem = mem if mem < center_left else index - 1 - mem = mem if mem < lower_left else index - 2 + match index: + case x if x in (top_left, center_left, lower_left): + mem = 0 + case x if x > lower_left: + mem = index - 2 + case x if x > center_left: + mem = index - 1 + case x if x < center_left: + mem = index + # mem = 0 if index in (top_left, center_left, lower_left) else index + # mem = mem if mem < center_left else index - 1 + # mem = mem if mem < lower_left else index - 2 # Create an object that holds all the fields for this map - map_fields = maps.MapFields( + map_fields = MapFields( grib_path=grib_path, grib_path2=dp2, fhr=fhr, @@ -136,16 +141,16 @@ def parallel_maps( level=level, name=variable, map_type=cla.graphic_type, - model=model, + model=cla.model_name, tile=tile, ) # Generate a map object - m = maps.Map( + m = Map( airport_fn=AIRPORTS, ax=current_ax, grid_info=map_fields.shaded.grid_info(), - model=model, + model=cla.model_name, plot_airports=spec.get("plot_airports", True), tile=tile, ) @@ -208,11 +213,10 @@ def parallel_maps( plt.clf() # Closes all the figure windows. plt.close("all") - del m gc.collect() -def parallel_skewt(cla: Namespace, fhr: int, ds: Dataset, site: str, workdir: Path): +def parallel_skewt(cla: Namespace, fhr: int, grib_path: Path, site: str, workdir: Path): """ Function that creates a single SkewT plot. @@ -225,7 +229,8 @@ def parallel_skewt(cla: Namespace, fhr: int, ds: Dataset, site: str, workdir: Pa site the string representation of the site from the sites file workdir output directory """ - + cf = cfgrib_spec(cla.specs["temp"]["ua"]["cfgrib"], cla.model_name) + ds = gribfile.GribFile(grib_path, cf).contents skew = skewt.SkewTDiagram( ds=ds, fhr=fhr, @@ -233,6 +238,8 @@ def parallel_skewt(cla: Namespace, fhr: int, ds: Dataset, site: str, workdir: Pa loc=site, max_plev=cla.max_plev, model_name=cla.model_name, + spec=cla.specs, + grib_path=grib_path, ) skew.create_diagram() outfile = f"{skew.site_code}_{skew.site_num}_skewt_f{fhr:03d}.png" diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 44fa7d2..4f46761 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -23,7 +23,7 @@ from mpl_toolkits.basemap import Basemap, shiftgrid from adb_graphics.datahandler import gribdata, gribfile -from adb_graphics.utils import cfgrib_spec, set_level +from adb_graphics.utils import cfgrib_spec, numeric_level, set_level # FULL_TILES is a list of strings that includes the labels GSL attaches to some of # the wgrib2 cutouts used for larger domains like RAP, RRFS NA, and global. @@ -500,6 +500,7 @@ def _colorbar(self, cc: QuadContourSet, ax: Axes): ticks=ticks, ) + tick_labels = [str(t) for t in ticks] if self.field.short_name == "flru": tick_labels = [label.rjust(30) for label in ["VFR", "MVFR", "IFR", "LIFR", ""]] @@ -811,7 +812,7 @@ def _title(self): loc="left", ) - level, lev_unit = f.numeric_level() + level, lev_unit = numeric_level(f.level) units = f"({f.units}, shaded)" if f.vspec.get("print_units", True) else "" # Title or Atmospheric level and unit in the high center @@ -892,8 +893,8 @@ def _xy_mesh(self, field: gribdata.FieldData): """Helper function to create mesh for various plot.""" lat, lon = field.latlons() - if self.map.model == "obs": - lat, lon = np.meshgrid(lat, lon, sparse=False, indexing="ij") + # if self.map.model == "obs": + # lat, lon = np.meshgrid(lat, lon, sparse=False, indexing="ij") adjust = 360 if np.any(lon < 0) else 0 return self.map.m(adjust + lon, lat) diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index bab7efb..a1eb25c 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -72,8 +72,6 @@ def __init__(self, ds: Dataset, loc: str, **kwargs): self.model_name = kwargs.get("model_name", "Analysis") def _add_hydrometeors(self, hydro_subplot: Axes): - # pylint: disable=too-many-locals - mixing_ratios: dict[str, HydroPlotSettings] = { "clwmr": { "color": "blue", @@ -112,11 +110,8 @@ def _add_hydrometeors(self, hydro_subplot: Axes): }, } - profiles = self.atmo_profiles # dictionary - pres = profiles.get("pres").get("data") - temp = profiles.get("temp").get("data") - nlevs = len(pres) # determine number of vertical levels - pres_sfc = pres[0] # need correct surface pressure value! + pres = self.atmo_profiles.get("pres").get("data") + temp = self.atmo_profiles.get("temp").get("data") handles = [] gravity = 9.81 # m/s^2 @@ -131,14 +126,17 @@ def _add_hydrometeors(self, hydro_subplot: Axes): scale = settings.get("scale", 1.0) try: profile = self.values(name=mixr) * 1000.0 * scale - except errors.GribReadError: - print(f"missing {mixr} for hydrometeor plot, skipping that field.") - continue + except (errors.NoGraphicsDefinitionForVariableError, IndexError): + try: + profile = self.values(name=mixr, level="uanat") * 1000.0 * scale + except errors.NoGraphicsDefinitionForVariableError: + print(f"missing {mixr} for hydrometeor plot, skipping that field.") + continue mixr_total: units = 0.0 - for n in range(nlevs): + for n in range(len(pres)): if n == 0: - pres_layer = 2 * (pres_sfc - pres[n]) # layer depth - pres_sigma = pres_sfc - pres_layer # pressure at next sigma level + pres_layer = 2 * (pres[0] - pres[n]) # layer depth + pres_sigma = pres[0] - pres_layer # pressure at next sigma level else: pres_layer = 2 * (pres_sigma - pres[n]) # layer depth pres_sigma = pres_sigma - pres_layer # pressure at next sigma level @@ -149,6 +147,7 @@ def _add_hydrometeors(self, hydro_subplot: Axes): profile = DataArray.where((profile > 10.0), 10.0, profile) # noqa: PLR2004 # plot line + profile = profile[: pres.shape[0]] hydro_subplot.plot( profile, pres, @@ -187,9 +186,7 @@ def _add_hydrometeors(self, hydro_subplot: Axes): layer = False # compute vertically integrated amount and add legend line - line = ( - f"{settings.get('label'):<7s} {mixr_total.magnitude:>10.3f} {settings.get('units')}" - ) + line = f"{settings.get('label'):<7s} {mixr_total:>10.3f} {settings.get('units')}" if scale != 1.0: line = ( f"{settings.get('label'):<5s}(x{scale}) {mixr_total.magnitude:.3f} " @@ -234,9 +231,8 @@ def _add_thermo_inset(self, skew: SkewT): for name, items in self.thermo_variables.items(): # Magic to get the desired number of decimals to appear. decimals = items.get("decimals", 0) - value = items["data"] - if value != "--": - value = int(value) if decimals == 0 else value.round(decimals=decimals).to_numpy() + data = items["data"] + value = int(data) if decimals == 0 else data.round(decimals=decimals).to_numpy() # Sure would have been nice to use a variable in the f string to # denote the format per variable. @@ -312,7 +308,7 @@ def atmo_profiles(self): # Only return values up to the maximum pressure level requested if var == "pres" and top is None: - top = np.sum(np.where(tmp.magnitude >= self.max_plev)) - 1 + top = np.where(tmp.magnitude >= self.max_plev)[0][-1] atmo_vars[var]["data"] = tmp[:top] @@ -408,7 +404,7 @@ def _write_profile(self, csv_path: str | Path): temp = profiles.get("temp").get("data").to("degC") sphum = profiles.get("sphum").get("data") - dewpt = np.array(mpcalc.dewpoint_from_specific_humidity(sphum, temp, pres).to("degC")) + dewpt = np.array(mpcalc.dewpoint_from_specific_humidity(pres, temp, sphum).to("degC")) wspd = np.array(mpcalc.wind_speed(u, v)) wdir = np.array(mpcalc.wind_direction(u, v)) @@ -433,7 +429,7 @@ def _plot_profile(self, skew: SkewT): temp = profiles.get("temp").get("data") sphum = profiles.get("sphum").get("data") - dewpt = mpcalc.dewpoint_from_specific_humidity(sphum, temp, pres).to("degF") + dewpt = mpcalc.dewpoint_from_specific_humidity(pres, temp, sphum).to("degF") # Pressure vs temperature skew.plot(pres, temp, "r", linewidth=1.5) @@ -539,8 +535,8 @@ def _setup_diagram(self): mixing_lines = np.array([1, 2, 3, 5, 8, 12, 16, 20]).reshape(-1, 1) / 1000 mix_pr = np.arange(1001, 400, -50) * units.hPa skew.plot_mixing_lines( - w=mixing_lines, - p=mix_pr, + mixing_ratio=mixing_lines, + pressure=mix_pr, colors="green", linestyles=(0, (5, 10)), linewidth=0.7, @@ -657,7 +653,6 @@ def thermo_variables(self): except errors.GribReadError: tmp = DataArray([]) - thermo[var]["data"] = tmp thermo[var]["units"] = spec.get("unit") diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 8bd960e..2e50317 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -263,6 +263,7 @@ def label_lines(ax: Axes, lines: Any, labels: np.ndarray, offset: float = 0, **k kwargs["color"] = lines.get_color()[0] for i, line in enumerate(lines.get_segments()): + assert not labels[i].ndim > 1 label = int(labels[i]) label_line(ax, str(label), line, align=True, offset=offset, **kwargs) diff --git a/create_graphics.py b/create_graphics.py index 04bd134..43a536f 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -68,8 +68,8 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): Generate arguments for parallel processing of Skew T graphics, and generate a pool of workers to complete the tasks. """ - - args = [(cla, fhr, grib_path, site, workdir) for site in cla.sites] + vspec = utils.cfgrib_spec(cla.spec["temp"]["ua"], cla.model_name) + args = [(cla, fhr, grib_path, site, vspec, workdir) for site in cla.sites] print(f"Queueing {len(args)} Skew Ts") with Pool(processes=cla.nprocs) as pool: @@ -108,7 +108,6 @@ def create_maps( grib_paths, level, model, - spec, variable, workdir, tile, diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index 65951b3..d8eb419 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -221,7 +221,7 @@ def test_uppdata_valid_dt(uppdata_obj): assert uppdata_obj.valid_dt == datetime(2025, 10, 6, 15) -def test_uppdata_vector_magnitude_cfkeys(prsfile, spec): +def test_uppdata_vector_magnitude(prsfile, spec): ds = gribfile.GribFile( prsfile, cfgrib_config={ @@ -238,62 +238,10 @@ def test_uppdata_vector_magnitude_cfkeys(prsfile, spec): short_name="u", spec=spec, ) - vm = fd.vector_magnitude( - field1=fd.ds, cfkeys={"shortName": "v", "typeOfLevel": "isobaricInhPa"} - ) + vm = fd.vector_magnitude(field1=fd.ds.u, field2_id="v_250mb") assert not np.array_equal(vm, ds.contents.u) -def test_uppdata_vector_magnitude_field2_id(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "u", - "typeOfLevel": "isobaricInhPa", - "level": 250, - }, - ) - fd = ConcreteUPPData( - ds=ds.contents, - fhr=15, - grib_path=prsfile, - level="250mb", - short_name="u", - spec=spec, - ) - vm = fd.vector_magnitude(field1=fd.ds, field2_id="v_250mb") - assert not np.array_equal(vm, ds.contents.u) - - -def test_uppdata_vector_magnitude_no_field2_args(uppdata_obj): - with raises(errors.ArgumentError): - uppdata_obj.vector_magnitude(uppdata_obj.ds) - - -def test_uppdata_vector_magnitude_options_equal(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "u", - "typeOfLevel": "isobaricInhPa", - "level": 250, - }, - ) - fd = ConcreteUPPData( - ds=ds.contents, - fhr=15, - grib_path=prsfile, - level="250mb", - short_name="u", - spec=spec, - ) - vm_cfkeys = fd.vector_magnitude( - field1=fd.ds, cfkeys={"shortName": "v", "typeOfLevel": "isobaricInhPa"} - ) - vm_field2 = fd.vector_magnitude(field1=fd.ds, field2_id="v_250mb") - assert np.array_equal(vm_cfkeys, vm_field2) - - def test_uppdata_vspec(uppdata_obj): expected = { "cfgrib": {"shortName": "t", "typeOfLevel": "hybrid"}, @@ -474,13 +422,13 @@ def test_fielddata_supercooled_liquid_water(natfile, spec): def test_fielddata_ticks_default(fielddata_obj): - assert fielddata_obj.ticks() == 10 + assert fielddata_obj.ticks == 10 def test_fielddata_ticks_in_vspec(fielddata_obj): ticks = 22 fielddata_obj.vspec["ticks"] = ticks - assert fielddata_obj.ticks() == ticks + assert fielddata_obj.ticks == ticks def test_fielddata_units_default(fielddata_obj): From 9d5647d5db41bf4f4c70d6a17ba2641d9a14b59e Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 29 Oct 2025 20:19:18 -0600 Subject: [PATCH 27/98] Probably should add the tests. --- tests/test_figure_builders.py | 180 ++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 tests/test_figure_builders.py diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py new file mode 100644 index 0000000..c794964 --- /dev/null +++ b/tests/test_figure_builders.py @@ -0,0 +1,180 @@ +import gc +import tracemalloc +from argparse import Namespace +from datetime import datetime +from unittest.mock import call, patch + +import numpy as np +from pytest import fixture + +from adb_graphics import figure_builders, utils +from adb_graphics.datahandler import gribdata, gribfile + + +@fixture +def hrrr_data(prsfile): + return gribfile.GribFile( + prsfile, + cfgrib_config={ + "shortName": "t", + "typeOfLevel": "surface", + }, + ) + + +@fixture +def fielddata_obj(hrrr_data, prsfile, spec): + return gribdata.FieldData( + ds=hrrr_data.contents, + fhr=15, + grib_path=prsfile, + level="cref", + short_name="temp", + spec=spec, + ) + + +@fixture +def parallel_maps_args(prsfile, spec, tmp_path): + cla = Namespace( + **{ # noqa: PIE804 + "ens_size": 0, + "graphic_type": "maps", + "img_res": 72, + "model_name": "hrrr", + "specs": spec, + } + ) + return { + "cla": cla, + "fhr": 15, + "grib_path": prsfile, + "level": "sfc", + "variable": "temp", + "workdir": tmp_path, + } + + +@fixture +def parallel_skewt_args(natfile, spec, tmp_path): + cla = Namespace( + **{ # noqa: PIE804 + "file_type": "nat", + "img_res": 72, + "max_plev": 100, + "model_name": "hrrr", + "start_time": datetime(2025, 10, 6, 0), + "specs": spec, + } + ) + return { + "cla": cla, + "fhr": 15, + "grib_path": natfile, + "site": " DNR 23062 72469 39.77 104.88 1611 Denver, CO", + "workdir": tmp_path, + } + + +@fixture +def spec(spec_file): + return utils.load_yaml(spec_file) + + +def test_add_obs_panel(fielddata_obj, spec): + fig, ax = figure_builders.set_figure("hrrr", "enspanel", "full") + # Overwriting this explicitly since the cfgrib should indefinitely come from the model data. + spec["cref"]["obs"]["cfgrib"] = spec["1ref"]["1000m"]["cfgrib"]["hrrr"] + args = { + "ax": ax[8], + "model_name": "hrrr", + "obs_file": fielddata_obj.grib_path, # fake it with model data + "proj_info": fielddata_obj.grid_info(), + "spec": spec, + "short_name": "cref", + "tile": "full", + } + dm = figure_builders.add_obs_panel(**args) + assert dm.figure == fig + assert np.array_equal(dm.levels, np.arange(5, 76, 5)) + + +def test_parallel_maps(parallel_maps_args, tmp_path): + figure_builders.parallel_maps(**parallel_maps_args) + assert (tmp_path / "temp_full_sfc_f015.png").is_file() + + +def test_parallel_maps_enspanel(parallel_maps_args, tmp_path): + parallel_maps_args["cla"].ens_size = 9 + parallel_maps_args["cla"].graphic_type = "enspanel" + parallel_maps_args["cla"].obs_file_path = tmp_path + parallel_maps_args["cla"].specs["temp"]["sfc"]["include_obs"] = True + + with ( + patch.object(figure_builders, "MapFields") as fields, + patch.object(figure_builders, "Map") as m, + patch.object(figure_builders, "MultiPanelDataMap") as mpdm, + patch.object(figure_builders, "add_obs_panel") as aop, + ): + mpdm_calls = [ + call( + **{ # noqa: PIE804 + "map_fields": fields(), + "map_": m(), + "member": mem, + "model_name": "hrrr", + "last_panel": mem == 9, + } + ) + for mem in [0, 1, 2, 3, 0, 4, 5, 6, 0, 7, 8, 9] + ] + figure_builders.parallel_maps(**parallel_maps_args) + assert mpdm.call_args_list == mpdm_calls + call.title().assert_called_once() + call.add_logo().assert_called_once() + aop.assert_called_once() + assert (tmp_path / "temp_full_sfc_f015.png").is_file() + + +def test_parallel_maps_mem_leak(parallel_maps_args): + gc.collect() + tracemalloc.start() + snapshot_before = tracemalloc.take_snapshot() + figure_builders.parallel_maps(**parallel_maps_args) + snapshot_after = tracemalloc.take_snapshot() + gc.collect() + tracemalloc.stop() + # Compare memory usage + stats_diff = snapshot_after.compare_to(snapshot_before, "lineno") + total_diff_mb = sum(stat.size_diff for stat in stats_diff) / (1024 * 1024) + assert total_diff_mb < 92 # Appropriate size when test was written + + +def test_parallel_skewt(parallel_skewt_args, tmp_path): + figure_builders.parallel_skewt(**parallel_skewt_args) + assert (tmp_path / "DNR_72469_skewt_f015.png").is_file() + assert (tmp_path / "DNR.72469.skewt.2025100600_f015.csv").is_file() + + +def test_set_figure_enspanel_full(): + fig, ax = figure_builders.set_figure("hrrr", "enspanel", "full") + assert len(ax) == 12 + assert list(fig.get_size_inches()) == [20.0, 10.0] + + +def test_set_figure_enspanel_other(): + fig, ax = figure_builders.set_figure("hrrr", "enspanel", "other") + assert len(ax) == 12 + assert list(fig.get_size_inches()) == [20.0, 16.0] + + +def test_set_figure_enspanel_se(): + fig, ax = figure_builders.set_figure("hrrr", "enspanel", "SE") + assert len(ax) == 12 + assert list(fig.get_size_inches()) == [20.0, 19.0] + + +def test_set_figure_maps_full(): + fig, ax = figure_builders.set_figure("hrrr", "maps", "full") + assert len(ax) == 1 + assert list(fig.get_size_inches()) == [10.0, 10.0] From c691a6441cfb3fe57f64ec660c676ff4f8019279 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 31 Oct 2025 16:19:08 -0600 Subject: [PATCH 28/98] WIP for utils tests. --- adb_graphics/utils.py | 19 ++---- tests/test_common.py | 2 +- tests/test_utils.py | 145 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 15 deletions(-) create mode 100644 tests/test_utils.py diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 2e50317..fe68ee5 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -1,4 +1,3 @@ -# pylint: disable=invalid-name """ A set of generic utilities available to all the adb_graphics components. """ @@ -11,6 +10,7 @@ import time from collections.abc import Callable from datetime import datetime, timedelta +from importlib import import_module from importlib.util import find_spec from math import atan2, degrees from multiprocessing import Process @@ -41,7 +41,7 @@ def create_zip(files_to_zip: list[str], zipf: Path | str): while True: if not lock_file.exists(): # Create the lock - fd = lock_file.open() + lock_file.touch() print(f"Writing to zip file {zipf} for files like: {files_to_zip[0][-10:]}") cmd = f"zip -uj {zipf} {' '.join(files_to_zip)}" @@ -63,7 +63,6 @@ def create_zip(files_to_zip: list[str], zipf: Path | str): Path(file_to_zip).unlink(missing_ok=True) finally: # Remove the lock - fd.close() lock_file.unlink(missing_ok=True) break # Wait before trying to obtain the lock on the file @@ -110,11 +109,7 @@ def get_func(val: str): dictionary of functions: how to load without converting to strings." """ - if "." in val: - module_name, fun_name = val.rsplit(".", 1) - else: - module_name = "__main__" - fun_name = val + module_name, fun_name = val.rsplit(".", 1) mod_spec = find_spec(module_name, package="adb_graphics") if mod_spec is None: @@ -123,11 +118,7 @@ def get_func(val: str): msg = "Could not find {module_name} in current environment." raise ValueError(msg) - try: - __import__(mod_spec.name) - except ImportError: - print(f"Could not load {module_name} while trying to locate function in get_func") - raise + import_module(mod_spec.name) module = sys.modules[mod_spec.name] return getattr(module, fun_name) @@ -160,7 +151,7 @@ def arange_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> Any: # noqa return np.arange(*[float(n.value) for n in node.value]) -def load_yaml(config: Path | str): +def load_yaml(config: Path | str) -> YAMLConfig: yaml.add_constructor("!join_ranges", join_ranges, Loader=uw_yaml_loader()) yaml.add_constructor("!arange", arange_constructor, Loader=uw_yaml_loader()) return YAMLConfig(config) diff --git a/tests/test_common.py b/tests/test_common.py index f5b5cb2..1311275 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -126,7 +126,7 @@ def check_kwargs(self, accepted_args, kwargs): assert self.is_a_key(short_name) if lev: - assert self.cfg.get(short_name).get(lev) is not None + assert self.cfg.get(short_name, {}).get(lev) is not None for arg in args: assert arg in accepted_args diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..5bc1ea2 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,145 @@ +import signal +from contextlib import contextmanager +from datetime import datetime + +import numpy as np +import yaml +from pytest import mark, raises + +from adb_graphics import conversions, utils + + +@contextmanager +def timeout(duration): + def timeout_handler(signum, frame): # noqa: ARG001 + raise TimeoutError + + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(duration) + try: + yield + finally: + signal.alarm(0) + + +def test_cfgrib_spec_no_model(): + config = {"foo": {"bar": "baz"}} + answer = utils.cfgrib_spec(config, "model") + assert answer == config + + +def test_cfgrib_spec_model(): + config = {"model": {"foo": {"bar": "baz"}}} + answer = utils.cfgrib_spec(config, "model") + assert answer == config["model"] + + +def test_create_zip(tmp_path): + afile = tmp_path / "a.txt" + bfile = tmp_path / "b.txt" + afile.touch() + bfile.touch() + zipf = tmp_path / "file.zip" + utils.create_zip([str(f) for f in [afile, bfile]], zipf) + assert zipf.is_file() + assert not afile.is_file() + assert not bfile.is_file() + + +def test_create_zip_locked(tmp_path): + afile = tmp_path / "a.txt" + bfile = tmp_path / "b.txt" + afile.touch() + bfile.touch() + zipf = tmp_path / "file.zip" + zipf_lock = tmp_path / "file.zip._lock" + zipf_lock.touch() + with raises(TimeoutError), timeout(3): + utils.create_zip([str(f) for f in [afile, bfile]], zipf) + assert not zipf.is_file() + assert afile.is_file() + assert bfile.is_file() + + +@mark.parametrize( + ("arg", "expected"), + [ + ([1], [1]), + ([1, 9], list(range(1, 10))), + ([1, 9, 3], list(range(1, 10, 3))), + ([3, 4, 7, 19], [3, 4, 7, 19]), + ], +) +def test_fhr_list(arg, expected): + assert utils.fhr_list(arg) == expected + + +@mark.parametrize( + ("arg", "expected"), + [ + (datetime(2025, 10, 31, 12), "2025103112"), + (datetime(2025, 10, 31), "2025103100"), + (datetime(2025, 10, 31, 12, 1, 2), "2025103112"), + ], +) +def test_from_datetime(arg, expected): + assert utils.from_datetime(arg) == expected + + +@mark.parametrize( + ("arg", "expected"), + [ + ("conversions.to_micro", conversions.to_micro), + ("utils.join_ranges", utils.join_ranges), + ], +) +def test_get_func(arg, expected): + assert utils.get_func(arg) == expected + + +def test_get_func_undefined(): + with raises(ValueError): # noqa: PT011 + utils.get_func("foo.bar") + + +def test_join_ranges(): + yaml_str = """ + a: !join_ranges [[0, 10, 0.1], [10, 51, 1.0]] + b: !join_ranges [[0, 5], [4]] + c: !join_ranges [[2, 17, 7]] + """ + yaml.add_constructor("!join_ranges", utils.join_ranges, Loader=yaml.SafeLoader) + + d = yaml.safe_load(yaml_str) + assert np.array_equal(d["a"], np.concatenate([np.arange(0, 10, 0.1), np.arange(10, 51, 1.0)])) + assert np.array_equal(d["b"], np.asarray([0, 1, 2, 3, 4, 0, 1, 2, 3])) + assert np.array_equal(d["c"], np.asarray([2, 9, 16])) + + +def test_arange_constructor(): + yaml_str = """ + a: !arange [0, 10, 0.1] + b: !arange [0, 5] + c: !arange [2, 17, 7] + """ + yaml.add_constructor("!arange", utils.arange_constructor, Loader=yaml.SafeLoader) + + d = yaml.safe_load(yaml_str) + assert np.array_equal(d["a"], np.arange(0, 10, 0.1)) + assert np.array_equal(d["b"], np.asarray([0, 1, 2, 3, 4])) + assert np.array_equal(d["c"], np.asarray([2, 9, 16])) + + +def test_load_yaml(tmp_path): + yaml_str = """ + a: !float '{{ c[1] - 2 }}' + b: !join_ranges [[0, 5], [4]] + c: !arange [2, 17, 7] + """ + cfg = tmp_path / "config.yaml" + cfg.write_text(yaml_str) + d = utils.load_yaml(cfg) + d.dereference() + assert d["a"] == 7 + assert np.array_equal(d["b"], np.asarray([0, 1, 2, 3, 4, 0, 1, 2, 3])) + assert np.array_equal(d["c"], np.asarray([2, 9, 16])) From f5b9676275696165645ca6e2c41cf954e6c06974 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 3 Nov 2025 17:21:05 -0700 Subject: [PATCH 29/98] utils tested. --- adb_graphics/datahandler/gribfile.py | 6 +- adb_graphics/figures/skewt.py | 113 +- adb_graphics/utils.py | 200 +-- tests/data/wgrib2_submsg1.txt | 1847 ++++++++++++++++++++++++++ tests/test_utils.py | 234 ++++ 5 files changed, 2241 insertions(+), 159 deletions(-) create mode 100644 tests/data/wgrib2_submsg1.txt diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 88f752e..4e37a2b 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -29,7 +29,11 @@ def _load(self) -> xr.Dataset: self.filename, engine="cfgrib", lock=False, - backend_kwargs=({"filter_by_keys": self.cfgrib_config}), + backend_kwargs=( + { + "filter_by_keys": self.cfgrib_config, + } + ), ) diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index a1eb25c..cb1773a 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -5,8 +5,9 @@ """ from functools import cached_property +from math import atan2, degrees from pathlib import Path -from typing import TYPE_CHECKING, TypedDict +from typing import TYPE_CHECKING, Any, TypedDict import matplotlib.font_manager as fm import matplotlib.lines as mlines @@ -22,7 +23,7 @@ from mpl_toolkits.axes_grid1.inset_locator import inset_axes from xarray import DataArray, Dataset -from adb_graphics import errors, utils +from adb_graphics import errors from adb_graphics.datahandler import gribdata if TYPE_CHECKING: @@ -508,7 +509,7 @@ def _setup_diagram(self): linestyles="solid", linewidth=0.7, ) - utils.label_lines( + label_lines( ax=skew.ax, lines=skew.dry_adiabats, labels=dry_adiabats.magnitude, @@ -525,7 +526,7 @@ def _setup_diagram(self): linestyles="solid", linewidth=0.7, ) - utils.label_lines( + label_lines( ax=skew.ax, lines=skew.moist_adiabats, labels=moist_adiabats.magnitude, @@ -541,7 +542,7 @@ def _setup_diagram(self): linestyles=(0, (5, 10)), linewidth=0.7, ) - utils.label_lines( + label_lines( ax=skew.ax, lines=skew.mixing_lines, labels=mixing_lines * 1000, @@ -690,3 +691,105 @@ def _title(self): loc="center", position=(-2.5, 1.0), ) + + +def label_line(ax: Axes, label: str, segment: np.ndarray, **kwargs): + """ + Label a single line with line2D label data. + + Input: + + ax the SkewT object axis + label label to be used for the current line + segment a list (array) of values for the current line + + Key Word Arguments + + align optional bool to enable the rotation of the label to line angle + end the end of the line at which to put the label. 'bottom' or 'top' + offset index to use for the "end" of the array + + Any kwargs accepted by matplotlib's text box. + """ + + # Strip non-text-box key word arguments and set default if they don't exist + align = kwargs.pop("align", True) + end = kwargs.pop("end", "bottom") + offset = kwargs.pop("offset", 0) + + # Label location + if end == "bottom": + x, y = segment[0 + offset, :] + ip = 1 + offset + elif end == "top": + x, y = segment[-1 - offset, :] + ip = -1 - offset + + if align: + # Compute the slope + dx = segment[ip, 0] - segment[ip - 1, 0] + dy = segment[ip, 1] - segment[ip - 1, 1] + ang = degrees(atan2(dy, dx)) + + # Transform to screen co-ordinates + pt = np.array([x, y]).reshape((1, 2)) + trans_angle = ax.transData.transform_angles(np.array((ang,)), pt)[0] + + if end == "top": + trans_angle -= 180 + + else: + trans_angle = 0 + + # Set a bunch of keyword arguments + if ("horizontalalignment" not in kwargs) and ("ha" not in kwargs): + kwargs["ha"] = "center" + + if ("verticalalignment" not in kwargs) and ("va" not in kwargs): + kwargs["va"] = "center" + + if "backgroundcolor" not in kwargs: + kwargs["backgroundcolor"] = ax.get_facecolor() + + if "clip_on" not in kwargs: + kwargs["clip_on"] = True + + if "fontsize" not in kwargs: + kwargs["fontsize"] = "larger" + + if "fontweight" not in kwargs: + kwargs["fontweight"] = "bold" + + # Larger value (e.g., 2.0) to move box in front of other diagram elements + if "zorder" not in kwargs: + kwargs["zorder"] = 1.50 + + # Place the text box label on the line. + ax.text(x, y, label, rotation=trans_angle, **kwargs) + + +def label_lines(ax: Axes, lines: Any, labels: np.ndarray, offset: float = 0, **kwargs): + """ + Plots labels on a set of lines from SkewT. + + Input: + + ax the SkewT object axis + lines the SkewT object special lines + labels list of labels to be used + offset index to use for the "end" of the array + + Key Word Arguments + + color line color + + Along with any other kwargs accepted by matplotlib's text box. + """ + + if "color" not in kwargs: + kwargs["color"] = lines.get_color()[0] + + for i, line in enumerate(lines.get_segments()): + assert not labels[i].ndim > 1 + label = int(labels[i]) + label_line(ax, str(label), line, align=True, offset=offset, **kwargs) diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index fe68ee5..5ee79fa 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -2,7 +2,6 @@ A set of generic utilities available to all the adb_graphics components. """ -import argparse import functools import glob import subprocess @@ -12,7 +11,6 @@ from datetime import datetime, timedelta from importlib import import_module from importlib.util import find_spec -from math import atan2, degrees from multiprocessing import Process from pathlib import Path from string import ascii_letters, digits @@ -20,7 +18,6 @@ import numpy as np import yaml -from matplotlib.axes import Axes from uwtools.api.config import YAMLConfig from uwtools.config.support import uw_yaml_loader @@ -52,19 +49,19 @@ def create_zip(files_to_zip: list[str], zipf: Path | str): check=True, shell=True, ) - except: # noqa: E722 - print(f"Error on writing zip file! {sys.exc_info()[0]}") + except Exception as e: count += 1 if count >= retry: - raise + msg = "Error on writing zip file!" + raise RuntimeError(msg) from e else: # Zipping was successful. Remove files that were zipped for file_to_zip in files_to_zip: Path(file_to_zip).unlink(missing_ok=True) + break finally: # Remove the lock lock_file.unlink(missing_ok=True) - break # Wait before trying to obtain the lock on the file time.sleep(5) @@ -157,151 +154,26 @@ def load_yaml(config: Path | str) -> YAMLConfig: return YAMLConfig(config) -def label_line(ax: Axes, label: str, segment: np.ndarray, **kwargs): - """ - Label a single line with line2D label data. - - Input: - - ax the SkewT object axis - label label to be used for the current line - segment a list (array) of values for the current line - - Key Word Arguments - - align optional bool to enable the rotation of the label to line angle - end the end of the line at which to put the label. 'bottom' or 'top' - offset index to use for the "end" of the array - - Any kwargs accepted by matplotlib's text box. - """ - - # Strip non-text-box key word arguments and set default if they don't exist - align = kwargs.pop("align", True) - end = kwargs.pop("end", "bottom") - offset = kwargs.pop("offset", 0) - - # Label location - if end == "bottom": - x, y = segment[0 + offset, :] - ip = 1 + offset - elif end == "top": - x, y = segment[-1 - offset, :] - ip = -1 - offset - - if align: - # Compute the slope - dx = segment[ip, 0] - segment[ip - 1, 0] - dy = segment[ip, 1] - segment[ip - 1, 1] - ang = degrees(atan2(dy, dx)) - - # Transform to screen co-ordinates - pt = np.array([x, y]).reshape((1, 2)) - trans_angle = ax.transData.transform_angles(np.array((ang,)), pt)[0] - - if end == "top": - trans_angle -= 180 - - else: - trans_angle = 0 - - # Set a bunch of keyword arguments - if ("horizontalalignment" not in kwargs) and ("ha" not in kwargs): - kwargs["ha"] = "center" - - if ("verticalalignment" not in kwargs) and ("va" not in kwargs): - kwargs["va"] = "center" - - if "backgroundcolor" not in kwargs: - kwargs["backgroundcolor"] = ax.get_facecolor() - - if "clip_on" not in kwargs: - kwargs["clip_on"] = True - - if "fontsize" not in kwargs: - kwargs["fontsize"] = "larger" - - if "fontweight" not in kwargs: - kwargs["fontweight"] = "bold" - - # Larger value (e.g., 2.0) to move box in front of other diagram elements - if "zorder" not in kwargs: - kwargs["zorder"] = 1.50 - - # Place the text box label on the line. - ax.text(x, y, label, rotation=trans_angle, **kwargs) - - -def label_lines(ax: Axes, lines: Any, labels: np.ndarray, offset: float = 0, **kwargs): - """ - Plots labels on a set of lines from SkewT. - - Input: - - ax the SkewT object axis - lines the SkewT object special lines - labels list of labels to be used - offset index to use for the "end" of the array - - Key Word Arguments - - color line color - - Along with any other kwargs accepted by matplotlib's text box. - """ - - if "color" not in kwargs: - kwargs["color"] = lines.get_color()[0] - - for i, line in enumerate(lines.get_segments()): - assert not labels[i].ndim > 1 - label = int(labels[i]) - label_line(ax, str(label), line, align=True, offset=offset, **kwargs) - - def load_sites(arg: str | Path) -> list[str]: """Check that the sites file exists, and return its contents.""" # Check that the file exists - path = path_exists(arg) + path = Path(arg) + path.exists() with path.open() as sites_file: sites: list[str] = sites_file.readlines() return sites -def uniq_wgrib2_list(inlist: list[str]): - """ - Given a list of wgrib2 output fields, returns a uniq list of fields for - simplifying a grib2 dataset. Uniqueness is defined by the wgrib output from - field 3 (colon delimted) onward, although the original full grib record must - be included in the wgrib2 command below. - """ - - uniq_field_set = set() - uniq_list = [] - for infield in inlist: - infield_info = infield.split(":") - if len(infield_info) <= 3: # noqa: PLR2004 - continue - infield_str = ":".join(infield_info[3:]) - if infield_str not in uniq_field_set: - uniq_list.append(infield) - uniq_field_set.add(infield_str) - - return uniq_list - - -def load_specs(arg: str | Path) -> dict: +def load_specs(arg: str | Path) -> YAMLConfig: """Check to make sure arg file exists. Return its contents.""" spec_file = Path(arg) - assert spec_file.exists() - - specs: dict - with spec_file.open() as fn: - specs = yaml.load(fn, Loader=yaml.Loader) - + if not spec_file.exists(): + msg = f"The spec file {spec_file} does not exist." + raise FileNotFoundError(msg) + specs = load_yaml(spec_file) specs["file"] = spec_file return specs @@ -360,11 +232,26 @@ def path_exists(path: Path | str): ret_path = Path(path) if not ret_path.exists(): msg = f"{path} does not exist!" - raise argparse.ArgumentTypeError(msg) + raise FileNotFoundError(msg) return ret_path +def set_level(level: str, model: str, spec: dict): + nlevel, _ = numeric_level(level=level) + level_info = any( + key + for keys in cfgrib_spec(spec["cfgrib"], model) + for key in ("level", "top", "bottom", "Surface") + if key in keys + ) + if nlevel and not level_info: + if spec["cfgrib"].get(model) is not None: + spec["cfgrib"][model]["level"] = nlevel + else: + spec["cfgrib"]["level"] = nlevel + + def timer(func: Callable): """Decorator function that provides an elapsed time for a method.""" @@ -386,19 +273,26 @@ def to_datetime(string: str): return datetime.strptime(string, "%Y%m%d%H") -def set_level(level: str, model: str, spec: dict): - nlevel, _ = numeric_level(level=level) - level_info = any( - key - for keys in cfgrib_spec(spec["cfgrib"], model) - for key in ("level", "top", "bottom", "Surface") - if key in keys - ) - if nlevel and not level_info: - if spec["cfgrib"].get(model): - spec["cfgrib"][model]["level"] = nlevel - else: - spec["cfgrib"]["level"] = nlevel +def uniq_wgrib2_list(inlist: list[str]): + """ + Given a list of wgrib2 output fields, returns a uniq list of fields for + simplifying a grib2 dataset. Uniqueness is defined by the wgrib output from + field 3 (colon delimted) onward, although the original full grib record must + be included in the wgrib2 command below. + """ + + uniq_field_set = set() + uniq_list = [] + for infield in inlist: + infield_info = infield.split(":") + if len(infield_info) <= 3: # noqa: PLR2004 + continue + infield_str = ":".join(infield_info[3:]) + if infield_str not in uniq_field_set: + uniq_list.append(infield) + uniq_field_set.add(infield_str) + + return uniq_list @timer diff --git a/tests/data/wgrib2_submsg1.txt b/tests/data/wgrib2_submsg1.txt new file mode 100644 index 0000000..53d3b76 --- /dev/null +++ b/tests/data/wgrib2_submsg1.txt @@ -0,0 +1,1847 @@ +1:0:d=2025100600:PRES:1 hybrid level:16 hour fcst: +2:2259236:d=2025100600:CLMR:1 hybrid level:16 hour fcst: +3:2275399:d=2025100600:CIMIXR:1 hybrid level:16 hour fcst: +4:2275766:d=2025100600:RWMR:1 hybrid level:16 hour fcst: +5:2589683:d=2025100600:SNMR:1 hybrid level:16 hour fcst: +6:2590771:d=2025100600:GRLE:1 hybrid level:16 hour fcst: +7:2591646:d=2025100600:NCONCD:1 hybrid level:16 hour fcst: +8:2612852:d=2025100600:NCCICE:1 hybrid level:16 hour fcst: +9:2613203:d=2025100600:SPNCR:1 hybrid level:16 hour fcst: +10:2928563:d=2025100600:PMTF:1 hybrid level:16 hour fcst: +11:3580683:d=2025100600:PMTC:1 hybrid level:16 hour fcst: +12:4221636:d=2025100600:FRACCC:1 hybrid level:16 hour fcst: +13:4271096:d=2025100600:HGT:1 hybrid level:16 hour fcst: +14:6575263:d=2025100600:TMP:1 hybrid level:16 hour fcst: +15:7737704:d=2025100600:SPFH:1 hybrid level:16 hour fcst: +16:9287179:d=2025100600:UGRD:1 hybrid level:16 hour fcst: +17:10495303:d=2025100600:VGRD:1 hybrid level:16 hour fcst: +18:11641941:d=2025100600:VVEL:1 hybrid level:16 hour fcst: +19:12856890:d=2025100600:TKE:1 hybrid level:16 hour fcst: +20:13469298:d=2025100600:MASSDEN:1 hybrid level:16 hour fcst: +21:14223354:d=2025100600:PRES:2 hybrid level:16 hour fcst: +22:16482381:d=2025100600:CLMR:2 hybrid level:16 hour fcst: +23:16498523:d=2025100600:CIMIXR:2 hybrid level:16 hour fcst: +24:16498892:d=2025100600:RWMR:2 hybrid level:16 hour fcst: +25:16814928:d=2025100600:SNMR:2 hybrid level:16 hour fcst: +26:16816097:d=2025100600:GRLE:2 hybrid level:16 hour fcst: +27:16817048:d=2025100600:NCONCD:2 hybrid level:16 hour fcst: +28:16838583:d=2025100600:NCCICE:2 hybrid level:16 hour fcst: +29:16838933:d=2025100600:SPNCR:2 hybrid level:16 hour fcst: +30:17158611:d=2025100600:PMTF:2 hybrid level:16 hour fcst: +31:17803941:d=2025100600:PMTC:2 hybrid level:16 hour fcst: +32:18444712:d=2025100600:FRACCC:2 hybrid level:16 hour fcst: +33:18493071:d=2025100600:HGT:2 hybrid level:16 hour fcst: +34:20867924:d=2025100600:TMP:2 hybrid level:16 hour fcst: +35:22010707:d=2025100600:SPFH:2 hybrid level:16 hour fcst: +36:23517214:d=2025100600:UGRD:2 hybrid level:16 hour fcst: +37:24740523:d=2025100600:VGRD:2 hybrid level:16 hour fcst: +38:25884357:d=2025100600:VVEL:2 hybrid level:16 hour fcst: +39:27311692:d=2025100600:TKE:2 hybrid level:16 hour fcst: +40:27873402:d=2025100600:MASSDEN:2 hybrid level:16 hour fcst: +41:28614767:d=2025100600:PRES:3 hybrid level:16 hour fcst: +42:30872983:d=2025100600:CLMR:3 hybrid level:16 hour fcst: +43:30893270:d=2025100600:CIMIXR:3 hybrid level:16 hour fcst: +44:30893645:d=2025100600:RWMR:3 hybrid level:16 hour fcst: +45:31214039:d=2025100600:SNMR:3 hybrid level:16 hour fcst: +46:31215405:d=2025100600:GRLE:3 hybrid level:16 hour fcst: +47:31216522:d=2025100600:NCONCD:3 hybrid level:16 hour fcst: +48:31244871:d=2025100600:NCCICE:3 hybrid level:16 hour fcst: +49:31245225:d=2025100600:SPNCR:3 hybrid level:16 hour fcst: +50:31572258:d=2025100600:PMTF:3 hybrid level:16 hour fcst: +51:32728250:d=2025100600:PMTC:3 hybrid level:16 hour fcst: +52:33358261:d=2025100600:FRACCC:3 hybrid level:16 hour fcst: +53:33432292:d=2025100600:HGT:3 hybrid level:16 hour fcst: +54:35834968:d=2025100600:TMP:3 hybrid level:16 hour fcst: +55:36965853:d=2025100600:SPFH:3 hybrid level:16 hour fcst: +56:38452065:d=2025100600:UGRD:3 hybrid level:16 hour fcst: +57:39658610:d=2025100600:VGRD:3 hybrid level:16 hour fcst: +58:40793223:d=2025100600:VVEL:3 hybrid level:16 hour fcst: +59:42312783:d=2025100600:TKE:3 hybrid level:16 hour fcst: +60:42870505:d=2025100600:MASSDEN:3 hybrid level:16 hour fcst: +61:43612659:d=2025100600:PRES:4 hybrid level:16 hour fcst: +62:45870590:d=2025100600:CLMR:4 hybrid level:16 hour fcst: +63:45900384:d=2025100600:CIMIXR:4 hybrid level:16 hour fcst: +64:45900766:d=2025100600:RWMR:4 hybrid level:16 hour fcst: +65:46227687:d=2025100600:SNMR:4 hybrid level:16 hour fcst: +66:46229485:d=2025100600:GRLE:4 hybrid level:16 hour fcst: +67:46230869:d=2025100600:NCONCD:4 hybrid level:16 hour fcst: +68:46274552:d=2025100600:NCCICE:4 hybrid level:16 hour fcst: +69:46274902:d=2025100600:SPNCR:4 hybrid level:16 hour fcst: +70:46736460:d=2025100600:PMTF:4 hybrid level:16 hour fcst: +71:47898688:d=2025100600:PMTC:4 hybrid level:16 hour fcst: +72:48536911:d=2025100600:FRACCC:4 hybrid level:16 hour fcst: +73:48664654:d=2025100600:HGT:4 hybrid level:16 hour fcst: +74:51105192:d=2025100600:TMP:4 hybrid level:16 hour fcst: +75:52219055:d=2025100600:SPFH:4 hybrid level:16 hour fcst: +76:53704102:d=2025100600:UGRD:4 hybrid level:16 hour fcst: +77:54870906:d=2025100600:VGRD:4 hybrid level:16 hour fcst: +78:55988172:d=2025100600:VVEL:4 hybrid level:16 hour fcst: +79:57592013:d=2025100600:TKE:4 hybrid level:16 hour fcst: +80:58155898:d=2025100600:MASSDEN:4 hybrid level:16 hour fcst: +81:58909005:d=2025100600:PRES:5 hybrid level:16 hour fcst: +82:61167057:d=2025100600:CLMR:5 hybrid level:16 hour fcst: +83:61215063:d=2025100600:CIMIXR:5 hybrid level:16 hour fcst: +84:61215444:d=2025100600:RWMR:5 hybrid level:16 hour fcst: +85:61552899:d=2025100600:SNMR:5 hybrid level:16 hour fcst: +86:61555598:d=2025100600:GRLE:5 hybrid level:16 hour fcst: +87:61557426:d=2025100600:NCONCD:5 hybrid level:16 hour fcst: +88:61631176:d=2025100600:NCCICE:5 hybrid level:16 hour fcst: +89:61631552:d=2025100600:SPNCR:5 hybrid level:16 hour fcst: +90:62113501:d=2025100600:PMTF:5 hybrid level:16 hour fcst: +91:63295398:d=2025100600:PMTC:5 hybrid level:16 hour fcst: +92:63965202:d=2025100600:FRACCC:5 hybrid level:16 hour fcst: +93:64205990:d=2025100600:HGT:5 hybrid level:16 hour fcst: +94:66682669:d=2025100600:TMP:5 hybrid level:16 hour fcst: +95:67776334:d=2025100600:SPFH:5 hybrid level:16 hour fcst: +96:69285663:d=2025100600:UGRD:5 hybrid level:16 hour fcst: +97:70402945:d=2025100600:VGRD:5 hybrid level:16 hour fcst: +98:71504531:d=2025100600:VVEL:5 hybrid level:16 hour fcst: +99:73179084:d=2025100600:TKE:5 hybrid level:16 hour fcst: +100:73743682:d=2025100600:MASSDEN:5 hybrid level:16 hour fcst: +101:74536683:d=2025100600:PRES:6 hybrid level:16 hour fcst: +102:76795229:d=2025100600:CLMR:6 hybrid level:16 hour fcst: +103:76873140:d=2025100600:CIMIXR:6 hybrid level:16 hour fcst: +104:76873637:d=2025100600:RWMR:6 hybrid level:16 hour fcst: +105:77221811:d=2025100600:SNMR:6 hybrid level:16 hour fcst: +106:77225972:d=2025100600:GRLE:6 hybrid level:16 hour fcst: +107:77228537:d=2025100600:NCONCD:6 hybrid level:16 hour fcst: +108:77310951:d=2025100600:NCCICE:6 hybrid level:16 hour fcst: +109:77311418:d=2025100600:SPNCR:6 hybrid level:16 hour fcst: +110:77816658:d=2025100600:PMTF:6 hybrid level:16 hour fcst: +111:79055142:d=2025100600:PMTC:6 hybrid level:16 hour fcst: +112:79750834:d=2025100600:FRACCC:6 hybrid level:16 hour fcst: +113:80148145:d=2025100600:HGT:6 hybrid level:16 hour fcst: +114:82656766:d=2025100600:TMP:6 hybrid level:16 hour fcst: +115:83736727:d=2025100600:SPFH:6 hybrid level:16 hour fcst: +116:85292207:d=2025100600:UGRD:6 hybrid level:16 hour fcst: +117:86379379:d=2025100600:VGRD:6 hybrid level:16 hour fcst: +118:87474107:d=2025100600:VVEL:6 hybrid level:16 hour fcst: +119:89194151:d=2025100600:TKE:6 hybrid level:16 hour fcst: +120:89745576:d=2025100600:MASSDEN:6 hybrid level:16 hour fcst: +121:90558716:d=2025100600:PRES:7 hybrid level:16 hour fcst: +122:92818049:d=2025100600:CLMR:7 hybrid level:16 hour fcst: +123:92912032:d=2025100600:CIMIXR:7 hybrid level:16 hour fcst: +124:92912949:d=2025100600:RWMR:7 hybrid level:16 hour fcst: +125:93265192:d=2025100600:SNMR:7 hybrid level:16 hour fcst: +126:93271327:d=2025100600:GRLE:7 hybrid level:16 hour fcst: +127:93273063:d=2025100600:NCONCD:7 hybrid level:16 hour fcst: +128:93373552:d=2025100600:NCCICE:7 hybrid level:16 hour fcst: +129:93374596:d=2025100600:SPNCR:7 hybrid level:16 hour fcst: +130:93770961:d=2025100600:PMTF:7 hybrid level:16 hour fcst: +131:95032830:d=2025100600:PMTC:7 hybrid level:16 hour fcst: +132:95754882:d=2025100600:FRACCC:7 hybrid level:16 hour fcst: +133:96311106:d=2025100600:HGT:7 hybrid level:16 hour fcst: +134:98842296:d=2025100600:TMP:7 hybrid level:16 hour fcst: +135:99918570:d=2025100600:SPFH:7 hybrid level:16 hour fcst: +136:101499179:d=2025100600:UGRD:7 hybrid level:16 hour fcst: +137:102583109:d=2025100600:VGRD:7 hybrid level:16 hour fcst: +138:103681320:d=2025100600:VVEL:7 hybrid level:16 hour fcst: +139:105434623:d=2025100600:TKE:7 hybrid level:16 hour fcst: +140:105938506:d=2025100600:MASSDEN:7 hybrid level:16 hour fcst: +141:107161342:d=2025100600:PRES:8 hybrid level:16 hour fcst: +142:109419689:d=2025100600:CLMR:8 hybrid level:16 hour fcst: +143:109547928:d=2025100600:CIMIXR:8 hybrid level:16 hour fcst: +144:109549399:d=2025100600:RWMR:8 hybrid level:16 hour fcst: +145:109907635:d=2025100600:SNMR:8 hybrid level:16 hour fcst: +146:109916027:d=2025100600:GRLE:8 hybrid level:16 hour fcst: +147:109920926:d=2025100600:NCONCD:8 hybrid level:16 hour fcst: +148:110056714:d=2025100600:NCCICE:8 hybrid level:16 hour fcst: +149:110057889:d=2025100600:SPNCR:8 hybrid level:16 hour fcst: +150:110465761:d=2025100600:PMTF:8 hybrid level:16 hour fcst: +151:111740944:d=2025100600:PMTC:8 hybrid level:16 hour fcst: +152:112484037:d=2025100600:FRACCC:8 hybrid level:16 hour fcst: +153:113135608:d=2025100600:HGT:8 hybrid level:16 hour fcst: +154:115680851:d=2025100600:TMP:8 hybrid level:16 hour fcst: +155:116762737:d=2025100600:SPFH:8 hybrid level:16 hour fcst: +156:118340814:d=2025100600:UGRD:8 hybrid level:16 hour fcst: +157:119429428:d=2025100600:VGRD:8 hybrid level:16 hour fcst: +158:120533965:d=2025100600:VVEL:8 hybrid level:16 hour fcst: +159:122264667:d=2025100600:TKE:8 hybrid level:16 hour fcst: +160:122691241:d=2025100600:MASSDEN:8 hybrid level:16 hour fcst: +161:124121792:d=2025100600:PRES:9 hybrid level:16 hour fcst: +162:126375635:d=2025100600:CLMR:9 hybrid level:16 hour fcst: +163:126550943:d=2025100600:CIMIXR:9 hybrid level:16 hour fcst: +164:126552635:d=2025100600:RWMR:9 hybrid level:16 hour fcst: +165:126911932:d=2025100600:SNMR:9 hybrid level:16 hour fcst: +166:126923047:d=2025100600:GRLE:9 hybrid level:16 hour fcst: +167:126929788:d=2025100600:NCONCD:9 hybrid level:16 hour fcst: +168:127103679:d=2025100600:NCCICE:9 hybrid level:16 hour fcst: +169:127105009:d=2025100600:SPNCR:9 hybrid level:16 hour fcst: +170:127522975:d=2025100600:PMTF:9 hybrid level:16 hour fcst: +171:128784043:d=2025100600:PMTC:9 hybrid level:16 hour fcst: +172:129539853:d=2025100600:FRACCC:9 hybrid level:16 hour fcst: +173:130171588:d=2025100600:HGT:9 hybrid level:16 hour fcst: +174:132727896:d=2025100600:TMP:9 hybrid level:16 hour fcst: +175:133812786:d=2025100600:SPFH:9 hybrid level:16 hour fcst: +176:135370771:d=2025100600:UGRD:9 hybrid level:16 hour fcst: +177:136451528:d=2025100600:VGRD:9 hybrid level:16 hour fcst: +178:137544390:d=2025100600:VVEL:9 hybrid level:16 hour fcst: +179:139252359:d=2025100600:TKE:9 hybrid level:16 hour fcst: +180:139578372:d=2025100600:MASSDEN:9 hybrid level:16 hour fcst: +181:141196166:d=2025100600:PRES:10 hybrid level:16 hour fcst: +182:143425289:d=2025100600:CLMR:10 hybrid level:16 hour fcst: +183:143586309:d=2025100600:CIMIXR:10 hybrid level:16 hour fcst: +184:143587576:d=2025100600:RWMR:10 hybrid level:16 hour fcst: +185:143928891:d=2025100600:SNMR:10 hybrid level:16 hour fcst: +186:143943503:d=2025100600:GRLE:10 hybrid level:16 hour fcst: +187:143948355:d=2025100600:NCONCD:10 hybrid level:16 hour fcst: +188:144098712:d=2025100600:NCCICE:10 hybrid level:16 hour fcst: +189:144100256:d=2025100600:SPNCR:10 hybrid level:16 hour fcst: +190:144493630:d=2025100600:PMTF:10 hybrid level:16 hour fcst: +191:145745709:d=2025100600:PMTC:10 hybrid level:16 hour fcst: +192:146503615:d=2025100600:FRACCC:10 hybrid level:16 hour fcst: +193:147063899:d=2025100600:HGT:10 hybrid level:16 hour fcst: +194:149626521:d=2025100600:TMP:10 hybrid level:16 hour fcst: +195:150702451:d=2025100600:SPFH:10 hybrid level:16 hour fcst: +196:152232961:d=2025100600:UGRD:10 hybrid level:16 hour fcst: +197:153297472:d=2025100600:VGRD:10 hybrid level:16 hour fcst: +198:154373776:d=2025100600:VVEL:10 hybrid level:16 hour fcst: +199:156052405:d=2025100600:TKE:10 hybrid level:16 hour fcst: +200:156258566:d=2025100600:MASSDEN:10 hybrid level:16 hour fcst: +201:157870227:d=2025100600:PRES:11 hybrid level:16 hour fcst: +202:160069255:d=2025100600:CLMR:11 hybrid level:16 hour fcst: +203:160181462:d=2025100600:CIMIXR:11 hybrid level:16 hour fcst: +204:160183128:d=2025100600:RWMR:11 hybrid level:16 hour fcst: +205:160490394:d=2025100600:SNMR:11 hybrid level:16 hour fcst: +206:160509280:d=2025100600:GRLE:11 hybrid level:16 hour fcst: +207:160517947:d=2025100600:NCONCD:11 hybrid level:16 hour fcst: +208:160625274:d=2025100600:NCCICE:11 hybrid level:16 hour fcst: +209:160627369:d=2025100600:SPNCR:11 hybrid level:16 hour fcst: +210:160969924:d=2025100600:PMTF:11 hybrid level:16 hour fcst: +211:162168406:d=2025100600:PMTC:11 hybrid level:16 hour fcst: +212:162923793:d=2025100600:FRACCC:11 hybrid level:16 hour fcst: +213:163409762:d=2025100600:HGT:11 hybrid level:16 hour fcst: +214:165972684:d=2025100600:TMP:11 hybrid level:16 hour fcst: +215:167021814:d=2025100600:SPFH:11 hybrid level:16 hour fcst: +216:168507484:d=2025100600:UGRD:11 hybrid level:16 hour fcst: +217:169551660:d=2025100600:VGRD:11 hybrid level:16 hour fcst: +218:170605892:d=2025100600:VVEL:11 hybrid level:16 hour fcst: +219:172258645:d=2025100600:TKE:11 hybrid level:16 hour fcst: +220:172358446:d=2025100600:MASSDEN:11 hybrid level:16 hour fcst: +221:174160630:d=2025100600:PRES:12 hybrid level:16 hour fcst: +222:176338548:d=2025100600:CLMR:12 hybrid level:16 hour fcst: +223:176448554:d=2025100600:CIMIXR:12 hybrid level:16 hour fcst: +224:176451111:d=2025100600:RWMR:12 hybrid level:16 hour fcst: +225:176658455:d=2025100600:SNMR:12 hybrid level:16 hour fcst: +226:176683532:d=2025100600:GRLE:12 hybrid level:16 hour fcst: +227:176697831:d=2025100600:NCONCD:12 hybrid level:16 hour fcst: +228:176804621:d=2025100600:NCCICE:12 hybrid level:16 hour fcst: +229:176807686:d=2025100600:SPNCR:12 hybrid level:16 hour fcst: +230:177132499:d=2025100600:PMTF:12 hybrid level:16 hour fcst: +231:178267701:d=2025100600:PMTC:12 hybrid level:16 hour fcst: +232:178989260:d=2025100600:FRACCC:12 hybrid level:16 hour fcst: +233:179452104:d=2025100600:HGT:12 hybrid level:16 hour fcst: +234:182010428:d=2025100600:TMP:12 hybrid level:16 hour fcst: +235:183026832:d=2025100600:SPFH:12 hybrid level:16 hour fcst: +236:184462619:d=2025100600:UGRD:12 hybrid level:16 hour fcst: +237:185484614:d=2025100600:VGRD:12 hybrid level:16 hour fcst: +238:186519978:d=2025100600:VVEL:12 hybrid level:16 hour fcst: +239:188157629:d=2025100600:TKE:12 hybrid level:16 hour fcst: +240:188201701:d=2025100600:MASSDEN:12 hybrid level:16 hour fcst: +241:189998383:d=2025100600:PRES:13 hybrid level:16 hour fcst: +242:192403129:d=2025100600:CLMR:13 hybrid level:16 hour fcst: +243:192511965:d=2025100600:CIMIXR:13 hybrid level:16 hour fcst: +244:192515437:d=2025100600:RWMR:13 hybrid level:16 hour fcst: +245:192718698:d=2025100600:SNMR:13 hybrid level:16 hour fcst: +246:192757294:d=2025100600:GRLE:13 hybrid level:16 hour fcst: +247:192767756:d=2025100600:NCONCD:13 hybrid level:16 hour fcst: +248:192871182:d=2025100600:NCCICE:13 hybrid level:16 hour fcst: +249:192875472:d=2025100600:SPNCR:13 hybrid level:16 hour fcst: +250:193197239:d=2025100600:PMTF:13 hybrid level:16 hour fcst: +251:194237341:d=2025100600:PMTC:13 hybrid level:16 hour fcst: +252:194936176:d=2025100600:FRACCC:13 hybrid level:16 hour fcst: +253:195371408:d=2025100600:HGT:13 hybrid level:16 hour fcst: +254:197920180:d=2025100600:TMP:13 hybrid level:16 hour fcst: +255:198900287:d=2025100600:SPFH:13 hybrid level:16 hour fcst: +256:200280195:d=2025100600:UGRD:13 hybrid level:16 hour fcst: +257:201279491:d=2025100600:VGRD:13 hybrid level:16 hour fcst: +258:202290408:d=2025100600:VVEL:13 hybrid level:16 hour fcst: +259:203930442:d=2025100600:TKE:13 hybrid level:16 hour fcst: +260:203950997:d=2025100600:MASSDEN:13 hybrid level:16 hour fcst: +261:205671795:d=2025100600:PRES:14 hybrid level:16 hour fcst: +262:208042580:d=2025100600:CLMR:14 hybrid level:16 hour fcst: +263:208171861:d=2025100600:CIMIXR:14 hybrid level:16 hour fcst: +264:208178895:d=2025100600:RWMR:14 hybrid level:16 hour fcst: +265:208369530:d=2025100600:SNMR:14 hybrid level:16 hour fcst: +266:208399861:d=2025100600:GRLE:14 hybrid level:16 hour fcst: +267:208417805:d=2025100600:NCONCD:14 hybrid level:16 hour fcst: +268:208531050:d=2025100600:NCCICE:14 hybrid level:16 hour fcst: +269:208538900:d=2025100600:SPNCR:14 hybrid level:16 hour fcst: +270:208732582:d=2025100600:PMTF:14 hybrid level:16 hour fcst: +271:209683732:d=2025100600:PMTC:14 hybrid level:16 hour fcst: +272:210350428:d=2025100600:FRACCC:14 hybrid level:16 hour fcst: +273:210714134:d=2025100600:HGT:14 hybrid level:16 hour fcst: +274:213247894:d=2025100600:TMP:14 hybrid level:16 hour fcst: +275:214234613:d=2025100600:SPFH:14 hybrid level:16 hour fcst: +276:215543299:d=2025100600:UGRD:14 hybrid level:16 hour fcst: +277:216514147:d=2025100600:VGRD:14 hybrid level:16 hour fcst: +278:217473521:d=2025100600:VVEL:14 hybrid level:16 hour fcst: +279:219088252:d=2025100600:TKE:14 hybrid level:16 hour fcst: +280:219099074:d=2025100600:MASSDEN:14 hybrid level:16 hour fcst: +281:220644193:d=2025100600:PRES:15 hybrid level:16 hour fcst: +282:222975012:d=2025100600:CLMR:15 hybrid level:16 hour fcst: +283:223067280:d=2025100600:CIMIXR:15 hybrid level:16 hour fcst: +284:223080495:d=2025100600:RWMR:15 hybrid level:16 hour fcst: +285:223224492:d=2025100600:SNMR:15 hybrid level:16 hour fcst: +286:223270365:d=2025100600:GRLE:15 hybrid level:16 hour fcst: +287:223298641:d=2025100600:NCONCD:15 hybrid level:16 hour fcst: +288:223382383:d=2025100600:NCCICE:15 hybrid level:16 hour fcst: +289:223396746:d=2025100600:SPNCR:15 hybrid level:16 hour fcst: +290:223603236:d=2025100600:PMTF:15 hybrid level:16 hour fcst: +291:224449524:d=2025100600:PMTC:15 hybrid level:16 hour fcst: +292:225097756:d=2025100600:FRACCC:15 hybrid level:16 hour fcst: +293:225351292:d=2025100600:HGT:15 hybrid level:16 hour fcst: +294:227864038:d=2025100600:TMP:15 hybrid level:16 hour fcst: +295:228830683:d=2025100600:SPFH:15 hybrid level:16 hour fcst: +296:230051170:d=2025100600:UGRD:15 hybrid level:16 hour fcst: +297:230995691:d=2025100600:VGRD:15 hybrid level:16 hour fcst: +298:231930835:d=2025100600:VVEL:15 hybrid level:16 hour fcst: +299:233512199:d=2025100600:TKE:15 hybrid level:16 hour fcst: +300:233517133:d=2025100600:MASSDEN:15 hybrid level:16 hour fcst: +301:234718560:d=2025100600:PRES:16 hybrid level:16 hour fcst: +302:236997902:d=2025100600:CLMR:16 hybrid level:16 hour fcst: +303:237075364:d=2025100600:CIMIXR:16 hybrid level:16 hour fcst: +304:237103785:d=2025100600:RWMR:16 hybrid level:16 hour fcst: +305:237216572:d=2025100600:SNMR:16 hybrid level:16 hour fcst: +306:237286101:d=2025100600:GRLE:16 hybrid level:16 hour fcst: +307:237323026:d=2025100600:NCONCD:16 hybrid level:16 hour fcst: +308:237391434:d=2025100600:NCCICE:16 hybrid level:16 hour fcst: +309:237421180:d=2025100600:SPNCR:16 hybrid level:16 hour fcst: +310:237542635:d=2025100600:PMTF:16 hybrid level:16 hour fcst: +311:238304783:d=2025100600:PMTC:16 hybrid level:16 hour fcst: +312:238913428:d=2025100600:FRACCC:16 hybrid level:16 hour fcst: +313:239126786:d=2025100600:HGT:16 hybrid level:16 hour fcst: +314:241610643:d=2025100600:TMP:16 hybrid level:16 hour fcst: +315:242536194:d=2025100600:SPFH:16 hybrid level:16 hour fcst: +316:244424473:d=2025100600:UGRD:16 hybrid level:16 hour fcst: +317:245337746:d=2025100600:VGRD:16 hybrid level:16 hour fcst: +318:246236852:d=2025100600:VVEL:16 hybrid level:16 hour fcst: +319:247780895:d=2025100600:TKE:16 hybrid level:16 hour fcst: +320:247782608:d=2025100600:MASSDEN:16 hybrid level:16 hour fcst: +321:249046038:d=2025100600:PRES:17 hybrid level:16 hour fcst: +322:251229111:d=2025100600:CLMR:17 hybrid level:16 hour fcst: +323:251287039:d=2025100600:CIMIXR:17 hybrid level:16 hour fcst: +324:251312077:d=2025100600:RWMR:17 hybrid level:16 hour fcst: +325:251373673:d=2025100600:SNMR:17 hybrid level:16 hour fcst: +326:251469200:d=2025100600:GRLE:17 hybrid level:16 hour fcst: +327:251513964:d=2025100600:NCONCD:17 hybrid level:16 hour fcst: +328:251563899:d=2025100600:NCCICE:17 hybrid level:16 hour fcst: +329:251591393:d=2025100600:SPNCR:17 hybrid level:16 hour fcst: +330:251667379:d=2025100600:PMTF:17 hybrid level:16 hour fcst: +331:252333066:d=2025100600:PMTC:17 hybrid level:16 hour fcst: +332:252914715:d=2025100600:FRACCC:17 hybrid level:16 hour fcst: +333:253109024:d=2025100600:HGT:17 hybrid level:16 hour fcst: +334:255537548:d=2025100600:TMP:17 hybrid level:16 hour fcst: +335:256442471:d=2025100600:SPFH:17 hybrid level:16 hour fcst: +336:258192619:d=2025100600:UGRD:17 hybrid level:16 hour fcst: +337:259078857:d=2025100600:VGRD:17 hybrid level:16 hour fcst: +338:259929093:d=2025100600:VVEL:17 hybrid level:16 hour fcst: +339:261413238:d=2025100600:TKE:17 hybrid level:16 hour fcst: +340:261413726:d=2025100600:MASSDEN:17 hybrid level:16 hour fcst: +341:262830486:d=2025100600:PRES:18 hybrid level:16 hour fcst: +342:265192764:d=2025100600:CLMR:18 hybrid level:16 hour fcst: +343:265237007:d=2025100600:CIMIXR:18 hybrid level:16 hour fcst: +344:265272527:d=2025100600:RWMR:18 hybrid level:16 hour fcst: +345:265338898:d=2025100600:SNMR:18 hybrid level:16 hour fcst: +346:265442023:d=2025100600:GRLE:18 hybrid level:16 hour fcst: +347:265481338:d=2025100600:NCONCD:18 hybrid level:16 hour fcst: +348:265537952:d=2025100600:NCCICE:18 hybrid level:16 hour fcst: +349:265577490:d=2025100600:SPNCR:18 hybrid level:16 hour fcst: +350:265674638:d=2025100600:PMTF:18 hybrid level:16 hour fcst: +351:266284308:d=2025100600:PMTC:18 hybrid level:16 hour fcst: +352:266852256:d=2025100600:FRACCC:18 hybrid level:16 hour fcst: +353:267056284:d=2025100600:HGT:18 hybrid level:16 hour fcst: +354:269439148:d=2025100600:TMP:18 hybrid level:16 hour fcst: +355:270325235:d=2025100600:SPFH:18 hybrid level:16 hour fcst: +356:271963195:d=2025100600:UGRD:18 hybrid level:16 hour fcst: +357:272828246:d=2025100600:VGRD:18 hybrid level:16 hour fcst: +358:273675097:d=2025100600:VVEL:18 hybrid level:16 hour fcst: +359:275113051:d=2025100600:TKE:18 hybrid level:16 hour fcst: +360:275113386:d=2025100600:MASSDEN:18 hybrid level:16 hour fcst: +361:276563973:d=2025100600:PRES:19 hybrid level:16 hour fcst: +362:278919289:d=2025100600:CLMR:19 hybrid level:16 hour fcst: +363:278956875:d=2025100600:CIMIXR:19 hybrid level:16 hour fcst: +364:278994546:d=2025100600:RWMR:19 hybrid level:16 hour fcst: +365:279055785:d=2025100600:SNMR:19 hybrid level:16 hour fcst: +366:279165894:d=2025100600:GRLE:19 hybrid level:16 hour fcst: +367:279197366:d=2025100600:NCONCD:19 hybrid level:16 hour fcst: +368:279244562:d=2025100600:NCCICE:19 hybrid level:16 hour fcst: +369:279284736:d=2025100600:SPNCR:19 hybrid level:16 hour fcst: +370:279364694:d=2025100600:PMTF:19 hybrid level:16 hour fcst: +371:279940446:d=2025100600:PMTC:19 hybrid level:16 hour fcst: +372:280488775:d=2025100600:FRACCC:19 hybrid level:16 hour fcst: +373:280706424:d=2025100600:HGT:19 hybrid level:16 hour fcst: +374:283042094:d=2025100600:TMP:19 hybrid level:16 hour fcst: +375:283907427:d=2025100600:SPFH:19 hybrid level:16 hour fcst: +376:285439024:d=2025100600:UGRD:19 hybrid level:16 hour fcst: +377:286312464:d=2025100600:VGRD:19 hybrid level:16 hour fcst: +378:287151296:d=2025100600:VVEL:19 hybrid level:16 hour fcst: +379:288549292:d=2025100600:TKE:19 hybrid level:16 hour fcst: +380:288549605:d=2025100600:MASSDEN:19 hybrid level:16 hour fcst: +381:289772340:d=2025100600:PRES:20 hybrid level:16 hour fcst: +382:292027986:d=2025100600:CLMR:20 hybrid level:16 hour fcst: +383:292048307:d=2025100600:CIMIXR:20 hybrid level:16 hour fcst: +384:292085072:d=2025100600:RWMR:20 hybrid level:16 hour fcst: +385:292123234:d=2025100600:SNMR:20 hybrid level:16 hour fcst: +386:292229013:d=2025100600:GRLE:20 hybrid level:16 hour fcst: +387:292253822:d=2025100600:NCONCD:20 hybrid level:16 hour fcst: +388:292278343:d=2025100600:NCCICE:20 hybrid level:16 hour fcst: +389:292319558:d=2025100600:SPNCR:20 hybrid level:16 hour fcst: +390:292370976:d=2025100600:PMTF:20 hybrid level:16 hour fcst: +391:292887521:d=2025100600:PMTC:20 hybrid level:16 hour fcst: +392:293436045:d=2025100600:FRACCC:20 hybrid level:16 hour fcst: +393:293659346:d=2025100600:HGT:20 hybrid level:16 hour fcst: +394:295907199:d=2025100600:TMP:20 hybrid level:16 hour fcst: +395:296745417:d=2025100600:SPFH:20 hybrid level:16 hour fcst: +396:298162232:d=2025100600:UGRD:20 hybrid level:16 hour fcst: +397:299027187:d=2025100600:VGRD:20 hybrid level:16 hour fcst: +398:299852191:d=2025100600:VVEL:20 hybrid level:16 hour fcst: +399:301219001:d=2025100600:TKE:20 hybrid level:16 hour fcst: +400:301219455:d=2025100600:MASSDEN:20 hybrid level:16 hour fcst: +401:302445099:d=2025100600:PRES:21 hybrid level:16 hour fcst: +402:304849718:d=2025100600:CLMR:21 hybrid level:16 hour fcst: +403:304858923:d=2025100600:CIMIXR:21 hybrid level:16 hour fcst: +404:304897978:d=2025100600:RWMR:21 hybrid level:16 hour fcst: +405:304922528:d=2025100600:SNMR:21 hybrid level:16 hour fcst: +406:305019923:d=2025100600:GRLE:21 hybrid level:16 hour fcst: +407:305040544:d=2025100600:NCONCD:21 hybrid level:16 hour fcst: +408:305051131:d=2025100600:NCCICE:21 hybrid level:16 hour fcst: +409:305094382:d=2025100600:SPNCR:21 hybrid level:16 hour fcst: +410:305122088:d=2025100600:PMTF:21 hybrid level:16 hour fcst: +411:305606977:d=2025100600:PMTC:21 hybrid level:16 hour fcst: +412:306130526:d=2025100600:FRACCC:21 hybrid level:16 hour fcst: +413:306329370:d=2025100600:HGT:21 hybrid level:16 hour fcst: +414:308520405:d=2025100600:TMP:21 hybrid level:16 hour fcst: +415:309325777:d=2025100600:SPFH:21 hybrid level:16 hour fcst: +416:310634912:d=2025100600:UGRD:21 hybrid level:16 hour fcst: +417:311475580:d=2025100600:VGRD:21 hybrid level:16 hour fcst: +418:312284885:d=2025100600:VVEL:21 hybrid level:16 hour fcst: +419:313636915:d=2025100600:TKE:21 hybrid level:16 hour fcst: +420:313637342:d=2025100600:MASSDEN:21 hybrid level:16 hour fcst: +421:314897023:d=2025100600:PRES:22 hybrid level:16 hour fcst: +422:317204080:d=2025100600:CLMR:22 hybrid level:16 hour fcst: +423:317208899:d=2025100600:CIMIXR:22 hybrid level:16 hour fcst: +424:317253096:d=2025100600:RWMR:22 hybrid level:16 hour fcst: +425:317270160:d=2025100600:SNMR:22 hybrid level:16 hour fcst: +426:317367986:d=2025100600:GRLE:22 hybrid level:16 hour fcst: +427:317385758:d=2025100600:NCONCD:22 hybrid level:16 hour fcst: +428:317390841:d=2025100600:NCCICE:22 hybrid level:16 hour fcst: +429:317441887:d=2025100600:SPNCR:22 hybrid level:16 hour fcst: +430:317461546:d=2025100600:PMTF:22 hybrid level:16 hour fcst: +431:317913050:d=2025100600:PMTC:22 hybrid level:16 hour fcst: +432:318416059:d=2025100600:FRACCC:22 hybrid level:16 hour fcst: +433:318625958:d=2025100600:HGT:22 hybrid level:16 hour fcst: +434:320756464:d=2025100600:TMP:22 hybrid level:16 hour fcst: +435:321528202:d=2025100600:SPFH:22 hybrid level:16 hour fcst: +436:322733684:d=2025100600:UGRD:22 hybrid level:16 hour fcst: +437:323577977:d=2025100600:VGRD:22 hybrid level:16 hour fcst: +438:324379373:d=2025100600:VVEL:22 hybrid level:16 hour fcst: +439:325687193:d=2025100600:TKE:22 hybrid level:16 hour fcst: +440:325687766:d=2025100600:MASSDEN:22 hybrid level:16 hour fcst: +441:327047674:d=2025100600:PRES:23 hybrid level:16 hour fcst: +442:329216624:d=2025100600:CLMR:23 hybrid level:16 hour fcst: +443:329220084:d=2025100600:CIMIXR:23 hybrid level:16 hour fcst: +444:329273016:d=2025100600:RWMR:23 hybrid level:16 hour fcst: +445:329285617:d=2025100600:SNMR:23 hybrid level:16 hour fcst: +446:329383365:d=2025100600:GRLE:23 hybrid level:16 hour fcst: +447:329399203:d=2025100600:NCONCD:23 hybrid level:16 hour fcst: +448:329402799:d=2025100600:NCCICE:23 hybrid level:16 hour fcst: +449:329469592:d=2025100600:SPNCR:23 hybrid level:16 hour fcst: +450:329484774:d=2025100600:PMTF:23 hybrid level:16 hour fcst: +451:329921750:d=2025100600:PMTC:23 hybrid level:16 hour fcst: +452:330403705:d=2025100600:FRACCC:23 hybrid level:16 hour fcst: +453:330620454:d=2025100600:HGT:23 hybrid level:16 hour fcst: +454:332670715:d=2025100600:TMP:23 hybrid level:16 hour fcst: +455:333410999:d=2025100600:SPFH:23 hybrid level:16 hour fcst: +456:334506608:d=2025100600:UGRD:23 hybrid level:16 hour fcst: +457:335331937:d=2025100600:VGRD:23 hybrid level:16 hour fcst: +458:336127102:d=2025100600:VVEL:23 hybrid level:16 hour fcst: +459:337402106:d=2025100600:TKE:23 hybrid level:16 hour fcst: +460:337403714:d=2025100600:MASSDEN:23 hybrid level:16 hour fcst: +461:338647038:d=2025100600:PRES:24 hybrid level:16 hour fcst: +462:340957009:d=2025100600:CLMR:24 hybrid level:16 hour fcst: +463:340959510:d=2025100600:CIMIXR:24 hybrid level:16 hour fcst: +464:340982622:d=2025100600:RWMR:24 hybrid level:16 hour fcst: +465:340991205:d=2025100600:SNMR:24 hybrid level:16 hour fcst: +466:341089454:d=2025100600:GRLE:24 hybrid level:16 hour fcst: +467:341103814:d=2025100600:NCONCD:24 hybrid level:16 hour fcst: +468:341106485:d=2025100600:NCCICE:24 hybrid level:16 hour fcst: +469:341144861:d=2025100600:SPNCR:24 hybrid level:16 hour fcst: +470:341157328:d=2025100600:PMTF:24 hybrid level:16 hour fcst: +471:341578989:d=2025100600:PMTC:24 hybrid level:16 hour fcst: +472:342032242:d=2025100600:FRACCC:24 hybrid level:16 hour fcst: +473:342264030:d=2025100600:HGT:24 hybrid level:16 hour fcst: +474:344244564:d=2025100600:TMP:24 hybrid level:16 hour fcst: +475:344954659:d=2025100600:SPFH:24 hybrid level:16 hour fcst: +476:345971643:d=2025100600:UGRD:24 hybrid level:16 hour fcst: +477:346795220:d=2025100600:VGRD:24 hybrid level:16 hour fcst: +478:347588646:d=2025100600:VVEL:24 hybrid level:16 hour fcst: +479:348829630:d=2025100600:TKE:24 hybrid level:16 hour fcst: +480:348832620:d=2025100600:MASSDEN:24 hybrid level:16 hour fcst: +481:349968049:d=2025100600:PRES:25 hybrid level:16 hour fcst: +482:352373585:d=2025100600:CLMR:25 hybrid level:16 hour fcst: +483:352375387:d=2025100600:CIMIXR:25 hybrid level:16 hour fcst: +484:352393014:d=2025100600:RWMR:25 hybrid level:16 hour fcst: +485:352400722:d=2025100600:SNMR:25 hybrid level:16 hour fcst: +486:352492827:d=2025100600:GRLE:25 hybrid level:16 hour fcst: +487:352505786:d=2025100600:NCONCD:25 hybrid level:16 hour fcst: +488:352507568:d=2025100600:NCCICE:25 hybrid level:16 hour fcst: +489:352558319:d=2025100600:SPNCR:25 hybrid level:16 hour fcst: +490:352567267:d=2025100600:PMTF:25 hybrid level:16 hour fcst: +491:352969293:d=2025100600:PMTC:25 hybrid level:16 hour fcst: +492:353813229:d=2025100600:FRACCC:25 hybrid level:16 hour fcst: +493:354043964:d=2025100600:HGT:25 hybrid level:16 hour fcst: +494:355950537:d=2025100600:TMP:25 hybrid level:16 hour fcst: +495:356630557:d=2025100600:SPFH:25 hybrid level:16 hour fcst: +496:357563033:d=2025100600:UGRD:25 hybrid level:16 hour fcst: +497:358382797:d=2025100600:VGRD:25 hybrid level:16 hour fcst: +498:359173426:d=2025100600:VVEL:25 hybrid level:16 hour fcst: +499:360381147:d=2025100600:TKE:25 hybrid level:16 hour fcst: +500:360385765:d=2025100600:MASSDEN:25 hybrid level:16 hour fcst: +501:361443519:d=2025100600:PRES:26 hybrid level:16 hour fcst: +502:363778329:d=2025100600:CLMR:26 hybrid level:16 hour fcst: +503:363779460:d=2025100600:CIMIXR:26 hybrid level:16 hour fcst: +504:363820545:d=2025100600:RWMR:26 hybrid level:16 hour fcst: +505:363825585:d=2025100600:SNMR:26 hybrid level:16 hour fcst: +506:363910655:d=2025100600:GRLE:26 hybrid level:16 hour fcst: +507:363922602:d=2025100600:NCONCD:26 hybrid level:16 hour fcst: +508:363924041:d=2025100600:NCCICE:26 hybrid level:16 hour fcst: +509:363986797:d=2025100600:SPNCR:26 hybrid level:16 hour fcst: +510:363992735:d=2025100600:PMTF:26 hybrid level:16 hour fcst: +511:364709154:d=2025100600:PMTC:26 hybrid level:16 hour fcst: +512:365523576:d=2025100600:FRACCC:26 hybrid level:16 hour fcst: +513:365745399:d=2025100600:HGT:26 hybrid level:16 hour fcst: +514:367568541:d=2025100600:TMP:26 hybrid level:16 hour fcst: +515:368218415:d=2025100600:SPFH:26 hybrid level:16 hour fcst: +516:369084921:d=2025100600:UGRD:26 hybrid level:16 hour fcst: +517:369913427:d=2025100600:VGRD:26 hybrid level:16 hour fcst: +518:370696426:d=2025100600:VVEL:26 hybrid level:16 hour fcst: +519:371871449:d=2025100600:TKE:26 hybrid level:16 hour fcst: +520:371877763:d=2025100600:MASSDEN:26 hybrid level:16 hour fcst: +521:372874408:d=2025100600:PRES:27 hybrid level:16 hour fcst: +522:375011540:d=2025100600:CLMR:27 hybrid level:16 hour fcst: +523:375011972:d=2025100600:CIMIXR:27 hybrid level:16 hour fcst: +524:375020852:d=2025100600:RWMR:27 hybrid level:16 hour fcst: +525:375023965:d=2025100600:SNMR:27 hybrid level:16 hour fcst: +526:375098971:d=2025100600:GRLE:27 hybrid level:16 hour fcst: +527:375109840:d=2025100600:NCONCD:27 hybrid level:16 hour fcst: +528:375110279:d=2025100600:NCCICE:27 hybrid level:16 hour fcst: +529:375182858:d=2025100600:SPNCR:27 hybrid level:16 hour fcst: +530:375186027:d=2025100600:PMTF:27 hybrid level:16 hour fcst: +531:375883223:d=2025100600:PMTC:27 hybrid level:16 hour fcst: +532:376645572:d=2025100600:FRACCC:27 hybrid level:16 hour fcst: +533:376860779:d=2025100600:HGT:27 hybrid level:16 hour fcst: +534:378572097:d=2025100600:TMP:27 hybrid level:16 hour fcst: +535:379189595:d=2025100600:SPFH:27 hybrid level:16 hour fcst: +536:380569226:d=2025100600:UGRD:27 hybrid level:16 hour fcst: +537:381392127:d=2025100600:VGRD:27 hybrid level:16 hour fcst: +538:382169077:d=2025100600:VVEL:27 hybrid level:16 hour fcst: +539:383312718:d=2025100600:TKE:27 hybrid level:16 hour fcst: +540:383321666:d=2025100600:MASSDEN:27 hybrid level:16 hour fcst: +541:384248285:d=2025100600:PRES:28 hybrid level:16 hour fcst: +542:386403899:d=2025100600:CLMR:28 hybrid level:16 hour fcst: +543:386404152:d=2025100600:CIMIXR:28 hybrid level:16 hour fcst: +544:386457336:d=2025100600:RWMR:28 hybrid level:16 hour fcst: +545:386457671:d=2025100600:SNMR:28 hybrid level:16 hour fcst: +546:386526687:d=2025100600:GRLE:28 hybrid level:16 hour fcst: +547:386536501:d=2025100600:NCONCD:28 hybrid level:16 hour fcst: +548:386536755:d=2025100600:NCCICE:28 hybrid level:16 hour fcst: +549:386619205:d=2025100600:SPNCR:28 hybrid level:16 hour fcst: +550:386619547:d=2025100600:PMTF:28 hybrid level:16 hour fcst: +551:387296797:d=2025100600:PMTC:28 hybrid level:16 hour fcst: +552:388013512:d=2025100600:FRACCC:28 hybrid level:16 hour fcst: +553:388236237:d=2025100600:HGT:28 hybrid level:16 hour fcst: +554:389836873:d=2025100600:TMP:28 hybrid level:16 hour fcst: +555:390448177:d=2025100600:SPFH:28 hybrid level:16 hour fcst: +556:391706260:d=2025100600:UGRD:28 hybrid level:16 hour fcst: +557:392520949:d=2025100600:VGRD:28 hybrid level:16 hour fcst: +558:393289835:d=2025100600:VVEL:28 hybrid level:16 hour fcst: +559:394402885:d=2025100600:TKE:28 hybrid level:16 hour fcst: +560:394414825:d=2025100600:MASSDEN:28 hybrid level:16 hour fcst: +561:395405671:d=2025100600:PRES:29 hybrid level:16 hour fcst: +562:397519682:d=2025100600:CLMR:29 hybrid level:16 hour fcst: +563:397519870:d=2025100600:CIMIXR:29 hybrid level:16 hour fcst: +564:397578458:d=2025100600:RWMR:29 hybrid level:16 hour fcst: +565:397578743:d=2025100600:SNMR:29 hybrid level:16 hour fcst: +566:397640283:d=2025100600:GRLE:29 hybrid level:16 hour fcst: +567:397648844:d=2025100600:NCONCD:29 hybrid level:16 hour fcst: +568:397649032:d=2025100600:NCCICE:29 hybrid level:16 hour fcst: +569:397736209:d=2025100600:SPNCR:29 hybrid level:16 hour fcst: +570:397736495:d=2025100600:PMTF:29 hybrid level:16 hour fcst: +571:398402833:d=2025100600:PMTC:29 hybrid level:16 hour fcst: +572:399085426:d=2025100600:FRACCC:29 hybrid level:16 hour fcst: +573:399300361:d=2025100600:HGT:29 hybrid level:16 hour fcst: +574:400772862:d=2025100600:TMP:29 hybrid level:16 hour fcst: +575:401387539:d=2025100600:SPFH:29 hybrid level:16 hour fcst: +576:402561099:d=2025100600:UGRD:29 hybrid level:16 hour fcst: +577:403377272:d=2025100600:VGRD:29 hybrid level:16 hour fcst: +578:404145300:d=2025100600:VVEL:29 hybrid level:16 hour fcst: +579:405232029:d=2025100600:TKE:29 hybrid level:16 hour fcst: +580:405244224:d=2025100600:MASSDEN:29 hybrid level:16 hour fcst: +581:406039144:d=2025100600:PRES:30 hybrid level:16 hour fcst: +582:408087658:d=2025100600:CLMR:30 hybrid level:16 hour fcst: +583:408087899:d=2025100600:CIMIXR:30 hybrid level:16 hour fcst: +584:408144759:d=2025100600:RWMR:30 hybrid level:16 hour fcst: +585:408145026:d=2025100600:SNMR:30 hybrid level:16 hour fcst: +586:408198508:d=2025100600:GRLE:30 hybrid level:16 hour fcst: +587:408206286:d=2025100600:NCONCD:30 hybrid level:16 hour fcst: +588:408206529:d=2025100600:NCCICE:30 hybrid level:16 hour fcst: +589:408293226:d=2025100600:SPNCR:30 hybrid level:16 hour fcst: +590:408293495:d=2025100600:PMTF:30 hybrid level:16 hour fcst: +591:408950994:d=2025100600:PMTC:30 hybrid level:16 hour fcst: +592:409606837:d=2025100600:FRACCC:30 hybrid level:16 hour fcst: +593:409779841:d=2025100600:HGT:30 hybrid level:16 hour fcst: +594:411134333:d=2025100600:TMP:30 hybrid level:16 hour fcst: +595:411747632:d=2025100600:SPFH:30 hybrid level:16 hour fcst: +596:412805569:d=2025100600:UGRD:30 hybrid level:16 hour fcst: +597:413613735:d=2025100600:VGRD:30 hybrid level:16 hour fcst: +598:414368292:d=2025100600:VVEL:30 hybrid level:16 hour fcst: +599:415428144:d=2025100600:TKE:30 hybrid level:16 hour fcst: +600:415439723:d=2025100600:MASSDEN:30 hybrid level:16 hour fcst: +601:416160068:d=2025100600:PRES:31 hybrid level:16 hour fcst: +602:417543303:d=2025100600:CLMR:31 hybrid level:16 hour fcst: +603:417543491:d=2025100600:CIMIXR:31 hybrid level:16 hour fcst: +604:417595057:d=2025100600:RWMR:31 hybrid level:16 hour fcst: +605:417595308:d=2025100600:SNMR:31 hybrid level:16 hour fcst: +606:417640030:d=2025100600:GRLE:31 hybrid level:16 hour fcst: +607:417647032:d=2025100600:NCONCD:31 hybrid level:16 hour fcst: +608:417647220:d=2025100600:NCCICE:31 hybrid level:16 hour fcst: +609:417729236:d=2025100600:SPNCR:31 hybrid level:16 hour fcst: +610:417729488:d=2025100600:PMTF:31 hybrid level:16 hour fcst: +611:418376961:d=2025100600:PMTC:31 hybrid level:16 hour fcst: +612:418995807:d=2025100600:FRACCC:31 hybrid level:16 hour fcst: +613:419133375:d=2025100600:HGT:31 hybrid level:16 hour fcst: +614:420436831:d=2025100600:TMP:31 hybrid level:16 hour fcst: +615:421055924:d=2025100600:SPFH:31 hybrid level:16 hour fcst: +616:421996628:d=2025100600:UGRD:31 hybrid level:16 hour fcst: +617:422798552:d=2025100600:VGRD:31 hybrid level:16 hour fcst: +618:423544946:d=2025100600:VVEL:31 hybrid level:16 hour fcst: +619:424576175:d=2025100600:TKE:31 hybrid level:16 hour fcst: +620:424586787:d=2025100600:MASSDEN:31 hybrid level:16 hour fcst: +621:425241168:d=2025100600:PRES:32 hybrid level:16 hour fcst: +622:426231667:d=2025100600:CLMR:32 hybrid level:16 hour fcst: +623:426231855:d=2025100600:CIMIXR:32 hybrid level:16 hour fcst: +624:426272516:d=2025100600:RWMR:32 hybrid level:16 hour fcst: +625:426272704:d=2025100600:SNMR:32 hybrid level:16 hour fcst: +626:426311400:d=2025100600:GRLE:32 hybrid level:16 hour fcst: +627:426317282:d=2025100600:NCONCD:32 hybrid level:16 hour fcst: +628:426317470:d=2025100600:NCCICE:32 hybrid level:16 hour fcst: +629:426386616:d=2025100600:SPNCR:32 hybrid level:16 hour fcst: +630:426386804:d=2025100600:PMTF:32 hybrid level:16 hour fcst: +631:427026405:d=2025100600:PMTC:32 hybrid level:16 hour fcst: +632:427605554:d=2025100600:FRACCC:32 hybrid level:16 hour fcst: +633:427725899:d=2025100600:HGT:32 hybrid level:16 hour fcst: +634:429035351:d=2025100600:TMP:32 hybrid level:16 hour fcst: +635:429660843:d=2025100600:SPFH:32 hybrid level:16 hour fcst: +636:430525796:d=2025100600:UGRD:32 hybrid level:16 hour fcst: +637:431312372:d=2025100600:VGRD:32 hybrid level:16 hour fcst: +638:432052215:d=2025100600:VVEL:32 hybrid level:16 hour fcst: +639:433039711:d=2025100600:TKE:32 hybrid level:16 hour fcst: +640:433049856:d=2025100600:MASSDEN:32 hybrid level:16 hour fcst: +641:433713688:d=2025100600:PRES:33 hybrid level:16 hour fcst: +642:434539386:d=2025100600:CLMR:33 hybrid level:16 hour fcst: +643:434539574:d=2025100600:CIMIXR:33 hybrid level:16 hour fcst: +644:434574286:d=2025100600:RWMR:33 hybrid level:16 hour fcst: +645:434574474:d=2025100600:SNMR:33 hybrid level:16 hour fcst: +646:434604195:d=2025100600:GRLE:33 hybrid level:16 hour fcst: +647:434608888:d=2025100600:NCONCD:33 hybrid level:16 hour fcst: +648:434609076:d=2025100600:NCCICE:33 hybrid level:16 hour fcst: +649:434667598:d=2025100600:SPNCR:33 hybrid level:16 hour fcst: +650:434667786:d=2025100600:PMTF:33 hybrid level:16 hour fcst: +651:435300922:d=2025100600:PMTC:33 hybrid level:16 hour fcst: +652:435829291:d=2025100600:FRACCC:33 hybrid level:16 hour fcst: +653:435939229:d=2025100600:HGT:33 hybrid level:16 hour fcst: +654:437252456:d=2025100600:TMP:33 hybrid level:16 hour fcst: +655:437878426:d=2025100600:SPFH:33 hybrid level:16 hour fcst: +656:439243367:d=2025100600:UGRD:33 hybrid level:16 hour fcst: +657:440026445:d=2025100600:VGRD:33 hybrid level:16 hour fcst: +658:440751955:d=2025100600:VVEL:33 hybrid level:16 hour fcst: +659:441707719:d=2025100600:TKE:33 hybrid level:16 hour fcst: +660:441715626:d=2025100600:MASSDEN:33 hybrid level:16 hour fcst: +661:442510459:d=2025100600:PRES:34 hybrid level:16 hour fcst: +662:443195325:d=2025100600:CLMR:34 hybrid level:16 hour fcst: +663:443195513:d=2025100600:CIMIXR:34 hybrid level:16 hour fcst: +664:443222500:d=2025100600:RWMR:34 hybrid level:16 hour fcst: +665:443222688:d=2025100600:SNMR:34 hybrid level:16 hour fcst: +666:443243555:d=2025100600:GRLE:34 hybrid level:16 hour fcst: +667:443246966:d=2025100600:NCONCD:34 hybrid level:16 hour fcst: +668:443247154:d=2025100600:NCCICE:34 hybrid level:16 hour fcst: +669:443295894:d=2025100600:SPNCR:34 hybrid level:16 hour fcst: +670:443296082:d=2025100600:PMTF:34 hybrid level:16 hour fcst: +671:443930893:d=2025100600:PMTC:34 hybrid level:16 hour fcst: +672:444420391:d=2025100600:FRACCC:34 hybrid level:16 hour fcst: +673:444505529:d=2025100600:HGT:34 hybrid level:16 hour fcst: +674:445820934:d=2025100600:TMP:34 hybrid level:16 hour fcst: +675:446447752:d=2025100600:SPFH:34 hybrid level:16 hour fcst: +676:447680459:d=2025100600:UGRD:34 hybrid level:16 hour fcst: +677:448460905:d=2025100600:VGRD:34 hybrid level:16 hour fcst: +678:449179643:d=2025100600:VVEL:34 hybrid level:16 hour fcst: +679:450088909:d=2025100600:TKE:34 hybrid level:16 hour fcst: +680:450093829:d=2025100600:MASSDEN:34 hybrid level:16 hour fcst: +681:450965292:d=2025100600:PRES:35 hybrid level:16 hour fcst: +682:451616482:d=2025100600:CLMR:35 hybrid level:16 hour fcst: +683:451616670:d=2025100600:CIMIXR:35 hybrid level:16 hour fcst: +684:451649571:d=2025100600:RWMR:35 hybrid level:16 hour fcst: +685:451649759:d=2025100600:SNMR:35 hybrid level:16 hour fcst: +686:451662641:d=2025100600:GRLE:35 hybrid level:16 hour fcst: +687:451664837:d=2025100600:NCONCD:35 hybrid level:16 hour fcst: +688:451665025:d=2025100600:NCCICE:35 hybrid level:16 hour fcst: +689:451699428:d=2025100600:SPNCR:35 hybrid level:16 hour fcst: +690:451699616:d=2025100600:PMTF:35 hybrid level:16 hour fcst: +691:452347699:d=2025100600:PMTC:35 hybrid level:16 hour fcst: +692:452784833:d=2025100600:FRACCC:35 hybrid level:16 hour fcst: +693:452819327:d=2025100600:HGT:35 hybrid level:16 hour fcst: +694:454136327:d=2025100600:TMP:35 hybrid level:16 hour fcst: +695:454755766:d=2025100600:SPFH:35 hybrid level:16 hour fcst: +696:455826972:d=2025100600:UGRD:35 hybrid level:16 hour fcst: +697:456609629:d=2025100600:VGRD:35 hybrid level:16 hour fcst: +698:457324011:d=2025100600:VVEL:35 hybrid level:16 hour fcst: +699:458195457:d=2025100600:TKE:35 hybrid level:16 hour fcst: +700:458198050:d=2025100600:MASSDEN:35 hybrid level:16 hour fcst: +701:459092190:d=2025100600:PRES:36 hybrid level:16 hour fcst: +702:459621344:d=2025100600:CLMR:36 hybrid level:16 hour fcst: +703:459621532:d=2025100600:CIMIXR:36 hybrid level:16 hour fcst: +704:459633566:d=2025100600:RWMR:36 hybrid level:16 hour fcst: +705:459633754:d=2025100600:SNMR:36 hybrid level:16 hour fcst: +706:459639851:d=2025100600:GRLE:36 hybrid level:16 hour fcst: +707:459640952:d=2025100600:NCONCD:36 hybrid level:16 hour fcst: +708:459641140:d=2025100600:NCCICE:36 hybrid level:16 hour fcst: +709:459655564:d=2025100600:SPNCR:36 hybrid level:16 hour fcst: +710:459655752:d=2025100600:PMTF:36 hybrid level:16 hour fcst: +711:460321290:d=2025100600:PMTC:36 hybrid level:16 hour fcst: +712:460663419:d=2025100600:FRACCC:36 hybrid level:16 hour fcst: +713:460664475:d=2025100600:HGT:36 hybrid level:16 hour fcst: +714:461984072:d=2025100600:TMP:36 hybrid level:16 hour fcst: +715:462607702:d=2025100600:SPFH:36 hybrid level:16 hour fcst: +716:463518051:d=2025100600:UGRD:36 hybrid level:16 hour fcst: +717:464307905:d=2025100600:VGRD:36 hybrid level:16 hour fcst: +718:465020937:d=2025100600:VVEL:36 hybrid level:16 hour fcst: +719:465846097:d=2025100600:TKE:36 hybrid level:16 hour fcst: +720:465846950:d=2025100600:MASSDEN:36 hybrid level:16 hour fcst: +721:467054867:d=2025100600:PRES:37 hybrid level:16 hour fcst: +722:467560558:d=2025100600:CLMR:37 hybrid level:16 hour fcst: +723:467560746:d=2025100600:CIMIXR:37 hybrid level:16 hour fcst: +724:467563694:d=2025100600:RWMR:37 hybrid level:16 hour fcst: +725:467563882:d=2025100600:SNMR:37 hybrid level:16 hour fcst: +726:467565574:d=2025100600:GRLE:37 hybrid level:16 hour fcst: +727:467566302:d=2025100600:NCONCD:37 hybrid level:16 hour fcst: +728:467566490:d=2025100600:NCCICE:37 hybrid level:16 hour fcst: +729:467570392:d=2025100600:SPNCR:37 hybrid level:16 hour fcst: +730:467570580:d=2025100600:PMTF:37 hybrid level:16 hour fcst: +731:468234162:d=2025100600:PMTC:37 hybrid level:16 hour fcst: +732:468458487:d=2025100600:FRACCC:37 hybrid level:16 hour fcst: +733:468458745:d=2025100600:HGT:37 hybrid level:16 hour fcst: +734:469777671:d=2025100600:TMP:37 hybrid level:16 hour fcst: +735:470422254:d=2025100600:SPFH:37 hybrid level:16 hour fcst: +736:471180676:d=2025100600:UGRD:37 hybrid level:16 hour fcst: +737:471979361:d=2025100600:VGRD:37 hybrid level:16 hour fcst: +738:472698260:d=2025100600:VVEL:37 hybrid level:16 hour fcst: +739:473455752:d=2025100600:TKE:37 hybrid level:16 hour fcst: +740:473456187:d=2025100600:MASSDEN:37 hybrid level:16 hour fcst: +741:474789531:d=2025100600:PRES:38 hybrid level:16 hour fcst: +742:478600001:d=2025100600:CLMR:38 hybrid level:16 hour fcst: +743:478600189:d=2025100600:CIMIXR:38 hybrid level:16 hour fcst: +744:478603075:d=2025100600:RWMR:38 hybrid level:16 hour fcst: +745:478603263:d=2025100600:SNMR:38 hybrid level:16 hour fcst: +746:478607278:d=2025100600:GRLE:38 hybrid level:16 hour fcst: +747:478609145:d=2025100600:NCONCD:38 hybrid level:16 hour fcst: +748:478609333:d=2025100600:NCCICE:38 hybrid level:16 hour fcst: +749:478613726:d=2025100600:SPNCR:38 hybrid level:16 hour fcst: +750:478613914:d=2025100600:PMTF:38 hybrid level:16 hour fcst: +751:479255251:d=2025100600:PMTC:38 hybrid level:16 hour fcst: +752:479843448:d=2025100600:FRACCC:38 hybrid level:16 hour fcst: +753:479843636:d=2025100600:HGT:38 hybrid level:16 hour fcst: +754:481154355:d=2025100600:TMP:38 hybrid level:16 hour fcst: +755:481819175:d=2025100600:SPFH:38 hybrid level:16 hour fcst: +756:482495341:d=2025100600:UGRD:38 hybrid level:16 hour fcst: +757:483286229:d=2025100600:VGRD:38 hybrid level:16 hour fcst: +758:484009914:d=2025100600:VVEL:38 hybrid level:16 hour fcst: +759:484692160:d=2025100600:TKE:38 hybrid level:16 hour fcst: +760:484692406:d=2025100600:MASSDEN:38 hybrid level:16 hour fcst: +761:486213704:d=2025100600:PRES:39 hybrid level:16 hour fcst: +762:490024174:d=2025100600:CLMR:39 hybrid level:16 hour fcst: +763:490024362:d=2025100600:CIMIXR:39 hybrid level:16 hour fcst: +764:490026434:d=2025100600:RWMR:39 hybrid level:16 hour fcst: +765:490026622:d=2025100600:SNMR:39 hybrid level:16 hour fcst: +766:490032366:d=2025100600:GRLE:39 hybrid level:16 hour fcst: +767:490035337:d=2025100600:NCONCD:39 hybrid level:16 hour fcst: +768:490035525:d=2025100600:NCCICE:39 hybrid level:16 hour fcst: +769:490036797:d=2025100600:SPNCR:39 hybrid level:16 hour fcst: +770:490036985:d=2025100600:PMTF:39 hybrid level:16 hour fcst: +771:490668586:d=2025100600:PMTC:39 hybrid level:16 hour fcst: +772:491191479:d=2025100600:FRACCC:39 hybrid level:16 hour fcst: +773:491191667:d=2025100600:HGT:39 hybrid level:16 hour fcst: +774:492473358:d=2025100600:TMP:39 hybrid level:16 hour fcst: +775:493132534:d=2025100600:SPFH:39 hybrid level:16 hour fcst: +776:493777408:d=2025100600:UGRD:39 hybrid level:16 hour fcst: +777:494568670:d=2025100600:VGRD:39 hybrid level:16 hour fcst: +778:495284717:d=2025100600:VVEL:39 hybrid level:16 hour fcst: +779:495898684:d=2025100600:TKE:39 hybrid level:16 hour fcst: +780:495991649:d=2025100600:MASSDEN:39 hybrid level:16 hour fcst: +781:497676176:d=2025100600:PRES:40 hybrid level:16 hour fcst: +782:501486646:d=2025100600:CLMR:40 hybrid level:16 hour fcst: +783:501486834:d=2025100600:CIMIXR:40 hybrid level:16 hour fcst: +784:501488582:d=2025100600:RWMR:40 hybrid level:16 hour fcst: +785:501488770:d=2025100600:SNMR:40 hybrid level:16 hour fcst: +786:501491130:d=2025100600:GRLE:40 hybrid level:16 hour fcst: +787:501492182:d=2025100600:NCONCD:40 hybrid level:16 hour fcst: +788:501492370:d=2025100600:NCCICE:40 hybrid level:16 hour fcst: +789:501493727:d=2025100600:SPNCR:40 hybrid level:16 hour fcst: +790:501493915:d=2025100600:PMTF:40 hybrid level:16 hour fcst: +791:502118539:d=2025100600:PMTC:40 hybrid level:16 hour fcst: +792:502590847:d=2025100600:FRACCC:40 hybrid level:16 hour fcst: +793:502591035:d=2025100600:HGT:40 hybrid level:16 hour fcst: +794:503845063:d=2025100600:TMP:40 hybrid level:16 hour fcst: +795:504526802:d=2025100600:SPFH:40 hybrid level:16 hour fcst: +796:505153030:d=2025100600:UGRD:40 hybrid level:16 hour fcst: +797:505945613:d=2025100600:VGRD:40 hybrid level:16 hour fcst: +798:506683846:d=2025100600:VVEL:40 hybrid level:16 hour fcst: +799:507235590:d=2025100600:TKE:40 hybrid level:16 hour fcst: +800:507295296:d=2025100600:MASSDEN:40 hybrid level:16 hour fcst: +801:509181361:d=2025100600:PRES:41 hybrid level:16 hour fcst: +802:512991831:d=2025100600:CLMR:41 hybrid level:16 hour fcst: +803:512992019:d=2025100600:CIMIXR:41 hybrid level:16 hour fcst: +804:512992718:d=2025100600:RWMR:41 hybrid level:16 hour fcst: +805:512992906:d=2025100600:SNMR:41 hybrid level:16 hour fcst: +806:512996688:d=2025100600:GRLE:41 hybrid level:16 hour fcst: +807:512997268:d=2025100600:NCONCD:41 hybrid level:16 hour fcst: +808:512997456:d=2025100600:NCCICE:41 hybrid level:16 hour fcst: +809:512998014:d=2025100600:SPNCR:41 hybrid level:16 hour fcst: +810:512998202:d=2025100600:PMTF:41 hybrid level:16 hour fcst: +811:513620259:d=2025100600:PMTC:41 hybrid level:16 hour fcst: +812:514063082:d=2025100600:FRACCC:41 hybrid level:16 hour fcst: +813:514063270:d=2025100600:HGT:41 hybrid level:16 hour fcst: +814:515315738:d=2025100600:TMP:41 hybrid level:16 hour fcst: +815:516033318:d=2025100600:SPFH:41 hybrid level:16 hour fcst: +816:516663644:d=2025100600:UGRD:41 hybrid level:16 hour fcst: +817:517504935:d=2025100600:VGRD:41 hybrid level:16 hour fcst: +818:518290355:d=2025100600:VVEL:41 hybrid level:16 hour fcst: +819:518823002:d=2025100600:TKE:41 hybrid level:16 hour fcst: +820:518869039:d=2025100600:MASSDEN:41 hybrid level:16 hour fcst: +821:520787497:d=2025100600:PRES:42 hybrid level:16 hour fcst: +822:524597967:d=2025100600:CLMR:42 hybrid level:16 hour fcst: +823:524598155:d=2025100600:CIMIXR:42 hybrid level:16 hour fcst: +824:524598462:d=2025100600:RWMR:42 hybrid level:16 hour fcst: +825:524598650:d=2025100600:SNMR:42 hybrid level:16 hour fcst: +826:524603681:d=2025100600:GRLE:42 hybrid level:16 hour fcst: +827:524603978:d=2025100600:NCONCD:42 hybrid level:16 hour fcst: +828:524604166:d=2025100600:NCCICE:42 hybrid level:16 hour fcst: +829:524604475:d=2025100600:SPNCR:42 hybrid level:16 hour fcst: +830:524604663:d=2025100600:PMTF:42 hybrid level:16 hour fcst: +831:525199020:d=2025100600:PMTC:42 hybrid level:16 hour fcst: +832:525944990:d=2025100600:FRACCC:42 hybrid level:16 hour fcst: +833:525945178:d=2025100600:HGT:42 hybrid level:16 hour fcst: +834:527239004:d=2025100600:TMP:42 hybrid level:16 hour fcst: +835:527956015:d=2025100600:SPFH:42 hybrid level:16 hour fcst: +836:528980909:d=2025100600:UGRD:42 hybrid level:16 hour fcst: +837:529811322:d=2025100600:VGRD:42 hybrid level:16 hour fcst: +838:530606647:d=2025100600:VVEL:42 hybrid level:16 hour fcst: +839:531093183:d=2025100600:TKE:42 hybrid level:16 hour fcst: +840:531123702:d=2025100600:MASSDEN:42 hybrid level:16 hour fcst: +841:533198200:d=2025100600:PRES:43 hybrid level:16 hour fcst: +842:537008670:d=2025100600:CLMR:43 hybrid level:16 hour fcst: +843:537008858:d=2025100600:CIMIXR:43 hybrid level:16 hour fcst: +844:537009046:d=2025100600:RWMR:43 hybrid level:16 hour fcst: +845:537009234:d=2025100600:SNMR:43 hybrid level:16 hour fcst: +846:537011159:d=2025100600:GRLE:43 hybrid level:16 hour fcst: +847:537011424:d=2025100600:NCONCD:43 hybrid level:16 hour fcst: +848:537011612:d=2025100600:NCCICE:43 hybrid level:16 hour fcst: +849:537011800:d=2025100600:SPNCR:43 hybrid level:16 hour fcst: +850:537011988:d=2025100600:PMTF:43 hybrid level:16 hour fcst: +851:537551932:d=2025100600:PMTC:43 hybrid level:16 hour fcst: +852:538268656:d=2025100600:FRACCC:43 hybrid level:16 hour fcst: +853:538268844:d=2025100600:HGT:43 hybrid level:16 hour fcst: +854:539548212:d=2025100600:TMP:43 hybrid level:16 hour fcst: +855:540261198:d=2025100600:SPFH:43 hybrid level:16 hour fcst: +856:541233371:d=2025100600:UGRD:43 hybrid level:16 hour fcst: +857:542073163:d=2025100600:VGRD:43 hybrid level:16 hour fcst: +858:542863134:d=2025100600:VVEL:43 hybrid level:16 hour fcst: +859:543283190:d=2025100600:TKE:43 hybrid level:16 hour fcst: +860:543297628:d=2025100600:MASSDEN:43 hybrid level:16 hour fcst: +861:545260568:d=2025100600:PRES:44 hybrid level:16 hour fcst: +862:549071038:d=2025100600:CLMR:44 hybrid level:16 hour fcst: +863:549071226:d=2025100600:CIMIXR:44 hybrid level:16 hour fcst: +864:549071414:d=2025100600:RWMR:44 hybrid level:16 hour fcst: +865:549071602:d=2025100600:SNMR:44 hybrid level:16 hour fcst: +866:549072751:d=2025100600:GRLE:44 hybrid level:16 hour fcst: +867:549072939:d=2025100600:NCONCD:44 hybrid level:16 hour fcst: +868:549073127:d=2025100600:NCCICE:44 hybrid level:16 hour fcst: +869:549073315:d=2025100600:SPNCR:44 hybrid level:16 hour fcst: +870:549073503:d=2025100600:PMTF:44 hybrid level:16 hour fcst: +871:549598300:d=2025100600:PMTC:44 hybrid level:16 hour fcst: +872:550300413:d=2025100600:FRACCC:44 hybrid level:16 hour fcst: +873:550300601:d=2025100600:HGT:44 hybrid level:16 hour fcst: +874:551575014:d=2025100600:TMP:44 hybrid level:16 hour fcst: +875:552281889:d=2025100600:SPFH:44 hybrid level:16 hour fcst: +876:553243988:d=2025100600:UGRD:44 hybrid level:16 hour fcst: +877:554077180:d=2025100600:VGRD:44 hybrid level:16 hour fcst: +878:554858318:d=2025100600:VVEL:44 hybrid level:16 hour fcst: +879:555251445:d=2025100600:TKE:44 hybrid level:16 hour fcst: +880:555255384:d=2025100600:MASSDEN:44 hybrid level:16 hour fcst: +881:557274170:d=2025100600:PRES:45 hybrid level:16 hour fcst: +882:561084640:d=2025100600:CLMR:45 hybrid level:16 hour fcst: +883:561084828:d=2025100600:CIMIXR:45 hybrid level:16 hour fcst: +884:561085016:d=2025100600:RWMR:45 hybrid level:16 hour fcst: +885:561085204:d=2025100600:SNMR:45 hybrid level:16 hour fcst: +886:561085502:d=2025100600:GRLE:45 hybrid level:16 hour fcst: +887:561085690:d=2025100600:NCONCD:45 hybrid level:16 hour fcst: +888:561085878:d=2025100600:NCCICE:45 hybrid level:16 hour fcst: +889:561086066:d=2025100600:SPNCR:45 hybrid level:16 hour fcst: +890:561086254:d=2025100600:PMTF:45 hybrid level:16 hour fcst: +891:561908566:d=2025100600:PMTC:45 hybrid level:16 hour fcst: +892:562586246:d=2025100600:FRACCC:45 hybrid level:16 hour fcst: +893:562586434:d=2025100600:HGT:45 hybrid level:16 hour fcst: +894:563868221:d=2025100600:TMP:45 hybrid level:16 hour fcst: +895:564559922:d=2025100600:SPFH:45 hybrid level:16 hour fcst: +896:566263216:d=2025100600:UGRD:45 hybrid level:16 hour fcst: +897:567094149:d=2025100600:VGRD:45 hybrid level:16 hour fcst: +898:567872075:d=2025100600:VVEL:45 hybrid level:16 hour fcst: +899:568245532:d=2025100600:TKE:45 hybrid level:16 hour fcst: +900:568246603:d=2025100600:MASSDEN:45 hybrid level:16 hour fcst: +901:570289487:d=2025100600:PRES:46 hybrid level:16 hour fcst: +902:574099957:d=2025100600:CLMR:46 hybrid level:16 hour fcst: +903:574100145:d=2025100600:CIMIXR:46 hybrid level:16 hour fcst: +904:574100333:d=2025100600:RWMR:46 hybrid level:16 hour fcst: +905:574100521:d=2025100600:SNMR:46 hybrid level:16 hour fcst: +906:574100709:d=2025100600:GRLE:46 hybrid level:16 hour fcst: +907:574100897:d=2025100600:NCONCD:46 hybrid level:16 hour fcst: +908:574101085:d=2025100600:NCCICE:46 hybrid level:16 hour fcst: +909:574101273:d=2025100600:SPNCR:46 hybrid level:16 hour fcst: +910:574101461:d=2025100600:PMTF:46 hybrid level:16 hour fcst: +911:574579926:d=2025100600:PMTC:46 hybrid level:16 hour fcst: +912:575222287:d=2025100600:FRACCC:46 hybrid level:16 hour fcst: +913:575222475:d=2025100600:HGT:46 hybrid level:16 hour fcst: +914:576508998:d=2025100600:TMP:46 hybrid level:16 hour fcst: +915:577207665:d=2025100600:SPFH:46 hybrid level:16 hour fcst: +916:578857809:d=2025100600:UGRD:46 hybrid level:16 hour fcst: +917:579675043:d=2025100600:VGRD:46 hybrid level:16 hour fcst: +918:580436280:d=2025100600:VVEL:46 hybrid level:16 hour fcst: +919:580706432:d=2025100600:TKE:46 hybrid level:16 hour fcst: +920:580706652:d=2025100600:MASSDEN:46 hybrid level:16 hour fcst: +921:582843766:d=2025100600:PRES:47 hybrid level:16 hour fcst: +922:586654236:d=2025100600:CLMR:47 hybrid level:16 hour fcst: +923:586654424:d=2025100600:CIMIXR:47 hybrid level:16 hour fcst: +924:586654612:d=2025100600:RWMR:47 hybrid level:16 hour fcst: +925:586654800:d=2025100600:SNMR:47 hybrid level:16 hour fcst: +926:586654988:d=2025100600:GRLE:47 hybrid level:16 hour fcst: +927:586655176:d=2025100600:NCONCD:47 hybrid level:16 hour fcst: +928:586655364:d=2025100600:NCCICE:47 hybrid level:16 hour fcst: +929:586655552:d=2025100600:SPNCR:47 hybrid level:16 hour fcst: +930:586655740:d=2025100600:PMTF:47 hybrid level:16 hour fcst: +931:587130102:d=2025100600:PMTC:47 hybrid level:16 hour fcst: +932:587759309:d=2025100600:FRACCC:47 hybrid level:16 hour fcst: +933:587759497:d=2025100600:HGT:47 hybrid level:16 hour fcst: +934:589044371:d=2025100600:TMP:47 hybrid level:16 hour fcst: +935:589732725:d=2025100600:SPFH:47 hybrid level:16 hour fcst: +936:591317486:d=2025100600:UGRD:47 hybrid level:16 hour fcst: +937:592125940:d=2025100600:VGRD:47 hybrid level:16 hour fcst: +938:592875069:d=2025100600:VVEL:47 hybrid level:16 hour fcst: +939:593130924:d=2025100600:TKE:47 hybrid level:16 hour fcst: +940:593131112:d=2025100600:MASSDEN:47 hybrid level:16 hour fcst: +941:595225541:d=2025100600:PRES:48 hybrid level:16 hour fcst: +942:599036011:d=2025100600:CLMR:48 hybrid level:16 hour fcst: +943:599036199:d=2025100600:CIMIXR:48 hybrid level:16 hour fcst: +944:599036387:d=2025100600:RWMR:48 hybrid level:16 hour fcst: +945:599036575:d=2025100600:SNMR:48 hybrid level:16 hour fcst: +946:599036763:d=2025100600:GRLE:48 hybrid level:16 hour fcst: +947:599036951:d=2025100600:NCONCD:48 hybrid level:16 hour fcst: +948:599037139:d=2025100600:NCCICE:48 hybrid level:16 hour fcst: +949:599037327:d=2025100600:SPNCR:48 hybrid level:16 hour fcst: +950:599037515:d=2025100600:PMTF:48 hybrid level:16 hour fcst: +951:599515584:d=2025100600:PMTC:48 hybrid level:16 hour fcst: +952:600128503:d=2025100600:FRACCC:48 hybrid level:16 hour fcst: +953:600128691:d=2025100600:HGT:48 hybrid level:16 hour fcst: +954:601414807:d=2025100600:TMP:48 hybrid level:16 hour fcst: +955:602103144:d=2025100600:SPFH:48 hybrid level:16 hour fcst: +956:603685731:d=2025100600:UGRD:48 hybrid level:16 hour fcst: +957:604494785:d=2025100600:VGRD:48 hybrid level:16 hour fcst: +958:605247464:d=2025100600:VVEL:48 hybrid level:16 hour fcst: +959:605448365:d=2025100600:TKE:48 hybrid level:16 hour fcst: +960:605448553:d=2025100600:MASSDEN:48 hybrid level:16 hour fcst: +961:607445851:d=2025100600:PRES:49 hybrid level:16 hour fcst: +962:611256321:d=2025100600:CLMR:49 hybrid level:16 hour fcst: +963:611256509:d=2025100600:CIMIXR:49 hybrid level:16 hour fcst: +964:611256697:d=2025100600:RWMR:49 hybrid level:16 hour fcst: +965:611256885:d=2025100600:SNMR:49 hybrid level:16 hour fcst: +966:611257073:d=2025100600:GRLE:49 hybrid level:16 hour fcst: +967:611257261:d=2025100600:NCONCD:49 hybrid level:16 hour fcst: +968:611257449:d=2025100600:NCCICE:49 hybrid level:16 hour fcst: +969:611257637:d=2025100600:SPNCR:49 hybrid level:16 hour fcst: +970:611257825:d=2025100600:PMTF:49 hybrid level:16 hour fcst: +971:611746656:d=2025100600:PMTC:49 hybrid level:16 hour fcst: +972:612365154:d=2025100600:FRACCC:49 hybrid level:16 hour fcst: +973:612365342:d=2025100600:HGT:49 hybrid level:16 hour fcst: +974:613650774:d=2025100600:TMP:49 hybrid level:16 hour fcst: +975:614311753:d=2025100600:SPFH:49 hybrid level:16 hour fcst: +976:615923227:d=2025100600:UGRD:49 hybrid level:16 hour fcst: +977:616698729:d=2025100600:VGRD:49 hybrid level:16 hour fcst: +978:617411418:d=2025100600:VVEL:49 hybrid level:16 hour fcst: +979:617617183:d=2025100600:TKE:49 hybrid level:16 hour fcst: +980:617617371:d=2025100600:MASSDEN:49 hybrid level:16 hour fcst: +981:619365828:d=2025100600:PRES:50 hybrid level:16 hour fcst: +982:623176298:d=2025100600:CLMR:50 hybrid level:16 hour fcst: +983:623176486:d=2025100600:CIMIXR:50 hybrid level:16 hour fcst: +984:623176674:d=2025100600:RWMR:50 hybrid level:16 hour fcst: +985:623176862:d=2025100600:SNMR:50 hybrid level:16 hour fcst: +986:623177050:d=2025100600:GRLE:50 hybrid level:16 hour fcst: +987:623177238:d=2025100600:NCONCD:50 hybrid level:16 hour fcst: +988:623177426:d=2025100600:NCCICE:50 hybrid level:16 hour fcst: +989:623177614:d=2025100600:SPNCR:50 hybrid level:16 hour fcst: +990:623177802:d=2025100600:PMTF:50 hybrid level:16 hour fcst: +991:623688737:d=2025100600:PMTC:50 hybrid level:16 hour fcst: +992:624314935:d=2025100600:FRACCC:50 hybrid level:16 hour fcst: +993:624315123:d=2025100600:HGT:50 hybrid level:16 hour fcst: +994:625578363:d=2025100600:TMP:50 hybrid level:16 hour fcst: +995:626253691:d=2025100600:SPFH:50 hybrid level:16 hour fcst: +996:627167649:d=2025100600:UGRD:50 hybrid level:16 hour fcst: +997:627803696:d=2025100600:VGRD:50 hybrid level:16 hour fcst: +998:628415609:d=2025100600:VVEL:50 hybrid level:16 hour fcst: +999:628642835:d=2025100600:TKE:50 hybrid level:16 hour fcst: +1000:628643023:d=2025100600:MASSDEN:50 hybrid level:16 hour fcst: +1001:630742305:d=2025100600:SOILW:0-0 m below ground:16 hour fcst: +1002:631983681:d=2025100600:SOILW:0.01-0.01 m below ground:16 hour fcst: +1003:633245010:d=2025100600:REFC:entire atmosphere:16 hour fcst: +1004:633774884:d=2025100600:RETOP:cloud top:16 hour fcst: +1005:633984358:d=2025100600:VIL:entire atmosphere:16 hour fcst: +1006:634294000:d=2025100600:VIS:surface:16 hour fcst: +1007:635792306:d=2025100600:REFD:1000 m above ground:16 hour fcst: +1008:636139549:d=2025100600:REFD:4000 m above ground:16 hour fcst: +1009:636323240:d=2025100600:REFD:263 K level:16 hour fcst: +1010:636504677:d=2025100600:GUST:surface:16 hour fcst: +1011:637690259:d=2025100600:MAXUVV:100-1000 mb above ground:15-16 hour max fcst: +1012:638512148:d=2025100600:MAXDVV:100-1000 mb above ground:15-16 hour max fcst: +1013:639270772:d=2025100600:DZDT:0.5-0.8 sigma layer:15-16 hour ave fcst: +1014:639876972:d=2025100600:MSLMA:mean sea level:16 hour fcst: +1015:640488257:d=2025100600:HGT:1000 mb:16 hour fcst: +1016:641194146:d=2025100600:MAXREF:1000 m above ground:15-16 hour max fcst: +1017:641510511:d=2025100600:REFD:263 K level:15-16 hour max fcst: +1018:641737366:d=2025100600:MXUPHL:5000-2000 m above ground:15-16 hour max fcst: +1019:641794809:d=2025100600:MNUPHL:5000-2000 m above ground:15-16 hour min fcst: +1020:641850333:d=2025100600:MXUPHL:2000-0 m above ground:15-16 hour max fcst: +1021:641898160:d=2025100600:MNUPHL:2000-0 m above ground:15-16 hour min fcst: +1022:641928000:d=2025100600:MXUPHL:3000-0 m above ground:15-16 hour max fcst: +1023:641983268:d=2025100600:MNUPHL:3000-0 m above ground:15-16 hour min fcst: +1024:642022379:d=2025100600:RELV:2000-0 m above ground:15-16 hour max fcst: +1025:644639823:d=2025100600:RELV:1000-0 m above ground:15-16 hour max fcst: +1026:647546850:d=2025100600:HAIL:entire atmosphere:15-16 hour max fcst: +1027:647781144:d=2025100600:HAIL:0.1 sigma level:15-16 hour max fcst: +1028:647782320:d=2025100600:TCOLG:entire atmosphere (considered as a single layer):15-16 hour max fcst: +1029:647836600:d=2025100600:LTNG:entire atmosphere:16 hour fcst: +1030:647869156:d=2025100600:UGRD:80 m above ground:16 hour fcst: +1031:649076871:d=2025100600:VGRD:80 m above ground:16 hour fcst: +1032:650212220:d=2025100600:PRES:surface:16 hour fcst: +1033:651723436:d=2025100600:HGT:surface:16 hour fcst: +1034:653877131:d=2025100600:TMP:surface:16 hour fcst: +1035:655233110:d=2025100600:ASNOW:surface:0-16 hour acc fcst: +1036:655277062:d=2025100600:MSTAV:0 m underground:16 hour fcst: +1037:656750350:d=2025100600:CNWAT:surface:16 hour fcst: +1038:656840459:d=2025100600:WEASD:surface:16 hour fcst: +1039:656858845:d=2025100600:SNOWC:surface:16 hour fcst: +1040:656872567:d=2025100600:SNOD:surface:16 hour fcst: +1041:656887952:d=2025100600:TMP:2 m above ground:16 hour fcst: +1042:658101664:d=2025100600:POT:2 m above ground:16 hour fcst: +1043:659241522:d=2025100600:SPFH:2 m above ground:16 hour fcst: +1044:660855796:d=2025100600:DPT:2 m above ground:16 hour fcst: +1045:662077418:d=2025100600:RH:2 m above ground:16 hour fcst: +1046:663722586:d=2025100600:MASSDEN:8 m above ground:16 hour fcst: +1047:664476642:d=2025100600:UGRD:10 m above ground:16 hour fcst: +1048:666858257:d=2025100600:VGRD:10 m above ground:16 hour fcst: +1049:669239872:d=2025100600:WIND:10 m above ground:15-16 hour max fcst: +1050:670466135:d=2025100600:MAXUW:10 m above ground:15-16 hour max fcst: +1051:671830867:d=2025100600:MAXVW:10 m above ground:15-16 hour max fcst: +1052:673140904:d=2025100600:CPOFP:surface:16 hour fcst: +1053:673270526:d=2025100600:PRATE:surface:16 hour fcst: +1054:673342198:d=2025100600:APCP:surface:0-16 hour acc fcst: +1055:674068507:d=2025100600:WEASD:surface:0-16 hour acc fcst: +1056:674105925:d=2025100600:FROZR:surface:0-16 hour acc fcst: +1057:674152239:d=2025100600:FRZR:surface:0-16 hour acc fcst: +1058:674170229:d=2025100600:SSRUN:surface:15-16 hour acc fcst: +1059:674188531:d=2025100600:BGRUN:surface:15-16 hour acc fcst: +1060:674188915:d=2025100600:APCP:surface:15-16 hour acc fcst: +1061:674431581:d=2025100600:WEASD:surface:15-16 hour acc fcst: +1062:674435094:d=2025100600:FROZR:surface:15-16 hour acc fcst: +1063:674437659:d=2025100600:CSNOW:surface:16 hour fcst: +1064:674438296:d=2025100600:CICEP:surface:16 hour fcst: +1065:674438537:d=2025100600:CFRZR:surface:16 hour fcst: +1066:674438981:d=2025100600:CRAIN:surface:16 hour fcst: +1067:674518622:d=2025100600:SFCR:surface:16 hour fcst: +1068:676436550:d=2025100600:FRICV:surface:16 hour fcst: +1069:677575086:d=2025100600:SHTFL:surface:16 hour fcst: +1070:679076095:d=2025100600:LHTFL:surface:16 hour fcst: +1071:680683350:d=2025100600:GFLUX:surface:16 hour fcst: +1072:681332960:d=2025100600:VGTYP:surface:16 hour fcst: +1073:682114139:d=2025100600:LFTX:500-1000 mb:16 hour fcst: +1074:683078298:d=2025100600:CAPE:surface:16 hour fcst: +1075:683680102:d=2025100600:CIN:surface:16 hour fcst: +1076:684368362:d=2025100600:PWAT:entire atmosphere (considered as a single layer):16 hour fcst: +1077:685411078:d=2025100600:AOTK:entire atmosphere (considered as a single layer):16 hour fcst: +1078:686533917:d=2025100600:COLMD:entire atmosphere (considered as a single layer):16 hour fcst: +1079:687438958:d=2025100600:LCDC:low cloud layer:16 hour fcst: +1080:688407266:d=2025100600:MCDC:middle cloud layer:16 hour fcst: +1081:688865146:d=2025100600:HCDC:high cloud layer:16 hour fcst: +1082:689242493:d=2025100600:TCDC:entire atmosphere:16 hour fcst: +1083:690188975:d=2025100600:HGT:cloud ceiling:16 hour fcst: +1084:691571070:d=2025100600:HGT:cloud base:16 hour fcst: +1085:694048623:d=2025100600:PRES:cloud base:16 hour fcst: +1086:695265646:d=2025100600:PRES:cloud top:16 hour fcst: +1087:695967301:d=2025100600:HGT:cloud top:16 hour fcst: +1088:697190089:d=2025100600:ULWRF:top of atmosphere:16 hour fcst: +1089:699199908:d=2025100600:DSWRF:surface:16 hour fcst: +1090:701852022:d=2025100600:DLWRF:surface:16 hour fcst: +1091:704032629:d=2025100600:USWRF:surface:16 hour fcst: +1092:706146169:d=2025100600:ULWRF:surface:16 hour fcst: +1093:707830726:d=2025100600:CFNSF:surface:16 hour fcst: +1094:707835605:d=2025100600:VBDSF:surface:16 hour fcst: +1095:709880154:d=2025100600:VDDSF:surface:16 hour fcst: +1096:712625728:d=2025100600:USWRF:top of atmosphere:16 hour fcst: +1097:715265027:d=2025100600:HLCY:3000-0 m above ground:16 hour fcst: +1098:716461643:d=2025100600:HLCY:1000-0 m above ground:16 hour fcst: +1099:718387051:d=2025100600:USTM:0-6000 m above ground:16 hour fcst: +1100:719496963:d=2025100600:VSTM:0-6000 m above ground:16 hour fcst: +1101:720562317:d=2025100600:VUCSH:0-1000 m above ground:16 hour fcst: +1102:722943932:d=2025100600:VVCSH:0-1000 m above ground:16 hour fcst: +1103:725325547:d=2025100600:VUCSH:0-6000 m above ground:16 hour fcst: +1104:727707162:d=2025100600:VVCSH:0-6000 m above ground:16 hour fcst: +1105:730326919:d=2025100600:HGT:0C isotherm:16 hour fcst: +1106:732618038:d=2025100600:RH:0C isotherm:16 hour fcst: +1107:733380863:d=2025100600:PRES:0C isotherm:16 hour fcst: +1108:734125282:d=2025100600:HGT:highest tropospheric freezing level:16 hour fcst: +1109:734896800:d=2025100600:RH:highest tropospheric freezing level:16 hour fcst: +1110:735646333:d=2025100600:PRES:highest tropospheric freezing level:16 hour fcst: +1111:736387892:d=2025100600:HGT:263 K level:16 hour fcst: +1112:737081661:d=2025100600:HGT:253 K level:16 hour fcst: +1113:737703986:d=2025100600:4LFTX:180-0 mb above ground:16 hour fcst: +1114:738674405:d=2025100600:CAPE:180-0 mb above ground:16 hour fcst: +1115:739274969:d=2025100600:CIN:180-0 mb above ground:16 hour fcst: +1116:739982133:d=2025100600:HPBL:surface:16 hour fcst: +1117:742993897:d=2025100600:HGT:level of adiabatic condensation from sfc:16 hour fcst: +1118:745978329:d=2025100600:CAPE:90-0 mb above ground:16 hour fcst: +1119:746474940:d=2025100600:CIN:90-0 mb above ground:16 hour fcst: +1120:747121176:d=2025100600:CAPE:255-0 mb above ground:16 hour fcst: +1121:747741125:d=2025100600:CIN:255-0 mb above ground:16 hour fcst: +1122:748441927:d=2025100600:HGT:equilibrium level:16 hour fcst: +1123:750765392:d=2025100600:PLPL:255-0 mb above ground:16 hour fcst: +1124:751809005:d=2025100600:CAPE:0-3000 m above ground:16 hour fcst: +1125:752837041:d=2025100600:HGT:level of free convection:16 hour fcst: +1126:755743823:d=2025100600:EFHL:surface:16 hour fcst: +1127:756814472:d=2025100600:CANGLE:0-500 m above ground:16 hour fcst: +1128:759105238:d=2025100600:LAYTH:261 K level - 256 K level:16 hour fcst: +1129:760437999:d=2025100600:ESP:0-3000 m above ground:16 hour fcst: +1130:761432482:d=2025100600:RHPW:entire atmosphere:16 hour fcst: +1131:762674516:d=2025100600:LAND:surface:16 hour fcst: +1132:762724992:d=2025100600:ICEC:surface:16 hour fcst: +1133:762725225:d=2025100600:SBT123:top of atmosphere:16 hour fcst: +1134:764382093:d=2025100600:SBT124:top of atmosphere:16 hour fcst: +1135:766169273:d=2025100600:SBT113:top of atmosphere:16 hour fcst: +1136:767706923:d=2025100600:SBT114:top of atmosphere:16 hour fcst: +1137:769413571:d=2025100600:HGT:50 mb:16 hour fcst: +1138:770117417:d=2025100600:TMP:50 mb:16 hour fcst: +1139:770663364:d=2025100600:RH:50 mb:16 hour fcst: +1140:770886421:d=2025100600:DPT:50 mb:16 hour fcst: +1141:770886609:d=2025100600:SPFH:50 mb:16 hour fcst: +1142:771801650:d=2025100600:VVEL:50 mb:16 hour fcst: +1143:772024945:d=2025100600:UGRD:50 mb:16 hour fcst: +1144:772595906:d=2025100600:VGRD:50 mb:16 hour fcst: +1145:773165770:d=2025100600:ABSV:50 mb:16 hour fcst: +1146:773889954:d=2025100600:CLMR:50 mb:16 hour fcst: +1147:773890142:d=2025100600:CIMIXR:50 mb:16 hour fcst: +1148:773890330:d=2025100600:RWMR:50 mb:16 hour fcst: +1149:773890518:d=2025100600:SNMR:50 mb:16 hour fcst: +1150:773891556:d=2025100600:GRLE:50 mb:16 hour fcst: +1151:773891744:d=2025100600:HGT:75 mb:16 hour fcst: +1152:774593111:d=2025100600:TMP:75 mb:16 hour fcst: +1153:775157691:d=2025100600:RH:75 mb:16 hour fcst: +1154:775645706:d=2025100600:DPT:75 mb:16 hour fcst: +1155:775646116:d=2025100600:SPFH:75 mb:16 hour fcst: +1156:776264759:d=2025100600:VVEL:75 mb:16 hour fcst: +1157:776580449:d=2025100600:UGRD:75 mb:16 hour fcst: +1158:777157853:d=2025100600:VGRD:75 mb:16 hour fcst: +1159:777730032:d=2025100600:ABSV:75 mb:16 hour fcst: +1160:778246433:d=2025100600:CLMR:75 mb:16 hour fcst: +1161:778246621:d=2025100600:CIMIXR:75 mb:16 hour fcst: +1162:778247319:d=2025100600:RWMR:75 mb:16 hour fcst: +1163:778247507:d=2025100600:SNMR:75 mb:16 hour fcst: +1164:778251230:d=2025100600:GRLE:75 mb:16 hour fcst: +1165:778251809:d=2025100600:HGT:100 mb:16 hour fcst: +1166:778961168:d=2025100600:TMP:100 mb:16 hour fcst: +1167:779514641:d=2025100600:RH:100 mb:16 hour fcst: +1168:780055101:d=2025100600:DPT:100 mb:16 hour fcst: +1169:780062507:d=2025100600:SPFH:100 mb:16 hour fcst: +1170:780703478:d=2025100600:VVEL:100 mb:16 hour fcst: +1171:781109854:d=2025100600:UGRD:100 mb:16 hour fcst: +1172:781692270:d=2025100600:VGRD:100 mb:16 hour fcst: +1173:782261475:d=2025100600:ABSV:100 mb:16 hour fcst: +1174:782922522:d=2025100600:CLMR:100 mb:16 hour fcst: +1175:782922710:d=2025100600:CIMIXR:100 mb:16 hour fcst: +1176:782924915:d=2025100600:RWMR:100 mb:16 hour fcst: +1177:782925103:d=2025100600:SNMR:100 mb:16 hour fcst: +1178:782931624:d=2025100600:GRLE:100 mb:16 hour fcst: +1179:782935076:d=2025100600:HGT:125 mb:16 hour fcst: +1180:783672207:d=2025100600:TMP:125 mb:16 hour fcst: +1181:784232112:d=2025100600:RH:125 mb:16 hour fcst: +1182:784792460:d=2025100600:DPT:125 mb:16 hour fcst: +1183:784817152:d=2025100600:SPFH:125 mb:16 hour fcst: +1184:785562763:d=2025100600:VVEL:125 mb:16 hour fcst: +1185:786060581:d=2025100600:UGRD:125 mb:16 hour fcst: +1186:786651350:d=2025100600:VGRD:125 mb:16 hour fcst: +1187:787218158:d=2025100600:ABSV:125 mb:16 hour fcst: +1188:787711110:d=2025100600:CLMR:125 mb:16 hour fcst: +1189:787711298:d=2025100600:CIMIXR:125 mb:16 hour fcst: +1190:787714122:d=2025100600:RWMR:125 mb:16 hour fcst: +1191:787714310:d=2025100600:SNMR:125 mb:16 hour fcst: +1192:787715940:d=2025100600:GRLE:125 mb:16 hour fcst: +1193:787716639:d=2025100600:HGT:150 mb:16 hour fcst: +1194:788454176:d=2025100600:TMP:150 mb:16 hour fcst: +1195:789012294:d=2025100600:RH:150 mb:16 hour fcst: +1196:789637836:d=2025100600:DPT:150 mb:16 hour fcst: +1197:789934807:d=2025100600:SPFH:150 mb:16 hour fcst: +1198:790949774:d=2025100600:VVEL:150 mb:16 hour fcst: +1199:791475565:d=2025100600:UGRD:150 mb:16 hour fcst: +1200:792060055:d=2025100600:VGRD:150 mb:16 hour fcst: +1201:792633025:d=2025100600:ABSV:150 mb:16 hour fcst: +1202:793121973:d=2025100600:CLMR:150 mb:16 hour fcst: +1203:793122161:d=2025100600:CIMIXR:150 mb:16 hour fcst: +1204:793152423:d=2025100600:RWMR:150 mb:16 hour fcst: +1205:793152611:d=2025100600:SNMR:150 mb:16 hour fcst: +1206:793164374:d=2025100600:GRLE:150 mb:16 hour fcst: +1207:793166371:d=2025100600:HGT:175 mb:16 hour fcst: +1208:793916149:d=2025100600:TMP:175 mb:16 hour fcst: +1209:794477803:d=2025100600:RH:175 mb:16 hour fcst: +1210:795139701:d=2025100600:DPT:175 mb:16 hour fcst: +1211:795703455:d=2025100600:SPFH:175 mb:16 hour fcst: +1212:796975376:d=2025100600:VVEL:175 mb:16 hour fcst: +1213:797524737:d=2025100600:UGRD:175 mb:16 hour fcst: +1214:798109516:d=2025100600:VGRD:175 mb:16 hour fcst: +1215:798678326:d=2025100600:ABSV:175 mb:16 hour fcst: +1216:799171133:d=2025100600:CLMR:175 mb:16 hour fcst: +1217:799171321:d=2025100600:CIMIXR:175 mb:16 hour fcst: +1218:799202613:d=2025100600:RWMR:175 mb:16 hour fcst: +1219:799202801:d=2025100600:SNMR:175 mb:16 hour fcst: +1220:799228477:d=2025100600:GRLE:175 mb:16 hour fcst: +1221:799232663:d=2025100600:HGT:200 mb:16 hour fcst: +1222:799973916:d=2025100600:TMP:200 mb:16 hour fcst: +1223:800537658:d=2025100600:RH:200 mb:16 hour fcst: +1224:801226291:d=2025100600:DPT:200 mb:16 hour fcst: +1225:801897047:d=2025100600:SPFH:200 mb:16 hour fcst: +1226:802757586:d=2025100600:VVEL:200 mb:16 hour fcst: +1227:803326890:d=2025100600:UGRD:200 mb:16 hour fcst: +1228:803912708:d=2025100600:VGRD:200 mb:16 hour fcst: +1229:804493061:d=2025100600:ABSV:200 mb:16 hour fcst: +1230:804997283:d=2025100600:CLMR:200 mb:16 hour fcst: +1231:804997471:d=2025100600:CIMIXR:200 mb:16 hour fcst: +1232:805043437:d=2025100600:RWMR:200 mb:16 hour fcst: +1233:805043681:d=2025100600:SNMR:200 mb:16 hour fcst: +1234:805083532:d=2025100600:GRLE:200 mb:16 hour fcst: +1235:805089699:d=2025100600:HGT:225 mb:16 hour fcst: +1236:805823608:d=2025100600:TMP:225 mb:16 hour fcst: +1237:806386515:d=2025100600:RH:225 mb:16 hour fcst: +1238:807107807:d=2025100600:DPT:225 mb:16 hour fcst: +1239:807909344:d=2025100600:SPFH:225 mb:16 hour fcst: +1240:808961616:d=2025100600:VVEL:225 mb:16 hour fcst: +1241:809545692:d=2025100600:UGRD:225 mb:16 hour fcst: +1242:810144022:d=2025100600:VGRD:225 mb:16 hour fcst: +1243:810729321:d=2025100600:ABSV:225 mb:16 hour fcst: +1244:811255731:d=2025100600:CLMR:225 mb:16 hour fcst: +1245:811255974:d=2025100600:CIMIXR:225 mb:16 hour fcst: +1246:811313403:d=2025100600:RWMR:225 mb:16 hour fcst: +1247:811313667:d=2025100600:SNMR:225 mb:16 hour fcst: +1248:811368267:d=2025100600:GRLE:225 mb:16 hour fcst: +1249:811376118:d=2025100600:HGT:250 mb:16 hour fcst: +1250:812102545:d=2025100600:TMP:250 mb:16 hour fcst: +1251:812656596:d=2025100600:RH:250 mb:16 hour fcst: +1252:813397497:d=2025100600:DPT:250 mb:16 hour fcst: +1253:814315449:d=2025100600:SPFH:250 mb:16 hour fcst: +1254:815539833:d=2025100600:VVEL:250 mb:16 hour fcst: +1255:816133635:d=2025100600:UGRD:250 mb:16 hour fcst: +1256:816733869:d=2025100600:VGRD:250 mb:16 hour fcst: +1257:817322137:d=2025100600:ABSV:250 mb:16 hour fcst: +1258:817841277:d=2025100600:CLMR:250 mb:16 hour fcst: +1259:817841530:d=2025100600:CIMIXR:250 mb:16 hour fcst: +1260:817894590:d=2025100600:RWMR:250 mb:16 hour fcst: +1261:817894873:d=2025100600:SNMR:250 mb:16 hour fcst: +1262:817962889:d=2025100600:GRLE:250 mb:16 hour fcst: +1263:817972515:d=2025100600:HGT:275 mb:16 hour fcst: +1264:818704505:d=2025100600:TMP:275 mb:16 hour fcst: +1265:819249135:d=2025100600:RH:275 mb:16 hour fcst: +1266:819984406:d=2025100600:DPT:275 mb:16 hour fcst: +1267:820932456:d=2025100600:SPFH:275 mb:16 hour fcst: +1268:822290698:d=2025100600:VVEL:275 mb:16 hour fcst: +1269:822890466:d=2025100600:UGRD:275 mb:16 hour fcst: +1270:823488640:d=2025100600:VGRD:275 mb:16 hour fcst: +1271:824076418:d=2025100600:ABSV:275 mb:16 hour fcst: +1272:824596507:d=2025100600:CLMR:275 mb:16 hour fcst: +1273:824597792:d=2025100600:CIMIXR:275 mb:16 hour fcst: +1274:824605176:d=2025100600:RWMR:275 mb:16 hour fcst: +1275:824606155:d=2025100600:SNMR:275 mb:16 hour fcst: +1276:824685208:d=2025100600:GRLE:275 mb:16 hour fcst: +1277:824696476:d=2025100600:HGT:300 mb:16 hour fcst: +1278:825420532:d=2025100600:TMP:300 mb:16 hour fcst: +1279:825963243:d=2025100600:RH:300 mb:16 hour fcst: +1280:826701621:d=2025100600:DPT:300 mb:16 hour fcst: +1281:827701354:d=2025100600:SPFH:300 mb:16 hour fcst: +1282:828579616:d=2025100600:VVEL:300 mb:16 hour fcst: +1283:829182740:d=2025100600:UGRD:300 mb:16 hour fcst: +1284:829774173:d=2025100600:VGRD:300 mb:16 hour fcst: +1285:830361811:d=2025100600:ABSV:300 mb:16 hour fcst: +1286:830882238:d=2025100600:CLMR:300 mb:16 hour fcst: +1287:830883982:d=2025100600:CIMIXR:300 mb:16 hour fcst: +1288:830919897:d=2025100600:RWMR:300 mb:16 hour fcst: +1289:830922399:d=2025100600:SNMR:300 mb:16 hour fcst: +1290:831011926:d=2025100600:GRLE:300 mb:16 hour fcst: +1291:831024515:d=2025100600:HGT:325 mb:16 hour fcst: +1292:831742917:d=2025100600:TMP:325 mb:16 hour fcst: +1293:832284018:d=2025100600:RH:325 mb:16 hour fcst: +1294:833027105:d=2025100600:DPT:325 mb:16 hour fcst: +1295:834048123:d=2025100600:SPFH:325 mb:16 hour fcst: +1296:835019175:d=2025100600:VVEL:325 mb:16 hour fcst: +1297:835625713:d=2025100600:UGRD:325 mb:16 hour fcst: +1298:836218126:d=2025100600:VGRD:325 mb:16 hour fcst: +1299:836807320:d=2025100600:ABSV:325 mb:16 hour fcst: +1300:837330521:d=2025100600:CLMR:325 mb:16 hour fcst: +1301:837332981:d=2025100600:CIMIXR:325 mb:16 hour fcst: +1302:837357463:d=2025100600:RWMR:325 mb:16 hour fcst: +1303:837359841:d=2025100600:SNMR:325 mb:16 hour fcst: +1304:837457881:d=2025100600:GRLE:325 mb:16 hour fcst: +1305:837471943:d=2025100600:HGT:350 mb:16 hour fcst: +1306:838184940:d=2025100600:TMP:350 mb:16 hour fcst: +1307:838725668:d=2025100600:RH:350 mb:16 hour fcst: +1308:839469986:d=2025100600:DPT:350 mb:16 hour fcst: +1309:840498033:d=2025100600:SPFH:350 mb:16 hour fcst: +1310:841543899:d=2025100600:VVEL:350 mb:16 hour fcst: +1311:842153105:d=2025100600:UGRD:350 mb:16 hour fcst: +1312:842744968:d=2025100600:VGRD:350 mb:16 hour fcst: +1313:843333611:d=2025100600:ABSV:350 mb:16 hour fcst: +1314:843867926:d=2025100600:CLMR:350 mb:16 hour fcst: +1315:843871513:d=2025100600:CIMIXR:350 mb:16 hour fcst: +1316:843928388:d=2025100600:RWMR:350 mb:16 hour fcst: +1317:843932237:d=2025100600:SNMR:350 mb:16 hour fcst: +1318:844033152:d=2025100600:GRLE:350 mb:16 hour fcst: +1319:844048692:d=2025100600:HGT:375 mb:16 hour fcst: +1320:844758503:d=2025100600:TMP:375 mb:16 hour fcst: +1321:845301162:d=2025100600:RH:375 mb:16 hour fcst: +1322:846041490:d=2025100600:DPT:375 mb:16 hour fcst: +1323:847058770:d=2025100600:SPFH:375 mb:16 hour fcst: +1324:848180377:d=2025100600:VVEL:375 mb:16 hour fcst: +1325:848781477:d=2025100600:UGRD:375 mb:16 hour fcst: +1326:849370278:d=2025100600:VGRD:375 mb:16 hour fcst: +1327:849955966:d=2025100600:ABSV:375 mb:16 hour fcst: +1328:850478252:d=2025100600:CLMR:375 mb:16 hour fcst: +1329:850483268:d=2025100600:CIMIXR:375 mb:16 hour fcst: +1330:850530719:d=2025100600:RWMR:375 mb:16 hour fcst: +1331:850535863:d=2025100600:SNMR:375 mb:16 hour fcst: +1332:850635912:d=2025100600:GRLE:375 mb:16 hour fcst: +1333:850653344:d=2025100600:HGT:400 mb:16 hour fcst: +1334:851363568:d=2025100600:TMP:400 mb:16 hour fcst: +1335:851897247:d=2025100600:RH:400 mb:16 hour fcst: +1336:852643705:d=2025100600:DPT:400 mb:16 hour fcst: +1337:853652027:d=2025100600:SPFH:400 mb:16 hour fcst: +1338:854863680:d=2025100600:VVEL:400 mb:16 hour fcst: +1339:855464411:d=2025100600:UGRD:400 mb:16 hour fcst: +1340:856051013:d=2025100600:VGRD:400 mb:16 hour fcst: +1341:856635840:d=2025100600:ABSV:400 mb:16 hour fcst: +1342:857155114:d=2025100600:CLMR:400 mb:16 hour fcst: +1343:857164118:d=2025100600:CIMIXR:400 mb:16 hour fcst: +1344:857205828:d=2025100600:RWMR:400 mb:16 hour fcst: +1345:857212559:d=2025100600:SNMR:400 mb:16 hour fcst: +1346:857312655:d=2025100600:GRLE:400 mb:16 hour fcst: +1347:857332547:d=2025100600:HGT:425 mb:16 hour fcst: +1348:858037995:d=2025100600:TMP:425 mb:16 hour fcst: +1349:858573022:d=2025100600:RH:425 mb:16 hour fcst: +1350:859324643:d=2025100600:DPT:425 mb:16 hour fcst: +1351:860341344:d=2025100600:SPFH:425 mb:16 hour fcst: +1352:861629177:d=2025100600:VVEL:425 mb:16 hour fcst: +1353:862229724:d=2025100600:UGRD:425 mb:16 hour fcst: +1354:862816701:d=2025100600:VGRD:425 mb:16 hour fcst: +1355:863401684:d=2025100600:ABSV:425 mb:16 hour fcst: +1356:863919880:d=2025100600:CLMR:425 mb:16 hour fcst: +1357:863939205:d=2025100600:CIMIXR:425 mb:16 hour fcst: +1358:863979255:d=2025100600:RWMR:425 mb:16 hour fcst: +1359:863988142:d=2025100600:SNMR:425 mb:16 hour fcst: +1360:864094098:d=2025100600:GRLE:425 mb:16 hour fcst: +1361:864116969:d=2025100600:HGT:450 mb:16 hour fcst: +1362:864817509:d=2025100600:TMP:450 mb:16 hour fcst: +1363:865361738:d=2025100600:RH:450 mb:16 hour fcst: +1364:866126043:d=2025100600:DPT:450 mb:16 hour fcst: +1365:867181272:d=2025100600:SPFH:450 mb:16 hour fcst: +1366:868560623:d=2025100600:VVEL:450 mb:16 hour fcst: +1367:869162118:d=2025100600:UGRD:450 mb:16 hour fcst: +1368:869753043:d=2025100600:VGRD:450 mb:16 hour fcst: +1369:870334107:d=2025100600:ABSV:450 mb:16 hour fcst: +1370:870855570:d=2025100600:CLMR:450 mb:16 hour fcst: +1371:870885628:d=2025100600:CIMIXR:450 mb:16 hour fcst: +1372:870924106:d=2025100600:RWMR:450 mb:16 hour fcst: +1373:870934847:d=2025100600:SNMR:450 mb:16 hour fcst: +1374:871044875:d=2025100600:GRLE:450 mb:16 hour fcst: +1375:871070601:d=2025100600:HGT:475 mb:16 hour fcst: +1376:871768756:d=2025100600:TMP:475 mb:16 hour fcst: +1377:872304619:d=2025100600:RH:475 mb:16 hour fcst: +1378:873069745:d=2025100600:DPT:475 mb:16 hour fcst: +1379:874137218:d=2025100600:SPFH:475 mb:16 hour fcst: +1380:875596192:d=2025100600:VVEL:475 mb:16 hour fcst: +1381:876195243:d=2025100600:UGRD:475 mb:16 hour fcst: +1382:876783412:d=2025100600:VGRD:475 mb:16 hour fcst: +1383:877369486:d=2025100600:ABSV:475 mb:16 hour fcst: +1384:877885658:d=2025100600:CLMR:475 mb:16 hour fcst: +1385:877925721:d=2025100600:CIMIXR:475 mb:16 hour fcst: +1386:877964921:d=2025100600:RWMR:475 mb:16 hour fcst: +1387:877979536:d=2025100600:SNMR:475 mb:16 hour fcst: +1388:878092307:d=2025100600:GRLE:475 mb:16 hour fcst: +1389:878123011:d=2025100600:HGT:500 mb:16 hour fcst: +1390:878821874:d=2025100600:TMP:500 mb:16 hour fcst: +1391:879358245:d=2025100600:RH:500 mb:16 hour fcst: +1392:880122503:d=2025100600:DPT:500 mb:16 hour fcst: +1393:881180052:d=2025100600:SPFH:500 mb:16 hour fcst: +1394:882692420:d=2025100600:VVEL:500 mb:16 hour fcst: +1395:883290792:d=2025100600:UGRD:500 mb:16 hour fcst: +1396:883876727:d=2025100600:VGRD:500 mb:16 hour fcst: +1397:884461225:d=2025100600:ABSV:500 mb:16 hour fcst: +1398:884972506:d=2025100600:CLMR:500 mb:16 hour fcst: +1399:885026091:d=2025100600:CIMIXR:500 mb:16 hour fcst: +1400:885064950:d=2025100600:RWMR:500 mb:16 hour fcst: +1401:885083767:d=2025100600:SNMR:500 mb:16 hour fcst: +1402:885193817:d=2025100600:GRLE:500 mb:16 hour fcst: +1403:885230009:d=2025100600:HGT:525 mb:16 hour fcst: +1404:885923010:d=2025100600:TMP:525 mb:16 hour fcst: +1405:886460603:d=2025100600:RH:525 mb:16 hour fcst: +1406:887242498:d=2025100600:DPT:525 mb:16 hour fcst: +1407:888347346:d=2025100600:SPFH:525 mb:16 hour fcst: +1408:889949726:d=2025100600:VVEL:525 mb:16 hour fcst: +1409:890549253:d=2025100600:UGRD:525 mb:16 hour fcst: +1410:891138395:d=2025100600:VGRD:525 mb:16 hour fcst: +1411:891716657:d=2025100600:ABSV:525 mb:16 hour fcst: +1412:892234186:d=2025100600:CLMR:525 mb:16 hour fcst: +1413:892293112:d=2025100600:CIMIXR:525 mb:16 hour fcst: +1414:892331068:d=2025100600:RWMR:525 mb:16 hour fcst: +1415:892353351:d=2025100600:SNMR:525 mb:16 hour fcst: +1416:892459746:d=2025100600:GRLE:525 mb:16 hour fcst: +1417:892500143:d=2025100600:HGT:550 mb:16 hour fcst: +1418:893196392:d=2025100600:TMP:550 mb:16 hour fcst: +1419:893731250:d=2025100600:RH:550 mb:16 hour fcst: +1420:894506147:d=2025100600:DPT:550 mb:16 hour fcst: +1421:895572573:d=2025100600:SPFH:550 mb:16 hour fcst: +1422:897211534:d=2025100600:VVEL:550 mb:16 hour fcst: +1423:897810227:d=2025100600:UGRD:550 mb:16 hour fcst: +1424:898396048:d=2025100600:VGRD:550 mb:16 hour fcst: +1425:898971855:d=2025100600:ABSV:550 mb:16 hour fcst: +1426:899484295:d=2025100600:CLMR:550 mb:16 hour fcst: +1427:899553758:d=2025100600:CIMIXR:550 mb:16 hour fcst: +1428:899588324:d=2025100600:RWMR:550 mb:16 hour fcst: +1429:899617587:d=2025100600:SNMR:550 mb:16 hour fcst: +1430:899719384:d=2025100600:GRLE:550 mb:16 hour fcst: +1431:899764049:d=2025100600:HGT:575 mb:16 hour fcst: +1432:900456055:d=2025100600:TMP:575 mb:16 hour fcst: +1433:900992543:d=2025100600:RH:575 mb:16 hour fcst: +1434:901781127:d=2025100600:DPT:575 mb:16 hour fcst: +1435:902870091:d=2025100600:SPFH:575 mb:16 hour fcst: +1436:904582308:d=2025100600:VVEL:575 mb:16 hour fcst: +1437:905184647:d=2025100600:UGRD:575 mb:16 hour fcst: +1438:905773988:d=2025100600:VGRD:575 mb:16 hour fcst: +1439:906352701:d=2025100600:ABSV:575 mb:16 hour fcst: +1440:906873149:d=2025100600:CLMR:575 mb:16 hour fcst: +1441:906954839:d=2025100600:CIMIXR:575 mb:16 hour fcst: +1442:906981083:d=2025100600:RWMR:575 mb:16 hour fcst: +1443:906995827:d=2025100600:SNMR:575 mb:16 hour fcst: +1444:907093721:d=2025100600:GRLE:575 mb:16 hour fcst: +1445:907140214:d=2025100600:HGT:600 mb:16 hour fcst: +1446:907832429:d=2025100600:TMP:600 mb:16 hour fcst: +1447:908366444:d=2025100600:RH:600 mb:16 hour fcst: +1448:909148753:d=2025100600:DPT:600 mb:16 hour fcst: +1449:910229378:d=2025100600:SPFH:600 mb:16 hour fcst: +1450:911977090:d=2025100600:VVEL:600 mb:16 hour fcst: +1451:912579378:d=2025100600:UGRD:600 mb:16 hour fcst: +1452:913166597:d=2025100600:VGRD:600 mb:16 hour fcst: +1453:913729023:d=2025100600:ABSV:600 mb:16 hour fcst: +1454:914245933:d=2025100600:CLMR:600 mb:16 hour fcst: +1455:914338473:d=2025100600:CIMIXR:600 mb:16 hour fcst: +1456:914356970:d=2025100600:RWMR:600 mb:16 hour fcst: +1457:914379966:d=2025100600:SNMR:600 mb:16 hour fcst: +1458:914467302:d=2025100600:GRLE:600 mb:16 hour fcst: +1459:914512123:d=2025100600:HGT:625 mb:16 hour fcst: +1460:915197980:d=2025100600:TMP:625 mb:16 hour fcst: +1461:915736105:d=2025100600:RH:625 mb:16 hour fcst: +1462:916529946:d=2025100600:DPT:625 mb:16 hour fcst: +1463:917639847:d=2025100600:SPFH:625 mb:16 hour fcst: +1464:919474874:d=2025100600:VVEL:625 mb:16 hour fcst: +1465:920080553:d=2025100600:UGRD:625 mb:16 hour fcst: +1466:920671432:d=2025100600:VGRD:625 mb:16 hour fcst: +1467:921238138:d=2025100600:ABSV:625 mb:16 hour fcst: +1468:921763646:d=2025100600:CLMR:625 mb:16 hour fcst: +1469:921860418:d=2025100600:CIMIXR:625 mb:16 hour fcst: +1470:921887961:d=2025100600:RWMR:625 mb:16 hour fcst: +1471:921917149:d=2025100600:SNMR:625 mb:16 hour fcst: +1472:921987865:d=2025100600:GRLE:625 mb:16 hour fcst: +1473:922025780:d=2025100600:HGT:650 mb:16 hour fcst: +1474:922711927:d=2025100600:TMP:650 mb:16 hour fcst: +1475:923251244:d=2025100600:RH:650 mb:16 hour fcst: +1476:924041125:d=2025100600:DPT:650 mb:16 hour fcst: +1477:925147792:d=2025100600:SPFH:650 mb:16 hour fcst: +1478:926262813:d=2025100600:VVEL:650 mb:16 hour fcst: +1479:926871283:d=2025100600:UGRD:650 mb:16 hour fcst: +1480:927460774:d=2025100600:VGRD:650 mb:16 hour fcst: +1481:928033617:d=2025100600:ABSV:650 mb:16 hour fcst: +1482:928558780:d=2025100600:CLMR:650 mb:16 hour fcst: +1483:928662353:d=2025100600:CIMIXR:650 mb:16 hour fcst: +1484:928680136:d=2025100600:RWMR:650 mb:16 hour fcst: +1485:928714196:d=2025100600:SNMR:650 mb:16 hour fcst: +1486:928772685:d=2025100600:GRLE:650 mb:16 hour fcst: +1487:928806439:d=2025100600:HGT:675 mb:16 hour fcst: +1488:929487192:d=2025100600:TMP:675 mb:16 hour fcst: +1489:930044405:d=2025100600:RH:675 mb:16 hour fcst: +1490:930851790:d=2025100600:DPT:675 mb:16 hour fcst: +1491:931985208:d=2025100600:SPFH:675 mb:16 hour fcst: +1492:933158096:d=2025100600:VVEL:675 mb:16 hour fcst: +1493:933780763:d=2025100600:UGRD:675 mb:16 hour fcst: +1494:934376207:d=2025100600:VGRD:675 mb:16 hour fcst: +1495:934962685:d=2025100600:ABSV:675 mb:16 hour fcst: +1496:935494548:d=2025100600:CLMR:675 mb:16 hour fcst: +1497:935617270:d=2025100600:CIMIXR:675 mb:16 hour fcst: +1498:935628049:d=2025100600:RWMR:675 mb:16 hour fcst: +1499:935666993:d=2025100600:SNMR:675 mb:16 hour fcst: +1500:935706748:d=2025100600:GRLE:675 mb:16 hour fcst: +1501:935732710:d=2025100600:HGT:700 mb:16 hour fcst: +1502:936412279:d=2025100600:TMP:700 mb:16 hour fcst: +1503:936970315:d=2025100600:RH:700 mb:16 hour fcst: +1504:937786720:d=2025100600:DPT:700 mb:16 hour fcst: +1505:938927861:d=2025100600:SPFH:700 mb:16 hour fcst: +1506:940150364:d=2025100600:VVEL:700 mb:16 hour fcst: +1507:940777796:d=2025100600:UGRD:700 mb:16 hour fcst: +1508:941380455:d=2025100600:VGRD:700 mb:16 hour fcst: +1509:941958604:d=2025100600:ABSV:700 mb:16 hour fcst: +1510:942489708:d=2025100600:CLMR:700 mb:16 hour fcst: +1511:942638591:d=2025100600:CIMIXR:700 mb:16 hour fcst: +1512:942646754:d=2025100600:RWMR:700 mb:16 hour fcst: +1513:942691369:d=2025100600:SNMR:700 mb:16 hour fcst: +1514:942723676:d=2025100600:GRLE:700 mb:16 hour fcst: +1515:942743986:d=2025100600:HGT:725 mb:16 hour fcst: +1516:943421160:d=2025100600:TMP:725 mb:16 hour fcst: +1517:943989125:d=2025100600:RH:725 mb:16 hour fcst: +1518:944821431:d=2025100600:DPT:725 mb:16 hour fcst: +1519:945971482:d=2025100600:SPFH:725 mb:16 hour fcst: +1520:947241342:d=2025100600:VVEL:725 mb:16 hour fcst: +1521:947877511:d=2025100600:UGRD:725 mb:16 hour fcst: +1522:948475872:d=2025100600:VGRD:725 mb:16 hour fcst: +1523:949048897:d=2025100600:ABSV:725 mb:16 hour fcst: +1524:949585367:d=2025100600:CLMR:725 mb:16 hour fcst: +1525:949750292:d=2025100600:CIMIXR:725 mb:16 hour fcst: +1526:949754923:d=2025100600:RWMR:725 mb:16 hour fcst: +1527:949802663:d=2025100600:SNMR:725 mb:16 hour fcst: +1528:949825708:d=2025100600:GRLE:725 mb:16 hour fcst: +1529:949840587:d=2025100600:HGT:750 mb:16 hour fcst: +1530:950523035:d=2025100600:TMP:750 mb:16 hour fcst: +1531:951091847:d=2025100600:RH:750 mb:16 hour fcst: +1532:951937079:d=2025100600:DPT:750 mb:16 hour fcst: +1533:953101629:d=2025100600:SPFH:750 mb:16 hour fcst: +1534:954416888:d=2025100600:VVEL:750 mb:16 hour fcst: +1535:955063256:d=2025100600:UGRD:750 mb:16 hour fcst: +1536:955674663:d=2025100600:VGRD:750 mb:16 hour fcst: +1537:956260419:d=2025100600:ABSV:750 mb:16 hour fcst: +1538:956801133:d=2025100600:CLMR:750 mb:16 hour fcst: +1539:956958633:d=2025100600:CIMIXR:750 mb:16 hour fcst: +1540:956964393:d=2025100600:RWMR:750 mb:16 hour fcst: +1541:957013017:d=2025100600:SNMR:750 mb:16 hour fcst: +1542:957046953:d=2025100600:GRLE:750 mb:16 hour fcst: +1543:957057450:d=2025100600:HGT:775 mb:16 hour fcst: +1544:957747364:d=2025100600:TMP:775 mb:16 hour fcst: +1545:958319840:d=2025100600:RH:775 mb:16 hour fcst: +1546:959169132:d=2025100600:DPT:775 mb:16 hour fcst: +1547:960331651:d=2025100600:SPFH:775 mb:16 hour fcst: +1548:961674010:d=2025100600:VVEL:775 mb:16 hour fcst: +1549:962348673:d=2025100600:UGRD:775 mb:16 hour fcst: +1550:962962247:d=2025100600:VGRD:775 mb:16 hour fcst: +1551:963560890:d=2025100600:ABSV:775 mb:16 hour fcst: +1552:964106273:d=2025100600:CLMR:775 mb:16 hour fcst: +1553:964243011:d=2025100600:CIMIXR:775 mb:16 hour fcst: +1554:964246648:d=2025100600:RWMR:775 mb:16 hour fcst: +1555:964295653:d=2025100600:SNMR:775 mb:16 hour fcst: +1556:964318515:d=2025100600:GRLE:775 mb:16 hour fcst: +1557:964325426:d=2025100600:HGT:800 mb:16 hour fcst: +1558:965024811:d=2025100600:TMP:800 mb:16 hour fcst: +1559:965601141:d=2025100600:RH:800 mb:16 hour fcst: +1560:966466025:d=2025100600:DPT:800 mb:16 hour fcst: +1561:967639957:d=2025100600:SPFH:800 mb:16 hour fcst: +1562:969023871:d=2025100600:VVEL:800 mb:16 hour fcst: +1563:969696898:d=2025100600:UGRD:800 mb:16 hour fcst: +1564:970315704:d=2025100600:VGRD:800 mb:16 hour fcst: +1565:970921088:d=2025100600:ABSV:800 mb:16 hour fcst: +1566:971473255:d=2025100600:CLMR:800 mb:16 hour fcst: +1567:971606285:d=2025100600:CIMIXR:800 mb:16 hour fcst: +1568:971607575:d=2025100600:RWMR:800 mb:16 hour fcst: +1569:971656046:d=2025100600:SNMR:800 mb:16 hour fcst: +1570:971672502:d=2025100600:GRLE:800 mb:16 hour fcst: +1571:971682949:d=2025100600:HGT:825 mb:16 hour fcst: +1572:972392988:d=2025100600:TMP:825 mb:16 hour fcst: +1573:972974306:d=2025100600:RH:825 mb:16 hour fcst: +1574:973853010:d=2025100600:DPT:825 mb:16 hour fcst: +1575:975033662:d=2025100600:SPFH:825 mb:16 hour fcst: +1576:976458620:d=2025100600:VVEL:825 mb:16 hour fcst: +1577:977136973:d=2025100600:UGRD:825 mb:16 hour fcst: +1578:977758057:d=2025100600:VGRD:825 mb:16 hour fcst: +1579:978365103:d=2025100600:ABSV:825 mb:16 hour fcst: +1580:978920564:d=2025100600:CLMR:825 mb:16 hour fcst: +1581:979046504:d=2025100600:CIMIXR:825 mb:16 hour fcst: +1582:979047174:d=2025100600:RWMR:825 mb:16 hour fcst: +1583:979163825:d=2025100600:SNMR:825 mb:16 hour fcst: +1584:979174188:d=2025100600:GRLE:825 mb:16 hour fcst: +1585:979180498:d=2025100600:HGT:850 mb:16 hour fcst: +1586:979904883:d=2025100600:TMP:850 mb:16 hour fcst: +1587:980490184:d=2025100600:RH:850 mb:16 hour fcst: +1588:981364544:d=2025100600:DPT:850 mb:16 hour fcst: +1589:982545621:d=2025100600:SPFH:850 mb:16 hour fcst: +1590:984006058:d=2025100600:VVEL:850 mb:16 hour fcst: +1591:984687503:d=2025100600:UGRD:850 mb:16 hour fcst: +1592:985302066:d=2025100600:VGRD:850 mb:16 hour fcst: +1593:985909603:d=2025100600:ABSV:850 mb:16 hour fcst: +1594:986466666:d=2025100600:CLMR:850 mb:16 hour fcst: +1595:986629144:d=2025100600:CIMIXR:850 mb:16 hour fcst: +1596:986696324:d=2025100600:RWMR:850 mb:16 hour fcst: +1597:986821901:d=2025100600:SNMR:850 mb:16 hour fcst: +1598:986827461:d=2025100600:GRLE:850 mb:16 hour fcst: +1599:986830648:d=2025100600:HGT:875 mb:16 hour fcst: +1600:987572478:d=2025100600:TMP:875 mb:16 hour fcst: +1601:988161305:d=2025100600:RH:875 mb:16 hour fcst: +1602:989053420:d=2025100600:DPT:875 mb:16 hour fcst: +1603:990227985:d=2025100600:SPFH:875 mb:16 hour fcst: +1604:991732351:d=2025100600:VVEL:875 mb:16 hour fcst: +1605:992418296:d=2025100600:UGRD:875 mb:16 hour fcst: +1606:993044472:d=2025100600:VGRD:875 mb:16 hour fcst: +1607:993652324:d=2025100600:ABSV:875 mb:16 hour fcst: +1608:994209842:d=2025100600:CLMR:875 mb:16 hour fcst: +1609:994393581:d=2025100600:CIMIXR:875 mb:16 hour fcst: +1610:994523720:d=2025100600:RWMR:875 mb:16 hour fcst: +1611:994650754:d=2025100600:SNMR:875 mb:16 hour fcst: +1612:994656302:d=2025100600:GRLE:875 mb:16 hour fcst: +1613:994657703:d=2025100600:HGT:900 mb:16 hour fcst: +1614:995414355:d=2025100600:TMP:900 mb:16 hour fcst: +1615:996015064:d=2025100600:RH:900 mb:16 hour fcst: +1616:996913508:d=2025100600:DPT:900 mb:16 hour fcst: +1617:998094137:d=2025100600:SPFH:900 mb:16 hour fcst: +1618:999632991:d=2025100600:VVEL:900 mb:16 hour fcst: +1619:1000307988:d=2025100600:UGRD:900 mb:16 hour fcst: +1620:1000932887:d=2025100600:VGRD:900 mb:16 hour fcst: +1621:1001539480:d=2025100600:ABSV:900 mb:16 hour fcst: +1622:1002096412:d=2025100600:CLMR:900 mb:16 hour fcst: +1623:1002264152:d=2025100600:CIMIXR:900 mb:16 hour fcst: +1624:1002389941:d=2025100600:RWMR:900 mb:16 hour fcst: +1625:1002514982:d=2025100600:SNMR:900 mb:16 hour fcst: +1626:1002517397:d=2025100600:GRLE:900 mb:16 hour fcst: +1627:1002518977:d=2025100600:HGT:925 mb:16 hour fcst: +1628:1003295386:d=2025100600:TMP:925 mb:16 hour fcst: +1629:1003900106:d=2025100600:RH:925 mb:16 hour fcst: +1630:1004795323:d=2025100600:DPT:925 mb:16 hour fcst: +1631:1005973024:d=2025100600:SPFH:925 mb:16 hour fcst: +1632:1007530300:d=2025100600:VVEL:925 mb:16 hour fcst: +1633:1008188969:d=2025100600:UGRD:925 mb:16 hour fcst: +1634:1008815981:d=2025100600:VGRD:925 mb:16 hour fcst: +1635:1009430969:d=2025100600:ABSV:925 mb:16 hour fcst: +1636:1009986498:d=2025100600:CLMR:925 mb:16 hour fcst: +1637:1010092641:d=2025100600:CIMIXR:925 mb:16 hour fcst: +1638:1010218694:d=2025100600:RWMR:925 mb:16 hour fcst: +1639:1010337105:d=2025100600:SNMR:925 mb:16 hour fcst: +1640:1010338399:d=2025100600:GRLE:925 mb:16 hour fcst: +1641:1010339852:d=2025100600:HGT:950 mb:16 hour fcst: +1642:1011123223:d=2025100600:TMP:950 mb:16 hour fcst: +1643:1011732999:d=2025100600:RH:950 mb:16 hour fcst: +1644:1012619346:d=2025100600:DPT:950 mb:16 hour fcst: +1645:1013787161:d=2025100600:SPFH:950 mb:16 hour fcst: +1646:1015351874:d=2025100600:VVEL:950 mb:16 hour fcst: +1647:1015996761:d=2025100600:UGRD:950 mb:16 hour fcst: +1648:1016626030:d=2025100600:VGRD:950 mb:16 hour fcst: +1649:1017241333:d=2025100600:ABSV:950 mb:16 hour fcst: +1650:1017795711:d=2025100600:CLMR:950 mb:16 hour fcst: +1651:1017853730:d=2025100600:CIMIXR:950 mb:16 hour fcst: +1652:1017987388:d=2025100600:RWMR:950 mb:16 hour fcst: +1653:1018097336:d=2025100600:SNMR:950 mb:16 hour fcst: +1654:1018097817:d=2025100600:GRLE:950 mb:16 hour fcst: +1655:1018098376:d=2025100600:HGT:975 mb:16 hour fcst: +1656:1018903251:d=2025100600:TMP:975 mb:16 hour fcst: +1657:1019519356:d=2025100600:RH:975 mb:16 hour fcst: +1658:1020400791:d=2025100600:DPT:975 mb:16 hour fcst: +1659:1021566265:d=2025100600:SPFH:975 mb:16 hour fcst: +1660:1023145309:d=2025100600:VVEL:975 mb:16 hour fcst: +1661:1023766661:d=2025100600:UGRD:975 mb:16 hour fcst: +1662:1024399773:d=2025100600:VGRD:975 mb:16 hour fcst: +1663:1025015539:d=2025100600:ABSV:975 mb:16 hour fcst: +1664:1025568606:d=2025100600:CLMR:975 mb:16 hour fcst: +1665:1025599439:d=2025100600:CIMIXR:975 mb:16 hour fcst: +1666:1025800280:d=2025100600:RWMR:975 mb:16 hour fcst: +1667:1025896499:d=2025100600:SNMR:975 mb:16 hour fcst: +1668:1025977876:d=2025100600:GRLE:975 mb:16 hour fcst: +1669:1025978571:d=2025100600:TMP:1000 mb:16 hour fcst: +1670:1026603921:d=2025100600:RH:1000 mb:16 hour fcst: +1671:1027474002:d=2025100600:DPT:1000 mb:16 hour fcst: +1672:1028643425:d=2025100600:SPFH:1000 mb:16 hour fcst: +1673:1030245802:d=2025100600:VVEL:1000 mb:16 hour fcst: +1674:1030821025:d=2025100600:UGRD:1000 mb:16 hour fcst: +1675:1031460645:d=2025100600:VGRD:1000 mb:16 hour fcst: +1676:1032076265:d=2025100600:ABSV:1000 mb:16 hour fcst: +1677:1032626421:d=2025100600:CLMR:1000 mb:16 hour fcst: +1678:1032640918:d=2025100600:CIMIXR:1000 mb:16 hour fcst: +1679:1032781407:d=2025100600:RWMR:1000 mb:16 hour fcst: +1680:1032850332:d=2025100600:SNMR:1000 mb:16 hour fcst: +1681:1032935480:d=2025100600:GRLE:1000 mb:16 hour fcst: +1682:1032935848:d=2025100600:HGT:1013.2 mb:16 hour fcst: +1683:1033783505:d=2025100600:TMP:1013.2 mb:16 hour fcst: +1684:1034412423:d=2025100600:RH:1013.2 mb:16 hour fcst: +1685:1035281592:d=2025100600:DPT:1013.2 mb:16 hour fcst: +1686:1036448937:d=2025100600:SPFH:1013.2 mb:16 hour fcst: +1687:1038057663:d=2025100600:VVEL:1013.2 mb:16 hour fcst: +1688:1038555681:d=2025100600:UGRD:1013.2 mb:16 hour fcst: +1689:1039195806:d=2025100600:VGRD:1013.2 mb:16 hour fcst: +1690:1039809497:d=2025100600:ABSV:1013.2 mb:16 hour fcst: +1691:1040357372:d=2025100600:CLMR:1013.2 mb:16 hour fcst: +1692:1040372100:d=2025100600:CIMIXR:1013.2 mb:16 hour fcst: +1693:1040450676:d=2025100600:RWMR:1013.2 mb:16 hour fcst: +1694:1040491986:d=2025100600:SNMR:1013.2 mb:16 hour fcst: +1695:1040570562:d=2025100600:GRLE:1013.2 mb:16 hour fcst: +1696:1040619661:d=2025100600:TSOIL:0-0 m below ground:16 hour fcst: +1697:1042504545:d=2025100600:SOILW:0-0 m below ground:16 hour fcst: +1698:1043745921:d=2025100600:TSOIL:0.01-0.01 m below ground:16 hour fcst: +1699:1045586734:d=2025100600:SOILW:0.01-0.01 m below ground:16 hour fcst: +1700:1046848063:d=2025100600:TSOIL:0.04-0.04 m below ground:16 hour fcst: +1701:1048664384:d=2025100600:SOILW:0.04-0.04 m below ground:16 hour fcst: +1702:1049923586:d=2025100600:TSOIL:0.1-0.1 m below ground:16 hour fcst: +1703:1051730291:d=2025100600:SOILW:0.1-0.1 m below ground:16 hour fcst: +1704:1052989010:d=2025100600:TSOIL:0.3-0.3 m below ground:16 hour fcst: +1705:1054823078:d=2025100600:SOILW:0.3-0.3 m below ground:16 hour fcst: +1706:1056079831:d=2025100600:TSOIL:0.6-0.6 m below ground:16 hour fcst: +1707:1057967812:d=2025100600:SOILW:0.6-0.6 m below ground:16 hour fcst: +1708:1059208197:d=2025100600:TSOIL:1-1 m below ground:16 hour fcst: +1709:1061152929:d=2025100600:SOILW:1-1 m below ground:16 hour fcst: +1710:1062367353:d=2025100600:TSOIL:1.6-1.6 m below ground:16 hour fcst: +1711:1064355673:d=2025100600:SOILW:1.6-1.6 m below ground:16 hour fcst: +1712:1065561972:d=2025100600:TSOIL:3-3 m below ground:16 hour fcst: +1713:1067127234:d=2025100600:SOILW:3-3 m below ground:16 hour fcst: +1714:1067959601:d=2025100600:REFC:entire atmosphere:16 hour fcst: +1715:1068489475:d=2025100600:RETOP:cloud top:16 hour fcst: +1716:1068698949:d=2025100600:VIL:entire atmosphere:16 hour fcst: +1717:1069008591:d=2025100600:VIS:surface:16 hour fcst: +1718:1070506897:d=2025100600:REFD:1000 m above ground:16 hour fcst: +1719:1070854140:d=2025100600:REFD:4000 m above ground:16 hour fcst: +1720:1071037831:d=2025100600:REFD:263 K level:16 hour fcst: +1721:1071219268:d=2025100600:GUST:surface:16 hour fcst: +1722:1072404850:d=2025100600:MAXUVV:100-1000 mb above ground:15-16 hour max fcst: +1723:1073226739:d=2025100600:MAXDVV:100-1000 mb above ground:15-16 hour max fcst: +1724:1073985363:d=2025100600:DZDT:0.5-0.8 sigma layer:15-16 hour ave fcst: +1725:1074591563:d=2025100600:MSLMA:mean sea level:16 hour fcst: +1726:1075202848:d=2025100600:HGT:1000 mb:16 hour fcst: +1727:1075908737:d=2025100600:MAXREF:1000 m above ground:15-16 hour max fcst: +1728:1076225102:d=2025100600:REFD:263 K level:15-16 hour max fcst: +1729:1076451957:d=2025100600:MXUPHL:5000-2000 m above ground:15-16 hour max fcst: +1730:1076509400:d=2025100600:MNUPHL:5000-2000 m above ground:15-16 hour min fcst: +1731:1076564924:d=2025100600:MXUPHL:2000-0 m above ground:15-16 hour max fcst: +1732:1076612751:d=2025100600:MNUPHL:2000-0 m above ground:15-16 hour min fcst: +1733:1076642591:d=2025100600:MXUPHL:3000-0 m above ground:15-16 hour max fcst: +1734:1076697859:d=2025100600:MNUPHL:3000-0 m above ground:15-16 hour min fcst: +1735:1076736970:d=2025100600:RELV:2000-0 m above ground:15-16 hour max fcst: +1736:1079354414:d=2025100600:RELV:1000-0 m above ground:15-16 hour max fcst: +1737:1082261441:d=2025100600:HAIL:entire atmosphere:15-16 hour max fcst: +1738:1082495735:d=2025100600:HAIL:0.1 sigma level:15-16 hour max fcst: +1739:1082496911:d=2025100600:TCOLG:entire atmosphere (considered as a single layer):15-16 hour max fcst: +1740:1082551191:d=2025100600:LTNG:entire atmosphere:16 hour fcst: +1741:1082583747:d=2025100600:UGRD:80 m above ground:16 hour fcst: +1742:1083791462:d=2025100600:VGRD:80 m above ground:16 hour fcst: +1743:1084926811:d=2025100600:PRES:surface:16 hour fcst: +1744:1086438027:d=2025100600:HGT:surface:16 hour fcst: +1745:1088591722:d=2025100600:TMP:surface:16 hour fcst: +1746:1089947701:d=2025100600:ASNOW:surface:0-16 hour acc fcst: +1747:1089991653:d=2025100600:MSTAV:0 m underground:16 hour fcst: +1748:1091464941:d=2025100600:CNWAT:surface:16 hour fcst: +1749:1091555050:d=2025100600:WEASD:surface:16 hour fcst: +1750:1091573436:d=2025100600:SNOWC:surface:16 hour fcst: +1751:1091587158:d=2025100600:SNOD:surface:16 hour fcst: +1752:1091602543:d=2025100600:TMP:2 m above ground:16 hour fcst: +1753:1092816255:d=2025100600:POT:2 m above ground:16 hour fcst: +1754:1093956113:d=2025100600:SPFH:2 m above ground:16 hour fcst: +1755:1095570387:d=2025100600:DPT:2 m above ground:16 hour fcst: +1756:1096792009:d=2025100600:RH:2 m above ground:16 hour fcst: +1757:1098437177:d=2025100600:MASSDEN:8 m above ground:16 hour fcst: +1758:1099191233:d=2025100600:UGRD:10 m above ground:16 hour fcst: +1759:1101572848:d=2025100600:VGRD:10 m above ground:16 hour fcst: +1760:1103954463:d=2025100600:WIND:10 m above ground:15-16 hour max fcst: +1761:1105180726:d=2025100600:MAXUW:10 m above ground:15-16 hour max fcst: +1762:1106545458:d=2025100600:MAXVW:10 m above ground:15-16 hour max fcst: +1763:1107855495:d=2025100600:CPOFP:surface:16 hour fcst: +1764:1107985117:d=2025100600:PRATE:surface:16 hour fcst: +1765:1108056789:d=2025100600:APCP:surface:0-16 hour acc fcst: +1766:1108783098:d=2025100600:WEASD:surface:0-16 hour acc fcst: +1767:1108820516:d=2025100600:FROZR:surface:0-16 hour acc fcst: +1768:1108866830:d=2025100600:FRZR:surface:0-16 hour acc fcst: +1769:1108884820:d=2025100600:SSRUN:surface:15-16 hour acc fcst: +1770:1108903122:d=2025100600:BGRUN:surface:15-16 hour acc fcst: +1771:1108903506:d=2025100600:APCP:surface:15-16 hour acc fcst: +1772:1109146172:d=2025100600:WEASD:surface:15-16 hour acc fcst: +1773:1109149685:d=2025100600:FROZR:surface:15-16 hour acc fcst: +1774:1109152250:d=2025100600:CSNOW:surface:16 hour fcst: +1775:1109152887:d=2025100600:CICEP:surface:16 hour fcst: +1776:1109153128:d=2025100600:CFRZR:surface:16 hour fcst: +1777:1109153572:d=2025100600:CRAIN:surface:16 hour fcst: +1778:1109233213:d=2025100600:SFCR:surface:16 hour fcst: +1779:1111151141:d=2025100600:FRICV:surface:16 hour fcst: +1780:1112289677:d=2025100600:SHTFL:surface:16 hour fcst: +1781:1113790686:d=2025100600:LHTFL:surface:16 hour fcst: +1782:1115397941:d=2025100600:GFLUX:surface:16 hour fcst: +1783:1116047551:d=2025100600:VGTYP:surface:16 hour fcst: +1784:1116828730:d=2025100600:LFTX:500-1000 mb:16 hour fcst: +1785:1117792889:d=2025100600:CAPE:surface:16 hour fcst: +1786:1118394693:d=2025100600:CIN:surface:16 hour fcst: +1787:1119082953:d=2025100600:PWAT:entire atmosphere (considered as a single layer):16 hour fcst: +1788:1120125669:d=2025100600:AOTK:entire atmosphere (considered as a single layer):16 hour fcst: +1789:1121248508:d=2025100600:COLMD:entire atmosphere (considered as a single layer):16 hour fcst: +1790:1122153549:d=2025100600:LCDC:low cloud layer:16 hour fcst: +1791:1123121857:d=2025100600:MCDC:middle cloud layer:16 hour fcst: +1792:1123579737:d=2025100600:HCDC:high cloud layer:16 hour fcst: +1793:1123957084:d=2025100600:TCDC:entire atmosphere:16 hour fcst: +1794:1124903566:d=2025100600:HGT:cloud ceiling:16 hour fcst: +1795:1126285661:d=2025100600:HGT:cloud base:16 hour fcst: +1796:1128763214:d=2025100600:PRES:cloud base:16 hour fcst: +1797:1129980237:d=2025100600:PRES:cloud top:16 hour fcst: +1798:1130681892:d=2025100600:HGT:cloud top:16 hour fcst: +1799:1131904680:d=2025100600:ULWRF:top of atmosphere:16 hour fcst: +1800:1133914499:d=2025100600:DSWRF:surface:16 hour fcst: +1801:1136566613:d=2025100600:DLWRF:surface:16 hour fcst: +1802:1138747220:d=2025100600:USWRF:surface:16 hour fcst: +1803:1140860760:d=2025100600:ULWRF:surface:16 hour fcst: +1804:1142545317:d=2025100600:CFNSF:surface:16 hour fcst: +1805:1142550196:d=2025100600:VBDSF:surface:16 hour fcst: +1806:1144594745:d=2025100600:VDDSF:surface:16 hour fcst: +1807:1147340319:d=2025100600:USWRF:top of atmosphere:16 hour fcst: +1808:1149979618:d=2025100600:HLCY:3000-0 m above ground:16 hour fcst: +1809:1151176234:d=2025100600:HLCY:1000-0 m above ground:16 hour fcst: +1810:1153101642:d=2025100600:USTM:0-6000 m above ground:16 hour fcst: +1811:1154211554:d=2025100600:VSTM:0-6000 m above ground:16 hour fcst: +1812:1155276908:d=2025100600:VUCSH:0-1000 m above ground:16 hour fcst: +1813:1157658523:d=2025100600:VVCSH:0-1000 m above ground:16 hour fcst: +1814:1160040138:d=2025100600:VUCSH:0-6000 m above ground:16 hour fcst: +1815:1162421753:d=2025100600:VVCSH:0-6000 m above ground:16 hour fcst: +1816:1165041510:d=2025100600:HGT:0C isotherm:16 hour fcst: +1817:1167332629:d=2025100600:RH:0C isotherm:16 hour fcst: +1818:1168095454:d=2025100600:PRES:0C isotherm:16 hour fcst: +1819:1168839873:d=2025100600:HGT:highest tropospheric freezing level:16 hour fcst: +1820:1169611391:d=2025100600:RH:highest tropospheric freezing level:16 hour fcst: +1821:1170360924:d=2025100600:PRES:highest tropospheric freezing level:16 hour fcst: +1822:1171102483:d=2025100600:HGT:263 K level:16 hour fcst: +1823:1171796252:d=2025100600:HGT:253 K level:16 hour fcst: +1824:1172418577:d=2025100600:4LFTX:180-0 mb above ground:16 hour fcst: +1825:1173388996:d=2025100600:CAPE:180-0 mb above ground:16 hour fcst: +1826:1173989560:d=2025100600:CIN:180-0 mb above ground:16 hour fcst: +1827:1174696724:d=2025100600:HPBL:surface:16 hour fcst: +1828:1177708488:d=2025100600:HGT:level of adiabatic condensation from sfc:16 hour fcst: +1829:1180692920:d=2025100600:CAPE:90-0 mb above ground:16 hour fcst: +1830:1181189531:d=2025100600:CIN:90-0 mb above ground:16 hour fcst: +1831:1181835767:d=2025100600:CAPE:255-0 mb above ground:16 hour fcst: +1832:1182455716:d=2025100600:CIN:255-0 mb above ground:16 hour fcst: +1833:1183156518:d=2025100600:HGT:equilibrium level:16 hour fcst: +1834:1185479983:d=2025100600:PLPL:255-0 mb above ground:16 hour fcst: +1835:1186523596:d=2025100600:CAPE:0-3000 m above ground:16 hour fcst: +1836:1187551632:d=2025100600:HGT:level of free convection:16 hour fcst: +1837:1190458414:d=2025100600:EFHL:surface:16 hour fcst: +1838:1191529063:d=2025100600:CANGLE:0-500 m above ground:16 hour fcst: +1839:1193819829:d=2025100600:LAYTH:261 K level - 256 K level:16 hour fcst: +1840:1195152590:d=2025100600:ESP:0-3000 m above ground:16 hour fcst: +1841:1196147073:d=2025100600:RHPW:entire atmosphere:16 hour fcst: +1842:1197389107:d=2025100600:LAND:surface:16 hour fcst: +1843:1197439583:d=2025100600:ICEC:surface:16 hour fcst: +1844:1197439816:d=2025100600:SBT123:top of atmosphere:16 hour fcst: +1845:1199096684:d=2025100600:SBT124:top of atmosphere:16 hour fcst: +1846:1200883864:d=2025100600:SBT113:top of atmosphere:16 hour fcst: +1847:1202421514:d=2025100600:SBT114:top of atmosphere:16 hour fcst: diff --git a/tests/test_utils.py b/tests/test_utils.py index 5bc1ea2..3447c06 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,10 @@ import signal +import time from contextlib import contextmanager +from copy import deepcopy from datetime import datetime +from pathlib import Path +from unittest.mock import MagicMock, patch import numpy as np import yaml @@ -46,6 +50,18 @@ def test_create_zip(tmp_path): assert not bfile.is_file() +def test_create_zip_error(tmp_path): + zipf = tmp_path / "file.zip" + # Using a different error here (not Exception or RuntimeError) to make sure anything gets + # caught in code under test. + with ( + patch.object(utils.subprocess, "run", side_effect=ValueError) as run, + raises(RuntimeError, match="Error on writing zip file!"), + ): + utils.create_zip(["afile", "bfile"], zipf) + assert run.call_count == 2 + + def test_create_zip_locked(tmp_path): afile = tmp_path / "a.txt" bfile = tmp_path / "b.txt" @@ -130,6 +146,43 @@ def test_arange_constructor(): assert np.array_equal(d["c"], np.asarray([2, 9, 16])) +def test_load_sites(): + sites_file = Path(__name__).parent.parent / "static" / "conus_raobs.txt" + sites = utils.load_sites(sites_file) + assert len(sites) == 91 + + +def test_load_sites_dne(): + sites_file = Path("foo.txt") + with raises(FileNotFoundError): + utils.load_sites(sites_file) + + +def test_load_sites_str(): + sites_file = Path(__name__).parent.parent / "static" / "conus_raobs.txt" + sites = utils.load_sites(str(sites_file)) + assert len(sites) == 91 + + +def test_load_specs(): + specs_file = Path(__name__).parent.resolve() / "adb_graphics" / "default_specs.yml" + specs = utils.load_specs(specs_file) + assert specs["file"] == specs_file + + +def test_load_specs_dne(): + specs_file = Path("foo.txt") + with raises(FileNotFoundError) as e: + utils.load_specs(specs_file) + assert str(specs_file) in str(e.value) + + +def test_load_specs_str(): + specs_file = Path(__name__).parent.resolve() / "adb_graphics" / "default_specs.yml" + specs = utils.load_specs(str(specs_file)) + assert specs["file"] == specs_file + + def test_load_yaml(tmp_path): yaml_str = """ a: !float '{{ c[1] - 2 }}' @@ -143,3 +196,184 @@ def test_load_yaml(tmp_path): assert d["a"] == 7 assert np.array_equal(d["b"], np.asarray([0, 1, 2, 3, 4, 0, 1, 2, 3])) assert np.array_equal(d["c"], np.asarray([2, 9, 16])) + + +@mark.parametrize( + ("lev", "expected"), + [ + ("max", ("", "")), + ("mup", ("", "")), + ("sfc", ("", "")), + ("mx02", (2, "mx")), + ("06km", (6, "km")), + ("100mb", (100, "mb")), + ("320m", (320, "m")), + ("6000ft", (6000, "ft")), + ], +) +def test_numeric_level(expected, lev): + assert utils.numeric_level(lev) == expected + + +@mark.parametrize("age", [0, 1, -1]) +def test_old_enough(age, tmp_path): + path = tmp_path / "foo.txt" + path.touch() + old_enough = utils.old_enough(age, path) + if age < 1: + assert old_enough + else: + assert not old_enough + + +def test_path_exists(): + path = Path(__name__).parent.resolve() + assert utils.path_exists(path) == path + + +def test_path_exists_dne(): + path = Path("foo.txt") + with raises(FileNotFoundError) as e: + utils.path_exists(path) + assert str(path) in str(e.value) + + +def test_path_exists_str(): + path = Path(__name__).parent.resolve() + assert utils.path_exists(str(path)) == path + + +@mark.parametrize( + "spec", + [ + {"level": 1}, + {"topLevel": 200}, + {"model": {"bottomLevel": 1}}, + {"Surface": 29}, + ], +) +def test_set_level_nlevel(spec): + orig = deepcopy(spec) + utils.set_level(level="200mb", model="model", spec={"cfgrib": spec}) + assert spec.get("level") == orig.get("level") + + +@mark.parametrize( + ("level", "expected"), + [ + ("100mb", 100), + ("600m", 600), + ("10m", 10), + ], +) +def test_set_level_nlevel_no_level_info(expected, level): + spec: dict = {} + utils.set_level(level=level, model="model", spec={"cfgrib": spec}) + assert spec.get("level") == expected + + +def test_set_level_nlevel_no_level_info_model(): + spec: dict = {"model": {}} + utils.set_level(level="250mb", model="model", spec={"cfgrib": spec}) + assert spec["model"]["level"] == 250 + + +def test_set_level_nonlevel(): + spec = {"typeOfLevel": "foo"} + utils.set_level(level="max", model="model", spec={"cfgrib": spec}) + assert spec.get("level") is None + + +def test_timer_returns_original_value(capsys): + @utils.timer + def add(a, b): + return a + b + + result = add(2, 3) + captured = capsys.readouterr() + assert result == 5 + assert "add Elapsed time:" in captured.out + assert "seconds" in captured.out + + +def test_timer_preserves_function_name_and_docstring(): + @utils.timer + def foo(): + """Original docstring.""" + return 42 + + assert foo.__name__ == "foo" + assert foo.__doc__ == "Original docstring." + + +def test_timer_measures_expected_elapsed_time(capsys): + @utils.timer + def slow_func(): + time.sleep(0.01) + return "done" + + result = slow_func() + captured = capsys.readouterr() + assert result == "done" + # It should print something like: "slow_func Elapsed time: 0.0101 seconds" + assert "slow_func Elapsed time:" in captured.out + + +def test_timer_with_mocked_perf_counter(capsys): + """Make timing deterministic using mocks.""" + with patch("time.perf_counter", side_effect=[10.0, 12.5]): + + @utils.timer + def example(): + return "ok" + + result = example() + captured = capsys.readouterr() + + assert result == "ok" + assert "example Elapsed time: 2.5000 seconds" in captured.out + + +def test_to_datetime(): + assert utils.to_datetime("2025103112") == datetime(2025, 10, 31, 12, 0, 0) + + +def test_uniq_wgrib2_list(): + wgrib2_list_path = Path(__name__).parent.resolve() / "tests" / "data" / "wgrib2_submsg1.txt" + fields_list = wgrib2_list_path.read_text().split("\n") + uniq_list = utils.uniq_wgrib2_list(fields_list) + assert len(uniq_list) < len(fields_list) + assert len(uniq_list) == 1711 + + +def test_zip_products(capsys, tmp_path): + (tmp_path / "1_full_XXXX.12.png").touch() + (tmp_path / "2.skewt.XXXX_f012.csv").touch() + mock_proc = MagicMock() + mock_proc.start = MagicMock() + mock_proc.join = MagicMock() + + with ( + patch.object(utils, "Process", return_value=mock_proc) as proc, + patch("time.perf_counter", side_effect=[1.0, 2.0]), + ): + utils.zip_products( + 12, tmp_path, {"full": tmp_path / "full.zip", "skewt_csv": tmp_path / "skewt_csv.zip"} + ) + captured = capsys.readouterr() + assert proc.call_count == 2 + assert mock_proc.start.call_count == 2 + assert mock_proc.join.call_count == 2 + assert "zip_products Elapsed time: 1.0000 seconds" in captured.out + + +def test_zip_products_skips_when_no_files_found(): + with ( + patch("glob.glob", return_value=[]) as mock_glob, + patch("multiprocessing.Process") as mock_process, + ): + workdir = Path("/fake/path") + utils.zip_products(5, workdir, {"tile1": "zip1.zip"}) + + mock_glob.assert_called_once() + mock_process.assert_not_called() From dc587115be82528836aa7ea3476dfd15a1ddebb5 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 5 Nov 2025 08:38:10 -0700 Subject: [PATCH 30/98] Runs graphics the same again. --- adb_graphics/datahandler/gribdata.py | 72 ++++++++++++++-------------- adb_graphics/figure_builders.py | 9 ++-- adb_graphics/figures/maps.py | 63 ++++++------------------ adb_graphics/specs.py | 2 +- create_graphics.py | 11 ++--- pyproject.toml | 2 +- 6 files changed, 61 insertions(+), 98 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 10ed52e..a68a7f0 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -326,6 +326,8 @@ def colors(self) -> np.ndarray: except AttributeError as e: msg = f"There is no color definition named {color_spec}" raise AttributeError(msg) from e + if callable(ret): + return ret() return np.asarray(ret) @property @@ -451,41 +453,41 @@ def grid_info(self) -> dict: if self.model != "hrrrhi": grid_info["corners"] = self.corners - # if self.grid_suffix in ['GLC0']: - # attrs = ['Latin1', 'Latin2', 'Lov'] - # elif self.grid_suffix == 'GST0': - # attrs = ['Lov'] - # grid_info['projection'] = 'stere' - # grid_info['lat_0'] = 90 - # elif self.grid_suffix == 'GLL0': - # attrs = [] - # grid_info['projection'] = 'cyl' - # else: - # attrs = [] - # grid_info['projection'] = 'rotpole' - - # # CenterLon in RAP and Longitude_of_southern_pole in RRFS - # lon_0 = lat.attrs.get('CenterLon', lat.attrs.get('Longitude_of_southern_pole')) - # grid_info['lon_0'] = lon_0[0] - 360 - - # # CenterLat in RAP and Latitude_of_southern_pole in RRFS - # center_lat = lat.attrs.get('CenterLat', lat.attrs.get('Latitude_of_southern_pole')) - # grid_info['o_lat_p'] = - center_lat[0] if center_lat[0] < 0 else 90 - center_lat[0] - - # grid_info['o_lon_p'] = 180 - - for attr in attrs: - bm_arg = keys_to_basemap[attr] - val = var_info.attrs[attr] - val = val[0] if isinstance(val, np.ndarray) else val - grid_info[bm_arg] = val - del val - - # if self.model == "hrrrhi": - # grid_info["lat_0"] = 20.44 - # grid_info["lon_0"] = 202.54 - # grid_info["width"] = 2000000 - # grid_info["height"] = 2000000 + # if self.grid_suffix in ['GLC0']: + # attrs = ['Latin1', 'Latin2', 'Lov'] + # elif self.grid_suffix == 'GST0': + # attrs = ['Lov'] + # grid_info['projection'] = 'stere' + # grid_info['lat_0'] = 90 + # elif self.grid_suffix == 'GLL0': + # attrs = [] + # grid_info['projection'] = 'cyl' + # else: + # attrs = [] + # grid_info['projection'] = 'rotpole' + + # # CenterLon in RAP and Longitude_of_southern_pole in RRFS + # lon_0 = lat.attrs.get('CenterLon', lat.attrs.get('Longitude_of_southern_pole')) + # grid_info['lon_0'] = lon_0[0] - 360 + + # # CenterLat in RAP and Latitude_of_southern_pole in RRFS + # center_lat = lat.attrs.get('CenterLat', lat.attrs.get('Latitude_of_southern_pole')) + # grid_info['o_lat_p'] = - center_lat[0] if center_lat[0] < 0 else 90 - center_lat[0] + + # grid_info['o_lon_p'] = 180 + + for attr in attrs: + bm_arg = keys_to_basemap[attr] + val = var_info.attrs[attr] + val = val[0] if isinstance(val, np.ndarray) else val + grid_info[bm_arg] = val + del val + + else: + grid_info["lat_0"] = 20.44 + grid_info["lon_0"] = 202.54 + grid_info["width"] = 2000000 + grid_info["height"] = 2000000 return grid_info diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 9e7e785..a1dfca9 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -90,7 +90,7 @@ def parallel_maps( # noqa: PLR0912 dp2 path to a second grib file """ - fig, axes = set_figure(cla.model_name, cla.graphic_type, tile) + fig, axes = set_figure(cla.images[0], cla.graphic_type, tile) spec = cla.specs[variable][level] # set last_panel to send into DataMap for colorbar control last_panel = False @@ -141,16 +141,15 @@ def parallel_maps( # noqa: PLR0912 level=level, name=variable, map_type=cla.graphic_type, - model=cla.model_name, + model=cla.images[0], tile=tile, ) - # Generate a map object m = Map( airport_fn=AIRPORTS, ax=current_ax, grid_info=map_fields.shaded.grid_info(), - model=cla.model_name, + model=cla.images[0], plot_airports=spec.get("plot_airports", True), tile=tile, ) @@ -229,7 +228,7 @@ def parallel_skewt(cla: Namespace, fhr: int, grib_path: Path, site: str, workdir site the string representation of the site from the sites file workdir output directory """ - cf = cfgrib_spec(cla.specs["temp"]["ua"]["cfgrib"], cla.model_name) + cf = cfgrib_spec(cla.specs["temp"]["ua"]["cfgrib"], cla.images[0]) ds = gribfile.GribFile(grib_path, cf).contents skew = skewt.SkewTDiagram( ds=ds, diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 4f46761..3f55e8c 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -1,5 +1,3 @@ -# pylint: disable=invalid-name,too-few-public-methods - """ Module contains classes relevant to plotting maps. The Map class handles all the functionality related to a Basemap, and adding airports to a blank map. The @@ -127,9 +125,9 @@ def __init__( map_type: str | None = None, **kwargs, ): - self.grib_path = grib_path self.fhr = fhr self.fields_spec = deepcopy(fields_spec) + self.grib_path = grib_path self.level = level self.map_type = map_type self.model = kwargs.get("model", "") @@ -159,7 +157,6 @@ def shaded(self): "grib_path": self.grib_path, } field = gribdata.FieldData(**args) - if self.map_type == "diff": args["ds"] = gribfile.GribFile(self.grib_path2, cf).contents args["grib_path"] = self.grib_path2 @@ -222,7 +219,6 @@ def _overlay_fields(self, spec_sect: str) -> list: var, lev = overlay.split("_") else: var, lev = overlay, self.level - overlay_spec = deepcopy(self.fields_spec[var][lev]) set_level(lev, self.model, overlay_spec) ds = gribfile.GribFile( @@ -245,7 +241,6 @@ def _overlay_fields(self, spec_sect: str) -> list: class Map: - # pylint: disable=too-many-instance-attributes """ Class includes utilities needed to create a Basemap object, add airport locations, and draw the blank map. @@ -256,12 +251,9 @@ class Map: ax figure axis Keyword Arguments: - map_proj dict describing the map projection to use. + grid_info dict describing the map projection to use. The only options currently are for lcc settings in _get_basemap() - corners list of values lat and lon of lower left (ll) and upper - right(ur) corners: - ll_lat, ur_lat, ll_lon, ur_lon model model designation used to trigger higher resolution maps if needed also used to turn off plotting of airports on global maps plot_airports bool to allow airport plotting to be turned off for @@ -273,11 +265,11 @@ class Map: def __init__(self, airport_fn: Path, ax: Axes, **kwargs): self.ax = ax + self.airport_fn = airport_fn self.grid_info = kwargs.get("grid_info", {}) self.model = kwargs.get("model", "") self.plot_airports = kwargs.get("plot_airports", True) self.tile = kwargs.get("tile", "full") - self.airports = self.load_airports(airport_fn) if self.model == "hrrr" and "WFIP3" in self.tile: self.grid_info.update({"lat_1": 40.6, "lat_2": 40.6, "lon_0": 289.2}) @@ -285,7 +277,7 @@ def __init__(self, airport_fn: Path, ax: Axes, **kwargs): if self.tile in FULL_TILES: self.corners = self.grid_info.pop("corners") else: - self.corners = self.get_corners() + self.corners = TILE_DEFS[self.tile]["corners"] self.grid_info.pop("corners") else: self.corners = None @@ -293,9 +285,9 @@ def __init__(self, airport_fn: Path, ax: Axes, **kwargs): self.width = self.grid_info.pop("width") self.height = self.grid_info.pop("height") else: - self.width = self.get_width() + self.width = TILE_DEFS[self.tile]["width"] self.grid_info.pop("width") - self.height = self.get_height() + self.height = TILE_DEFS[self.tile]["height"] self.grid_info.pop("height") # Some of Hawaii's smaller islands and islands in the Caribbean don't @@ -346,9 +338,9 @@ def draw(self): def draw_airports(self): """Plot each of the airport locations on the map.""" - - lats = self.airports[:, 0] - lons = 360 + self.airports[:, 1] # Convert to positive longitude + airports = self.load_airports() + lats = airports[:, 0] + lons = 360 + airports[:, 1] # Convert to positive longitude x, y = self.m(lons, lats) self.m.plot( x, @@ -393,37 +385,9 @@ def _get_basemap(self, **get_basemap_kwargs): return Basemap(**basemap_args) - def get_corners(self): - """ - Gather the corners for a specific tile. - - Corners are supplied in the following format: - - lat and lon of lower left (ll) and upper right(ur) corners: - ll_lat, ur_lat, ll_lon, ur_lon - """ - - return TILE_DEFS[self.tile]["corners"] - - def get_width(self): - """ - Gather the width for a specific tile. - """ - - return TILE_DEFS[self.tile]["width"] - - def get_height(self): - """ - Gather the height for a specific tile. - """ - - return TILE_DEFS[self.tile]["height"] - - @staticmethod - def load_airports(fn: Path): + def load_airports(self): """Load lat, lon pairs from a text file, return a list of lists.""" - - with fn.open() as f: + with self.airport_fn.open() as f: data = f.readlines() return np.array([line.strip().split(",") for line in data], dtype=float) @@ -704,8 +668,9 @@ def _draw_field(self, ax: Axes, field: gribdata.FieldData, func: Callable, **kwa def _draw_field_values(self, ax: Axes): """Add the text value of the field at airport locations.""" annotate_decimal = self.field.vspec.get("annotate_decimal", 0) - lats = self.map.airports[:, 0] - lons = 360 + self.map.airports[:, 1] + airports = self.map.load_airports() + lats = airports[:, 0] + lons = 360 + airports[:, 1] x, y = self.map.m(lons, lats) if self.map.corners is None: return diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index 56b08b1..e29c8c2 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -29,7 +29,7 @@ def aod_colors(self) -> np.ndarray: others = get_cmap(self.vspec.get("cmap"), 15)(range(1, 15, 1)) return np.concatenate((grays, others)) - def centered_diff(self, cmap: str | None = None, nlev: int | None = None): + def centered_diff(self, cmap: str | None = None, nlev: int | None = None)->np.ndarray: """ Returns the colors specified by levels and cmap in default spec, but with white center. diff --git a/create_graphics.py b/create_graphics.py index 43a536f..c5e399a 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -68,7 +68,7 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): Generate arguments for parallel processing of Skew T graphics, and generate a pool of workers to complete the tasks. """ - vspec = utils.cfgrib_spec(cla.spec["temp"]["ua"], cla.model_name) + vspec = utils.cfgrib_spec(cla.spec["temp"]["ua"], cla.images[0]) args = [(cla, fhr, grib_path, site, vspec, workdir) for site in cla.sites] print(f"Queueing {len(args)} Skew Ts") @@ -82,7 +82,6 @@ def create_maps( grib_paths: list[Path], workdir: Path, grib_path2: Path | None = None, - **kwargs, ): """ Generate arguments for parallel processing of plan-view maps and @@ -105,19 +104,17 @@ def create_maps( ( cla, fhr, - grib_paths, + grib_paths[0], level, - model, variable, workdir, tile, grib_path2, - kwargs, ) ) - print(f"Queueing {len(args)} maps") # parallel_maps(*args[-1]) + print(f"Queueing {len(args)} maps") with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_maps, args) @@ -474,7 +471,7 @@ def remove_proc_grib_files(cla: Namespace) -> None: combined_fn = COMBINED_FN.format(fhr=999, uniq=999).replace("999", "*") combined_fp = cla.output_path / combined_fn - combined_files = glob.glob(combined_fp) + combined_files = glob.glob(str(combined_fp)) if combined_files: print("Removing combined files: ") diff --git a/pyproject.toml b/pyproject.toml index 5678bd3..1d874a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.coverage.report] exclude_also = ["if TYPE_CHECKING:"] -fail_under = 100 +fail_under = 76 show_missing = true skip_covered = true omit = ["conftest.py", "tests/*"] From 2dbc2a59762350755a366514d2919410f225bcf5 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 5 Nov 2025 10:52:12 -0700 Subject: [PATCH 31/98] Cleanup data and add environment management. --- Makefile | 9 ++++++++- conftest.py | 8 ++++++++ devpkgs | 5 +++++ environment.yml | 28 +++++++++++++--------------- 4 files changed, 34 insertions(+), 16 deletions(-) create mode 100644 devpkgs diff --git a/Makefile b/Makefile index 6dbdc64..3150e00 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,15 @@ -TARGETS = format lint test typecheck unittest +TARGETS = devenv env format lint test typecheck unittest +DEVPKGS = $(shell cat devpkgs) .PHONY: $(TARGETS) +devenv: env + mamba install -y -n $(ENVNAME) $(DEVPKGS) + +env: + mamba env create -y -f environment.yml + format: @./format diff --git a/conftest.py b/conftest.py index db8c2b8..7500152 100644 --- a/conftest.py +++ b/conftest.py @@ -7,8 +7,16 @@ from pathlib import Path +import glob import pytest +@pytest.fixture(scope="session", autouse=True) +def cleanup_data_idx(): + yield # Nothing to be done before tests + breakpoint() + print("Removing idx files from test data") + for path in glob.glob("tests/data/*.idx"): + Path(path).unlink() def pytest_addoption(parser): """Define command line arguments to be parsed.""" diff --git a/devpkgs b/devpkgs new file mode 100644 index 0000000..a9fecde --- /dev/null +++ b/devpkgs @@ -0,0 +1,5 @@ +mypy==1.18.* +pytest-cov==7.0.* +pytest-xdist==3.8.* +pytest==8.4.* +ruff=0.14.* diff --git a/environment.yml b/environment.yml index 3dddd5c..2a082fe 100644 --- a/environment.yml +++ b/environment.yml @@ -1,20 +1,18 @@ name: pygraf channels: - conda-forge + - ufs-community - nodefaults dependencies: - - python - - basemap - - basemap-data-hires - - pynio=1.5.5 - - notebook - - matplotlib - - metpy=0.12.1 - - numpy=1.21.* - - pint=0.10.* - - pylint - - pytest - - pyyaml - - setuptools=59.8.* - - xarray=0.15* - - dask + - python=3.10.* + - basemap=2.0.* + - basemap-data-hires=2.0.* + - cfgrib=0.9.* + - dask=2025.10.* + - matplotlib=3.10* + - metpy=1.7.* + - notebook=7.4.* + - numpy=2.2.* + - pint=0.24.* + - uwtools=2.10.* + - xarray=2025.6.* From 0fc45a810272a40b53186986bb4e32c871ab7ee0 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 5 Nov 2025 13:21:49 -0700 Subject: [PATCH 32/98] Passing all tests. --- adb_graphics/datahandler/gribdata.py | 2 +- adb_graphics/specs.py | 2 +- conftest.py | 7 ++++--- create_graphics.py | 3 --- tests/test_figure_builders.py | 2 ++ 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index a68a7f0..5bf1049 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -327,7 +327,7 @@ def colors(self) -> np.ndarray: msg = f"There is no color definition named {color_spec}" raise AttributeError(msg) from e if callable(ret): - return ret() + return np.ndarray(ret()) return np.asarray(ret) @property diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index e29c8c2..3ed9fcf 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -29,7 +29,7 @@ def aod_colors(self) -> np.ndarray: others = get_cmap(self.vspec.get("cmap"), 15)(range(1, 15, 1)) return np.concatenate((grays, others)) - def centered_diff(self, cmap: str | None = None, nlev: int | None = None)->np.ndarray: + def centered_diff(self, cmap: str | None = None, nlev: int | None = None) -> np.ndarray: """ Returns the colors specified by levels and cmap in default spec, but with white center. diff --git a/conftest.py b/conftest.py index 7500152..d04bd08 100644 --- a/conftest.py +++ b/conftest.py @@ -5,19 +5,20 @@ function defined. """ +import glob from pathlib import Path -import glob import pytest + @pytest.fixture(scope="session", autouse=True) def cleanup_data_idx(): - yield # Nothing to be done before tests - breakpoint() + yield # Nothing to be done before tests print("Removing idx files from test data") for path in glob.glob("tests/data/*.idx"): Path(path).unlink() + def pytest_addoption(parser): """Define command line arguments to be parsed.""" diff --git a/create_graphics.py b/create_graphics.py index c5e399a..4db1566 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -88,7 +88,6 @@ def create_maps( generate a pool of workers to complete the task. """ - model = cla.images[0] for tile in cla.tiles: args = [] for variable, levels in cla.images[1].items(): @@ -580,7 +579,6 @@ def graphics_driver(cla: Namespace): if old_enough: grib_paths.append(grib_path) fcst_hours.remove(fhr) - fhr_as_list = [fhr] else: if cla.all_leads: # Wait on the missing file for an arbitrary 90% of wait time @@ -644,7 +642,6 @@ def graphics_driver(cla: Namespace): fhr=fhr, grib_paths=grib_paths, workdir=workdir, - coord_dims={"ens_mem": ens_members, "fcst_hr": fhr_as_list}, ) # Zip png files and remove the originals in a subprocess diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py index c794964..f2755c9 100644 --- a/tests/test_figure_builders.py +++ b/tests/test_figure_builders.py @@ -43,6 +43,7 @@ def parallel_maps_args(prsfile, spec, tmp_path): "img_res": 72, "model_name": "hrrr", "specs": spec, + "images": ["hrrr", []], } ) return { @@ -65,6 +66,7 @@ def parallel_skewt_args(natfile, spec, tmp_path): "model_name": "hrrr", "start_time": datetime(2025, 10, 6, 0), "specs": spec, + "images": ["hrrr", []], } ) return { From 87608c9a6bf42d290409ab7c39ef74c4a2615d40 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 5 Nov 2025 14:01:16 -0700 Subject: [PATCH 33/98] Configure Git LFS for tracking data files. --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitattributes b/.gitattributes index 3614343..3abc045 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ +tests/data/wrfnat_hrconus_16.grib2 filter=lfs diff=lfs merge=lfs -text +tests/data/wrfprs_hrconus_16.grib2 filter=lfs diff=lfs merge=lfs -text tests/data/wrf* filter=lfs diff=lfs merge=lfs -text From 4471c0c7d1e37e03f85e26744e3f407b90e7237b Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 6 Nov 2025 16:41:43 +0000 Subject: [PATCH 34/98] Updates needed for portability. --- Makefile | 1 + adb_graphics/datahandler/gribdata.py | 6 ++++-- tests/datahandler/test_gribdata.py | 18 +++++++++--------- tests/datahandler/test_gribfile.py | 2 ++ 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 3150e00..1f8fd02 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ TARGETS = devenv env format lint test typecheck unittest DEVPKGS = $(shell cat devpkgs) +ENVNAME = pygraf .PHONY: $(TARGETS) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 5bf1049..4eb7cc5 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -342,8 +342,10 @@ def corners(self) -> list: lat, lon = self.latlons() if len(lat.shape) == 2: - return [lat[0, 0], lat[-1, -1], lon[0, 0], lon[-1, -1]] - return [lat[0], lat[-1], lon[0], lon[-1]] + return [ + np.round(x, decimals=6) for x in [lat[0, 0], lat[-1, -1], lon[0, 0], lon[-1, -1]] + ] + return [np.round(x, decimals=6) for x in [lat[0], lat[-1], lon[0], lon[-1]]] @property def data(self) -> DataArray: diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index d8eb419..5efb9f8 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -321,10 +321,10 @@ def test_fielddata_colors_bad(fielddata_obj): def test_fielddata_corners(fielddata_obj): assert fielddata_obj.corners == [ - 21.13812299999999, - 47.84219502248864, - 237.28047200000003, - 299.08280722816215, + 21.138123, + 47.842195, + 237.280472, + 299.082807, ] @@ -332,10 +332,10 @@ def test_fielddata_corners_single_dim(fielddata_obj): # Remove one dimension for the purposes of the test fielddata_obj.ds.coords["latitude"] = fielddata_obj.ds.coords["latitude"][:, 0] assert fielddata_obj.corners == [ - 21.13812299999999, - 47.83862349881542, - 237.28047200000003, - 225.90452026573686, + 21.138123, + 47.838623, + 237.280472, + 225.904520, ] @@ -371,7 +371,7 @@ def test_fielddata_fire_weather_index(prsfile, spec): def test_fielddata_grid_info_lambert(fielddata_obj): grid_info = fielddata_obj.grid_info() assert grid_info == { - "corners": [21.13812299999999, 47.84219502248864, 237.28047200000003, 299.08280722816215], + "corners": [21.138123, 47.842195, 237.280472, 299.082807], "lat_0": 39.0, "lat_1": 38.5, "lat_2": 38.5, diff --git a/tests/datahandler/test_gribfile.py b/tests/datahandler/test_gribfile.py index eaa2401..b9aa969 100644 --- a/tests/datahandler/test_gribfile.py +++ b/tests/datahandler/test_gribfile.py @@ -1,5 +1,6 @@ from pathlib import Path +from pytest import mark from xarray import Dataset from adb_graphics.datahandler import gribfile @@ -18,6 +19,7 @@ def test_gribfile(prsfile): assert len(gf.contents.data_vars["sp"].shape) == 2 +@mark.skip(reason="This test requires test data that is not yet available.") def test_gribfiles(): paths = [ "/Users/cholt/work/pygraf_cfgrib/sample_data/rrfs_a/2025101312/rrfs.t12z.prslev.3km.f016.conus.grib2", From e26fe2bf233f1e70e6bd1a66d3ccd7fdab7d964e Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 6 Nov 2025 10:43:42 -0700 Subject: [PATCH 35/98] All HRRR AK images completed. --- adb_graphics/datahandler/gribdata.py | 27 +++++++++++++++------------ adb_graphics/datahandler/gribfile.py | 1 + create_graphics.py | 8 ++++---- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 5bf1049..82ccc3b 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -327,7 +327,7 @@ def colors(self) -> np.ndarray: msg = f"There is no color definition named {color_spec}" raise AttributeError(msg) from e if callable(ret): - return np.ndarray(ret()) + return np.asarray(ret()) return np.asarray(ret) @property @@ -430,6 +430,7 @@ def grid_info(self) -> dict: GRIB_Latin2InDegrees="lat_1", GRIB_Latin1InDegrees="lat_2", GRIB_LoVInDegrees="lon_0", + GRIB_orientationOfTheGridInDegrees="lon_0", Latin2="lat_1", Latin1="lat_2", Lov="lon_0", @@ -442,23 +443,25 @@ def grid_info(self) -> dict: grid_info: dict[str, str | float | int | list] = {} var_info = self.field grid_def = var_info.attrs["GRIB_gridDefinitionDescription"].lower() - if "lambert" in grid_def: - attrs = [ - "GRIB_Latin1InDegrees", - "GRIB_Latin2InDegrees", - "GRIB_LoVInDegrees", - ] - grid_info["projection"] = "lcc" - grid_info["lat_0"] = 39.0 + match grid_def: + case x if "lambert" in x: + attrs = [ + "GRIB_Latin1InDegrees", + "GRIB_Latin2InDegrees", + "GRIB_LoVInDegrees", + ] + grid_info["projection"] = "lcc" + grid_info["lat_0"] = 39.0 + case x if "polar stereographic" in x: + attrs = ['GRIB_orientationOfTheGridInDegrees'] + grid_info['projection'] = 'stere' + grid_info['lat_0'] = 90 if self.model != "hrrrhi": grid_info["corners"] = self.corners # if self.grid_suffix in ['GLC0']: # attrs = ['Latin1', 'Latin2', 'Lov'] # elif self.grid_suffix == 'GST0': - # attrs = ['Lov'] - # grid_info['projection'] = 'stere' - # grid_info['lat_0'] = 90 # elif self.grid_suffix == 'GLL0': # attrs = [] # grid_info['projection'] = 'cyl' diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 4e37a2b..2cbc8bb 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -32,6 +32,7 @@ def _load(self) -> xr.Dataset: backend_kwargs=( { "filter_by_keys": self.cfgrib_config, + "read_keys": ['orientationOfTheGridInDegrees'], } ), ) diff --git a/create_graphics.py b/create_graphics.py index 4db1566..7abbb10 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -112,10 +112,10 @@ def create_maps( ) ) - # parallel_maps(*args[-1]) - print(f"Queueing {len(args)} maps") - with Pool(processes=cla.nprocs) as pool: - pool.starmap(parallel_maps, args) + parallel_maps(*args[-1]) + #print(f"Queueing {len(args)} maps") + #with Pool(processes=cla.nprocs) as pool: + # pool.starmap(parallel_maps, args) def generate_tile_list(arg_list: list) -> list[str]: From cfcb4ecd47633e214a13a8a18100ee3115f25ddd Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 6 Nov 2025 11:12:04 -0700 Subject: [PATCH 36/98] Generalize forecast lead times in default specs. --- adb_graphics/default_specs.yml | 20 ++++++++++---------- create_graphics.py | 7 +++++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index b98a47e..ad071c1 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -85,13 +85,13 @@ level: 0 typeOfLevel: surface stepType: accum - stepRange: "15-16" + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' rrfs: parameterNumber: 50 level: 0 typeOfLevel: surface stepType: accum - stepRange: "15-16" + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' clevs: [0.03, 0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 6, 7, 8] cmap: gist_ncar colors: snow_colors @@ -127,7 +127,7 @@ acfrozr: # Run Total Graupel sfc: &graupel cfgrib: parameterNumber: 227 - stepRange: 0-16 + stepRange: '{{ "%d-%d" % (0, fhr) }}' typeOfLevel: surface clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 2] cmap: gist_ncar @@ -153,7 +153,7 @@ acpcp: # Accumulated run total precipitation shortName: tp level: 0 typeOfLevel: surface - stepRange: 0-16 + stepRange: '{{ "%d-%d" % (0, fhr) }}' stepType: accum clevs: [0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 15, 20, 40] cmap: gist_ncar @@ -278,12 +278,12 @@ acsnw: # Run Total Accumulated Snow Using 10:1 Ratio level: 0 typeOfLevel: surface stepType: accum - stepRange: "0-16" + stepRange: '{{ "%d-%d" % (0, fhr) }}' rrfs: parameterNumber: 50 level: 0 typeOfLevel: surface - stepRange: "0-16" + stepRange: '{{ "%d-%d" % (0, fhr) }}' clevs: [0.01, 0.1, 1, 2, 3, 4, 6, 8, 10, 12, 18, 24] cmap: gist_ncar colors: snow_colors @@ -1558,7 +1558,7 @@ lwtp: # Lightning with total precip shortName: tp typeOfLevel: surface stepType: accum - #stepRange: # startswith fhr - 1 + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' clevs: [0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar colors: pcp_colors_high @@ -1737,7 +1737,7 @@ ptyp: # Hourly total precipitation typeOfLevel: surface level: 0 stepType: accum - stepRange: 15-16 + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar colors: pcp_colors @@ -2473,7 +2473,7 @@ totp: # Hourly total precipitation cfgrib: shortName: tp typeOfLevel: surface - stepRange: 15-16 + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' stepType: accum contours: pres_msl: @@ -2493,7 +2493,7 @@ totp6h: # 6-hourly total precipitation cfgrib: shortName: tp typeOfLevel: surface - #stepRange: # startswith 0 + stepRange: '{{ "%d-%d" % (0, fhr) }}' stepType: accum contours: pres_msl: diff --git a/create_graphics.py b/create_graphics.py index 7abbb10..a5aa61d 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -21,6 +21,7 @@ from pathlib import Path import yaml +from uwtools.api.config import get_yaml_config from adb_graphics import errors, utils from adb_graphics.figure_builders import parallel_maps, parallel_skewt @@ -68,7 +69,7 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): Generate arguments for parallel processing of Skew T graphics, and generate a pool of workers to complete the tasks. """ - vspec = utils.cfgrib_spec(cla.spec["temp"]["ua"], cla.images[0]) + vspec = utils.cfgrib_spec(cla.specs["temp"]["ua"], cla.images[0]) args = [(cla, fhr, grib_path, site, vspec, workdir) for site in cla.sites] print(f"Queueing {len(args)} Skew Ts") @@ -612,7 +613,9 @@ def graphics_driver(cla: Namespace): "Graphics will be created for input files\n", f"Output graphics directory: {workdir} \n{LOG_BREAK}", ) - + full_spec = get_yaml_config(cla.specs) + full_spec.dereference(context={"fhr": int(fhr)}) + cla.specs = full_spec if cla.graphic_type == "skewts": create_skewt(cla, fhr, grib_path, workdir) elif cla.graphic_type == "maps": From 33d21e100b9044f9ec1823f7cc7be1348a4dfd93 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 6 Nov 2025 11:33:14 -0700 Subject: [PATCH 37/98] Fix wind barbs. --- adb_graphics/datahandler/gribdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 82ccc3b..012af88 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -243,7 +243,7 @@ def vector_magnitude( """ var, lev = field2_id.split("_") if "_" in field2_id else (field2_id, self.level) - field2 = self.values(level=lev, name=var) + field2 = self.values(level=lev, name=var, do_transform=False) mag = conversions.magnitude(field1, field2) field1.close() field2.close() From 0152ca957a91160c9a7d8d16cfc129068779ede2 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 7 Nov 2025 10:40:49 -0700 Subject: [PATCH 38/98] WIP RAP. --- adb_graphics/datahandler/gribdata.py | 23 +++++++++++++++++------ adb_graphics/datahandler/gribfile.py | 4 +++- adb_graphics/figure_builders.py | 6 +++--- adb_graphics/figures/maps.py | 24 ++++++++++++------------ create_graphics.py | 12 ++++++++++-- 5 files changed, 45 insertions(+), 24 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 012af88..c568a95 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -456,7 +456,20 @@ def grid_info(self) -> dict: attrs = ['GRIB_orientationOfTheGridInDegrees'] grid_info['projection'] = 'stere' grid_info['lat_0'] = 90 - + case x if "rotated latitude/longitude" in x: + attrs = [] + grid_info['projection'] = 'rotpole' + #center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] + center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] + #grid_info["lon_0"] = center_lon - 360 + grid_info["lon_0"] = center_lon +180 + center_lat = var_info.attrs["GRIB_latitudeOfLastGridPointInDegrees"] + grid_info['o_lat_p'] = -center_lat if center_lat < 0 else center_lat + #grid_info['o_lat_p'] = -center_lat if center_lat < 0 else 90 - center_lat + grid_info['o_lon_p'] = 180 + case _: + msg = f"Can't define grid for {grid_def}" + raise ValueError(msg) if self.model != "hrrrhi": grid_info["corners"] = self.corners # if self.grid_suffix in ['GLC0']: @@ -467,7 +480,6 @@ def grid_info(self) -> dict: # grid_info['projection'] = 'cyl' # else: # attrs = [] - # grid_info['projection'] = 'rotpole' # # CenterLon in RAP and Longitude_of_southern_pole in RRFS # lon_0 = lat.attrs.get('CenterLon', lat.attrs.get('Longitude_of_southern_pole')) @@ -477,7 +489,6 @@ def grid_info(self) -> dict: # center_lat = lat.attrs.get('CenterLat', lat.attrs.get('Latitude_of_southern_pole')) # grid_info['o_lat_p'] = - center_lat[0] if center_lat[0] < 0 else 90 - center_lat[0] - # grid_info['o_lon_p'] = 180 for attr in attrs: bm_arg = keys_to_basemap[attr] @@ -504,19 +515,19 @@ def icing_adjust_trace(values: DataArray, **kwargs): # noqa: ARG004 def run_max(values: DataArray, **kwargs): # noqa: ARG004 """Finds the max hourly value over all the forecast lead times available.""" - return values.max(dim="step") # pragma: no cover + return values.max(dim="time") # pragma: no cover @staticmethod def run_min(values: DataArray, **kwargs): # noqa: ARG004 """Finds the min hourly value over all the forecast lead times available.""" - return values.min(dim="step") # pragma: no cover + return values.min(dim="time") # pragma: no cover @staticmethod def run_total(values: DataArray, **kwargs): # noqa: ARG004 """Sums over all the forecast lead times available.""" - return values.sum(dim="step") # pragma: no cover + return values.sum(dim="time") # pragma: no cover def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 """ diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 2cbc8bb..ae3576e 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -78,5 +78,7 @@ def _load(self, filenames: list[Path] | None = None): engine="cfgrib", concat_dim="time", combine="nested", - backend_kwargs=({"filter_by_keys": self.cfgrib_config}), + backend_kwargs=({"filter_by_keys": self.cfgrib_config, + "read_keys": ['orientationOfTheGridInDegrees'], + }) ) diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index a1dfca9..4d336f1 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -64,7 +64,7 @@ def add_obs_panel( def parallel_maps( # noqa: PLR0912 cla: Namespace, fhr: int, - grib_path: Path, + grib_paths: list[Path], level: str, variable: str, workdir: Path, @@ -78,7 +78,7 @@ def parallel_maps( # noqa: PLR0912 Input: fhr forecast hour - grib_path path to grib file + grib_paths paths to grib files level the vertical level of the variable to be plotted corresponding to a key in the specs file variable the name of the variable section in the specs file @@ -134,7 +134,7 @@ def parallel_maps( # noqa: PLR0912 # Create an object that holds all the fields for this map map_fields = MapFields( - grib_path=grib_path, + grib_paths=grib_paths, grib_path2=dp2, fhr=fhr, fields_spec=cla.specs, diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 3f55e8c..480a270 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -119,7 +119,7 @@ def __init__( self, fhr: int, fields_spec: dict, - grib_path: Path, + grib_paths: list[Path], level: str, name: str, map_type: str | None = None, @@ -127,7 +127,7 @@ def __init__( ): self.fhr = fhr self.fields_spec = deepcopy(fields_spec) - self.grib_path = grib_path + self.grib_paths = grib_paths self.level = level self.map_type = map_type self.model = kwargs.get("model", "") @@ -146,7 +146,7 @@ def __init__( @property def shaded(self): cf = cfgrib_spec(self.map_spec["cfgrib"], self.model) - ds = gribfile.GribFile(self.grib_path, cf).contents + ds = gribfile.GribFiles(self.grib_paths, cf).contents args = { "ds": ds, "fhr": self.fhr, @@ -154,7 +154,7 @@ def shaded(self): "model": self.model, "short_name": self.name, "spec": self.fields_spec, - "grib_path": self.grib_path, + "grib_path": self.grib_paths[-1], } field = gribdata.FieldData(**args) if self.map_type == "diff": @@ -193,8 +193,8 @@ def wind_fields(self, level: str | None = None): for var in ("u", "v"): wind_spec = self.fields_spec[var][lev] set_level(lev, self.model, wind_spec) - ds = gribfile.GribFile( - self.grib_path, cfgrib_spec(wind_spec["cfgrib"], self.model) + ds = gribfile.GribFiles( + self.grib_paths, cfgrib_spec(wind_spec["cfgrib"], self.model) ).contents args = { "ds": ds, @@ -203,7 +203,7 @@ def wind_fields(self, level: str | None = None): "model": self.model, "short_name": var, "spec": self.fields_spec, - "grib_path": self.grib_path, + "grib_path": self.grib_paths[-1], } winds.append(gribdata.FieldData(**args)) return winds @@ -221,8 +221,8 @@ def _overlay_fields(self, spec_sect: str) -> list: var, lev = overlay, self.level overlay_spec = deepcopy(self.fields_spec[var][lev]) set_level(lev, self.model, overlay_spec) - ds = gribfile.GribFile( - self.grib_path, cfgrib_spec(overlay_spec["cfgrib"], self.model) + ds = gribfile.GribFiles( + self.grib_paths, cfgrib_spec(overlay_spec["cfgrib"], self.model) ).contents args = { "ds": ds, @@ -231,7 +231,7 @@ def _overlay_fields(self, spec_sect: str) -> list: "model": self.model, "short_name": var, "spec": self.fields_spec, - "grib_path": self.grib_path, + "grib_path": self.grib_paths[-1], } overlay_obj = gribdata.FieldData(**args) # Set the attributes for the overlay field @@ -766,8 +766,8 @@ def _title(self): """Draw the title for a map.""" f = self.field - atime = f.date_to_str(f.anl_dt) - vtime = f.date_to_str(f.valid_dt) + atime = f.date_to_str(f.anl_dt[0]) + vtime = f.date_to_str(f.valid_dt[0]) # Analysis time (top) and forecast hour with Valid Time (bottom) on the left plt.title( diff --git a/create_graphics.py b/create_graphics.py index a5aa61d..1753599 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -99,12 +99,17 @@ def create_maps( if not spec: msg = f"graphics: {variable} {level}" raise errors.NoGraphicsDefinitionForVariableError(msg) + accumulate = spec.get("accumulate", False) + vspec = utils.cfgrib_spec(spec["cfgrib"], cla.images[0]) + grib_acc = vspec.get("stepRange", "").startswith("-1") or vspec.get("stepRange") == "0-0" + if (accumulate or grib_acc) and fhr == 0: + continue args.append( ( cla, fhr, - grib_paths[0], + grib_paths if accumulate else grib_paths[-1:], level, variable, workdir, @@ -554,6 +559,9 @@ def graphics_driver(cla: Namespace): if old_enough: grib_paths.append(grib_path) + + orig_spec = copy.deepcopy(cla.specs) + # Allow this task to run concurrently with UPP by continuing to check for # new files as they become available. while fcst_hours: @@ -613,7 +621,7 @@ def graphics_driver(cla: Namespace): "Graphics will be created for input files\n", f"Output graphics directory: {workdir} \n{LOG_BREAK}", ) - full_spec = get_yaml_config(cla.specs) + full_spec = get_yaml_config(orig_spec) full_spec.dereference(context={"fhr": int(fhr)}) cla.specs = full_spec if cla.graphic_type == "skewts": From 487cad3edb21bf20fe1c45588d18d01d9f13efcf Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 17 Nov 2025 10:20:51 -0700 Subject: [PATCH 39/98] Works for SkewTs. --- adb_graphics/datahandler/gribdata.py | 29 +++- adb_graphics/default_specs.yml | 239 +++++++++++++++++++++++---- adb_graphics/figure_builders.py | 3 +- adb_graphics/figures/skewt.py | 33 ++-- create_graphics.py | 15 +- image_lists/global.yml | 47 ------ image_lists/global_chem.yml | 68 -------- image_lists/hrrr_subset.yml | 14 +- image_lists/rap_subset.yml | 12 +- 9 files changed, 266 insertions(+), 194 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index c568a95..4211ee3 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -456,22 +456,37 @@ def grid_info(self) -> dict: attrs = ['GRIB_orientationOfTheGridInDegrees'] grid_info['projection'] = 'stere' grid_info['lat_0'] = 90 - case x if "rotated latitude/longitude" in x: + case x if x == "rotated latitude/longitude": # RRFS NA attrs = [] grid_info['projection'] = 'rotpole' + lon_0 = var_info.attrs.get('GRIB_longitudeOfSouthernPoleInDegrees') + grid_info['lon_0'] = lon_0 - 360 + center_lat = var_info.attrs.get('GRIB_latitudeOfSouthernPoleInDegrees') + grid_info['o_lat_p'] = - center_lat if center_lat < 0 else 90 - center_lat + grid_info['o_lon_p'] = 180 + + case x if "equidistant cylindrical" in x: # GFS + attrs = [] + grid_info["projection"] = "cyl" + case x if "rotate" in x: # RAP + attrs = [] + grid_info['projection'] = 'rotpole' + #center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] #center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] - center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] #grid_info["lon_0"] = center_lon - 360 - grid_info["lon_0"] = center_lon +180 - center_lat = var_info.attrs["GRIB_latitudeOfLastGridPointInDegrees"] - grid_info['o_lat_p'] = -center_lat if center_lat < 0 else center_lat + grid_info["lon_0"] = - 106. + #center_lat = var_info.attrs["GRIB_latitudeOfLastGridPointInDegrees"] + center_lat = 54 + grid_info['o_lat_p'] = 90 - center_lat #grid_info['o_lat_p'] = -center_lat if center_lat < 0 else 90 - center_lat grid_info['o_lon_p'] = 180 + #grid_info["corners"] = [-10.590603, 46.591938, -139.08585, 22.66102] case _: msg = f"Can't define grid for {grid_def}" raise ValueError(msg) if self.model != "hrrrhi": - grid_info["corners"] = self.corners + if not grid_info.get("corners"): + grid_info["corners"] = self.corners # if self.grid_suffix in ['GLC0']: # attrs = ['Latin1', 'Latin2', 'Lov'] # elif self.grid_suffix == 'GST0': @@ -632,7 +647,7 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> if transforms and do_transform: vals = self.get_transform(transforms, vals) - return vals + return vals if "global" not in self.model else vals[::-1, :] class ProfileData(UPPData): diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index ad071c1..a1e5450 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -112,6 +112,10 @@ shortName: refd typeOfLevel: heightAboveGround level: 1000 + global: + shortName: refd + typeOfLevel: heightAboveGround + level: 1000 rrfs: shortName: rare typeOfLevel: heightAboveGround @@ -299,6 +303,10 @@ acsnw: # Run Total Accumulated Snow Using 10:1 Ratio unit: in aodbc: # Black Carbon AOD sfc: &aod + cfgrib: + parameterNumber: 102 + typeOfLevel: atmosphere + constituentType: 62009 clevs: [.1, .16, .23, .29, .36, .42, .49, .55, .61, .68, .74, .81, .87, 1] cmap: jet colors: aod_colors @@ -309,35 +317,64 @@ aodbc: # Black Carbon AOD aodfd: # Fine Dust AOD sfc: <<: *aod + cfgrib: + parameterNumber: 102 + typeOfLevel: atmosphere + constituentType: 62001 ncl_name: AOTK_P48_L10_{grid}_A62001 title: Fine Dust Aerosol Optical Depth aodhg: # OPT2 AOD sfc: <<: *aod + cfgrib: + parameterNumber: 102 + typeOfLevel: atmosphere + constituentType: 62007 ncl_name: AOTK_P48_L10_{grid}_A62007 title: Total OPT2 Aerosol Optical Depth aodoc: # Organic Carbon AOD sfc: <<: *aod + cfgrib: + parameterNumber: 102 + typeOfLevel: atmosphere + constituentType: 62010 ncl_name: AOTK_P48_L10_{grid}_A62010 title: Organic Carbon Aerosol Optical Depth aodss: # Sea Salt AOD sfc: <<: *aod + cfgrib: + parameterNumber: 102 + typeOfLevel: atmosphere + constituentType: 62008 ncl_name: AOTK_P48_L10_{grid}_A62008 title: Sea Salt Aerosol Optical Depth aodsulf: # Sulfate AOD sfc: <<: *aod + cfgrib: + parameterNumber: 102 + typeOfLevel: atmosphere + constituentType: 62006 ncl_name: AOTK_P48_L10_{grid}_A62006 title: Sulfate Aerosol Optical Depth aodtot: # Total AOD sfc: <<: *aod + cfgrib: + parameterNumber: 102 + typeOfLevel: atmosphere + constituentType: 62000 ncl_name: AOTK_P48_L10_{grid}_A62000 title: Total Aerosol Optical Depth bc: # Black Carbon sfc: &bcsfc + cfgrib: + shortName: pmtf + typeOfLevel: hybrid + constituentType: 62014 + level: 1 clevs: [0.05, 0.1, 0.2, 0.5, 1, 2, 3, 4, 5, 7, 10, 15, 20, 30] cmap: jet colors: aod_colors @@ -354,12 +391,22 @@ bc: # Black Carbon bc1: # Black Carbon 1 sfc: <<: *bcsfc + cfgrib: + shortName: pmtf + typeOfLevel: hybrid + constituentType: 62014 + level: 1 ncl_name: PMTF_P48_L105_{grid}_A62014 title: Surface Black Carbon 1 transform: [] bc2: # Black Carbon 2 sfc: <<: *bcsfc + cfgrib: + shortName: pmtf + typeOfLevel: hybrid + constituentType: 62013 + level: 1 ncl_name: PMTF_P48_L105_{grid}_A62013 title: Surface Black Carbon 2 transform: [] @@ -672,6 +719,10 @@ coarsedust: # Coarse dust colbc: # Column Black Carbon sfc: &col # clevs: [0.00000005, 0.0000001, 0.00000015, 0.0000002, 0.0000005, 0.000001, 0.000002, 0.000003, 0.000004, 0.000005, 0.0000075, 0.00001, 0.000015, 0.00002] + cfgrib: + parameterNumber: 1 + typeOfLevel: atmosphere + constituentType: 62009 clevs: [0.05, 0.1, 0.15, 0.2, 0.5, 1, 2, 3, 4, 5, 7, 10, 15, 20] cmap: jet colors: aod_colors @@ -684,36 +735,62 @@ colbc: # Column Black Carbon colfd: # Column Fine Dust sfc: <<: *col + cfgrib: + parameterNumber: 1 + typeOfLevel: atmosphere + constituentType: 62001 clevs: [2.5, 5, 7.5, 10, 25, 50, 100, 150, 200, 250, 375, 500, 750, 1000] ncl_name: COLMD_P48_L10_{grid}_A62001 title: Column Fine Dust coloc: # Column Organic Carbon sfc: <<: *col + cfgrib: + parameterNumber: 1 + typeOfLevel: atmosphere + constituentType: 62010 clevs: [0.5, 1, 5, 10, 20, 30, 40, 50, 80, 100, 120, 150, 200, 250] ncl_name: COLMD_P48_L10_{grid}_A62010 title: Column Organic Carbon colpm10: # Column PM10 sfc: <<: *col + cfgrib: + parameterNumber: 1 + typeOfLevel: atmosphere + constituentType: 62000 + scaledValueOfFirstSize: 10 clevs: [4, 5, 10, 20, 30, 40, 50, 100, 200, 300, 400, 500, 1000, 2000] ncl_name: COLMD_P48_L10_{grid}_A62000 title: Column PM10 colpm25: # Column PM25 sfc: <<: *col + cfgrib: + parameterNumber: 1 + typeOfLevel: atmosphere + constituentType: 62000 + scaledValueOfFirstSize: 25 clevs: [0.25, 0.5, 0.75, 1, 2.5, 5, 10, 15, 20, 25, 30, 50, 75, 100] ncl_name: COLMD_P48_L10_{grid}_A62000_1 title: Column PM25 colss: # Column Sea Salt sfc: <<: *col + cfgrib: + parameterNumber: 1 + typeOfLevel: atmosphere + constituentType: 62008 clevs: [0.1, 0.6, 3, 6, 10, 13, 16, 20, 23, 26, 30, 33, 36, 100] ncl_name: COLMD_P48_L10_{grid}_A62008 title: Column Sea Salt colsu: # Column Sulfate sfc: <<: *col + cfgrib: + parameterNumber: 1 + typeOfLevel: atmosphere + constituentType: 62006 clevs: [2, 3, 4, 5, 7.5, 10, 15, 20, 30, 40, 50, 100, 200, 300] ncl_name: COLMD_P48_L10_{grid}_A62006 title: Column Sulfate @@ -759,6 +836,10 @@ cref: # Composite reflectivity shortName: refc typeOfLevel: atmosphere level: 0 + global: + shortName: refc + typeOfLevel: atmosphereSingleLayer + level: 0 rrfs: parameterNumber: 5 parameterCategory: 16 @@ -838,7 +919,7 @@ dlwrfavg: # Downward Longwave Radiation Flux Average sfc: <<: *radiation_flux cfgrib: - shortName: avg_sdlwrf + shortName: sdlwrf typeOfLevel: surface clevs: !arange [200, 501, 12] cmap: gist_ncar @@ -870,7 +951,7 @@ dswrfavg: # Downward Shortwave Radiation Flux Average sfc: <<: *radiation_flux cfgrib: - shortName: avg_sdswrf + shortName: sdswrf typeOfLevel: surface clevs: [0, 50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] colors: rainbow12_colors @@ -879,6 +960,11 @@ dswrfavg: # Downward Shortwave Radiation Flux Average title: Downward SW Radiation Flux 6h Avg, Surface fd: # Fine dust, global chem sfc: + cfgrib: + shortName: pmtf + typeOfLevel: hybrid + level: 1 + constituentType: 62001 clevs: [1, 2, 4, 6, 8, 12, 16, 20, 25, 30, 40, 60, 100, 200] cmap: jet colors: aod_colors @@ -1127,12 +1213,15 @@ ghtfl: # Ground Heat Flux grle: # Graupel ua: cfgrib: - prs: - shortName: grle - typeOfLevel: isobaricInhPa + shortName: grle + typeOfLevel: isobaricInhPa ncl_name: nat: GRLE_P0_L105_{grid} prs: GRLE_P0_L100_{grid} + uanat: + cfgrib: + shortName: grle + typeOfLevel: hybrid gust: 10m: cfgrib: @@ -1360,10 +1449,18 @@ hlcytot: title: Run Total 2-5km Max Updraft Helicity transform: run_max hpbl: # Height of Planetary Boundary Layer - sfc: + sfc: cfgrib: - shortName: blh - typeOfLevel: surface + hrrr: + shortName: blh + typeOfLevel: surface + global: + parameterNumber: 196 + parameterCategory: 3 + typeOfLevel: surface + rap: + shortName: blh + typeOfLevel: surface clevs: [25, 50, 100, 200, 300, 500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000] cmap: ir_rgbv colors: pbl_colors @@ -1374,14 +1471,17 @@ hpbl: # Height of Planetary Boundary Layer icmr: # Ice Water Mixing Ratio ua: cfgrib: - prs: - shortName: icmr - typeOfLevel: isobaricInhPa + shortName: icmr + typeOfLevel: isobaricInhPa ncl_name: nat: - ICMR_P0_L105_{grid} - CIMIXR_P0_L105_{grid} prs: ICMR_P0_L100_{grid} + uanat: + cfgrib: + parameterNumber: 82 + typeOfLevel: hybrid icprb: # Icing Probability # levels chosen are arbitrary based on initial plot samples # additional levels may be requested in the future. @@ -1596,6 +1696,11 @@ mref: # Maximum reflectivity for past hour at 1 km AGL title: Max 1km agl Reflectivity (over prev hr) oc: # Organic Carbon sfc: &ocsfc + cfgrib: + shortName: pmtf + typeOfLevel: hybrid + level: 1 + constituentType: 62014 clevs: [0.05, 0.1, 0.5, 1, 2, 3, 5, 7, 10, 15, 20, 30, 50, 100] cmap: jet colors: aod_colors @@ -1612,17 +1717,32 @@ oc: # Organic Carbon oc1: # Organic Carbon 1 sfc: <<: *ocsfc + cfgrib: + shortName: pmtf + typeOfLevel: hybrid + level: 1 + constituentType: 62016 ncl_name: PMTF_P48_L105_{grid}_A62016 title: Surface Organic Carbon 1 transform: [] oc2: # Organic Carbon 2 sfc: <<: *ocsfc + cfgrib: + shortName: pmtf + typeOfLevel: hybrid + level: 1 + constituentType: 62015 ncl_name: PMTF_P48_L105_{grid}_A62015 title: Surface Organic Carbon 2 transform: [] PM10: # PM10, global chem sfc: &pmsfc + cfgrib: + shortName: pmtc + typeOfLevel: surface + constituentType: 62000 + scaledValueOfFirstSize: 10 clevs: [0.5, 1, 2, 5, 10, 20, 30, 50, 70, 100, 150, 200, 500, 1000] cmap: jet colors: aod_colors @@ -1634,6 +1754,11 @@ PM10: # PM10, global chem PM25: # PM25, global chem sfc: <<: *pmsfc + cfgrib: + shortName: pmtf + typeOfLevel: surface + constituentType: 62000 + scaledValueOfFirstSize: 25 ncl_name: PMTF_P48_L1_{grid}_A62000 title: PM25 pres: @@ -1656,10 +1781,15 @@ pres: shortName: mslma typeOfLevel: meanSea level: 0 + global: + shortName: prmsl + typeOfLevel: meanSea + level: 0 rrfs: shortName: mslet typeOfLevel: meanSea level: 0 + parameterNumber: 192 clevs: !arange [976, 1051, 4] cmap: Spectral_r colors: pmsl_colors @@ -1970,6 +2100,10 @@ rwmr: # Rain Mixing Ratio ncl_name: RWMR_P0_L100_{grid} seasalt: # Fine dust, global chem sfc: + cfgrib: + shortName: pmtf + typeOfLevel: surface + constituentType: 62008 clevs: [0.05, 0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 30, 40, 50, 100] cmap: jet colors: aod_colors @@ -2096,12 +2230,15 @@ slw: # Supercooled Liquid Water -- requires nat data snmr: # Snow Mixing Ratio ua: cfgrib: - prs: - shortName: snmr - typeOfLevel: isobaricInhPa + shortName: snmr + typeOfLevel: isobaricInhPa ncl_name: nat: SNMR_P0_L105_{grid} prs: SNMR_P0_L100_{grid} + uanat: + cfgrib: + shortName: snmr + typeOfLevel: hybrid snod: # Snow Depth sfc: <<: *snow @@ -2162,10 +2299,21 @@ soilt: # Soil Temperature 10cm: <<: *soilt_levs cfgrib: - shortName: st - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 10 - scaledValueOfSecondFixedSurface: 10 + hrrr: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 10 + scaledValueOfSecondFixedSurface: 10 + global: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 0 + scaledValueOfSecondFixedSurface: 10 + rrfs: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 10 + scaledValueOfSecondFixedSurface: 10 title: Soil Temperature at 10cm 30cm: <<: *soilt_levs @@ -2189,10 +2337,21 @@ soilt: # Soil Temperature 1m: <<: *soilt_levs cfgrib: - shortName: st - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 100 - scaledValueOfSecondFixedSurface: 100 + hrrr: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 100 + scaledValueOfSecondFixedSurface: 100 + global: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 40 + scaledValueOfSecondFixedSurface: 100 + rrfs: + shortName: st + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 100 + scaledValueOfSecondFixedSurface: 100 title: Soil Temperature at 1m 1.6m: <<: *soilt_levs @@ -2243,10 +2402,16 @@ soilw: # Soil Moisture 10cm: <<: *soilw_levs cfgrib: - shortName: soilw - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 10 - scaledValueOfSecondFixedSurface: 10 + hrrr: &soilw_cfgrib_10 + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 10 + scaledValueOfSecondFixedSurface: 10 + global: + <<: *soilw_cfgrib_10 + scaledValueOfFirstFixedSurface: 0 + rrfs: + <<: *soilw_cfgrib_10 title: Soil Moisture at 10cm 30cm: <<: *soilw_levs @@ -2275,10 +2440,16 @@ soilw: # Soil Moisture 1m: <<: *soilw_levs cfgrib: - shortName: soilw - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 100 - scaledValueOfSecondFixedSurface: 100 + hrrr: &soilw_cfgrib_100 + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 100 + scaledValueOfSecondFixedSurface: 100 + global: + <<: *soilw_cfgrib_100 + scaledValueOfFirstFixedSurface: 40 + rrfs: + <<: *soilw_cfgrib_100 title: Soil Moisture at 1m 1.6m: <<: *soilw_levs @@ -2342,6 +2513,11 @@ ssrun: # Storm Surface Runoff unit: in sulf: # Sulfate, global chem sfc: + cfgrib: + shortName: pmtf + typeOfLevel: hybrid + level: 1 + constituentType: 62006 clevs: [0.05, 0.1, 0.2, 0.5, 1, 2, 3, 5, 10, 15, 20, 30 ,40 ,50] cmap: jet colors: aod_colors @@ -2852,6 +3028,9 @@ vort: # Absolute vorticity vvel: # Vertical velocity 700mb: cfgrib: + global: + shortName: w + typeOfLevel: isobaricInhPa hrrr: shortName: w typeOfLevel: isobaricInhPa diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 4d336f1..c3af76f 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -228,8 +228,7 @@ def parallel_skewt(cla: Namespace, fhr: int, grib_path: Path, site: str, workdir site the string representation of the site from the sites file workdir output directory """ - cf = cfgrib_spec(cla.specs["temp"]["ua"]["cfgrib"], cla.images[0]) - ds = gribfile.GribFile(grib_path, cf).contents + ds = gribfile.GribFile(grib_path, cla.specs["temp"]["ua"]["cfgrib"]).contents skew = skewt.SkewTDiagram( ds=ds, fhr=fhr, diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index cb1773a..3d81eb0 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -21,7 +21,7 @@ from metpy.plots import Hodograph, SkewT from metpy.units import units from mpl_toolkits.axes_grid1.inset_locator import inset_axes -from xarray import DataArray, Dataset +from xarray import DataArray, Dataset, where from adb_graphics import errors from adb_graphics.datahandler import gribdata @@ -144,11 +144,11 @@ def _add_hydrometeors(self, hydro_subplot: Axes): mixr_total = mixr_total + pres_layer / gravity * profile[n] # limit values to upper and lower values of plotting range - profile = DataArray.where((profile > 0.0) & (profile < 1.0e-4), 1.0e-4, profile) # noqa: PLR2004 - profile = DataArray.where((profile > 10.0), 10.0, profile) # noqa: PLR2004 + profile = where((profile > 0.0) & (profile < 1.0e-4), 1.0e-4, profile) # noqa: PLR2004 + profile = where((profile > 10.0), 10.0, profile) # noqa: PLR2004 # plot line - profile = profile[: pres.shape[0]] + profile = profile[:pres.shape[0]] hydro_subplot.plot( profile, pres, @@ -187,10 +187,10 @@ def _add_hydrometeors(self, hydro_subplot: Axes): layer = False # compute vertically integrated amount and add legend line - line = f"{settings.get('label'):<7s} {mixr_total:>10.3f} {settings.get('units')}" + line = f"{settings.get('label'):<7s} {mixr_total.values:>10.3f} {settings.get('units')}" if scale != 1.0: line = ( - f"{settings.get('label'):<5s}(x{scale}) {mixr_total.magnitude:.3f} " + f"{settings.get('label'):<5s}(x{scale}) {mixr_total.values:.3f} " f"{settings.get('units')}" ) lines.append(line) @@ -217,7 +217,7 @@ def _add_hydrometeors(self, hydro_subplot: Axes): # Draw the vertically integrated amounts box hydro_subplot.text( 0.02, - 0.95, + 0.98, contents, bbox=dict(facecolor="white", edgecolor="black", alpha=0.7), fontproperties=fm.FontProperties(family="monospace"), @@ -297,21 +297,13 @@ def atmo_profiles(self): }, } - top = None for var, items in atmo_vars.items(): # Get the profile values and attach MetPy units tmp = np.asarray(self.values(name=var)) * items["units"] # Apply any needed transdecimals transform = items.get("transform") - if transform: - tmp = tmp.to(transform) - - # Only return values up to the maximum pressure level requested - if var == "pres" and top is None: - top = np.where(tmp.magnitude >= self.max_plev)[0][-1] - - atmo_vars[var]["data"] = tmp[:top] + atmo_vars[var]["data"] = tmp.to(transform) if transform else tmp return atmo_vars @@ -670,7 +662,8 @@ def _title(self): f"{self.model_name}: {atime}\nFcst Hr: {self.fhr}", fontsize=16, loc="left", - position=(-4.8, 1.03), + x=-4.8, + y=1.03, ) # Top Right @@ -678,7 +671,8 @@ def _title(self): f"Valid: {vtime}", fontsize=16, loc="right", - position=(-0.20, 1.03), + x=-0.20, + y=1.03, ) # Center @@ -689,7 +683,8 @@ def _title(self): site_title, fontsize=12, loc="center", - position=(-2.5, 1.0), + x=-2.5, + y=1.0, ) diff --git a/create_graphics.py b/create_graphics.py index 1753599..35d23fa 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -21,7 +21,6 @@ from pathlib import Path import yaml -from uwtools.api.config import get_yaml_config from adb_graphics import errors, utils from adb_graphics.figure_builders import parallel_maps, parallel_skewt @@ -69,10 +68,10 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): Generate arguments for parallel processing of Skew T graphics, and generate a pool of workers to complete the tasks. """ - vspec = utils.cfgrib_spec(cla.specs["temp"]["ua"], cla.images[0]) - args = [(cla, fhr, grib_path, site, vspec, workdir) for site in cla.sites] + args = [(cla, fhr, grib_path, site, workdir) for site in cla.sites] print(f"Queueing {len(args)} Skew Ts") + #parallel_skewt(*args[0]) with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_skewt, args) @@ -118,10 +117,10 @@ def create_maps( ) ) - parallel_maps(*args[-1]) - #print(f"Queueing {len(args)} maps") - #with Pool(processes=cla.nprocs) as pool: - # pool.starmap(parallel_maps, args) + # parallel_maps(*args[-1]) + print(f"Queueing {len(args)} maps") + with Pool(processes=cla.nprocs) as pool: + pool.starmap(parallel_maps, args) def generate_tile_list(arg_list: list) -> list[str]: @@ -621,7 +620,7 @@ def graphics_driver(cla: Namespace): "Graphics will be created for input files\n", f"Output graphics directory: {workdir} \n{LOG_BREAK}", ) - full_spec = get_yaml_config(orig_spec) + full_spec = utils.load_yaml(orig_spec) full_spec.dereference(context={"fhr": int(fhr)}) cla.specs = full_spec if cla.graphic_type == "skewts": diff --git a/image_lists/global.yml b/image_lists/global.yml index c99d9b9..8105b8c 100644 --- a/image_lists/global.yml +++ b/image_lists/global.yml @@ -18,19 +18,14 @@ hourly: - high - low - mid - - total cpofp: - sfc cref: - sfc dewp: - 2m - dlwrf: - - sfc dlwrfavg: - sfc - dswrf: - - sfc dswrfavg: - sfc gh: @@ -42,48 +37,14 @@ hourly: - sr03 hpbl: - sfc - lhtfl: - - sfc - lhtflavg: - - sfc - pres: - - msl pwtr: - sfc - rh: - - 2m - - 850mb - - mean - shtfl: - - sfc - shtflavg: - - sfc snod: - sfc soilt: &soilt_levs - 10cm - 1m soilw: *soilt_levs - temp: - - 2ds - - 2m - - 500mb - - 700mb - - 850mb - - 925mb - - sfc - totp6h: - - sfc - ulwrf: - - sfc - ulwrfavg: - - sfc - - top - uswrf: - - sfc - uswrfavg: - - sfc - - top vis: - sfc vort: @@ -92,11 +53,3 @@ hourly: - 700mb weasd: - sfc - wspeed: - - 10m - - 10mb - - 20mb - - 250mb - - 5mb - - 80m - - 850mb diff --git a/image_lists/global_chem.yml b/image_lists/global_chem.yml index 0900933..96d5bc3 100644 --- a/image_lists/global_chem.yml +++ b/image_lists/global_chem.yml @@ -7,8 +7,6 @@ hourly: - sfc aodfd: - sfc - aodhg: - - sfc aodoc: - sfc aodss: @@ -23,20 +21,6 @@ hourly: - sfc bc2: - sfc - cape: - - mu - - mul - - mx90mb - - sfc - ceil: - - ua - cin: - - sfc - cloudcover: - - high - - low - - mid - - total colbc: - sfc colfd: @@ -51,29 +35,10 @@ hourly: - sfc colsu: - sfc - colto: - - sfc - cpofp: - - sfc cref: - sfc - ctop: - - ua - dewp: - - 2m fd: - sfc - flru: - - sfc - gust: - - 10m - hlcy: - - sr01 - - sr03 - hpbl: - - sfc - lhtfl: - - sfc oc: - sfc oc1: @@ -86,52 +51,19 @@ hourly: - sfc pres: - msl - ptyp: - - sfc - pwtr: - - sfc rh: - 850mb seasalt: - sfc - shtfl: - - sfc - snod: - - sfc - soilt: &soilt_levs - - 10cm - - 1m - soilw: *soilt_levs sulf: - sfc temp: - - 2ds - - 2m - - 500mb - 700mb - 850mb - 925mb - sfc totp6h: - sfc - totp: - - sfc - ulwrf: - - sfc - - top - uswrf: - - sfc - - top - vis: - - sfc - vort: - - 500mb - vvel: - - 700mb - weasd: - - sfc wspeed: - 10m - - 80m - - 250mb - 850mb diff --git a/image_lists/hrrr_subset.yml b/image_lists/hrrr_subset.yml index 840c476..1530944 100644 --- a/image_lists/hrrr_subset.yml +++ b/image_lists/hrrr_subset.yml @@ -67,13 +67,13 @@ hourly: - mx25 - sr01 - sr03 -# hlcytot: -# - mn02 -# - mn03 -# - mn25 -# - mx02 -# - mx03 -# - mx25 + hlcytot: + - mn02 + - mn03 + - mn25 + - mx02 + - mx03 + - mx25 hpbl: - sfc lcl: diff --git a/image_lists/rap_subset.yml b/image_lists/rap_subset.yml index 261b8fe..17b178b 100644 --- a/image_lists/rap_subset.yml +++ b/image_lists/rap_subset.yml @@ -1,18 +1,18 @@ hourly: model: rap variables: - 1hsnw: - - sfc - acsnod: - - sfc - acsnw: - - sfc +# 1hsnw: +# - sfc +# acsnod: +# - sfc acfrozr: - sfc acfrzr: - sfc acpcp: - sfc + acsnw: + - sfc cape: - mu - mul From eecbeabce2c943656a134efa0e018855e4d3dfae Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 17 Nov 2025 11:46:21 -0700 Subject: [PATCH 40/98] Passing tests. --- Makefile | 2 +- adb_graphics/datahandler/gribdata.py | 49 ++++++++++++++-------------- adb_graphics/datahandler/gribfile.py | 11 ++++--- adb_graphics/figure_builders.py | 3 +- adb_graphics/figures/skewt.py | 15 +++------ create_graphics.py | 7 ++-- tests/datahandler/test_gribdata.py | 25 +++++++------- tests/test_figure_builders.py | 4 +-- 8 files changed, 58 insertions(+), 58 deletions(-) diff --git a/Makefile b/Makefile index 3150e00..e740fd9 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ typecheck: mypy --install-types --non-interactive . unittest: - pytest --cov -k "not hrrr" -n 4 . + pytest --cov -k "not hrrr" -n 8 . memtest: pytest --memray -k"not hrrr" . diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 4211ee3..a0f1757 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -37,7 +37,7 @@ class UPPData(specs.VarSpec): def __init__(self, ds: Dataset, short_name: str, spec: dict, **kwargs): # Parse kwargs first self.model = kwargs.get("model", "") - self.grib_path = Path(kwargs.get("grib_path", "")) + self.grib_path = Path(kwargs.get("grib_paths", [kwargs.get("grib_path")])[0]) self.spec = spec self.short_name = short_name @@ -453,34 +453,34 @@ def grid_info(self) -> dict: grid_info["projection"] = "lcc" grid_info["lat_0"] = 39.0 case x if "polar stereographic" in x: - attrs = ['GRIB_orientationOfTheGridInDegrees'] - grid_info['projection'] = 'stere' - grid_info['lat_0'] = 90 - case x if x == "rotated latitude/longitude": # RRFS NA + attrs = ["GRIB_orientationOfTheGridInDegrees"] + grid_info["projection"] = "stere" + grid_info["lat_0"] = 90 + case x if x == "rotated latitude/longitude": # RRFS NA attrs = [] - grid_info['projection'] = 'rotpole' - lon_0 = var_info.attrs.get('GRIB_longitudeOfSouthernPoleInDegrees') - grid_info['lon_0'] = lon_0 - 360 - center_lat = var_info.attrs.get('GRIB_latitudeOfSouthernPoleInDegrees') - grid_info['o_lat_p'] = - center_lat if center_lat < 0 else 90 - center_lat - grid_info['o_lon_p'] = 180 - - case x if "equidistant cylindrical" in x: # GFS + grid_info["projection"] = "rotpole" + lon_0: float = var_info.attrs["GRIB_longitudeOfSouthernPoleInDegrees"] + grid_info["lon_0"] = lon_0 - 360 + center_lat: float = var_info.attrs["GRIB_latitudeOfSouthernPoleInDegrees"] + grid_info["o_lat_p"] = -center_lat if center_lat < 0 else 90 - center_lat + grid_info["o_lon_p"] = 180 + + case x if "equidistant cylindrical" in x: # GFS attrs = [] grid_info["projection"] = "cyl" - case x if "rotate" in x: # RAP + case x if "rotate" in x: # RAP attrs = [] - grid_info['projection'] = 'rotpole' - #center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] - #center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] - #grid_info["lon_0"] = center_lon - 360 - grid_info["lon_0"] = - 106. - #center_lat = var_info.attrs["GRIB_latitudeOfLastGridPointInDegrees"] + grid_info["projection"] = "rotpole" + # center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] + # center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] + # grid_info["lon_0"] = center_lon - 360 + grid_info["lon_0"] = -106.0 + # center_lat = var_info.attrs["GRIB_latitudeOfLastGridPointInDegrees"] center_lat = 54 - grid_info['o_lat_p'] = 90 - center_lat - #grid_info['o_lat_p'] = -center_lat if center_lat < 0 else 90 - center_lat - grid_info['o_lon_p'] = 180 - #grid_info["corners"] = [-10.590603, 46.591938, -139.08585, 22.66102] + grid_info["o_lat_p"] = 90 - center_lat + # grid_info['o_lat_p'] = -center_lat if center_lat < 0 else 90 - center_lat + grid_info["o_lon_p"] = 180 + # grid_info["corners"] = [-10.590603, 46.591938, -139.08585, 22.66102] case _: msg = f"Can't define grid for {grid_def}" raise ValueError(msg) @@ -504,7 +504,6 @@ def grid_info(self) -> dict: # center_lat = lat.attrs.get('CenterLat', lat.attrs.get('Latitude_of_southern_pole')) # grid_info['o_lat_p'] = - center_lat[0] if center_lat[0] < 0 else 90 - center_lat[0] - for attr in attrs: bm_arg = keys_to_basemap[attr] val = var_info.attrs[attr] diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index ae3576e..dd0c9d2 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -32,7 +32,7 @@ def _load(self) -> xr.Dataset: backend_kwargs=( { "filter_by_keys": self.cfgrib_config, - "read_keys": ['orientationOfTheGridInDegrees'], + "read_keys": ["orientationOfTheGridInDegrees"], } ), ) @@ -78,7 +78,10 @@ def _load(self, filenames: list[Path] | None = None): engine="cfgrib", concat_dim="time", combine="nested", - backend_kwargs=({"filter_by_keys": self.cfgrib_config, - "read_keys": ['orientationOfTheGridInDegrees'], - }) + backend_kwargs=( + { + "filter_by_keys": self.cfgrib_config, + "read_keys": ["orientationOfTheGridInDegrees"], + } + ), ) diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index c3af76f..ef09013 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -15,7 +15,6 @@ from adb_graphics.datahandler import gribfile from adb_graphics.figures import skewt from adb_graphics.figures.maps import DataMap, DiffMap, Map, MapFields, MultiPanelDataMap -from adb_graphics.utils import cfgrib_spec AIRPORTS = Path("static/Airports_locs.txt") @@ -38,7 +37,7 @@ def add_obs_panel( map_fields = MapFields( fhr=0, fields_spec=spec, - grib_path=obs_file, + grib_paths=[obs_file], level="obs", model="obs", name=short_name, diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 3d81eb0..e63e943 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -148,7 +148,7 @@ def _add_hydrometeors(self, hydro_subplot: Axes): profile = where((profile > 10.0), 10.0, profile) # noqa: PLR2004 # plot line - profile = profile[:pres.shape[0]] + profile = profile[: pres.shape[0]] hydro_subplot.plot( profile, pres, @@ -187,24 +187,19 @@ def _add_hydrometeors(self, hydro_subplot: Axes): layer = False # compute vertically integrated amount and add legend line - line = f"{settings.get('label'):<7s} {mixr_total.values:>10.3f} {settings.get('units')}" + label = settings.get("label") + line = f"{label:<7s} {mixr_total.to_numpy():>10.3f} {settings.get('units')}" if scale != 1.0: - line = ( - f"{settings.get('label'):<5s}(x{scale}) {mixr_total.values:.3f} " - f"{settings.get('units')}" - ) + line = f"{label:<5s}(x{scale}) {mixr_total.to_numpy():.3f} {settings.get('units')}" lines.append(line) - label = f"{settings.get('label'):<7s}" - if scale != 1.0: - label = f"{settings.get('label'):<5s}(x{scale})" handles.append( mlines.Line2D( [], [], color=settings.get("color"), fillstyle="none", - label=label, + label=f"{label:<5s}(x{scale})" if scale != 1.0 else f"{label:<7s}", linewidth=1.0, marker=settings.get("marker"), markersize=8, diff --git a/create_graphics.py b/create_graphics.py index 35d23fa..fbac644 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -71,7 +71,7 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): args = [(cla, fhr, grib_path, site, workdir) for site in cla.sites] print(f"Queueing {len(args)} Skew Ts") - #parallel_skewt(*args[0]) + # parallel_skewt(*args[0]) with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_skewt, args) @@ -100,7 +100,9 @@ def create_maps( raise errors.NoGraphicsDefinitionForVariableError(msg) accumulate = spec.get("accumulate", False) vspec = utils.cfgrib_spec(spec["cfgrib"], cla.images[0]) - grib_acc = vspec.get("stepRange", "").startswith("-1") or vspec.get("stepRange") == "0-0" + grib_acc = ( + vspec.get("stepRange", "").startswith("-1") or vspec.get("stepRange") == "0-0" + ) if (accumulate or grib_acc) and fhr == 0: continue @@ -558,7 +560,6 @@ def graphics_driver(cla: Namespace): if old_enough: grib_paths.append(grib_path) - orig_spec = copy.deepcopy(cla.specs) # Allow this task to run concurrently with UPP by continuing to check for diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index d8eb419..ddfd2e8 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -29,7 +29,7 @@ def hrrr_data(prsfile): def fielddata_obj(hrrr_data, prsfile, spec): return gribdata.FieldData( ds=hrrr_data, - fhr=15, + fhr=16, grib_path=prsfile, level="sfc", short_name="temp", @@ -48,7 +48,7 @@ def profiledata_obj(natfile, spec): ) return gribdata.ProfileData( ds=ds.contents, - fhr=15, + fhr=16, grib_path=natfile, loc=" DNR 23062 72469 39.77 104.88 1611 Denver, CO", short_name="temp", @@ -58,7 +58,9 @@ def profiledata_obj(natfile, spec): @fixture def spec(spec_file): - return utils.load_yaml(spec_file) + spec = utils.load_yaml(spec_file) + spec.dereference(context={"fhr": 16}) + return spec @fixture @@ -67,7 +69,7 @@ def uppdata_obj(hrrr_data, prsfile, spec): ds=hrrr_data, short_name="temp", spec=spec, - fhr=15, + fhr=16, grib_path=prsfile, ) @@ -85,7 +87,8 @@ def uppdata_multilev_obj(prsfile, spec): ds=ds, short_name="temp", spec=spec, - fhr=15, + fhr=16, + grib_path=prsfile, ) @@ -218,7 +221,7 @@ def test_uppdata_opposite(factor, uppdata_obj): def test_uppdata_valid_dt(uppdata_obj): - assert uppdata_obj.valid_dt == datetime(2025, 10, 6, 15) + assert uppdata_obj.valid_dt == datetime(2025, 10, 6, 16) def test_uppdata_vector_magnitude(prsfile, spec): @@ -232,7 +235,7 @@ def test_uppdata_vector_magnitude(prsfile, spec): ) fd = ConcreteUPPData( ds=ds.contents, - fhr=15, + fhr=16, grib_path=prsfile, level="250mb", short_name="u", @@ -283,7 +286,7 @@ def test_fielddata_aviation_flight_rules(prsfile, spec): ) fd = gribdata.FieldData( ds=ds.contents, - fhr=15, + fhr=16, grib_path=prsfile, level="sfc", short_name="flru", @@ -356,7 +359,7 @@ def test_fielddata_fire_weather_index(prsfile, spec): ) fd = gribdata.FieldData( ds=ds.contents, - fhr=15, + fhr=16, grib_path=prsfile, level="sfc", short_name="firewxtransform", @@ -390,7 +393,7 @@ def test_fielddata_icing_adjust_trace(prsfile, spec): ) fd = gribdata.FieldData( ds=ds.contents, - fhr=15, + fhr=16, grib_path=prsfile, level="sfc", short_name="flru", @@ -411,7 +414,7 @@ def test_fielddata_supercooled_liquid_water(natfile, spec): ) fd = gribdata.FieldData( ds=ds.contents, - fhr=15, + fhr=16, grib_path=natfile, level="sfc", short_name="slw", diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py index f2755c9..ab1964f 100644 --- a/tests/test_figure_builders.py +++ b/tests/test_figure_builders.py @@ -27,7 +27,7 @@ def fielddata_obj(hrrr_data, prsfile, spec): return gribdata.FieldData( ds=hrrr_data.contents, fhr=15, - grib_path=prsfile, + grib_paths=[prsfile], level="cref", short_name="temp", spec=spec, @@ -49,7 +49,7 @@ def parallel_maps_args(prsfile, spec, tmp_path): return { "cla": cla, "fhr": 15, - "grib_path": prsfile, + "grib_paths": [prsfile], "level": "sfc", "variable": "temp", "workdir": tmp_path, From d9eca5bd13ce9f56db6f7129fe474f41a4b04a00 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 18 Nov 2025 12:23:18 -0700 Subject: [PATCH 41/98] All current tests passing. --- adb_graphics/datahandler/gribdata.py | 159 ++++++++++++----------- adb_graphics/datahandler/gribfile.py | 4 - adb_graphics/figure_builders.py | 11 +- adb_graphics/figures/maps.py | 28 ++--- adb_graphics/figures/skewt.py | 50 ++++---- tests/datahandler/test_gribdata.py | 182 +++++++++++---------------- tests/datahandler/test_gribfile.py | 1 - tests/test_figure_builders.py | 26 ++-- 8 files changed, 215 insertions(+), 246 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index a0f1757..b0e00ee 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -7,16 +7,17 @@ import abc from copy import deepcopy from datetime import datetime, timedelta +from functools import cached_property from pathlib import Path import numpy as np from matplotlib.pyplot import get_cmap from pandas import to_datetime -from xarray import DataArray, Dataset +from uwtools.api.config import YAMLConfig +from xarray import DataArray from adb_graphics import conversions, errors, specs, utils from adb_graphics.datahandler import gribfile -from adb_graphics.utils import cfgrib_spec class UPPData(specs.VarSpec): @@ -26,26 +27,29 @@ class UPPData(specs.VarSpec): Input: ds: xarray dataset from grib file + model: name of the model from the image list short_name: name of variable corresponding to entry in specs configuration - - kwargs: - config: path to a user-specified configuration file - model: string describing the model type - + spec: full specs dictionary """ - def __init__(self, ds: Dataset, short_name: str, spec: dict, **kwargs): - # Parse kwargs first - self.model = kwargs.get("model", "") - self.grib_path = Path(kwargs.get("grib_paths", [kwargs.get("grib_path")])[0]) - + def __init__( + self, + fhr: int, + grib_paths: list[Path], + model: str, + short_name: str, + spec: dict | YAMLConfig, + level: str | None = "ua", + ): + self.grib_paths = grib_paths + self.model = model self.spec = spec self.short_name = short_name - self.level = "ua" - - self.fhr = str(kwargs["fhr"]) + self.level = level - self.ds = ds + self.fhr = fhr + cf = utils.cfgrib_spec(self.vspec["cfgrib"], self.model) + self.ds = gribfile.GribFiles(self.grib_paths, cf).contents.squeeze() @property def anl_dt(self) -> datetime: @@ -75,13 +79,12 @@ def date_to_str(date: datetime) -> str: return date.strftime("%Y%m%d %H UTC") - @property + @cached_property def field(self) -> DataArray: """ Get the first DataArray out of the Dataset. """ - first_variable_name = list(self.ds.data_vars)[0] - return DataArray(self.ds[first_variable_name]) + return self._get_field(self.vspec["cfgrib"].get(self.model, self.vspec["cfgrib"])) def field_column_max(self, **kwargs): # noqa: ARG002 """Returns the column max of the values.""" @@ -127,16 +130,15 @@ def _get_data_levels(self, vertical_dim: str): dim = [str(coord) for coord in self.ds.coords if vertical_dim in str(coord)][0] return self.ds.coords[dim].to_numpy() - def _get_field(self, spec: dict) -> DataArray: + def _get_field(self, cfgribspec: dict) -> DataArray: """ Given a cfgrib block, return the DataArray. Arg: - spec the specifications dictionary to use for the variable in + cfgribspec the specifications dictionary to use for the variable in question """ - - ds = gribfile.GribFile(self.grib_path, spec).contents + ds = gribfile.GribFiles(self.grib_paths, cfgribspec).contents.squeeze() first_variable_name = list(ds.data_vars)[0] return DataArray(ds[first_variable_name]) @@ -226,7 +228,9 @@ def valid_dt(self) -> datetime: return self.anl_dt + fh @abc.abstractmethod - def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: + def values( + self, level: str | None = None, name: str | None = None, do_transform: bool = True + ) -> DataArray: """Returns the values of a given variable.""" def vector_magnitude( @@ -277,12 +281,28 @@ class FieldData(UPPData): """ - def __init__(self, ds: Dataset, level: str, short_name: str, **kwargs): - super().__init__(ds, short_name, **kwargs) - + def __init__( + self, + fhr: int, + grib_paths: list[Path], + level: str, + model: str, + short_name: str, + spec: dict | YAMLConfig, + member: str | None = None, + contour_kwargs: dict | None = None, + ): + super().__init__( + fhr=fhr, + grib_paths=grib_paths, + level=level, + model=model, + short_name=short_name, + spec=spec, + ) self.level = level - self.contour_kwargs = kwargs.get("contour_kwargs", {}) - self.mem = kwargs.get("member") + self.contour_kwargs = {} if contour_kwargs is None else contour_kwargs + self.mem = member def aviation_flight_rules(self, values: DataArray, **kwargs): # noqa: ARG002 """ @@ -369,18 +389,14 @@ def fire_weather_index(self, values: DataArray, **kwargs): # noqa: ARG002 """ def _load_field(level: str, short_name: str): - spec = cfgrib_spec(self.spec[short_name][level]["cfgrib"], self.model) - ds = gribfile.GribFile(self.grib_path, spec).contents - args = { - "ds": ds, - "fhr": self.fhr, - "level": level, - "model": self.model, - "short_name": short_name, - "spec": self.spec, - "grib_path": self.grib_path, - } - return FieldData(**args).values(do_transform=False) + return FieldData( + fhr=int(self.fhr), + grib_paths=self.grib_paths, + level=level, + model=self.model, + short_name=short_name, + spec=self.spec, + ).values(do_transform=False) # Gather fields from the input veg = np.asarray(values) @@ -560,10 +576,10 @@ def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 """ pres_sfc = self.values(name="pres", level="sfc") * 100.0 # convert back to Pa - pres_nat_lev = self.values(name="pres", level="ua", one_lev=False) - temp = self.values(name="temp", level="ua", one_lev=False) - cloud_mixing_ratio = self.values(name="clwmr", level="uanat", one_lev=False) - rain_mixing_ratio = self.values(name="rwmr", level="uanat", one_lev=False) + pres_nat_lev = self.values(name="pres", level="ua") + temp = self.values(name="temp", level="ua") + cloud_mixing_ratio = self.values(name="clwmr", level="uanat") + rain_mixing_ratio = self.values(name="rwmr", level="uanat") gravity = 9.81 slw = pres_sfc * 0.0 # start with array of zero values @@ -609,31 +625,24 @@ def units(self) -> str: return str(self.vspec.get("unit", self.field.units)) - def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: + def values( + self, level: str | None = None, name: str | None = None, do_transform: bool = True + ) -> DataArray: """ Returns the numpy array of values at the requested level for the variable after applying any unit conversion to the original data. Optional Input: - name the name of a field other than defined in self - level the desired level of the named field - - kwargs: - do_transform bool flag. to call, or not, the transform specified - in specs (default: True) - ncl_name the NCL-assigned Grib2 name (default: '') - one_lev bool flag. if True, get the single level of the variable - (default: True) - vertical_index the index (int) of the desired vertical level + level the desired level of the named field + name the name of a field other than defined in self + do_transform apply a standard transformation of units, etc.? """ - level = level or self.level + level = str(level or self.level) vals: DataArray = self.ds.to_dataarray().squeeze() spec = self.vspec - do_transform = kwargs.get("do_transform", True) - if name is not None: # Get the spec dict and ncl_name for the given variable name spec = deepcopy(self.spec.get(name, {}).get(level, {})) @@ -668,8 +677,18 @@ class ProfileData(UPPData): """ - def __init__(self, ds: Dataset, loc: str, short_name: str, **kwargs): - super().__init__(ds, short_name, **kwargs) + def __init__( + self, + fhr: int, + grib_paths: list[Path], + loc: str, + model: str, + short_name: str, + spec: dict | YAMLConfig, + ): + super().__init__( + fhr=fhr, grib_paths=grib_paths, model=model, short_name=short_name, spec=spec + ) # The first 31 columns are space delimted self.site_code, _, self.site_num, lat, lon = loc[:31].split() @@ -688,7 +707,12 @@ def __init__(self, ds: Dataset, loc: str, short_name: str, **kwargs): if self.site_lon < 0: self.site_lon = self.site_lon + 360.0 - def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: # noqa: ARG002 + def values( + self, + level: str | None = None, + name: str | None = None, + do_transform: bool = True, # noqa: ARG002 + ) -> DataArray: """ Returns the numpy array of values at the object's x, y location for the requested variable. @@ -698,13 +722,6 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> level the level of the alternate field to use, default='ua' for upper air - kwargs: - ncl_name the NCL name of the variable to be retrieved - one_lev bool flag. if True, get the single level of the variable - split bool flag. if True, level string numbers are split - into a list, e.g. used to get [0, 6000] from 06km - vertical_index the index of the required level - """ # Set the defaults here since this is an instance of an abstract method @@ -717,9 +734,9 @@ def values(self, level: str | None = None, name: str | None = None, **kwargs) -> if not spec: raise errors.NoGraphicsDefinitionForVariableError(name, level) utils.set_level(level=level, model=self.model, spec=spec) - profile = self._get_field(spec["cfgrib"].get(self.model, spec["cfgrib"])) + profile = self._get_field(spec["cfgrib"].get(self.model, spec["cfgrib"])).squeeze() else: - profile = self.field[::] + profile = self.field.squeeze() # Retrive the location for the profile x, y = self.get_xypoint(self.site_lat, self.site_lon) diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index dd0c9d2..132b60c 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -48,7 +48,6 @@ def __init__( self, filenames: list[Path], cfgrib_config: dict, - **kwargs, ): """ Initialize GribFiles object. @@ -60,12 +59,9 @@ def __init__( forecast lead times ('01fcst'), and all the free forecast hours after that ('free_fcst'). filetype key to use for dict when setting variable_names - - kwargs: model string describing the model type """ - self.model = kwargs.get("model", "") self.filenames = filenames self.cfgrib_config = cfgrib_config self.contents = self._load() diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index ef09013..103d6ea 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -12,7 +12,6 @@ import numpy as np from matplotlib import axes -from adb_graphics.datahandler import gribfile from adb_graphics.figures import skewt from adb_graphics.figures.maps import DataMap, DiffMap, Map, MapFields, MultiPanelDataMap @@ -214,7 +213,7 @@ def parallel_maps( # noqa: PLR0912 gc.collect() -def parallel_skewt(cla: Namespace, fhr: int, grib_path: Path, site: str, workdir: Path): +def parallel_skewt(cla: Namespace, fhr: int, grib_paths: list[Path], site: str, workdir: Path): """ Function that creates a single SkewT plot. @@ -227,16 +226,14 @@ def parallel_skewt(cla: Namespace, fhr: int, grib_path: Path, site: str, workdir site the string representation of the site from the sites file workdir output directory """ - ds = gribfile.GribFile(grib_path, cla.specs["temp"]["ua"]["cfgrib"]).contents skew = skewt.SkewTDiagram( - ds=ds, fhr=fhr, - filetype=cla.file_type, + grib_paths=grib_paths, loc=site, + model=cla.images[0], + spec=cla.specs, max_plev=cla.max_plev, model_name=cla.model_name, - spec=cla.specs, - grib_path=grib_path, ) skew.create_diagram() outfile = f"{skew.site_code}_{skew.site_num}_skewt_f{fhr:03d}.png" diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 480a270..40df7f0 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -20,8 +20,8 @@ from matplotlib.contour import QuadContourSet from mpl_toolkits.basemap import Basemap, shiftgrid -from adb_graphics.datahandler import gribdata, gribfile -from adb_graphics.utils import cfgrib_spec, numeric_level, set_level +from adb_graphics.datahandler import gribdata +from adb_graphics.utils import numeric_level, set_level # FULL_TILES is a list of strings that includes the labels GSL attaches to some of # the wgrib2 cutouts used for larger domains like RAP, RRFS NA, and global. @@ -145,21 +145,17 @@ def __init__( @property def shaded(self): - cf = cfgrib_spec(self.map_spec["cfgrib"], self.model) - ds = gribfile.GribFiles(self.grib_paths, cf).contents args = { - "ds": ds, "fhr": self.fhr, "level": self.level, "model": self.model, "short_name": self.name, "spec": self.fields_spec, - "grib_path": self.grib_paths[-1], + "grib_paths": self.grib_paths, } field = gribdata.FieldData(**args) if self.map_type == "diff": - args["ds"] = gribfile.GribFile(self.grib_path2, cf).contents - args["grib_path"] = self.grib_path2 + args["grib_paths"] = [self.grib_path2] field2 = gribdata.FieldData(**args) field.data = field.values() - field2.values() @@ -193,17 +189,13 @@ def wind_fields(self, level: str | None = None): for var in ("u", "v"): wind_spec = self.fields_spec[var][lev] set_level(lev, self.model, wind_spec) - ds = gribfile.GribFiles( - self.grib_paths, cfgrib_spec(wind_spec["cfgrib"], self.model) - ).contents args = { - "ds": ds, "fhr": self.fhr, "level": lev, "model": self.model, "short_name": var, "spec": self.fields_spec, - "grib_path": self.grib_paths[-1], + "grib_paths": self.grib_paths, } winds.append(gribdata.FieldData(**args)) return winds @@ -221,17 +213,13 @@ def _overlay_fields(self, spec_sect: str) -> list: var, lev = overlay, self.level overlay_spec = deepcopy(self.fields_spec[var][lev]) set_level(lev, self.model, overlay_spec) - ds = gribfile.GribFiles( - self.grib_paths, cfgrib_spec(overlay_spec["cfgrib"], self.model) - ).contents args = { - "ds": ds, "fhr": self.fhr, "level": lev, "model": self.model, "short_name": var, "spec": self.fields_spec, - "grib_path": self.grib_paths[-1], + "grib_paths": self.grib_paths, } overlay_obj = gribdata.FieldData(**args) # Set the attributes for the overlay field @@ -766,8 +754,8 @@ def _title(self): """Draw the title for a map.""" f = self.field - atime = f.date_to_str(f.anl_dt[0]) - vtime = f.date_to_str(f.valid_dt[0]) + atime = f.date_to_str(f.anl_dt) + vtime = f.date_to_str(f.valid_dt) # Analysis time (top) and forecast hour with Valid Time (bottom) on the left plt.title( diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index e63e943..3fe1968 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -21,7 +21,8 @@ from metpy.plots import Hodograph, SkewT from metpy.units import units from mpl_toolkits.axes_grid1.inset_locator import inset_axes -from xarray import DataArray, Dataset, where +from uwtools.api.config import YAMLConfig +from xarray import DataArray, where from adb_graphics import errors from adb_graphics.datahandler import gribdata @@ -58,19 +59,25 @@ class SkewTDiagram(gribdata.ProfileData): be included. """ - def __init__(self, ds: Dataset, loc: str, **kwargs): + def __init__( + self, + fhr: int, + grib_paths: list[Path], + loc: str, + model: str, + spec: dict | YAMLConfig, + max_plev: int | None = 0, + model_name: str | None = "Analysis", + ): # Initialize on the temperature field since we need to gather # field-specific data from this object, e.g. dates, lat, lon, etc. super().__init__( - ds=ds, - loc=loc, - short_name="temp", - **kwargs, + fhr=fhr, grib_paths=grib_paths, loc=loc, model=model, short_name="temp", spec=spec ) - self.max_plev = kwargs.get("max_plev", 0) - self.model_name = kwargs.get("model_name", "Analysis") + self.max_plev = max_plev + self.model_name = model_name def _add_hydrometeors(self, hydro_subplot: Axes): mixing_ratios: dict[str, HydroPlotSettings] = { @@ -392,20 +399,19 @@ def _write_profile(self, csv_path: str | Path): temp = profiles.get("temp").get("data").to("degC") sphum = profiles.get("sphum").get("data") - dewpt = np.array(mpcalc.dewpoint_from_specific_humidity(pres, temp, sphum).to("degC")) - wspd = np.array(mpcalc.wind_speed(u, v)) - wdir = np.array(mpcalc.wind_direction(u, v)) - - pres = np.array(pres) - temp = np.array(temp) + dewpt = mpcalc.dewpoint_from_specific_humidity(pressure=pres, specific_humidity=sphum).to( + "degC" + ) + wspd = mpcalc.wind_speed(u, v) + wdir = mpcalc.wind_direction(u, v) profile = pd.DataFrame( { - "LEVEL": pres, - "TEMP": temp, - "DWPT": dewpt, - "WDIR": wdir, - "WSPD": wspd, + "LEVEL": pres.magnitude, + "TEMP": temp.magnitude, + "DWPT": dewpt.magnitude, + "WDIR": wdir.magnitude, + "WSPD": wspd.magnitude, } ) @@ -417,7 +423,9 @@ def _plot_profile(self, skew: SkewT): temp = profiles.get("temp").get("data") sphum = profiles.get("sphum").get("data") - dewpt = mpcalc.dewpoint_from_specific_humidity(pres, temp, sphum).to("degF") + dewpt = mpcalc.dewpoint_from_specific_humidity(pressure=pres, specific_humidity=sphum).to( + "degF" + ) # Pressure vs temperature skew.plot(pres, temp, "r", linewidth=1.5) @@ -633,7 +641,7 @@ def thermo_variables(self): raise errors.NoGraphicsDefinitionForVariableError(varname, lev) try: - tmp = self.values(level=lev, name=varname, one_lev=True) + tmp = self.values(level=lev, name=varname) transforms = spec.get("transform") if transforms: diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index ddfd2e8..cd82c34 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -6,32 +6,26 @@ from xarray import DataArray, ones_like, zeros_like from adb_graphics import errors, utils -from adb_graphics.datahandler import gribdata, gribfile +from adb_graphics.datahandler import gribdata class ConcreteUPPData(gribdata.UPPData): - def values(self, level: str | None = None, name: str | None = None, **kwargs) -> DataArray: # noqa: ARG002 - return self.ds.to_dataarray().squeeze() + def values( + self, + level: str | None = None, # noqa: ARG002 + name: str | None = None, # noqa: ARG002 + do_transform: bool = True, # noqa: ARG002 + ) -> DataArray: + return self.ds.to_dataarray().squeeze() # type: ignore[no-any-return] @fixture -def hrrr_data(prsfile): - return gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "t", - "typeOfLevel": "surface", - }, - ).contents - - -@fixture -def fielddata_obj(hrrr_data, prsfile, spec): +def fielddata_obj(prsfile, spec): return gribdata.FieldData( - ds=hrrr_data, fhr=16, - grib_path=prsfile, + grib_paths=[prsfile], level="sfc", + model="hrrr", short_name="temp", spec=spec, ) @@ -39,18 +33,11 @@ def fielddata_obj(hrrr_data, prsfile, spec): @fixture def profiledata_obj(natfile, spec): - ds = gribfile.GribFile( - natfile, - cfgrib_config={ - "shortName": "t", - "typeOfLevel": "hybrid", - }, - ) return gribdata.ProfileData( - ds=ds.contents, fhr=16, - grib_path=natfile, + grib_paths=[natfile], loc=" DNR 23062 72469 39.77 104.88 1611 Denver, CO", + model="hrrr", short_name="temp", spec=spec, ) @@ -64,31 +51,36 @@ def spec(spec_file): @fixture -def uppdata_obj(hrrr_data, prsfile, spec): +def uppdata_obj(natfile, spec): return ConcreteUPPData( - ds=hrrr_data, + model="hrrr", short_name="temp", spec=spec, fhr=16, - grib_path=prsfile, + grib_paths=[natfile], ) @fixture -def uppdata_multilev_obj(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "t", - "typeOfLevel": "isobaricInhPa", - }, - ).contents +def uppdata_multilev_obj(natfile, spec): return ConcreteUPPData( - ds=ds, + model="hrrr", short_name="temp", spec=spec, fhr=16, - grib_path=prsfile, + grib_paths=[natfile], + ) + + +@fixture +def uppdata_multilev_prs_obj(prsfile, spec): + return ConcreteUPPData( + level="500mb", + model="hrrr", + short_name="temp", + spec=spec, + fhr=16, + grib_paths=[prsfile], ) @@ -111,50 +103,54 @@ def test_uppdata_date_to_str(uppdata_obj): def test_uppdata_field(uppdata_obj): - assert np.array_equal(uppdata_obj.field, uppdata_obj.ds.t) + assert np.array_equal(uppdata_obj.field, uppdata_obj.ds.t.squeeze()) def test_uppdata_field_column_max(uppdata_multilev_obj): assert np.array_equal( - uppdata_multilev_obj.field_column_max(), uppdata_multilev_obj.ds.t.max(axis=0) + uppdata_multilev_obj.field_column_max(), uppdata_multilev_obj.ds.t.squeeze().max(axis=0) ) assert uppdata_multilev_obj.field_column_max().shape == (1059, 1799) def test_uppdata_field_diff(uppdata_obj): summed_field = uppdata_obj.field_diff(values=uppdata_obj.field, variable2="temp", level2="sfc") - assert np.array_equal(summed_field, uppdata_obj.ds.t * 0) + assert np.array_equal(summed_field, uppdata_obj.ds.t.squeeze() * 0) -def test_uppdata_field_mean(uppdata_multilev_obj): +def test_uppdata_field_mean(prsfile, spec): + dataobj = ConcreteUPPData( + level="mean", + model="hrrr", + short_name="rh", + spec=spec, + fhr=16, + grib_paths=[prsfile], + ) levels = ["500mb", "800mb"] - mean = uppdata_multilev_obj.field_mean(values=uppdata_multilev_obj.field, levels=levels) + mean = dataobj.field_mean(values=dataobj.field, levels=levels) assert np.array_equal( - mean, uppdata_multilev_obj.ds.t.sel(isobaricInhPa=[500, 800]).mean("isobaricInhPa") + mean, dataobj.ds.r.squeeze().sel(isobaricInhPa=[500, 800]).mean("isobaricInhPa") ) assert mean.shape == (1059, 1799) def test_uppdata_field_sum(uppdata_obj): summed_field = uppdata_obj.field_sum(values=uppdata_obj.field, variable2="temp", level2="sfc") - assert np.array_equal(summed_field, uppdata_obj.ds.t * 2) + assert np.array_equal(summed_field, uppdata_obj.ds.t.squeeze() * 2) -def test_uppdata__get_data_levels(uppdata_multilev_obj): +def test_uppdata__get_data_levels(uppdata_multilev_prs_obj): assert np.array_equal( - uppdata_multilev_obj._get_data_levels("isobaricInhPa"), - uppdata_multilev_obj.ds.coords["isobaricInhPa"].to_numpy(), + uppdata_multilev_prs_obj._get_data_levels("isobaricInhPa"), + uppdata_multilev_prs_obj.ds.coords["isobaricInhPa"].to_numpy(), ) -def test_uppdata__get_field(prsfile, uppdata_obj): +def test_uppdata__get_field(uppdata_multilev_prs_obj): spec = {"shortName": "t", "typeOfLevel": "isobaricInhPa", "level": 500} - field = uppdata_obj._get_field(spec=spec) - ds = gribfile.GribFile( - prsfile, - cfgrib_config=spec, - ).contents - assert np.array_equal(field, ds.t) + field = uppdata_multilev_prs_obj._get_field(cfgribspec=spec) + assert np.array_equal(field, uppdata_multilev_prs_obj.ds.t.sel(isobaricInhPa=500).squeeze()) @mark.parametrize( @@ -225,24 +221,16 @@ def test_uppdata_valid_dt(uppdata_obj): def test_uppdata_vector_magnitude(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "u", - "typeOfLevel": "isobaricInhPa", - "level": 250, - }, - ) fd = ConcreteUPPData( - ds=ds.contents, - fhr=16, - grib_path=prsfile, + model="hrrr", level="250mb", + fhr=16, + grib_paths=[prsfile], short_name="u", spec=spec, ) vm = fd.vector_magnitude(field1=fd.ds.u, field2_id="v_250mb") - assert not np.array_equal(vm, ds.contents.u) + assert not np.array_equal(vm, fd.ds.u) def test_uppdata_vspec(uppdata_obj): @@ -277,18 +265,11 @@ def test_uppdata_vspec_bad(uppdata_obj): def test_fielddata_aviation_flight_rules(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "gh", - "typeOfLevel": "cloudCeiling", - }, - ) fd = gribdata.FieldData( - ds=ds.contents, fhr=16, - grib_path=prsfile, + grib_paths=[prsfile], level="sfc", + model="hrrr", short_name="flru", spec=spec, ) @@ -350,18 +331,11 @@ def test_fielddata_data_getter_and_setter(fielddata_obj): def test_fielddata_fire_weather_index(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "vgtyp", - "typeOfLevel": "surface", - }, - ) fd = gribdata.FieldData( - ds=ds.contents, fhr=16, - grib_path=prsfile, + grib_paths=[prsfile], level="sfc", + model="hrrr", short_name="firewxtransform", spec=spec, ) @@ -384,18 +358,11 @@ def test_fielddata_grid_info_lambert(fielddata_obj): def test_fielddata_icing_adjust_trace(prsfile, spec): - ds = gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "gh", - "typeOfLevel": "cloudCeiling", - }, - ) fd = gribdata.FieldData( - ds=ds.contents, fhr=16, - grib_path=prsfile, + grib_paths=[prsfile], level="sfc", + model="hrrr", short_name="flru", spec=spec, ) @@ -405,23 +372,16 @@ def test_fielddata_icing_adjust_trace(prsfile, spec): def test_fielddata_supercooled_liquid_water(natfile, spec): - ds = gribfile.GribFile( - natfile, - cfgrib_config={ - "shortName": "t", - "typeOfLevel": "surface", - }, - ) fd = gribdata.FieldData( - ds=ds.contents, fhr=16, - grib_path=natfile, + grib_paths=[natfile], level="sfc", + model="hrrr", short_name="slw", spec=spec, ) slw = fd.supercooled_liquid_water() - assert not np.array_equal(slw, ds.contents.t) + assert not np.array_equal(slw, fd.ds.t.squeeze()) def test_fielddata_ticks_default(fielddata_obj): @@ -450,27 +410,31 @@ def test_fielddata_units_in_vspec(fielddata_obj): def test_fielddata_values_args_no_transform(fielddata_obj, lev, var): fielddata_obj.vspec["transform"] = None fielddata_obj.model = "hrrr" - assert not np.array_equal(fielddata_obj.values(level=lev, name=var), fielddata_obj.ds.t) + assert not np.array_equal( + fielddata_obj.values(level=lev, name=var), fielddata_obj.ds.t.squeeze() + ) def test_fielddata_values_args_transform(fielddata_obj): fielddata_obj.vspec["transform"] = "opposite" fielddata_obj.model = "hrrr" - assert np.array_equal(fielddata_obj.values(level="sfc", name="temp"), -fielddata_obj.ds.t) + assert np.array_equal( + fielddata_obj.values(level="sfc", name="temp"), -fielddata_obj.ds.t.squeeze() + ) def test_fielddata_values_no_args_no_transform(fielddata_obj): field = ones_like(fielddata_obj.ds) fielddata_obj.ds = field fielddata_obj.vspec["transform"] = None - assert np.array_equal(fielddata_obj.values(), field.t) + assert np.array_equal(fielddata_obj.values(), field.t.squeeze()) def test_fielddata_values_no_args_transform(fielddata_obj): field = ones_like(fielddata_obj.ds) fielddata_obj.ds = field fielddata_obj.vspec["transform"] = "opposite" - assert np.array_equal(fielddata_obj.values(), -field.t) + assert np.array_equal(fielddata_obj.values(), -field.t.squeeze()) def test_fielddata_values_bad_name_level(fielddata_obj): diff --git a/tests/datahandler/test_gribfile.py b/tests/datahandler/test_gribfile.py index eaa2401..7ce4f43 100644 --- a/tests/datahandler/test_gribfile.py +++ b/tests/datahandler/test_gribfile.py @@ -30,7 +30,6 @@ def test_gribfiles(): "shortName": "sp", "typeOfLevel": "surface", }, - model="rrfs", ) assert isinstance(gf.contents, Dataset) assert len(gf.contents.data_vars) == 1 diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py index ab1964f..0854a7d 100644 --- a/tests/test_figure_builders.py +++ b/tests/test_figure_builders.py @@ -23,13 +23,13 @@ def hrrr_data(prsfile): @fixture -def fielddata_obj(hrrr_data, prsfile, spec): +def fielddata_obj(prsfile, spec): return gribdata.FieldData( - ds=hrrr_data.contents, - fhr=15, + model="hrrr", + fhr=16, grib_paths=[prsfile], - level="cref", - short_name="temp", + level="sfc", + short_name="cref", spec=spec, ) @@ -48,7 +48,7 @@ def parallel_maps_args(prsfile, spec, tmp_path): ) return { "cla": cla, - "fhr": 15, + "fhr": 16, "grib_paths": [prsfile], "level": "sfc", "variable": "temp", @@ -71,8 +71,8 @@ def parallel_skewt_args(natfile, spec, tmp_path): ) return { "cla": cla, - "fhr": 15, - "grib_path": natfile, + "fhr": 16, + "grib_paths": [natfile], "site": " DNR 23062 72469 39.77 104.88 1611 Denver, CO", "workdir": tmp_path, } @@ -90,7 +90,7 @@ def test_add_obs_panel(fielddata_obj, spec): args = { "ax": ax[8], "model_name": "hrrr", - "obs_file": fielddata_obj.grib_path, # fake it with model data + "obs_file": fielddata_obj.grib_paths[0], # fake it with model data "proj_info": fielddata_obj.grid_info(), "spec": spec, "short_name": "cref", @@ -103,7 +103,7 @@ def test_add_obs_panel(fielddata_obj, spec): def test_parallel_maps(parallel_maps_args, tmp_path): figure_builders.parallel_maps(**parallel_maps_args) - assert (tmp_path / "temp_full_sfc_f015.png").is_file() + assert (tmp_path / "temp_full_sfc_f016.png").is_file() def test_parallel_maps_enspanel(parallel_maps_args, tmp_path): @@ -135,7 +135,7 @@ def test_parallel_maps_enspanel(parallel_maps_args, tmp_path): call.title().assert_called_once() call.add_logo().assert_called_once() aop.assert_called_once() - assert (tmp_path / "temp_full_sfc_f015.png").is_file() + assert (tmp_path / "temp_full_sfc_f016.png").is_file() def test_parallel_maps_mem_leak(parallel_maps_args): @@ -154,8 +154,8 @@ def test_parallel_maps_mem_leak(parallel_maps_args): def test_parallel_skewt(parallel_skewt_args, tmp_path): figure_builders.parallel_skewt(**parallel_skewt_args) - assert (tmp_path / "DNR_72469_skewt_f015.png").is_file() - assert (tmp_path / "DNR.72469.skewt.2025100600_f015.csv").is_file() + assert (tmp_path / "DNR_72469_skewt_f016.png").is_file() + assert (tmp_path / "DNR.72469.skewt.2025100600_f016.csv").is_file() def test_set_figure_enspanel_full(): From f945a6b601881b245dff6eb83795b50641a01b63 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 19 Nov 2025 09:18:32 -0700 Subject: [PATCH 42/98] Speedup unittests to ~1 min. Using 8 procs. 1:14 with 4 procs Before changes: 3:22 with 8 procs, 3:44 with 4. --- Makefile | 2 +- adb_graphics/datahandler/gribdata.py | 148 ++++++++++++---------- adb_graphics/default_specs.yml | 32 ++--- adb_graphics/figures/skewt.py | 2 +- adb_graphics/utils.py | 2 +- conftest.py | 6 +- create_graphics.py | 2 +- tests/datahandler/test_gribdata.py | 179 ++++++++++++++++++--------- tests/test_common.py | 3 +- tests/test_figure_builders.py | 13 +- tests/test_utils.py | 2 +- 11 files changed, 221 insertions(+), 170 deletions(-) diff --git a/Makefile b/Makefile index e740fd9..3150e00 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ typecheck: mypy --install-types --non-interactive . unittest: - pytest --cov -k "not hrrr" -n 8 . + pytest --cov -k "not hrrr" -n 4 . memtest: pytest --memray -k"not hrrr" . diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index b0e00ee..0da84ba 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -14,7 +14,7 @@ from matplotlib.pyplot import get_cmap from pandas import to_datetime from uwtools.api.config import YAMLConfig -from xarray import DataArray +from xarray import DataArray, ufuncs, where from adb_graphics import conversions, errors, specs, utils from adb_graphics.datahandler import gribfile @@ -49,6 +49,10 @@ def __init__( self.fhr = fhr cf = utils.cfgrib_spec(self.vspec["cfgrib"], self.model) + self.cf_name = cf.pop("shortName", "unknown") + self.vertical_dim = cf["typeOfLevel"] + if not cf.get("stepType"): + cf["stepType"] = "instant" self.ds = gribfile.GribFiles(self.grib_paths, cf).contents.squeeze() @property @@ -86,40 +90,6 @@ def field(self) -> DataArray: """ return self._get_field(self.vspec["cfgrib"].get(self.model, self.vspec["cfgrib"])) - def field_column_max(self, **kwargs): # noqa: ARG002 - """Returns the column max of the values.""" - - return self.values().max(axis=0) - - def field_diff(self, values: DataArray, variable2: str, level2: str, **kwargs): # noqa: ARG002 - """Subtracts the values from variable2 from self.field.""" - - value2 = self.values(name=variable2, level=level2) - diff = values - value2 - value2.close() - - return diff - - def field_mean( - self, - values: DataArray, - levels: list, - **kwargs, # noqa: ARG002 - ): - """Returns the mean of the values over the vertical dimension.""" - - levs = [int(x[:-2]) for x in levels] - return values.sel(isobaricInhPa=levs).mean("isobaricInhPa") - - def field_sum(self, values: DataArray, variable2: str, level2: str, **kwargs): # noqa: ARG002 - """Return the sum of the values.""" - - value2 = self.values(name=variable2, level=level2) - sum2 = values + value2 - value2.close() - - return sum2 - def _get_data_levels(self, vertical_dim: str): """ Values of the vertical dimension. @@ -138,9 +108,21 @@ def _get_field(self, cfgribspec: dict) -> DataArray: cfgribspec the specifications dictionary to use for the variable in question """ - ds = gribfile.GribFiles(self.grib_paths, cfgribspec).contents.squeeze() - first_variable_name = list(ds.data_vars)[0] - return DataArray(ds[first_variable_name]) + if cfgribspec["typeOfLevel"] != self.vertical_dim: + ds = gribfile.GribFiles(self.grib_paths, cfgribspec).contents.squeeze() + else: + ds = self.ds + + var_name = cfgribspec.get("shortName", self.cf_name) + if (field := ds.get(var_name)) is not None: + return DataArray(field) + + for var in ds: + if ds[var].attrs["GRIB_shortName"] == var_name: + return DataArray(ds[var]) + + msg = f"Variable {var_name} not found in dataset." + raise ValueError(msg) def get_transform(self, transforms: dict | list | str, val: DataArray) -> DataArray: """ @@ -312,12 +294,12 @@ def aviation_flight_rules(self, values: DataArray, **kwargs): # noqa: ARG002 ceil = values vis = self.values(name="vis", level="sfc") - flru = np.where((ceil > 1.0) & (ceil < 3.0), 1.01, 0.0) - flru = np.where((vis > 3.0) & (vis < 5.0), 1.01, flru) - flru = np.where((ceil > 0.5) & (ceil < 1.0), 2.01, flru) - flru = np.where((vis > 1.0) & (vis < 3.0), 2.01, flru) - flru = np.where((ceil > 0.0) & (ceil < 0.5), 3.01, flru) - flru = np.where((vis < 1.0), 3.01, flru) + flru = where((ceil > 1.0) & (ceil < 3.0), 1.01, 0.0) + flru = where((vis > 3.0) & (vis < 5.0), 1.01, flru) + flru = where((ceil > 0.5) & (ceil < 1.0), 2.01, flru) + flru = where((vis > 1.0) & (vis < 3.0), 2.01, flru) + flru = where((ceil > 0.0) & (ceil < 0.5), 3.01, flru) + flru = where((vis < 1.0), 3.01, flru) vis.close() @@ -379,6 +361,45 @@ def data(self) -> DataArray: def data(self, value: DataArray): self._data = value + def field_column_max(self, values: DataArray, **kwargs): # noqa: ARG002 + """Returns the column max of the values.""" + + return values.max(dim=self.vertical_dim) + + def field_diff(self, values: DataArray, variable2: str, level2: str, **kwargs): + """Subtracts the values from variable2 from self.field.""" + + value2 = self.values( + name=variable2, level=level2, do_transform=kwargs.get("do_transform", True) + ) + diff = values - value2 + value2.close() + + return diff + + def field_mean( + self, + values: DataArray, + levels: list, + **kwargs, + ): + """Returns the mean of the values over the vertical dimension.""" + + levels = kwargs["global_levels"] if "global" in self.model else levels + levs = [int(x[:-2]) for x in levels] + return values.sel(isobaricInhPa=levs).mean("isobaricInhPa") + + def field_sum(self, values: DataArray, variable2: str, level2: str, **kwargs): + """Return the sum of the values.""" + + value2 = self.values( + name=variable2, level=level2, do_transform=kwargs.get("do_transform", True) + ) + sum2 = values + value2 + value2.close() + + return sum2 + def fire_weather_index(self, values: DataArray, **kwargs): # noqa: ARG002 """ Generates a field of Fire Weather Index. @@ -399,30 +420,31 @@ def _load_field(level: str, short_name: str): ).values(do_transform=False) # Gather fields from the input - veg = np.asarray(values) - temp = _load_field(level="2m", short_name="temp") - dewpt = _load_field(level="2m", short_name="dewp") - weasd = _load_field(level="sfc", short_name="weasd") - gust = _load_field(level="10m", short_name="gust") - soilm = _load_field(level="sfc", short_name="soilm") + veg = values + + temp = self.values(level="2m", name="temp", do_transform=False) + dewpt = self.values(level="2m", name="dewp", do_transform=False) + weasd = self.values(level="sfc", name="weasd", do_transform=False) + gust = self.values(level="10m", name="gust", do_transform=False) + soilm = self.values(level="sfc", name="soilm", do_transform=False) # A few derived fields dewpt_depression = temp - dewpt - dewpt_depression = np.where(dewpt_depression < 0, 0, dewpt_depression) - dewpt_depression = np.maximum(15.0, dewpt_depression) + dewpt_depression = where(dewpt_depression < 0, 0, dewpt_depression) + dewpt_depression = ufuncs.maximum(15.0, dewpt_depression) gust_max = np.maximum(3.0, gust) snowc = (25.0 - weasd) / 25.0 - snowc = np.where(snowc > 0.0, snowc, 0.0) + snowc = where(snowc > 0.0, snowc, 0.0) mois = 0.01 * (100.0 - soilm) # Set urban (13), snow/ice (15), barren (16), and water (17) to 0. for vegtype in [13, 15, 16, 17]: - veg = np.where(veg == vegtype, 0, veg) + veg = where(veg == vegtype, 0, veg) # Set all others vegetation types to 1 - veg = np.where(veg > 0, 1, veg) + veg = where(veg > 0, 1, veg) fwi = veg * (2.37 * (gust_max**1.11) * (dewpt_depression**0.92) * (mois**6.95) * snowc) @@ -539,7 +561,7 @@ def grid_info(self) -> dict: def icing_adjust_trace(values: DataArray, **kwargs): # noqa: ARG004 """Changes the value of ICSEV trace from 4.0 to 0.5, to maintain ascending order.""" - return np.where(values == 4.0, 0.5, values) + return where(values == 4.0, 0.5, values) @staticmethod def run_max(values: DataArray, **kwargs): # noqa: ARG004 @@ -574,12 +596,11 @@ def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 The process is iterative to the topof the atmosphere. """ - pres_sfc = self.values(name="pres", level="sfc") * 100.0 # convert back to Pa pres_nat_lev = self.values(name="pres", level="ua") temp = self.values(name="temp", level="ua") - cloud_mixing_ratio = self.values(name="clwmr", level="uanat") - rain_mixing_ratio = self.values(name="rwmr", level="uanat") + cloud_mixing_ratio = self.values(name="clwmr", level="ua") + rain_mixing_ratio = self.values(name="rwmr", level="ua") gravity = 9.81 slw = pres_sfc * 0.0 # start with array of zero values @@ -593,9 +614,9 @@ def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 pres_layer = 2 * (pres_sigma[:, :] - pres_nat_lev[n, :, :]) # layer depth pres_sigma = pres_sigma - pres_layer # pressure at next sigma level # compute supercooled water in layer and add to previous values - supercool_locs = np.where( - (temp[n, :, :] < 0.0), - cloud_mixing_ratio[n, :, :] + rain_mixing_ratio[n, :, :], + supercool_locs = where( + (temp[n, ::] < 0.0), + cloud_mixing_ratio[n, ::] + rain_mixing_ratio[n, ::], 0.0, ) slw = slw + pres_layer / gravity * supercool_locs @@ -650,6 +671,8 @@ def values( raise errors.NoGraphicsDefinitionForVariableError(name, level) utils.set_level(level=level, model=self.model, spec=spec) vals = self._get_field(spec["cfgrib"].get(self.model, spec["cfgrib"])) + else: + vals = self.ds.get(self.cf_name) transforms = spec.get("transform") if transforms and do_transform: @@ -689,7 +712,6 @@ def __init__( super().__init__( fhr=fhr, grib_paths=grib_paths, model=model, short_name=short_name, spec=spec ) - # The first 31 columns are space delimted self.site_code, _, self.site_num, lat, lon = loc[:31].split() diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index a1e5450..f8b7a0d 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -686,16 +686,11 @@ cloudcover: ncl_name: TCDC_P0_L{level_type}_{grid} title: Total Cloud Cover clwmr: # Cloud water Mixing Ratio - uanat: + ua: cfgrib: shortName: clwmr - typeOfLevel: hybrid + typeOfLevel: '{{ "isobaricInhPa" if file_type == "prs" else "hybrid" }}' ncl_name: CLWMR_P0_L105_{grid} - uaprs: - cfgrib: - shortName: clwmr - typeOfLevel: isobaricInhPa - ncl_name: CLWMR_P0_L100_{grid} coarsedust: # Coarse dust int: &coldust clevs: [1, 4, 7, 11, 15, 20, 25, 30, 40, 50, 75, 150, 250, 500] @@ -1214,14 +1209,10 @@ grle: # Graupel ua: cfgrib: shortName: grle - typeOfLevel: isobaricInhPa + typeOfLevel: '{{ "isobaricInhPa" if file_type == "prs" else "hybrid" }}' ncl_name: nat: GRLE_P0_L105_{grid} prs: GRLE_P0_L100_{grid} - uanat: - cfgrib: - shortName: grle - typeOfLevel: hybrid gust: 10m: cfgrib: @@ -2088,16 +2079,11 @@ rvil: # Radar-derived Vertically Integrated Liquid title: VIL (Reflectivity Derived) unit: kg/m$^{2}$ rwmr: # Rain Mixing Ratio - uanat: + ua: cfgrib: shortName: rwmr - typeOfLevel: hybrid + typeOfLevel: '{{ "isobaricInhPa" if file_type == "prs" else "hybrid" }}' ncl_name: RWMR_P0_L105_{grid} - uaprs: - cfgrib: - shortName: rwmr - typeOfLevel: isobaricInhPa - ncl_name: RWMR_P0_L100_{grid} seasalt: # Fine dust, global chem sfc: cfgrib: @@ -2218,7 +2204,7 @@ slw: # Supercooled Liquid Water -- requires nat data sfc: cfgrib: shortName: t - typeOfLevel: surface + typeOfLevel: hybrid clevs: [0.05, 0.1, 0.2, .3, .4, .5, .75, 1., 1.25, 1.5, 2., 3., 4., 5., 6.] cmap: gist_ncar colors: slw_colors @@ -2231,14 +2217,10 @@ snmr: # Snow Mixing Ratio ua: cfgrib: shortName: snmr - typeOfLevel: isobaricInhPa + typeOfLevel: '{{ "isobaricInhPa" if file_type == "prs" else "hybrid" }}' ncl_name: nat: SNMR_P0_L105_{grid} prs: SNMR_P0_L100_{grid} - uanat: - cfgrib: - shortName: snmr - typeOfLevel: hybrid snod: # Snow Depth sfc: <<: *snow diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 3fe1968..b820105 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -134,7 +134,7 @@ def _add_hydrometeors(self, hydro_subplot: Axes): scale = settings.get("scale", 1.0) try: profile = self.values(name=mixr) * 1000.0 * scale - except (errors.NoGraphicsDefinitionForVariableError, IndexError): + except (errors.NoGraphicsDefinitionForVariableError, IndexError, ValueError): try: profile = self.values(name=mixr, level="uanat") * 1000.0 * scale except errors.NoGraphicsDefinitionForVariableError: diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 5ee79fa..eaee0b8 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -63,7 +63,7 @@ def create_zip(files_to_zip: list[str], zipf: Path | str): # Remove the lock lock_file.unlink(missing_ok=True) # Wait before trying to obtain the lock on the file - time.sleep(5) + time.sleep(1) def fhr_list(args: list[int]) -> list[int]: diff --git a/conftest.py b/conftest.py index d04bd08..9381d81 100644 --- a/conftest.py +++ b/conftest.py @@ -35,20 +35,20 @@ def pytest_addoption(parser): ) -@pytest.fixture +@pytest.fixture(scope="session") def natfile(): """Interface to pass a grib file to pytest.""" return Path("tests", "data", "wrfnat_hrconus_16.grib2") -@pytest.fixture +@pytest.fixture(scope="session") def prsfile(): """Interface to pass a grib file to pytest.""" return Path("tests", "data", "wrfprs_hrconus_16.grib2") -@pytest.fixture +@pytest.fixture(scope="session") def spec_file(): """Interface to pass a grib file to pytest.""" return Path("adb_graphics", "default_specs.yml") diff --git a/create_graphics.py b/create_graphics.py index fbac644..bd86711 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -622,7 +622,7 @@ def graphics_driver(cla: Namespace): f"Output graphics directory: {workdir} \n{LOG_BREAK}", ) full_spec = utils.load_yaml(orig_spec) - full_spec.dereference(context={"fhr": int(fhr)}) + full_spec.dereference(context={"fhr": int(fhr), "file_type": cla.file_type}) cla.specs = full_spec if cla.graphic_type == "skewts": create_skewt(cla, fhr, grib_path, workdir) diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index cd82c34..642af11 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -3,6 +3,7 @@ import numpy as np from matplotlib.pyplot import get_cmap from pytest import fixture, mark, raises +from uwtools.api.config import get_yaml_config from xarray import DataArray, ones_like, zeros_like from adb_graphics import errors, utils @@ -21,6 +22,8 @@ def values( @fixture def fielddata_obj(prsfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "prs"}) return gribdata.FieldData( fhr=16, grib_paths=[prsfile], @@ -31,8 +34,24 @@ def fielddata_obj(prsfile, spec): ) -@fixture +@fixture(scope="module") +def fielddata_obj_ro(prsfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "prs"}) + return gribdata.FieldData( + fhr=16, + grib_paths=[prsfile], + level="sfc", + model="hrrr", + short_name="temp", + spec=spec, + ) + + +@fixture(scope="module") def profiledata_obj(natfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "nat"}) return gribdata.ProfileData( fhr=16, grib_paths=[natfile], @@ -43,7 +62,7 @@ def profiledata_obj(natfile, spec): ) -@fixture +@fixture(scope="module") def spec(spec_file): spec = utils.load_yaml(spec_file) spec.dereference(context={"fhr": 16}) @@ -52,7 +71,24 @@ def spec(spec_file): @fixture def uppdata_obj(natfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "nat"}) return ConcreteUPPData( + level="ua", + model="hrrr", + short_name="temp", + spec=spec, + fhr=16, + grib_paths=[natfile], + ) + + +@fixture(scope="module") +def uppdata_obj_ro(natfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "nat"}) + return ConcreteUPPData( + level="ua", model="hrrr", short_name="temp", spec=spec, @@ -63,6 +99,8 @@ def uppdata_obj(natfile, spec): @fixture def uppdata_multilev_obj(natfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "nat"}) return ConcreteUPPData( model="hrrr", short_name="temp", @@ -74,6 +112,8 @@ def uppdata_multilev_obj(natfile, spec): @fixture def uppdata_multilev_prs_obj(prsfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "prs"}) return ConcreteUPPData( level="500mb", model="hrrr", @@ -84,13 +124,13 @@ def uppdata_multilev_prs_obj(prsfile, spec): ) -def test_uppdata_anl_dt(uppdata_obj): - dt = uppdata_obj.anl_dt +def test_uppdata_anl_dt(uppdata_obj_ro): + dt = uppdata_obj_ro.anl_dt assert dt == datetime(2025, 10, 6, 0) -def test_uppdata_clevs_array(uppdata_obj): - assert np.array_equal(uppdata_obj.clevs, np.arange(-40, 40, 2.5)) +def test_uppdata_clevs_array(uppdata_obj_ro): + assert np.array_equal(uppdata_obj_ro.clevs, np.arange(-40, 40, 2.5)) def test_uppdata_clevs_list(uppdata_obj): @@ -98,28 +138,37 @@ def test_uppdata_clevs_list(uppdata_obj): assert np.array_equal(uppdata_obj.clevs, np.asarray([1, 2, 3])) -def test_uppdata_date_to_str(uppdata_obj): - assert uppdata_obj.date_to_str(uppdata_obj.anl_dt) == "20251006 00 UTC" +def test_uppdata_date_to_str(uppdata_obj_ro): + assert uppdata_obj_ro.date_to_str(uppdata_obj_ro.anl_dt) == "20251006 00 UTC" -def test_uppdata_field(uppdata_obj): - assert np.array_equal(uppdata_obj.field, uppdata_obj.ds.t.squeeze()) +def test_uppdata_field(uppdata_obj_ro): + assert np.array_equal(uppdata_obj_ro.field, uppdata_obj_ro.ds.t.squeeze()) -def test_uppdata_field_column_max(uppdata_multilev_obj): - assert np.array_equal( - uppdata_multilev_obj.field_column_max(), uppdata_multilev_obj.ds.t.squeeze().max(axis=0) +def test_uppdata_field_column_max(prsfile, spec): + fd = gribdata.FieldData( + fhr=16, + grib_paths=[prsfile], + level="500mb", + model="hrrr", + short_name="temp", + spec=spec, ) - assert uppdata_multilev_obj.field_column_max().shape == (1059, 1799) + column_max = fd.field_column_max(values=fd.ds.t) + assert np.array_equal(column_max, fd.ds.t.squeeze().max(axis=0)) + assert column_max.shape == (1059, 1799) -def test_uppdata_field_diff(uppdata_obj): - summed_field = uppdata_obj.field_diff(values=uppdata_obj.field, variable2="temp", level2="sfc") - assert np.array_equal(summed_field, uppdata_obj.ds.t.squeeze() * 0) +def test_uppdata_field_diff(fielddata_obj_ro): + summed_field = fielddata_obj_ro.field_diff( + values=fielddata_obj_ro.field, variable2="temp", level2="sfc", do_transform=False + ) + assert np.array_equal(summed_field, fielddata_obj_ro.ds.t.squeeze() * 0) def test_uppdata_field_mean(prsfile, spec): - dataobj = ConcreteUPPData( + dataobj = gribdata.FieldData( level="mean", model="hrrr", short_name="rh", @@ -135,9 +184,11 @@ def test_uppdata_field_mean(prsfile, spec): assert mean.shape == (1059, 1799) -def test_uppdata_field_sum(uppdata_obj): - summed_field = uppdata_obj.field_sum(values=uppdata_obj.field, variable2="temp", level2="sfc") - assert np.array_equal(summed_field, uppdata_obj.ds.t.squeeze() * 2) +def test_uppdata_field_sum(fielddata_obj_ro): + summed_field = fielddata_obj_ro.field_sum( + values=fielddata_obj_ro.field, variable2="temp", level2="sfc", do_transform=False + ) + assert np.array_equal(summed_field, fielddata_obj_ro.ds.t.squeeze() * 2) def test_uppdata__get_data_levels(uppdata_multilev_prs_obj): @@ -150,7 +201,7 @@ def test_uppdata__get_data_levels(uppdata_multilev_prs_obj): def test_uppdata__get_field(uppdata_multilev_prs_obj): spec = {"shortName": "t", "typeOfLevel": "isobaricInhPa", "level": 500} field = uppdata_multilev_prs_obj._get_field(cfgribspec=spec) - assert np.array_equal(field, uppdata_multilev_prs_obj.ds.t.sel(isobaricInhPa=500).squeeze()) + assert np.array_equal(field, uppdata_multilev_prs_obj.ds.t.squeeze()) @mark.parametrize( @@ -158,16 +209,20 @@ def test_uppdata__get_field(uppdata_multilev_prs_obj): [ "conversions.percent", ["conversions.percent", "opposite"], - {"funcs": "field_diff", "kwargs": {"variable2": "temp", "level2": "sfc"}}, + { + "funcs": "field_diff", + "kwargs": {"variable2": "temp", "level2": "sfc", "do_transform": False}, + }, ], ) -def test_uppdata_get_transform(transforms, uppdata_obj): - val = ones_like(uppdata_obj.ds.t) if not isinstance(transforms, dict) else uppdata_obj.ds.t - field = uppdata_obj.get_transform(transforms, val) +def test_uppdata_get_transform(fielddata_obj_ro, transforms): + temp = fielddata_obj_ro.ds.t + val = ones_like(temp) if not isinstance(transforms, dict) else temp + field = fielddata_obj_ro.get_transform(transforms, val) expected = 0 match transforms: case dict(): - expected = zeros_like(uppdata_obj.ds.t) + expected = zeros_like(temp) case list(): expected = val * -100.0 case str(): @@ -179,21 +234,21 @@ def test_uppdata_get_transform(transforms, uppdata_obj): ("lat", "lon", "expected"), [(40.019, 360 - 105.2747, (595, 679)), (25.7617, 360 - 80.1918, (109, 1487))], ) -def test_uppdata_get_xypoint(expected, lat, lon, uppdata_obj): - assert uppdata_obj.get_xypoint(lat, lon) == expected +def test_uppdata_get_xypoint(expected, lat, lon, uppdata_obj_ro): + assert uppdata_obj_ro.get_xypoint(lat, lon) == expected @mark.parametrize(("lat", "lon"), [(88.0, 270.0), (40, 180), (10, 330), (30, 345)]) -def test_uppdata_get_xypoint_outside(lat, lon, uppdata_obj): - assert uppdata_obj.get_xypoint(lat, lon) == (-1, -1) +def test_uppdata_get_xypoint_outside(lat, lon, uppdata_obj_ro): + assert uppdata_obj_ro.get_xypoint(lat, lon) == (-1, -1) -def test_uppdata_latlons(uppdata_obj): - lats = uppdata_obj.ds.coords["latitude"].to_numpy() - lons = uppdata_obj.ds.coords["longitude"].to_numpy() +def test_uppdata_latlons(uppdata_obj_ro): + lats = uppdata_obj_ro.ds.coords["latitude"].to_numpy() + lons = uppdata_obj_ro.ds.coords["longitude"].to_numpy() assert [ np.array_equal(act, exp) - for act, exp in zip(uppdata_obj.latlons(), [lats, lons], strict=True) + for act, exp in zip(uppdata_obj_ro.latlons(), [lats, lons], strict=True) ] @@ -211,13 +266,13 @@ def test_uppdata_latlons_lats_flipped(uppdata_obj): @mark.parametrize("factor", [1, -1, 0, -20.0, 6543.0]) -def test_uppdata_opposite(factor, uppdata_obj): - ds = ones_like(uppdata_obj.field) * factor - assert np.array_equal(uppdata_obj.opposite(ds), -ds) +def test_uppdata_opposite(factor, uppdata_obj_ro): + ds = ones_like(uppdata_obj_ro.field) * factor + assert np.array_equal(uppdata_obj_ro.opposite(ds), -ds) -def test_uppdata_valid_dt(uppdata_obj): - assert uppdata_obj.valid_dt == datetime(2025, 10, 6, 16) +def test_uppdata_valid_dt(uppdata_obj_ro): + assert uppdata_obj_ro.valid_dt == datetime(2025, 10, 6, 16) def test_uppdata_vector_magnitude(prsfile, spec): @@ -233,9 +288,9 @@ def test_uppdata_vector_magnitude(prsfile, spec): assert not np.array_equal(vm, fd.ds.u) -def test_uppdata_vspec(uppdata_obj): +def test_uppdata_vspec(uppdata_obj_ro): expected = { - "cfgrib": {"shortName": "t", "typeOfLevel": "hybrid"}, + "cfgrib": {"stepType": "instant", "typeOfLevel": "hybrid"}, "clevs": np.arange(-40, 40, 2.5), "cmap": "jet", "colors": "ua_temp_colors", @@ -250,12 +305,12 @@ def test_uppdata_vspec(uppdata_obj): "unit": "C", "wind": True, } - vspec = uppdata_obj.vspec - # Can't test the array items with ==, so check them separately and then remove. - assert np.array_equal(vspec["clevs"], np.asarray(expected["clevs"])) - vspec.pop("clevs") - expected.pop("clevs") - assert uppdata_obj.vspec == expected + vspec = uppdata_obj_ro.vspec + # Can't test the array items with ==, so check them separately. + actual_clevs = vspec.pop("clevs") + expected_clevs = expected.pop("clevs") + assert np.array_equal(actual_clevs, np.asarray(expected_clevs)) + assert uppdata_obj_ro.vspec == expected def test_uppdata_vspec_bad(uppdata_obj): @@ -279,8 +334,8 @@ def test_fielddata_aviation_flight_rules(prsfile, spec): assert flru.min() == 0.0 -def test_fielddata_cmap(fielddata_obj): - assert fielddata_obj.cmap == get_cmap("gist_ncar") +def test_fielddata_cmap(fielddata_obj_ro): + assert fielddata_obj_ro.cmap == get_cmap("gist_ncar") @mark.parametrize("color_def", ["aod_colors", "shear_colors", "vvel_colors"]) @@ -345,8 +400,8 @@ def test_fielddata_fire_weather_index(prsfile, spec): assert firewx.min() == 0 -def test_fielddata_grid_info_lambert(fielddata_obj): - grid_info = fielddata_obj.grid_info() +def test_fielddata_grid_info_lambert(fielddata_obj_ro): + grid_info = fielddata_obj_ro.grid_info() assert grid_info == { "corners": [21.13812299999999, 47.84219502248864, 237.28047200000003, 299.08280722816215], "lat_0": 39.0, @@ -358,6 +413,8 @@ def test_fielddata_grid_info_lambert(fielddata_obj): def test_fielddata_icing_adjust_trace(prsfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "prs"}) fd = gribdata.FieldData( fhr=16, grib_paths=[prsfile], @@ -372,6 +429,8 @@ def test_fielddata_icing_adjust_trace(prsfile, spec): def test_fielddata_supercooled_liquid_water(natfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "nat"}) fd = gribdata.FieldData( fhr=16, grib_paths=[natfile], @@ -384,8 +443,8 @@ def test_fielddata_supercooled_liquid_water(natfile, spec): assert not np.array_equal(slw, fd.ds.t.squeeze()) -def test_fielddata_ticks_default(fielddata_obj): - assert fielddata_obj.ticks == 10 +def test_fielddata_ticks_default(fielddata_obj_ro): + assert fielddata_obj_ro.ticks == 10 def test_fielddata_ticks_in_vspec(fielddata_obj): @@ -394,8 +453,8 @@ def test_fielddata_ticks_in_vspec(fielddata_obj): assert fielddata_obj.ticks == ticks -def test_fielddata_units_default(fielddata_obj): - assert fielddata_obj.units == "F" +def test_fielddata_units_default(fielddata_obj_ro): + assert fielddata_obj_ro.units == "F" def test_fielddata_units_in_vspec(fielddata_obj): @@ -437,13 +496,13 @@ def test_fielddata_values_no_args_transform(fielddata_obj): assert np.array_equal(fielddata_obj.values(), -field.t.squeeze()) -def test_fielddata_values_bad_name_level(fielddata_obj): +def test_fielddata_values_bad_name_level(fielddata_obj_ro): with raises(errors.NoGraphicsDefinitionForVariableError): - fielddata_obj.values(level="foo", name="temp") + fielddata_obj_ro.values(level="foo", name="temp") with raises(errors.NoGraphicsDefinitionForVariableError): - fielddata_obj.values(level="sfc", name="foo") + fielddata_obj_ro.values(level="sfc", name="foo") with raises(errors.NoGraphicsDefinitionForVariableError): - fielddata_obj.values(level="bar", name="foo") + fielddata_obj_ro.values(level="bar", name="foo") def test_profiledata_values(profiledata_obj): diff --git a/tests/test_common.py b/tests/test_common.py index 1311275..e825c44 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -195,7 +195,7 @@ def check_transform(self, entry): for key in kwargs: if key not in all_params: msg = ( - f"Function key {key} is not an explicit parameter" + f"Function key {key} is not an explicit parameter " f"in any of the transforms: {funcs}!" ) warnings.warn(msg, UserWarning, stacklevel=2) @@ -373,7 +373,6 @@ def is_a_level(key): "total", # total clouds "ua", # upper air "uanat", # upper air native file - "uaprs", # upper air prs file ] allowed_lev_type = [ diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py index 0854a7d..d61c565 100644 --- a/tests/test_figure_builders.py +++ b/tests/test_figure_builders.py @@ -8,18 +8,7 @@ from pytest import fixture from adb_graphics import figure_builders, utils -from adb_graphics.datahandler import gribdata, gribfile - - -@fixture -def hrrr_data(prsfile): - return gribfile.GribFile( - prsfile, - cfgrib_config={ - "shortName": "t", - "typeOfLevel": "surface", - }, - ) +from adb_graphics.datahandler import gribdata @fixture diff --git a/tests/test_utils.py b/tests/test_utils.py index 3447c06..6ef87df 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -70,7 +70,7 @@ def test_create_zip_locked(tmp_path): zipf = tmp_path / "file.zip" zipf_lock = tmp_path / "file.zip._lock" zipf_lock.touch() - with raises(TimeoutError), timeout(3): + with raises(TimeoutError), timeout(2): utils.create_zip([str(f) for f in [afile, bfile]], zipf) assert not zipf.is_file() assert afile.is_file() From 6a89eded1d9fc2c14cb6e8485a6fb3c2aeba3b6c Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 19 Nov 2025 13:26:42 -0700 Subject: [PATCH 43/98] Tests and maps pass. SkewTs need some speed work. --- adb_graphics/conversions.py | 1 - adb_graphics/datahandler/gribdata.py | 26 +++++++++++++------------- adb_graphics/default_specs.yml | 24 +++++++++++++++++++++++- adb_graphics/figure_builders.py | 10 +++++++++- adb_graphics/figures/maps.py | 6 +++--- create_graphics.py | 10 +++++----- tests/datahandler/test_gribdata.py | 6 ++++-- tests/test_figure_builders.py | 15 +++++++++++++-- 8 files changed, 70 insertions(+), 28 deletions(-) diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index a5d562b..16f2c7f 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -18,7 +18,6 @@ def k_to_c(field: ndarray, **kwargs): def k_to_f(field: ndarray, **kwargs): """Conversion from Kelvin to Farenheit.""" - return (field - 273.15) * 9 / 5 + 32 diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 0da84ba..29b76fb 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -1,5 +1,3 @@ -# pylint: disable=invalid-name, too-many-public-methods, too-many-lines - """ Classes that handle the specifics of grib files from UPP. """ @@ -11,6 +9,7 @@ from pathlib import Path import numpy as np +from cfgrib import DatasetBuildError from matplotlib.pyplot import get_cmap from pandas import to_datetime from uwtools.api.config import YAMLConfig @@ -48,12 +47,17 @@ def __init__( self.level = level self.fhr = fhr - cf = utils.cfgrib_spec(self.vspec["cfgrib"], self.model) + cf = deepcopy(self.vspec) + utils.set_level(level=str(level), model=self.model, spec=cf) + cf = utils.cfgrib_spec(cf["cfgrib"], self.model) self.cf_name = cf.pop("shortName", "unknown") self.vertical_dim = cf["typeOfLevel"] - if not cf.get("stepType"): - cf["stepType"] = "instant" - self.ds = gribfile.GribFiles(self.grib_paths, cf).contents.squeeze() + try: + self.ds = gribfile.GribFiles(self.grib_paths, cf).contents.squeeze() + except DatasetBuildError as e: + if "stepType" in str(e): + cf["stepType"] = "instant" + self.ds = gribfile.GribFiles(self.grib_paths, cf).contents.squeeze() @property def anl_dt(self) -> datetime: @@ -120,7 +124,6 @@ def _get_field(self, cfgribspec: dict) -> DataArray: for var in ds: if ds[var].attrs["GRIB_shortName"] == var_name: return DataArray(ds[var]) - msg = f"Variable {var_name} not found in dataset." raise ValueError(msg) @@ -354,7 +357,7 @@ def data(self) -> DataArray: the values associated with a given object -- helpful for differences. """ if not hasattr(self, "_data"): - return self.values() + self._data = self.values() return self._data @data.setter @@ -670,9 +673,7 @@ def values( if not spec: raise errors.NoGraphicsDefinitionForVariableError(name, level) utils.set_level(level=level, model=self.model, spec=spec) - vals = self._get_field(spec["cfgrib"].get(self.model, spec["cfgrib"])) - else: - vals = self.ds.get(self.cf_name) + vals = self._get_field(spec["cfgrib"].get(self.model, spec["cfgrib"])) transforms = spec.get("transform") if transforms and do_transform: @@ -704,8 +705,8 @@ def __init__( self, fhr: int, grib_paths: list[Path], - loc: str, model: str, + loc: str, short_name: str, spec: dict | YAMLConfig, ): @@ -745,7 +746,6 @@ def values( upper air """ - # Set the defaults here since this is an instance of an abstract method # level refers to the level key in the specs file. level = level if level is not None else "ua" diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index f8b7a0d..10f93ae 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -1244,9 +1244,11 @@ hail: # Max 1h Hail diameter shortName: hail typeOfLevel: sigma level: 0 + stepType: max rrfs: shortName: hail typeOfLevel: surface + stepType: max clevs: [0.10, 0.25, 0.50, 0.75, 1.0, 1.5, 2.0, 2.5, 3.0] cmap: gist_ncar colors: hail_colors @@ -1262,9 +1264,11 @@ hail: # Max 1h Hail diameter shortName: hail typeOfLevel: atmosphere level: 0 + stepType: max rrfs: shortName: hail typeOfLevel: surface + stepType: max ncl_name: HAIL_P8_L10_{grid}_max1h title: Max 1h Hail/Graupel Diameter, Entire Column hailcast: # Max 1h Hail diameter @@ -1298,6 +1302,7 @@ hlcy: # Helicity typeOfLevel: heightAboveGroundLayer topLevel: 2000 bottomLevel: 0 + stepType: min clevs: !arange [-300, -24, 25] cmap: gist_ncar colors: rainbow12_reverse @@ -1313,6 +1318,7 @@ hlcy: # Helicity typeOfLevel: heightAboveGroundLayer topLevel: 3000 bottomLevel: 0 + stepType: min title: 0-3km Min Updraft Helicity (over prv hr) mn16: &hlcy_mn16 # Hourly minimum of updraft helicity over 1-6 km layer <<: *hlcy_mn @@ -1324,6 +1330,7 @@ hlcy: # Helicity typeOfLevel: heightAboveGroundLayer topLevel: 5000 bottomLevel: 2000 + stepType: min title: 2-5km Min Updraft Helicity (over prv hr) mx02: &hlcy_mx02 # Hourly maximum of updraft helicity over 0-2 km layer <<: *hlcy @@ -1332,6 +1339,7 @@ hlcy: # Helicity typeOfLevel: heightAboveGroundLayer topLevel: 2000 bottomLevel: 0 + stepType: max clevs: !join_ranges [[12.5, 87.6, 12.5], [100, 301, 25]] colors: rainbow16_colors ncl_name: MXUPHL_P8_2L103_{grid}_max1h @@ -1345,6 +1353,7 @@ hlcy: # Helicity typeOfLevel: heightAboveGroundLayer topLevel: 3000 bottomLevel: 0 + stepType: max clevs: !join_ranges [[12.5, 87.6, 12.5], [100, 301, 25]] colors: rainbow16_colors ncl_name: MXUPHL_P8_2L103_{grid}_max1h @@ -1366,6 +1375,7 @@ hlcy: # Helicity typeOfLevel: heightAboveGroundLayer topLevel: 5000 bottomLevel: 2000 + stepType: max clevs: !join_ranges [[25, 176, 25], [200, 601, 50]] colors: rainbow16_colors ncl_name: MXUPHL_P8_2L103_{grid}_max1h @@ -2615,7 +2625,7 @@ temp: # Temperature <<: *ua_temp cfgrib: shortName: t - typeOfLevel: hybrid + typeOfLevel: '{{ "isobaricInhPa" if file_type == "prs" else "hybrid" }}' thick: 500mb: <<: *ua_gh @@ -3047,6 +3057,7 @@ vvort: # Vertical vorticity shortName: max_vo typeOfLevel: heightAboveGroundLayer level: 1000 + stepType: max clevs: !arange [0.0025, 0.0301, 0.0025] cmap: gist_ncar colors: vort_colors @@ -3060,6 +3071,7 @@ vvort: # Vertical vorticity shortName: max_vo typeOfLevel: heightAboveGroundLayer level: 2000 + stepType: max title: 0-2km Max Vertical Vorticity (over prev hour) weasd: # Water equivalent of accumulated snow depth sfc: @@ -3092,6 +3104,7 @@ windmax: shortName: max_10si typeOfLevel: heightAboveGround level: 10 + stepType: max clevs: !arange [5, 95, 5] cmap: gist_ncar colors: wind_colors @@ -3160,6 +3173,10 @@ wspeed: # Wind Speed typeOfLevel: heightAboveGround level: 80 title: 80m Wind + transform: + funcs: [vector_magnitude, conversions.ms_to_kt] + kwargs: + field2_id: v_80m 850mb: <<: *ua_wspeed cfgrib: @@ -3181,6 +3198,7 @@ wspeed: # Wind Speed shortName: max_10si typeOfLevel: heightAboveGround level: 10 + stepType: max ncl_name: WIND_P8_L103_{grid}_max1h title: Max 10m Wind (over prev hour) transform: conversions.ms_to_kt @@ -3192,11 +3210,13 @@ wspeed: # Wind Speed typeOfLevel: pressureFromGroundLayer topLevel: 10000 bottomLevel: 100000 + stepType: max rrfs: parameterNumber: 221 typeOfLevel: isobaricLayer topLevel: 100 bottomLevel: 1000 + stepType: max clevs: [-40, -35, -30, -25, -22.5, -20, -17.5, -15, -12.5, -10, -7.5, -5, -2.5, -2, -1.5, -1, -0.5] cmap: jet colors: mdn_colors @@ -3211,11 +3231,13 @@ wspeed: # Wind Speed typeOfLevel: pressureFromGroundLayer topLevel: 10000 bottomLevel: 100000 + stepType: max rrfs: parameterNumber: 220 typeOfLevel: isobaricLayer topLevel: 100 bottomLevel: 1000 + stepType: max clevs: [0.5, 1, 1.5, 2, 2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20, 22.5, 25, 30, 35, 40] cmap: jet colors: mup_colors diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 103d6ea..1e57380 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -226,11 +226,19 @@ def parallel_skewt(cla: Namespace, fhr: int, grib_paths: list[Path], site: str, site the string representation of the site from the sites file workdir output directory """ + # deduce model name + possible_namers = (cla.model_name, cla.data_root, cla.file_tmpl) + model = "" + for name in ("global", "hrrr", "rrfs"): + if any(name in namer.lower() if isinstance(namer, str) else name in str(namer[0]).lower() for namer in possible_namers): + + model = name + break skew = skewt.SkewTDiagram( fhr=fhr, grib_paths=grib_paths, loc=site, - model=cla.images[0], + model=model, spec=cla.specs, max_plev=cla.max_plev, model_name=cla.model_name, diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 40df7f0..ca84e74 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -157,7 +157,7 @@ def shaded(self): if self.map_type == "diff": args["grib_paths"] = [self.grib_path2] field2 = gribdata.FieldData(**args) - field.data = field.values() - field2.values() + field.data = field.data - field2.data return field @@ -578,7 +578,7 @@ def _draw_scatter(self, ax: Axes): field = self.field levels = self.field.clevs colors = self.field.colors - vals = self.field.values() + vals = self.field.data value_to_color = np.full_like(vals, colors[0], dtype="object") num_levels = len(levels) @@ -596,7 +596,7 @@ def _draw_scatter(self, ax: Axes): # Scatter plot dots are sized by value. Doing this here alters the size # without altering the colors we just set. - field.data = np.log10(field.values()) * 20 + field.data = np.log10(field.data) * 20 self._draw_field( ax=ax, diff --git a/create_graphics.py b/create_graphics.py index bd86711..ef8eeeb 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -71,7 +71,7 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): args = [(cla, fhr, grib_path, site, workdir) for site in cla.sites] print(f"Queueing {len(args)} Skew Ts") - # parallel_skewt(*args[0]) + #parallel_skewt(*args[0]) with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_skewt, args) @@ -119,10 +119,10 @@ def create_maps( ) ) - # parallel_maps(*args[-1]) - print(f"Queueing {len(args)} maps") - with Pool(processes=cla.nprocs) as pool: - pool.starmap(parallel_maps, args) + parallel_maps(*args[-1]) + # print(f"Queueing {len(args)} maps") + # with Pool(processes=cla.nprocs) as pool: + # pool.starmap(parallel_maps, args) def generate_tile_list(arg_list: list) -> list[str]: diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index 642af11..47b08e8 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -147,10 +147,12 @@ def test_uppdata_field(uppdata_obj_ro): def test_uppdata_field_column_max(prsfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "prs"}) fd = gribdata.FieldData( fhr=16, grib_paths=[prsfile], - level="500mb", + level="ua", model="hrrr", short_name="temp", spec=spec, @@ -290,7 +292,7 @@ def test_uppdata_vector_magnitude(prsfile, spec): def test_uppdata_vspec(uppdata_obj_ro): expected = { - "cfgrib": {"stepType": "instant", "typeOfLevel": "hybrid"}, + "cfgrib": {"shortName": "t", "typeOfLevel": "hybrid"}, "clevs": np.arange(-40, 40, 2.5), "cmap": "jet", "colors": "ua_temp_colors", diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py index d61c565..141c692 100644 --- a/tests/test_figure_builders.py +++ b/tests/test_figure_builders.py @@ -6,6 +6,7 @@ import numpy as np from pytest import fixture +from uwtools.api.config import get_yaml_config from adb_graphics import figure_builders, utils from adb_graphics.datahandler import gribdata @@ -13,6 +14,8 @@ @fixture def fielddata_obj(prsfile, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "prs"}) return gribdata.FieldData( model="hrrr", fhr=16, @@ -25,6 +28,8 @@ def fielddata_obj(prsfile, spec): @fixture def parallel_maps_args(prsfile, spec, tmp_path): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "prs"}) cla = Namespace( **{ # noqa: PIE804 "ens_size": 0, @@ -47,6 +52,8 @@ def parallel_maps_args(prsfile, spec, tmp_path): @fixture def parallel_skewt_args(natfile, spec, tmp_path): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "nat"}) cla = Namespace( **{ # noqa: PIE804 "file_type": "nat", @@ -67,12 +74,16 @@ def parallel_skewt_args(natfile, spec, tmp_path): } -@fixture +@fixture(scope="module") def spec(spec_file): - return utils.load_yaml(spec_file) + spec = utils.load_yaml(spec_file) + spec.dereference(context={"fhr": 16}) + return spec def test_add_obs_panel(fielddata_obj, spec): + spec = get_yaml_config(spec) + spec.dereference(context={"file_type": "nat"}) fig, ax = figure_builders.set_figure("hrrr", "enspanel", "full") # Overwriting this explicitly since the cfgrib should indefinitely come from the model data. spec["cref"]["obs"]["cfgrib"] = spec["1ref"]["1000m"]["cfgrib"]["hrrr"] From e8523a169868d045d306148dc4be8d2248fe512f Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 19 Nov 2025 14:03:36 -0700 Subject: [PATCH 44/98] Okay, NOW the tests pass. --- adb_graphics/figure_builders.py | 6 ++++-- tests/test_figure_builders.py | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 1e57380..5b880d0 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -230,8 +230,10 @@ def parallel_skewt(cla: Namespace, fhr: int, grib_paths: list[Path], site: str, possible_namers = (cla.model_name, cla.data_root, cla.file_tmpl) model = "" for name in ("global", "hrrr", "rrfs"): - if any(name in namer.lower() if isinstance(namer, str) else name in str(namer[0]).lower() for namer in possible_namers): - + if any( + name in namer.lower() if isinstance(namer, str) else name in str(namer[0]).lower() + for namer in possible_namers + ): model = name break skew = skewt.SkewTDiagram( diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py index 141c692..63d2edf 100644 --- a/tests/test_figure_builders.py +++ b/tests/test_figure_builders.py @@ -60,6 +60,8 @@ def parallel_skewt_args(natfile, spec, tmp_path): "img_res": 72, "max_plev": 100, "model_name": "hrrr", + "data_root": ["path"], + "file_tmpl": ["filename"], "start_time": datetime(2025, 10, 6, 0), "specs": spec, "images": ["hrrr", []], From 6a2dbcbeee405a38f6fc730008db36f3e1e314a4 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 20 Nov 2025 08:56:46 -0700 Subject: [PATCH 45/98] 30s skewts. --- adb_graphics/datahandler/gribdata.py | 33 +++++++++--- adb_graphics/datahandler/gribfile.py | 1 + adb_graphics/figures/skewt.py | 80 +++++++++++++++------------- create_graphics.py | 4 +- 4 files changed, 73 insertions(+), 45 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 29b76fb..52bd1e9 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -112,18 +112,31 @@ def _get_field(self, cfgribspec: dict) -> DataArray: cfgribspec the specifications dictionary to use for the variable in question """ + + def _find_var(): + if ds.get(var_name) is not None: + return var_name + + for var in ds: + if ds[var].attrs["GRIB_shortName"] == var_name: + return var + return None + if cfgribspec["typeOfLevel"] != self.vertical_dim: ds = gribfile.GribFiles(self.grib_paths, cfgribspec).contents.squeeze() else: ds = self.ds var_name = cfgribspec.get("shortName", self.cf_name) - if (field := ds.get(var_name)) is not None: - return DataArray(field) + var = _find_var() + if var is not None: + return DataArray(ds[var]) + + ds = gribfile.GribFiles(self.grib_paths, cfgribspec).contents.squeeze() + var = _find_var() + if var is not None: + return DataArray(ds[var]) - for var in ds: - if ds[var].attrs["GRIB_shortName"] == var_name: - return DataArray(ds[var]) msg = f"Variable {var_name} not found in dataset." raise ValueError(msg) @@ -709,10 +722,18 @@ def __init__( loc: str, short_name: str, spec: dict | YAMLConfig, + level: str | None = "ua", ): super().__init__( - fhr=fhr, grib_paths=grib_paths, model=model, short_name=short_name, spec=spec + fhr=fhr, + grib_paths=grib_paths, + level=level, + model=model, + short_name=short_name, + spec=spec, ) + + self.loc = loc # The first 31 columns are space delimted self.site_code, _, self.site_num, lat, lon = loc[:31].split() diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 132b60c..90e4beb 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -77,6 +77,7 @@ def _load(self, filenames: list[Path] | None = None): backend_kwargs=( { "filter_by_keys": self.cfgrib_config, + "indexpath": "", "read_keys": ["orientationOfTheGridInDegrees"], } ), diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index b820105..97dc9c7 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -140,15 +140,15 @@ def _add_hydrometeors(self, hydro_subplot: Axes): except errors.NoGraphicsDefinitionForVariableError: print(f"missing {mixr} for hydrometeor plot, skipping that field.") continue - mixr_total: units = 0.0 - for n in range(len(pres)): - if n == 0: - pres_layer = 2 * (pres[0] - pres[n]) # layer depth - pres_sigma = pres[0] - pres_layer # pressure at next sigma level - else: - pres_layer = 2 * (pres_sigma - pres[n]) # layer depth - pres_sigma = pres_sigma - pres_layer # pressure at next sigma level - mixr_total = mixr_total + pres_layer / gravity * profile[n] + if profile.any(): + mixr_total: units = 0.0 + for n in range(len(pres)): + if n == 0: + pres_sigma = pres[0] + else: + pres_layer = 2 * (pres_sigma - pres[n]) # layer depth + pres_sigma = pres_sigma - pres_layer # pressure at next sigma level + mixr_total = mixr_total + pres_layer / gravity * profile[n] # limit values to upper and lower values of plotting range profile = where((profile > 0.0) & (profile < 1.0e-4), 1.0e-4, profile) # noqa: PLR2004 @@ -166,32 +166,28 @@ def _add_hydrometeors(self, hydro_subplot: Axes): markersize=6, ) if mixr in ["clwmr", "rwmr"]: - hydro_subplot.plot( - profile[temp.magnitude < freezing_f], - pres[temp.magnitude < freezing_f], - settings.get("color", ""), - fillstyle="full", - linewidth=0.5, - marker=settings.get("marker"), - markersize=6, - ) - layer = False - for i, profile_lev in enumerate(profile): - if (profile_lev > 0.0 and temp[i].magnitude < freezing_f) and not layer: - layer = True - p_base = pres[i].magnitude - elif (profile_lev <= 0.0 or temp[i].magnitude > freezing_f) and layer: - # Shade the supercooled water depth - p_top = pres[i - 1].magnitude - rect = plt.Rectangle( - (0, p_top), - 100, - (p_base - p_top), - facecolor=settings.get("color"), - alpha=0.1, - ) - hydro_subplot.add_patch(rect) - layer = False + freezing_levs = profile.where( + (profile > 0.0) & (temp.magnitude < freezing_f), profile, 0 + ).to_numpy() + if freezing_levs.any(): + hydro_subplot.plot( + profile[temp.magnitude < freezing_f], + pres[temp.magnitude < freezing_f], + settings.get("color", ""), + fillstyle="full", + linewidth=0.5, + marker=settings.get("marker"), + markersize=6, + ) + pres_levs = pres[freezing_levs > 0].magnitude + rect = plt.Rectangle( + (0, pres_levs[-1]), + 100, + (pres_levs[0] - pres_levs[-1]), + facecolor=settings.get("color"), + alpha=0.1, + ) + hydro_subplot.add_patch(rect) # compute vertically integrated amount and add legend line label = settings.get("label") @@ -301,7 +297,7 @@ def atmo_profiles(self): for var, items in atmo_vars.items(): # Get the profile values and attach MetPy units - tmp = np.asarray(self.values(name=var)) * items["units"] + tmp = self.values(name=var).to_numpy() * items["units"] # Apply any needed transdecimals transform = items.get("transform") @@ -632,6 +628,16 @@ def thermo_variables(self): }, } + profile = gribdata.ProfileData( + fhr=self.fhr, + grib_paths=self.grib_paths, + level="mu", + loc=self.loc, + model=self.model, + short_name="cape", + spec=self.spec, + ) + for var, items in thermo.items(): varname = items.get("variable", var) lev = items.get("level", "ua") @@ -641,7 +647,7 @@ def thermo_variables(self): raise errors.NoGraphicsDefinitionForVariableError(varname, lev) try: - tmp = self.values(level=lev, name=varname) + tmp = profile.values(level=lev, name=varname) transforms = spec.get("transform") if transforms: diff --git a/create_graphics.py b/create_graphics.py index ef8eeeb..c929c23 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -68,10 +68,10 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): Generate arguments for parallel processing of Skew T graphics, and generate a pool of workers to complete the tasks. """ - args = [(cla, fhr, grib_path, site, workdir) for site in cla.sites] + args = [(cla, fhr, [grib_path], site, workdir) for site in cla.sites] print(f"Queueing {len(args)} Skew Ts") - #parallel_skewt(*args[0]) + # parallel_skewt(*args[0]) with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_skewt, args) From 22c301eaf5afc14ca0d8246a0d956248aaedc398 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 20 Nov 2025 09:46:46 -0700 Subject: [PATCH 46/98] Remove errant pylint directives. --- adb_graphics/conversions.py | 1 - adb_graphics/figure_builders.py | 2 -- adb_graphics/figures/maps.py | 1 - adb_graphics/specs.py | 1 - 4 files changed, 5 deletions(-) diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index 16f2c7f..fa303f5 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -1,4 +1,3 @@ -# pylint: disable=unused-argument,invalid-name """ This module contains functions for converting the units of a field. The interface requires a single atmospheric field in a Numpy array, and returns the diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 5b880d0..a7de6f1 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -1,4 +1,3 @@ -# pylint: disable=invalid-name """ This module is where pieces of the figures are put together. Data is compbined with maps and skewts to provide the final product. @@ -252,7 +251,6 @@ def parallel_skewt(cla: Namespace, fhr: int, grib_paths: list[Path], site: str, print(f"Creating image file: {png_path}") print("*" * 80) - # pylint: disable=duplicate-code plt.savefig( png_path, bbox_inches="tight", diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index ca84e74..f378779 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -381,7 +381,6 @@ def load_airports(self): class DataMap: - # pylint: disable=too-many-arguments """ Class that combines the input data and the chosen map to plot both together. diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index 3ed9fcf..fca30c6 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-public-methods """ This module sets the specifications for certain atmospheric variables. Typically this is related to a spec that needs some level of computation, i.e. a set of From af7c49e48f6846940bf425c30ee6897e718bb2a1 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 21 Nov 2025 14:46:05 -0700 Subject: [PATCH 47/98] Open file into single datastructure. --- adb_graphics/datahandler/gribdata.py | 114 +++++++++++---------------- adb_graphics/datahandler/gribfile.py | 41 +++++++++- adb_graphics/default_specs.yml | 12 +-- adb_graphics/figure_builders.py | 4 +- adb_graphics/figures/skewt.py | 32 +++++--- create_graphics.py | 21 +++-- tests/datahandler/test_gribdata.py | 58 +++++++------- tests/datahandler/test_gribfile.py | 8 +- tests/test_figure_builders.py | 2 +- 9 files changed, 162 insertions(+), 130 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 52bd1e9..336419c 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -9,7 +9,6 @@ from pathlib import Path import numpy as np -from cfgrib import DatasetBuildError from matplotlib.pyplot import get_cmap from pandas import to_datetime from uwtools.api.config import YAMLConfig @@ -50,14 +49,11 @@ def __init__( cf = deepcopy(self.vspec) utils.set_level(level=str(level), model=self.model, spec=cf) cf = utils.cfgrib_spec(cf["cfgrib"], self.model) - self.cf_name = cf.pop("shortName", "unknown") - self.vertical_dim = cf["typeOfLevel"] - try: - self.ds = gribfile.GribFiles(self.grib_paths, cf).contents.squeeze() - except DatasetBuildError as e: - if "stepType" in str(e): - cf["stepType"] = "instant" - self.ds = gribfile.GribFiles(self.grib_paths, cf).contents.squeeze() + self.vertical_coord = cf["typeOfLevel"] + if len(grib_paths) == 1: + self.ds = gribfile.WholeGribFile(self.grib_paths[0]).contents + else: + self.ds = gribfile.GribFiles(self.grib_paths, cf).contents @property def anl_dt(self) -> datetime: @@ -94,15 +90,15 @@ def field(self) -> DataArray: """ return self._get_field(self.vspec["cfgrib"].get(self.model, self.vspec["cfgrib"])) - def _get_data_levels(self, vertical_dim: str): + def _get_data_levels(self, vertical_coord: str): """ Values of the vertical dimension. Arg: - vertical_dim the name of the vertical dimension + vertical_coord the name of the vertical dimension """ - dim = [str(coord) for coord in self.ds.coords if vertical_dim in str(coord)][0] - return self.ds.coords[dim].to_numpy() + dim = [str(coord) for coord in self.field.coords if vertical_coord in str(coord)][0] + return self.field.coords[dim].to_numpy() def _get_field(self, cfgribspec: dict) -> DataArray: """ @@ -114,30 +110,41 @@ def _get_field(self, cfgribspec: dict) -> DataArray: """ def _find_var(): - if ds.get(var_name) is not None: - return var_name + if ds.get(short_name) is not None: + return short_name for var in ds: - if ds[var].attrs["GRIB_shortName"] == var_name: + if ds[var].attrs["GRIB_shortName"] == short_name: return var return None - if cfgribspec["typeOfLevel"] != self.vertical_dim: - ds = gribfile.GribFiles(self.grib_paths, cfgribspec).contents.squeeze() - else: - ds = self.ds - - var_name = cfgribspec.get("shortName", self.cf_name) - var = _find_var() - if var is not None: - return DataArray(ds[var]) - - ds = gribfile.GribFiles(self.grib_paths, cfgribspec).contents.squeeze() + short_name = cfgribspec.get("shortName", "unknown") + vertical_coord = cfgribspec["typeOfLevel"] + step_type = cfgribspec.get("stepType", "instant") + var_id = f"{short_name}_{vertical_coord}_{step_type}" + ds = self.ds.get(var_id) + if ds is None: + msg = f"{var_id} is not a valid key for the dataset" + raise ValueError(msg) var = _find_var() if var is not None: - return DataArray(ds[var]) - - msg = f"Variable {var_name} not found in dataset." + field = ds[var] + top = cfgribspec.get("topLevel", cfgribspec.get("scaledValueOfFirstFixedSurface")) + bottom = cfgribspec.get( + "bottomLevel", cfgribspec.get("scaledValueOfSecondFixedSurface") + ) + layered = top is not None or bottom is not None + level = top if top in field.coords[vertical_coord] else bottom + if level is None: + level = cfgribspec.get("level", utils.numeric_level(self.level)[0]) + level = None if level == "" else level + leveled = level is not None and vertical_coord != "hybrid" + if len(field.coords[vertical_coord].shape) > 0 and (layered or leveled): + if vertical_coord == "depthBelowLandLayer" and level: + level = level / 100.0 + field = field.sel(**{vertical_coord: level}) + return DataArray(field) + msg = f"Variable {short_name} not found in dataset." raise ValueError(msg) def get_transform(self, transforms: dict | list | str, val: DataArray) -> DataArray: @@ -201,12 +208,16 @@ def latlons(self) -> list[np.ndarray]: """Returns the set of latitudes and longitudes.""" coords = sorted( - [str(c) for c in list(self.ds.coords) if any(ele in str(c) for ele in ["lat", "lon"])] + [ + str(c) + for c in list(self.field.coords) + if any(ele in str(c) for ele in ["lat", "lon"]) + ] ) - lat = self.ds.coords[coords[0]].to_numpy() + lat = self.field.coords[coords[0]].to_numpy() if len(lat.shape) == 1 and lat[-1] < lat[0]: lat = lat[::-1] - lon = self.ds.coords[coords[-1]].to_numpy() + lon = self.field.coords[coords[-1]].to_numpy() return [lat, lon] @staticmethod @@ -380,7 +391,7 @@ def data(self, value: DataArray): def field_column_max(self, values: DataArray, **kwargs): # noqa: ARG002 """Returns the column max of the values.""" - return values.max(dim=self.vertical_dim) + return values.max(dim=self.vertical_coord) def field_diff(self, values: DataArray, variable2: str, level2: str, **kwargs): """Subtracts the values from variable2 from self.field.""" @@ -510,7 +521,7 @@ def grid_info(self) -> dict: attrs = ["GRIB_orientationOfTheGridInDegrees"] grid_info["projection"] = "stere" grid_info["lat_0"] = 90 - case x if x == "rotated latitude/longitude": # RRFS NA + case "rotated latitude/longitude": # RRFS NA attrs = [] grid_info["projection"] = "rotpole" lon_0: float = var_info.attrs["GRIB_longitudeOfSouthernPoleInDegrees"] @@ -522,41 +533,12 @@ def grid_info(self) -> dict: case x if "equidistant cylindrical" in x: # GFS attrs = [] grid_info["projection"] = "cyl" - case x if "rotate" in x: # RAP - attrs = [] - grid_info["projection"] = "rotpole" - # center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] - # center_lon = var_info.attrs["GRIB_longitudeOfLastGridPointInDegrees"] - # grid_info["lon_0"] = center_lon - 360 - grid_info["lon_0"] = -106.0 - # center_lat = var_info.attrs["GRIB_latitudeOfLastGridPointInDegrees"] - center_lat = 54 - grid_info["o_lat_p"] = 90 - center_lat - # grid_info['o_lat_p'] = -center_lat if center_lat < 0 else 90 - center_lat - grid_info["o_lon_p"] = 180 - # grid_info["corners"] = [-10.590603, 46.591938, -139.08585, 22.66102] case _: msg = f"Can't define grid for {grid_def}" raise ValueError(msg) if self.model != "hrrrhi": if not grid_info.get("corners"): grid_info["corners"] = self.corners - # if self.grid_suffix in ['GLC0']: - # attrs = ['Latin1', 'Latin2', 'Lov'] - # elif self.grid_suffix == 'GST0': - # elif self.grid_suffix == 'GLL0': - # attrs = [] - # grid_info['projection'] = 'cyl' - # else: - # attrs = [] - - # # CenterLon in RAP and Longitude_of_southern_pole in RRFS - # lon_0 = lat.attrs.get('CenterLon', lat.attrs.get('Longitude_of_southern_pole')) - # grid_info['lon_0'] = lon_0[0] - 360 - - # # CenterLat in RAP and Latitude_of_southern_pole in RRFS - # center_lat = lat.attrs.get('CenterLat', lat.attrs.get('Latitude_of_southern_pole')) - # grid_info['o_lat_p'] = - center_lat[0] if center_lat[0] < 0 else 90 - center_lat[0] for attr in attrs: bm_arg = keys_to_basemap[attr] @@ -666,7 +648,7 @@ def values( self, level: str | None = None, name: str | None = None, do_transform: bool = True ) -> DataArray: """ - Returns the numpy array of values at the requested level for the + Returns the FieldData array of values at the requested level for the variable after applying any unit conversion to the original data. Optional Input: @@ -677,7 +659,7 @@ def values( """ level = str(level or self.level) - vals: DataArray = self.ds.to_dataarray().squeeze() + vals = self.field spec = self.vspec if name is not None: diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 90e4beb..a90fa87 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -6,6 +6,7 @@ from pathlib import Path +import cfgrib import xarray as xr @@ -69,7 +70,7 @@ def __init__( def _load(self, filenames: list[Path] | None = None): """Load the set of files into a single XArray structure.""" filenames = self.filenames if filenames is None else filenames - return xr.open_mfdataset( + ds = xr.open_mfdataset( filenames, engine="cfgrib", concat_dim="time", @@ -77,8 +78,44 @@ def _load(self, filenames: list[Path] | None = None): backend_kwargs=( { "filter_by_keys": self.cfgrib_config, - "indexpath": "", + "indexpath": "", # create a temp file here or pyfakefs to hold it in mem. check + # unittests in wxvx "read_keys": ["orientationOfTheGridInDegrees"], } ), ) + return {_var_id(ds, list(ds.data_vars)[0]): ds} + + +class WholeGribFile: + def __init__( + self, + filename: Path, + ): + self.filename = filename + self.contents = self._load(filename) + + def _load(self, filename: Path): + datasets = cfgrib.open_datasets( + str(filename), read_keys=["orientationOfTheGridInDegrees", "parameterNumber"] + ) + + all_fields: dict = {} + for ds in datasets: + for var in ds.data_vars: + # var_name = var if var != "unknown" else ds[var].attrs.get("GRIB_cfName", + # ds[var].attrs.get("GRIB_shortName")) + var_id = _var_id(ds, str(var)) + if all_fields.get(var_id) is None: + all_fields[var_id] = ds + else: + msg = f"Multiple entries for {var_id} when opening {filename}" + raise ValueError(msg) + return all_fields + + +def _var_id(ds: xr.Dataset, var: str): + vertical_dim = ds[list(ds.data_vars)[0]].attrs.get("GRIB_typeOfLevel") + var_name = ds[var].attrs.get("GRIB_shortName") + step_type = ds[var].attrs.get("GRIB_stepType", "nostepType") + return f"{var_name}_{vertical_dim}_{step_type}" diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 10f93ae..60f34ae 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -133,6 +133,7 @@ acfrozr: # Run Total Graupel parameterNumber: 227 stepRange: '{{ "%d-%d" % (0, fhr) }}' typeOfLevel: surface + stepType: accum clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 2] cmap: gist_ncar colors: pcp_colors @@ -414,7 +415,7 @@ cape: mu: &cape # Most Unstable CAPE cfgrib: shortName: cape - level: 25500 + topLevel: 25500 typeOfLevel: pressureFromGroundLayer clevs: [1, 100, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000] cmap: gist_ncar @@ -442,7 +443,7 @@ cape: <<: *cape cfgrib: shortName: cape - level: 18000 + topLevel: 18000 typeOfLevel: pressureFromGroundLayer contours: cape: @@ -465,7 +466,7 @@ cape: <<: *cape cfgrib: shortName: cape - level: 9000 + topLevel: 9000 typeOfLevel: pressureFromGroundLayer contours: cape: @@ -3043,6 +3044,7 @@ vvel: # Vertical velocity cfgrib: shortName: wz typeOfLevel: sigmaLayer + stepType: avg level: 1 clevs: [-20, -15, -10, -5, -1, -0.75, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 0.75, 1, 5, 10, 15, 20] cmap: Spectral_r @@ -3056,7 +3058,7 @@ vvort: # Vertical vorticity cfgrib: shortName: max_vo typeOfLevel: heightAboveGroundLayer - level: 1000 + topLevel: 1000 stepType: max clevs: !arange [0.0025, 0.0301, 0.0025] cmap: gist_ncar @@ -3070,7 +3072,7 @@ vvort: # Vertical vorticity cfgrib: shortName: max_vo typeOfLevel: heightAboveGroundLayer - level: 2000 + topLevel: 2000 stepType: max title: 0-2km Max Vertical Vorticity (over prev hour) weasd: # Water equivalent of accumulated snow depth diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index a7de6f1..75d6349 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -212,7 +212,7 @@ def parallel_maps( # noqa: PLR0912 gc.collect() -def parallel_skewt(cla: Namespace, fhr: int, grib_paths: list[Path], site: str, workdir: Path): +def parallel_skewt(cla: Namespace, fhr: int, grib_path: Path, site: str, workdir: Path): """ Function that creates a single SkewT plot. @@ -237,7 +237,7 @@ def parallel_skewt(cla: Namespace, fhr: int, grib_paths: list[Path], site: str, break skew = skewt.SkewTDiagram( fhr=fhr, - grib_paths=grib_paths, + grib_paths=[grib_path], loc=site, model=model, spec=cla.specs, diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 97dc9c7..2452b12 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -140,8 +140,8 @@ def _add_hydrometeors(self, hydro_subplot: Axes): except errors.NoGraphicsDefinitionForVariableError: print(f"missing {mixr} for hydrometeor plot, skipping that field.") continue + mixr_total: units = 0.0 if profile.any(): - mixr_total: units = 0.0 for n in range(len(pres)): if n == 0: pres_sigma = pres[0] @@ -149,6 +149,7 @@ def _add_hydrometeors(self, hydro_subplot: Axes): pres_layer = 2 * (pres_sigma - pres[n]) # layer depth pres_sigma = pres_sigma - pres_layer # pressure at next sigma level mixr_total = mixr_total + pres_layer / gravity * profile[n] + mixr_total = mixr_total.to_numpy() # limit values to upper and lower values of plotting range profile = where((profile > 0.0) & (profile < 1.0e-4), 1.0e-4, profile) # noqa: PLR2004 @@ -191,9 +192,9 @@ def _add_hydrometeors(self, hydro_subplot: Axes): # compute vertically integrated amount and add legend line label = settings.get("label") - line = f"{label:<7s} {mixr_total.to_numpy():>10.3f} {settings.get('units')}" + line = f"{label:<7s} {mixr_total:>10.3f} {settings.get('units')}" if scale != 1.0: - line = f"{label:<5s}(x{scale}) {mixr_total.to_numpy():.3f} {settings.get('units')}" + line = f"{label:<5s}(x{scale}) {mixr_total:.3f} {settings.get('units')}" lines.append(line) handles.append( @@ -313,6 +314,7 @@ def create_diagram(self): skew, hydro_subplot = self._setup_diagram() self._title() + # breakpoint() self._plot_profile(skew) self._plot_wind_barbs(skew) self._plot_labels(skew) @@ -628,15 +630,15 @@ def thermo_variables(self): }, } - profile = gribdata.ProfileData( - fhr=self.fhr, - grib_paths=self.grib_paths, - level="mu", - loc=self.loc, - model=self.model, - short_name="cape", - spec=self.spec, - ) + # profile = gribdata.ProfileData( + # fhr=self.fhr, + # grib_paths=self.grib_paths, + # level="mu", + # loc=self.loc, + # model=self.model, + # short_name="cape", + # spec=self.spec, + # ) for var, items in thermo.items(): varname = items.get("variable", var) @@ -647,7 +649,11 @@ def thermo_variables(self): raise errors.NoGraphicsDefinitionForVariableError(varname, lev) try: - tmp = profile.values(level=lev, name=varname) + tmp = self.values(level=lev, name=varname) + # if varname == "hlcy": + # tmp = tmp.sel(heightAboveGroundLayer=3000 if var == "srh03" else 1000) + # elif var in ("mucin", "mucape"): + # tmp = tmp.sel(pressureFromGroundLayer=25500) transforms = spec.get("transform") if transforms: diff --git a/create_graphics.py b/create_graphics.py index c929c23..5227913 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -68,12 +68,17 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): Generate arguments for parallel processing of Skew T graphics, and generate a pool of workers to complete the tasks. """ - args = [(cla, fhr, [grib_path], site, workdir) for site in cla.sites] - + args = [(cla, fhr, grib_path, site, workdir) for site in cla.sites] + # Load global variable here with the full dataset? Process pool only. Try passing pickled object + # as preferred method. + # Concurrent futures library -- map a function over a pool and wait for it. + # global DS + # DS = gribfile.Gribfiles(grib_path).as_dict() print(f"Queueing {len(args)} Skew Ts") - # parallel_skewt(*args[0]) + #parallel_skewt(*args[0]) + # Maybe try a ThreadPool here? Threads help with io bound processes. CPU bound may be processes. with Pool(processes=cla.nprocs) as pool: - pool.starmap(parallel_skewt, args) + pool.starmap(parallel_skewt, args) def create_maps( @@ -119,10 +124,10 @@ def create_maps( ) ) - parallel_maps(*args[-1]) - # print(f"Queueing {len(args)} maps") - # with Pool(processes=cla.nprocs) as pool: - # pool.starmap(parallel_maps, args) + # parallel_maps(*args[-1]) + print(f"Queueing {len(args)} maps") + with Pool(processes=cla.nprocs) as pool: + pool.starmap(parallel_maps, args) def generate_tile_list(arg_list: list) -> list[str]: diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index 47b08e8..f41c2a0 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -17,7 +17,7 @@ def values( name: str | None = None, # noqa: ARG002 do_transform: bool = True, # noqa: ARG002 ) -> DataArray: - return self.ds.to_dataarray().squeeze() # type: ignore[no-any-return] + return self.field # type: ignore[no-any-return] @fixture @@ -143,7 +143,7 @@ def test_uppdata_date_to_str(uppdata_obj_ro): def test_uppdata_field(uppdata_obj_ro): - assert np.array_equal(uppdata_obj_ro.field, uppdata_obj_ro.ds.t.squeeze()) + assert np.array_equal(uppdata_obj_ro.field, uppdata_obj_ro.ds["t_hybrid_instant"].t) def test_uppdata_field_column_max(prsfile, spec): @@ -157,8 +157,8 @@ def test_uppdata_field_column_max(prsfile, spec): short_name="temp", spec=spec, ) - column_max = fd.field_column_max(values=fd.ds.t) - assert np.array_equal(column_max, fd.ds.t.squeeze().max(axis=0)) + column_max = fd.field_column_max(values=fd.field) + assert np.array_equal(column_max, fd.field.max(axis=0)) assert column_max.shape == (1059, 1799) @@ -166,7 +166,7 @@ def test_uppdata_field_diff(fielddata_obj_ro): summed_field = fielddata_obj_ro.field_diff( values=fielddata_obj_ro.field, variable2="temp", level2="sfc", do_transform=False ) - assert np.array_equal(summed_field, fielddata_obj_ro.ds.t.squeeze() * 0) + assert np.array_equal(summed_field, fielddata_obj_ro.field * 0) def test_uppdata_field_mean(prsfile, spec): @@ -181,7 +181,7 @@ def test_uppdata_field_mean(prsfile, spec): levels = ["500mb", "800mb"] mean = dataobj.field_mean(values=dataobj.field, levels=levels) assert np.array_equal( - mean, dataobj.ds.r.squeeze().sel(isobaricInhPa=[500, 800]).mean("isobaricInhPa") + mean, dataobj.field.squeeze().sel(isobaricInhPa=[500, 800]).mean("isobaricInhPa") ) assert mean.shape == (1059, 1799) @@ -190,20 +190,22 @@ def test_uppdata_field_sum(fielddata_obj_ro): summed_field = fielddata_obj_ro.field_sum( values=fielddata_obj_ro.field, variable2="temp", level2="sfc", do_transform=False ) - assert np.array_equal(summed_field, fielddata_obj_ro.ds.t.squeeze() * 2) + assert np.array_equal(summed_field, fielddata_obj_ro.field * 2) def test_uppdata__get_data_levels(uppdata_multilev_prs_obj): assert np.array_equal( uppdata_multilev_prs_obj._get_data_levels("isobaricInhPa"), - uppdata_multilev_prs_obj.ds.coords["isobaricInhPa"].to_numpy(), + uppdata_multilev_prs_obj.field.coords["isobaricInhPa"].to_numpy(), ) def test_uppdata__get_field(uppdata_multilev_prs_obj): spec = {"shortName": "t", "typeOfLevel": "isobaricInhPa", "level": 500} field = uppdata_multilev_prs_obj._get_field(cfgribspec=spec) - assert np.array_equal(field, uppdata_multilev_prs_obj.ds.t.squeeze()) + assert np.array_equal( + field, uppdata_multilev_prs_obj.ds["t_isobaricInhPa_instant"].t.sel(isobaricInhPa=500) + ) @mark.parametrize( @@ -218,7 +220,7 @@ def test_uppdata__get_field(uppdata_multilev_prs_obj): ], ) def test_uppdata_get_transform(fielddata_obj_ro, transforms): - temp = fielddata_obj_ro.ds.t + temp = fielddata_obj_ro.field val = ones_like(temp) if not isinstance(transforms, dict) else temp field = fielddata_obj_ro.get_transform(transforms, val) expected = 0 @@ -246,8 +248,8 @@ def test_uppdata_get_xypoint_outside(lat, lon, uppdata_obj_ro): def test_uppdata_latlons(uppdata_obj_ro): - lats = uppdata_obj_ro.ds.coords["latitude"].to_numpy() - lons = uppdata_obj_ro.ds.coords["longitude"].to_numpy() + lats = uppdata_obj_ro.field.coords["latitude"].to_numpy() + lons = uppdata_obj_ro.field.coords["longitude"].to_numpy() assert [ np.array_equal(act, exp) for act, exp in zip(uppdata_obj_ro.latlons(), [lats, lons], strict=True) @@ -256,11 +258,11 @@ def test_uppdata_latlons(uppdata_obj_ro): def test_uppdata_latlons_lats_flipped(uppdata_obj): # Test a 1D latitude option (like in Global, etc.) - ds = uppdata_obj.ds.sel(y=500) + ds = uppdata_obj.ds["t_hybrid_instant"].sel(y=500) lats = ds.coords["latitude"].to_numpy() ds.coords["latitude"] = (("x"), lats[::-1]) lons = ds.coords["longitude"].to_numpy() - uppdata_obj.ds = ds + uppdata_obj.ds = {"t_hybrid_instant": ds} assert [ np.array_equal(act, exp) for act, exp in zip(uppdata_obj.latlons(), [lats, lons], strict=True) @@ -286,8 +288,8 @@ def test_uppdata_vector_magnitude(prsfile, spec): short_name="u", spec=spec, ) - vm = fd.vector_magnitude(field1=fd.ds.u, field2_id="v_250mb") - assert not np.array_equal(vm, fd.ds.u) + vm = fd.vector_magnitude(field1=fd.field, field2_id="v_250mb") + assert not np.array_equal(vm, fd.field) def test_uppdata_vspec(uppdata_obj_ro): @@ -371,7 +373,7 @@ def test_fielddata_corners(fielddata_obj): def test_fielddata_corners_single_dim(fielddata_obj): # Remove one dimension for the purposes of the test - fielddata_obj.ds.coords["latitude"] = fielddata_obj.ds.coords["latitude"][:, 0] + fielddata_obj.field.coords["latitude"] = fielddata_obj.field.coords["latitude"][:, 0] assert fielddata_obj.corners == [ 21.13812299999999, 47.83862349881542, @@ -382,7 +384,7 @@ def test_fielddata_corners_single_dim(fielddata_obj): def test_fielddata_data_getter_and_setter(fielddata_obj): assert np.array_equal(fielddata_obj.data, fielddata_obj.values()) - new_data = ones_like(fielddata_obj.ds.t) + new_data = ones_like(fielddata_obj.field) fielddata_obj.data = new_data assert np.array_equal(fielddata_obj.data, new_data) @@ -442,7 +444,7 @@ def test_fielddata_supercooled_liquid_water(natfile, spec): spec=spec, ) slw = fd.supercooled_liquid_water() - assert not np.array_equal(slw, fd.ds.t.squeeze()) + assert not np.array_equal(slw, fd.field) def test_fielddata_ticks_default(fielddata_obj_ro): @@ -471,29 +473,25 @@ def test_fielddata_units_in_vspec(fielddata_obj): def test_fielddata_values_args_no_transform(fielddata_obj, lev, var): fielddata_obj.vspec["transform"] = None fielddata_obj.model = "hrrr" - assert not np.array_equal( - fielddata_obj.values(level=lev, name=var), fielddata_obj.ds.t.squeeze() - ) + assert not np.array_equal(fielddata_obj.values(level=lev, name=var), fielddata_obj.field) def test_fielddata_values_args_transform(fielddata_obj): fielddata_obj.vspec["transform"] = "opposite" fielddata_obj.model = "hrrr" - assert np.array_equal( - fielddata_obj.values(level="sfc", name="temp"), -fielddata_obj.ds.t.squeeze() - ) + assert np.array_equal(fielddata_obj.values(level="sfc", name="temp"), -fielddata_obj.field) def test_fielddata_values_no_args_no_transform(fielddata_obj): - field = ones_like(fielddata_obj.ds) - fielddata_obj.ds = field + field = ones_like(fielddata_obj.ds["t_surface_instant"]) + fielddata_obj.ds = {"t_surface_instant": field} fielddata_obj.vspec["transform"] = None - assert np.array_equal(fielddata_obj.values(), field.t.squeeze()) + assert np.array_equal(fielddata_obj.values(), field.t) def test_fielddata_values_no_args_transform(fielddata_obj): - field = ones_like(fielddata_obj.ds) - fielddata_obj.ds = field + field = ones_like(fielddata_obj.ds["t_surface_instant"]) + fielddata_obj.ds = {"t_surface_instant": field} fielddata_obj.vspec["transform"] = "opposite" assert np.array_equal(fielddata_obj.values(), -field.t.squeeze()) diff --git a/tests/datahandler/test_gribfile.py b/tests/datahandler/test_gribfile.py index 7ce4f43..94b2941 100644 --- a/tests/datahandler/test_gribfile.py +++ b/tests/datahandler/test_gribfile.py @@ -31,6 +31,8 @@ def test_gribfiles(): "typeOfLevel": "surface", }, ) - assert isinstance(gf.contents, Dataset) - assert len(gf.contents.data_vars) == 1 - assert len(gf.contents.data_vars["sp"].shape) == 3 + assert isinstance(gf.contents, dict) + assert isinstance(gf.contents["sp_surface_instant"], Dataset) + assert len(gf.contents) == 1 + assert len(gf.contents["sp_surface_instant"].data_vars) == 1 + assert len(gf.contents["sp_surface_instant"].data_vars["sp"].shape) == 3 diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py index 63d2edf..0b95eac 100644 --- a/tests/test_figure_builders.py +++ b/tests/test_figure_builders.py @@ -70,7 +70,7 @@ def parallel_skewt_args(natfile, spec, tmp_path): return { "cla": cla, "fhr": 16, - "grib_paths": [natfile], + "grib_path": natfile, "site": " DNR 23062 72469 39.77 104.88 1611 Denver, CO", "workdir": tmp_path, } From f1b4204851a3305d365abdf721bdd7a7cfc23709 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 25 Nov 2025 14:35:31 -0700 Subject: [PATCH 48/98] Open the file only once for most cases. --- adb_graphics/datahandler/gribdata.py | 34 +++++------------ adb_graphics/figure_builders.py | 44 +++++++++++----------- adb_graphics/figures/maps.py | 19 +++++----- adb_graphics/figures/skewt.py | 37 +++++------------- conftest.py | 22 ++++++++--- create_graphics.py | 19 +++++----- tests/datahandler/test_gribdata.py | 56 ++++++++++++++-------------- tests/test_figure_builders.py | 20 +++++----- 8 files changed, 116 insertions(+), 135 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 336419c..f103064 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -6,16 +6,14 @@ from copy import deepcopy from datetime import datetime, timedelta from functools import cached_property -from pathlib import Path import numpy as np from matplotlib.pyplot import get_cmap from pandas import to_datetime from uwtools.api.config import YAMLConfig -from xarray import DataArray, ufuncs, where +from xarray import DataArray, Dataset, ufuncs, where from adb_graphics import conversions, errors, specs, utils -from adb_graphics.datahandler import gribfile class UPPData(specs.VarSpec): @@ -33,13 +31,12 @@ class UPPData(specs.VarSpec): def __init__( self, fhr: int, - grib_paths: list[Path], + ds: dict[str, Dataset], model: str, short_name: str, spec: dict | YAMLConfig, level: str | None = "ua", ): - self.grib_paths = grib_paths self.model = model self.spec = spec self.short_name = short_name @@ -50,10 +47,7 @@ def __init__( utils.set_level(level=str(level), model=self.model, spec=cf) cf = utils.cfgrib_spec(cf["cfgrib"], self.model) self.vertical_coord = cf["typeOfLevel"] - if len(grib_paths) == 1: - self.ds = gribfile.WholeGribFile(self.grib_paths[0]).contents - else: - self.ds = gribfile.GribFiles(self.grib_paths, cf).contents + self.ds = ds @property def anl_dt(self) -> datetime: @@ -122,8 +116,8 @@ def _find_var(): vertical_coord = cfgribspec["typeOfLevel"] step_type = cfgribspec.get("stepType", "instant") var_id = f"{short_name}_{vertical_coord}_{step_type}" - ds = self.ds.get(var_id) - if ds is None: + ds: Dataset | dict = self.ds.get(var_id, {}) + if ds == {}: msg = f"{var_id} is not a valid key for the dataset" raise ValueError(msg) var = _find_var() @@ -293,7 +287,7 @@ class FieldData(UPPData): def __init__( self, fhr: int, - grib_paths: list[Path], + ds: dict[str, Dataset], level: str, model: str, short_name: str, @@ -303,7 +297,7 @@ def __init__( ): super().__init__( fhr=fhr, - grib_paths=grib_paths, + ds=ds, level=level, model=model, short_name=short_name, @@ -436,16 +430,6 @@ def fire_weather_index(self, values: DataArray, **kwargs): # noqa: ARG002 """ - def _load_field(level: str, short_name: str): - return FieldData( - fhr=int(self.fhr), - grib_paths=self.grib_paths, - level=level, - model=self.model, - short_name=short_name, - spec=self.spec, - ).values(do_transform=False) - # Gather fields from the input veg = values @@ -699,7 +683,7 @@ class ProfileData(UPPData): def __init__( self, fhr: int, - grib_paths: list[Path], + ds: dict[str, Dataset], model: str, loc: str, short_name: str, @@ -708,7 +692,7 @@ def __init__( ): super().__init__( fhr=fhr, - grib_paths=grib_paths, + ds=ds, level=level, model=model, short_name=short_name, diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 75d6349..9f01fc5 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -10,7 +10,9 @@ import matplotlib.pyplot as plt import numpy as np from matplotlib import axes +from xarray import Dataset +from adb_graphics.datahandler import gribfile from adb_graphics.figures import skewt from adb_graphics.figures.maps import DataMap, DiffMap, Map, MapFields, MultiPanelDataMap @@ -20,14 +22,14 @@ def add_obs_panel( ax: axes.Axes, model_name: str, - obs_file: Path, + dataset: dict[str, Dataset], proj_info: dict, spec: dict, short_name: str, tile: str, ): """ - Plot observation data provided by the obs_file + Plot observation data provided by the dataset path using the assigned projection. """ @@ -35,7 +37,7 @@ def add_obs_panel( map_fields = MapFields( fhr=0, fields_spec=spec, - grib_paths=[obs_file], + ds=dataset, level="obs", model="obs", name=short_name, @@ -58,15 +60,15 @@ def add_obs_panel( return dm.draw() -def parallel_maps( # noqa: PLR0912 +def parallel_maps( # noqa: PLR0915, PLR0912 cla: Namespace, fhr: int, - grib_paths: list[Path], + dataset: dict[str, Dataset], level: str, variable: str, workdir: Path, tile: str = "full", - dp2: Path | None = None, + ds2: Path | None = None, ): """ Function that creates plan-view maps, either a single panel, or @@ -75,16 +77,15 @@ def parallel_maps( # noqa: PLR0912 Input: fhr forecast hour - grib_paths paths to grib files + dataset loaded data level the vertical level of the variable to be plotted corresponding to a key in the specs file variable the name of the variable section in the specs file workdir output directory - tile Optional: tile the label of the tile being plotted - dp2 path to a second grib file + ds2 second dataset """ fig, axes = set_figure(cla.images[0], cla.graphic_type, tile) @@ -111,11 +112,14 @@ def parallel_maps( # noqa: PLR0912 if index in (top_left, lower_left): current_ax.axis("off") - ## If we have less than 10 members, skip the remaining panels. - # if index > cla.ens_size: - # continue - # Shenanigans to match ensemble member to panel index + # ---------------- + # | | 1 | 2 | 3 | + # ---------------- + # | 0 | 4 | 5 | 6 | + # ---------------- + # | o | 7 | 8 | 9 | + # ---------------- match index: case x if x in (top_left, center_left, lower_left): mem = 0 @@ -125,14 +129,11 @@ def parallel_maps( # noqa: PLR0912 mem = index - 1 case x if x < center_left: mem = index - # mem = 0 if index in (top_left, center_left, lower_left) else index - # mem = mem if mem < center_left else index - 1 - # mem = mem if mem < lower_left else index - 2 # Create an object that holds all the fields for this map map_fields = MapFields( - grib_paths=grib_paths, - grib_path2=dp2, + ds=dataset, + ds2=ds2, fhr=fhr, fields_spec=cla.specs, level=level, @@ -169,10 +170,11 @@ def parallel_maps( # noqa: PLR0912 if spec.get("include_obs", False) and cla.obs_file_path: # Add observation panel to lower left. Currently only # supported for composite reflectivity. + obs_ds = gribfile.WholeGribFile(cla.obs_file_path).contents add_obs_panel( ax=axes[8], model_name=cla.model_name, - obs_file=cla.obs_file_path, + dataset=obs_ds, proj_info=map_fields.shaded.grid_info(), short_name=variable, spec=cla.specs, @@ -212,7 +214,7 @@ def parallel_maps( # noqa: PLR0912 gc.collect() -def parallel_skewt(cla: Namespace, fhr: int, grib_path: Path, site: str, workdir: Path): +def parallel_skewt(cla: Namespace, fhr: int, dataset: dict[str, Dataset], site: str, workdir: Path): """ Function that creates a single SkewT plot. @@ -237,7 +239,7 @@ def parallel_skewt(cla: Namespace, fhr: int, grib_path: Path, site: str, workdir break skew = skewt.SkewTDiagram( fhr=fhr, - grib_paths=[grib_path], + ds=dataset, loc=site, model=model, spec=cla.specs, diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index f378779..a69e39c 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -19,6 +19,7 @@ from matplotlib.axes import Axes from matplotlib.contour import QuadContourSet from mpl_toolkits.basemap import Basemap, shiftgrid +from xarray import Dataset from adb_graphics.datahandler import gribdata from adb_graphics.utils import numeric_level, set_level @@ -119,7 +120,7 @@ def __init__( self, fhr: int, fields_spec: dict, - grib_paths: list[Path], + ds: dict[str, Dataset], level: str, name: str, map_type: str | None = None, @@ -127,7 +128,7 @@ def __init__( ): self.fhr = fhr self.fields_spec = deepcopy(fields_spec) - self.grib_paths = grib_paths + self.ds = ds self.level = level self.map_type = map_type self.model = kwargs.get("model", "") @@ -138,9 +139,9 @@ def __init__( set_level(self.level, self.model, self.map_spec) # Required if map_type is "diff" if map_type == "diff": - self.grib_path2: Path | str = kwargs.get("grib_path2", "") - if not self.grib_path2: - msg = "Diff map requires a second grib path. Provide grib_path2 argument!" + self.ds2: Path | str = kwargs.get("ds2", "") + if not self.ds2: + msg = "Diff map requires a second grib path. Provide ds2 argument!" raise ValueError(msg) @property @@ -151,11 +152,11 @@ def shaded(self): "model": self.model, "short_name": self.name, "spec": self.fields_spec, - "grib_paths": self.grib_paths, + "ds": self.ds, } field = gribdata.FieldData(**args) if self.map_type == "diff": - args["grib_paths"] = [self.grib_path2] + args["ds"] = self.ds2 field2 = gribdata.FieldData(**args) field.data = field.data - field2.data @@ -195,7 +196,7 @@ def wind_fields(self, level: str | None = None): "model": self.model, "short_name": var, "spec": self.fields_spec, - "grib_paths": self.grib_paths, + "ds": self.ds, } winds.append(gribdata.FieldData(**args)) return winds @@ -219,7 +220,7 @@ def _overlay_fields(self, spec_sect: str) -> list: "model": self.model, "short_name": var, "spec": self.fields_spec, - "grib_paths": self.grib_paths, + "ds": self.ds, } overlay_obj = gribdata.FieldData(**args) # Set the attributes for the overlay field diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 2452b12..87266b2 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -22,7 +22,7 @@ from metpy.units import units from mpl_toolkits.axes_grid1.inset_locator import inset_axes from uwtools.api.config import YAMLConfig -from xarray import DataArray, where +from xarray import DataArray, Dataset, where from adb_graphics import errors from adb_graphics.datahandler import gribdata @@ -62,7 +62,7 @@ class SkewTDiagram(gribdata.ProfileData): def __init__( self, fhr: int, - grib_paths: list[Path], + ds: dict[str, Dataset], loc: str, model: str, spec: dict | YAMLConfig, @@ -72,9 +72,7 @@ def __init__( # Initialize on the temperature field since we need to gather # field-specific data from this object, e.g. dates, lat, lon, etc. - super().__init__( - fhr=fhr, grib_paths=grib_paths, loc=loc, model=model, short_name="temp", spec=spec - ) + super().__init__(fhr=fhr, ds=ds, loc=loc, model=model, short_name="temp", spec=spec) self.max_plev = max_plev self.model_name = model_name @@ -298,11 +296,11 @@ def atmo_profiles(self): for var, items in atmo_vars.items(): # Get the profile values and attach MetPy units - tmp = self.values(name=var).to_numpy() * items["units"] + vals = self.values(name=var).to_numpy() * items["units"] - # Apply any needed transdecimals + # Apply any needed transformations transform = items.get("transform") - atmo_vars[var]["data"] = tmp.to(transform) if transform else tmp + atmo_vars[var]["data"] = vals.to(transform) if transform else vals return atmo_vars @@ -630,16 +628,6 @@ def thermo_variables(self): }, } - # profile = gribdata.ProfileData( - # fhr=self.fhr, - # grib_paths=self.grib_paths, - # level="mu", - # loc=self.loc, - # model=self.model, - # short_name="cape", - # spec=self.spec, - # ) - for var, items in thermo.items(): varname = items.get("variable", var) lev = items.get("level", "ua") @@ -649,19 +637,14 @@ def thermo_variables(self): raise errors.NoGraphicsDefinitionForVariableError(varname, lev) try: - tmp = self.values(level=lev, name=varname) - # if varname == "hlcy": - # tmp = tmp.sel(heightAboveGroundLayer=3000 if var == "srh03" else 1000) - # elif var in ("mucin", "mucape"): - # tmp = tmp.sel(pressureFromGroundLayer=25500) - + vals = self.values(level=lev, name=varname) transforms = spec.get("transform") if transforms: - tmp = self.get_transform(transforms, tmp) + vals = self.get_transform(transforms, vals) except errors.GribReadError: - tmp = DataArray([]) - thermo[var]["data"] = tmp + vals = DataArray([]) + thermo[var]["data"] = vals thermo[var]["units"] = spec.get("unit") return thermo diff --git a/conftest.py b/conftest.py index 9381d81..3cd24c5 100644 --- a/conftest.py +++ b/conftest.py @@ -8,10 +8,12 @@ import glob from pathlib import Path -import pytest +from pytest import fixture +from adb_graphics.datahandler import gribfile -@pytest.fixture(scope="session", autouse=True) + +@fixture(scope="session", autouse=True) def cleanup_data_idx(): yield # Nothing to be done before tests print("Removing idx files from test data") @@ -35,20 +37,30 @@ def pytest_addoption(parser): ) -@pytest.fixture(scope="session") +@fixture(scope="session") def natfile(): """Interface to pass a grib file to pytest.""" return Path("tests", "data", "wrfnat_hrconus_16.grib2") -@pytest.fixture(scope="session") +@fixture(scope="session") def prsfile(): """Interface to pass a grib file to pytest.""" return Path("tests", "data", "wrfprs_hrconus_16.grib2") -@pytest.fixture(scope="session") +@fixture(scope="session") def spec_file(): """Interface to pass a grib file to pytest.""" return Path("adb_graphics", "default_specs.yml") + + +@fixture(scope="session") +def prs_ds(prsfile): + return gribfile.WholeGribFile(prsfile).contents + + +@fixture(scope="session") +def nat_ds(natfile): + return gribfile.WholeGribFile(natfile).contents diff --git a/create_graphics.py b/create_graphics.py index 5227913..6d0b504 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -1,13 +1,10 @@ -# pylint: disable=invalid-name """ Driver for creating all the SkewT diagrams needed for a specific input dataset. """ -# pylint: disable=wrong-import-position, wrong-import-order import matplotlib as mpl mpl.use("Agg") -# pylint: enable=wrong-import-position, wrong-import-order import copy import glob @@ -23,6 +20,7 @@ import yaml from adb_graphics import errors, utils +from adb_graphics.datahandler import gribfile from adb_graphics.figure_builders import parallel_maps, parallel_skewt from adb_graphics.figures import maps @@ -68,17 +66,15 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): Generate arguments for parallel processing of Skew T graphics, and generate a pool of workers to complete the tasks. """ - args = [(cla, fhr, grib_path, site, workdir) for site in cla.sites] + ds = gribfile.WholeGribFile(grib_path).contents + args = [(cla, fhr, ds, site, workdir) for site in cla.sites] # Load global variable here with the full dataset? Process pool only. Try passing pickled object # as preferred method. # Concurrent futures library -- map a function over a pool and wait for it. - # global DS - # DS = gribfile.Gribfiles(grib_path).as_dict() print(f"Queueing {len(args)} Skew Ts") - #parallel_skewt(*args[0]) - # Maybe try a ThreadPool here? Threads help with io bound processes. CPU bound may be processes. + # parallel_skewt(*args[0]) with Pool(processes=cla.nprocs) as pool: - pool.starmap(parallel_skewt, args) + pool.starmap(parallel_skewt, args) def create_maps( @@ -93,6 +89,7 @@ def create_maps( generate a pool of workers to complete the task. """ + ds = gribfile.WholeGribFile(grib_paths[-1]).contents for tile in cla.tiles: args = [] for variable, levels in cla.images[1].items(): @@ -110,12 +107,14 @@ def create_maps( ) if (accumulate or grib_acc) and fhr == 0: continue + if accumulate: + ads = gribfile.GribFiles(grib_paths, vspec).contents args.append( ( cla, fhr, - grib_paths if accumulate else grib_paths[-1:], + ads if accumulate else ds, level, variable, workdir, diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index f41c2a0..0f1a37e 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -21,12 +21,12 @@ def values( @fixture -def fielddata_obj(prsfile, spec): +def fielddata_obj(prs_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "prs"}) return gribdata.FieldData( fhr=16, - grib_paths=[prsfile], + ds=prs_ds, level="sfc", model="hrrr", short_name="temp", @@ -35,12 +35,12 @@ def fielddata_obj(prsfile, spec): @fixture(scope="module") -def fielddata_obj_ro(prsfile, spec): +def fielddata_obj_ro(prs_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "prs"}) return gribdata.FieldData( fhr=16, - grib_paths=[prsfile], + ds=prs_ds, level="sfc", model="hrrr", short_name="temp", @@ -49,12 +49,12 @@ def fielddata_obj_ro(prsfile, spec): @fixture(scope="module") -def profiledata_obj(natfile, spec): +def profiledata_obj(nat_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "nat"}) return gribdata.ProfileData( fhr=16, - grib_paths=[natfile], + ds=nat_ds, loc=" DNR 23062 72469 39.77 104.88 1611 Denver, CO", model="hrrr", short_name="temp", @@ -70,7 +70,7 @@ def spec(spec_file): @fixture -def uppdata_obj(natfile, spec): +def uppdata_obj(nat_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "nat"}) return ConcreteUPPData( @@ -79,12 +79,12 @@ def uppdata_obj(natfile, spec): short_name="temp", spec=spec, fhr=16, - grib_paths=[natfile], + ds=nat_ds, ) @fixture(scope="module") -def uppdata_obj_ro(natfile, spec): +def uppdata_obj_ro(nat_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "nat"}) return ConcreteUPPData( @@ -93,12 +93,12 @@ def uppdata_obj_ro(natfile, spec): short_name="temp", spec=spec, fhr=16, - grib_paths=[natfile], + ds=nat_ds, ) @fixture -def uppdata_multilev_obj(natfile, spec): +def uppdata_multilev_obj(nat_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "nat"}) return ConcreteUPPData( @@ -106,12 +106,12 @@ def uppdata_multilev_obj(natfile, spec): short_name="temp", spec=spec, fhr=16, - grib_paths=[natfile], + ds=nat_ds, ) @fixture -def uppdata_multilev_prs_obj(prsfile, spec): +def uppdata_multilev_prs_obj(prs_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "prs"}) return ConcreteUPPData( @@ -120,7 +120,7 @@ def uppdata_multilev_prs_obj(prsfile, spec): short_name="temp", spec=spec, fhr=16, - grib_paths=[prsfile], + ds=prs_ds, ) @@ -146,12 +146,12 @@ def test_uppdata_field(uppdata_obj_ro): assert np.array_equal(uppdata_obj_ro.field, uppdata_obj_ro.ds["t_hybrid_instant"].t) -def test_uppdata_field_column_max(prsfile, spec): +def test_uppdata_field_column_max(prs_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "prs"}) fd = gribdata.FieldData( fhr=16, - grib_paths=[prsfile], + ds=prs_ds, level="ua", model="hrrr", short_name="temp", @@ -169,14 +169,14 @@ def test_uppdata_field_diff(fielddata_obj_ro): assert np.array_equal(summed_field, fielddata_obj_ro.field * 0) -def test_uppdata_field_mean(prsfile, spec): +def test_uppdata_field_mean(prs_ds, spec): dataobj = gribdata.FieldData( level="mean", model="hrrr", short_name="rh", spec=spec, fhr=16, - grib_paths=[prsfile], + ds=prs_ds, ) levels = ["500mb", "800mb"] mean = dataobj.field_mean(values=dataobj.field, levels=levels) @@ -279,12 +279,12 @@ def test_uppdata_valid_dt(uppdata_obj_ro): assert uppdata_obj_ro.valid_dt == datetime(2025, 10, 6, 16) -def test_uppdata_vector_magnitude(prsfile, spec): +def test_uppdata_vector_magnitude(prs_ds, spec): fd = ConcreteUPPData( model="hrrr", level="250mb", fhr=16, - grib_paths=[prsfile], + ds=prs_ds, short_name="u", spec=spec, ) @@ -323,10 +323,10 @@ def test_uppdata_vspec_bad(uppdata_obj): uppdata_obj.vspec # noqa: B018 -def test_fielddata_aviation_flight_rules(prsfile, spec): +def test_fielddata_aviation_flight_rules(prs_ds, spec): fd = gribdata.FieldData( fhr=16, - grib_paths=[prsfile], + ds=prs_ds, level="sfc", model="hrrr", short_name="flru", @@ -389,10 +389,10 @@ def test_fielddata_data_getter_and_setter(fielddata_obj): assert np.array_equal(fielddata_obj.data, new_data) -def test_fielddata_fire_weather_index(prsfile, spec): +def test_fielddata_fire_weather_index(prs_ds, spec): fd = gribdata.FieldData( fhr=16, - grib_paths=[prsfile], + ds=prs_ds, level="sfc", model="hrrr", short_name="firewxtransform", @@ -416,12 +416,12 @@ def test_fielddata_grid_info_lambert(fielddata_obj_ro): } -def test_fielddata_icing_adjust_trace(prsfile, spec): +def test_fielddata_icing_adjust_trace(prs_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "prs"}) fd = gribdata.FieldData( fhr=16, - grib_paths=[prsfile], + ds=prs_ds, level="sfc", model="hrrr", short_name="flru", @@ -432,12 +432,12 @@ def test_fielddata_icing_adjust_trace(prsfile, spec): assert np.array_equal(icing_adjust_trace, ones_like(field) * 0.5) -def test_fielddata_supercooled_liquid_water(natfile, spec): +def test_fielddata_supercooled_liquid_water(nat_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "nat"}) fd = gribdata.FieldData( fhr=16, - grib_paths=[natfile], + ds=nat_ds, level="sfc", model="hrrr", short_name="slw", diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py index 0b95eac..ac9096c 100644 --- a/tests/test_figure_builders.py +++ b/tests/test_figure_builders.py @@ -13,13 +13,13 @@ @fixture -def fielddata_obj(prsfile, spec): +def fielddata_obj(prs_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "prs"}) return gribdata.FieldData( model="hrrr", fhr=16, - grib_paths=[prsfile], + ds=prs_ds, level="sfc", short_name="cref", spec=spec, @@ -27,7 +27,7 @@ def fielddata_obj(prsfile, spec): @fixture -def parallel_maps_args(prsfile, spec, tmp_path): +def parallel_maps_args(prs_ds, spec, tmp_path): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "prs"}) cla = Namespace( @@ -43,7 +43,7 @@ def parallel_maps_args(prsfile, spec, tmp_path): return { "cla": cla, "fhr": 16, - "grib_paths": [prsfile], + "dataset": prs_ds, "level": "sfc", "variable": "temp", "workdir": tmp_path, @@ -51,7 +51,7 @@ def parallel_maps_args(prsfile, spec, tmp_path): @fixture -def parallel_skewt_args(natfile, spec, tmp_path): +def parallel_skewt_args(nat_ds, spec, tmp_path): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "nat"}) cla = Namespace( @@ -70,7 +70,7 @@ def parallel_skewt_args(natfile, spec, tmp_path): return { "cla": cla, "fhr": 16, - "grib_path": natfile, + "dataset": nat_ds, "site": " DNR 23062 72469 39.77 104.88 1611 Denver, CO", "workdir": tmp_path, } @@ -83,7 +83,7 @@ def spec(spec_file): return spec -def test_add_obs_panel(fielddata_obj, spec): +def test_add_obs_panel(fielddata_obj, nat_ds, spec): spec = get_yaml_config(spec) spec.dereference(context={"file_type": "nat"}) fig, ax = figure_builders.set_figure("hrrr", "enspanel", "full") @@ -92,7 +92,7 @@ def test_add_obs_panel(fielddata_obj, spec): args = { "ax": ax[8], "model_name": "hrrr", - "obs_file": fielddata_obj.grib_paths[0], # fake it with model data + "dataset": nat_ds, # fake it with model data "proj_info": fielddata_obj.grid_info(), "spec": spec, "short_name": "cref", @@ -108,10 +108,10 @@ def test_parallel_maps(parallel_maps_args, tmp_path): assert (tmp_path / "temp_full_sfc_f016.png").is_file() -def test_parallel_maps_enspanel(parallel_maps_args, tmp_path): +def test_parallel_maps_enspanel(parallel_maps_args, prsfile, tmp_path): parallel_maps_args["cla"].ens_size = 9 parallel_maps_args["cla"].graphic_type = "enspanel" - parallel_maps_args["cla"].obs_file_path = tmp_path + parallel_maps_args["cla"].obs_file_path = prsfile parallel_maps_args["cla"].specs["temp"]["sfc"]["include_obs"] = True with ( From 4a3d78941603b643a6f5f5fa1fb238971be7609f Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 1 Dec 2025 16:54:28 -0700 Subject: [PATCH 49/98] small fixes --- README.md | 2 +- adb_graphics/datahandler/gribdata.py | 6 +++++- adb_graphics/default_specs.yml | 4 +++- pyproject.toml | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 18c70ac..ab20ba9 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ activate this environment, do the following: ``` module use -a /contrib/miniconda3/modulefiles -module load miniconda3 +module load miniconda3/25.11.0 conda activate pygraf ``` diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 4d8844a..07ad2a4 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -46,7 +46,11 @@ def __init__( cf = deepcopy(self.vspec) utils.set_level(level=str(level), model=self.model, spec=cf) cf = utils.cfgrib_spec(cf["cfgrib"], self.model) - self.vertical_coord = cf["typeOfLevel"] + try: + self.vertical_coord = cf["typeOfLevel"] + except KeyError: + msg = f"typOfLevel is not a key for {short_name} at {level}. cf: {cf}" + raise KeyError(msg) self.ds = ds @property diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 60f34ae..59150e2 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -1291,6 +1291,7 @@ hlcy: # Helicity <<: *hlcy cfgrib: parameterNumber: 15 + typeOfLevel: heightAboveGroundLayer topLevel: 5000 bottomLevel: 2000 clevs: !arange [25, 301, 25] @@ -1460,7 +1461,7 @@ hpbl: # Height of Planetary Boundary Layer parameterNumber: 196 parameterCategory: 3 typeOfLevel: surface - rap: + rrfs: shortName: blh typeOfLevel: surface clevs: [25, 50, 100, 200, 300, 500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000] @@ -1648,6 +1649,7 @@ ltng: # Lightning shortName: ltng typeOfLevel: atmosphere level: 0 + stepType: max clevs: !arange [5, 91, 5] cmap: jet colors: graupel_colors diff --git a/pyproject.toml b/pyproject.toml index 1d874a6..14a142c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.coverage.report] exclude_also = ["if TYPE_CHECKING:"] -fail_under = 76 +fail_under = 75 show_missing = true skip_covered = true omit = ["conftest.py", "tests/*"] From deac2bc53e1d6f32b47f188d664da4b2d56dbee6 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 2 Dec 2025 12:51:27 -0700 Subject: [PATCH 50/98] Changes needed from self review. --- .github/workflows/graphics_tests.yml | 4 +- .github/workflows/hrrr_maps_tests.yml | 4 +- adb_graphics/datahandler/gribdata.py | 2 +- adb_graphics/datahandler/gribfile.py | 38 ++------- adb_graphics/default_specs.yml | 2 +- adb_graphics/figure_builders.py | 3 +- adb_graphics/figures/maps.py | 10 +-- adb_graphics/figures/skewt.py | 27 +++--- adb_graphics/utils.py | 6 ++ create_graphics.py | 5 -- image_lists/rap.yml | 114 -------------------------- image_lists/rap_subset.yml | 106 ------------------------ image_lists/rrfs_subset.yml | 10 +-- tests/datahandler/test_gribdata.py | 2 +- tests/datahandler/test_gribfile.py | 13 --- 15 files changed, 42 insertions(+), 304 deletions(-) delete mode 100644 image_lists/rap.yml delete mode 100644 image_lists/rap_subset.yml diff --git a/.github/workflows/graphics_tests.yml b/.github/workflows/graphics_tests.yml index 3fae546..b61efdf 100644 --- a/.github/workflows/graphics_tests.yml +++ b/.github/workflows/graphics_tests.yml @@ -25,8 +25,8 @@ jobs: cache-downloads: true cache-environment: true - name: Lint code - run: find . -type f -name "*.py" | xargs pylint + run: make lint shell: bash -el {0} - name: Test code - run: python -m pytest --nat-file tests/data/wrfnat_hrconus_07.grib2 --prs-file tests/data/wrfprs_hrconus_07.grib2 --ignore=tests/test_hrrr_maps.py + run: make test shell: bash -el {0} diff --git a/.github/workflows/hrrr_maps_tests.yml b/.github/workflows/hrrr_maps_tests.yml index a0d24ad..7bc8bb9 100644 --- a/.github/workflows/hrrr_maps_tests.yml +++ b/.github/workflows/hrrr_maps_tests.yml @@ -50,5 +50,5 @@ jobs: - name: Test code run: | export GITHUB_WORKSPACE=$(pwd) - python -m pytest tests/test_hrrr_maps.py - shell: bash -el {0} + pytest tests/test_hrrr_maps.py + shell: bash -el {0} diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 07ad2a4..a99b0d9 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -50,7 +50,7 @@ def __init__( self.vertical_coord = cf["typeOfLevel"] except KeyError: msg = f"typOfLevel is not a key for {short_name} at {level}. cf: {cf}" - raise KeyError(msg) + raise KeyError(msg) from None self.ds = ds @property diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index a90fa87..7a879e2 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -1,5 +1,3 @@ -# pylint: disable=invalid-name,too-few-public-methods,too-many-locals,too-many-branches,too-many-statements - """ Classes that load grib files. """ @@ -10,35 +8,6 @@ import xarray as xr -class GribFile: - """Wrappers and helper functions for interfacing with cfgrib.""" - - def __init__(self, filename: Path | str, cfgrib_config: dict): - # pylint: disable=unused-argument - - self.filename = filename - self.cfgrib_config = cfgrib_config - self.contents = self._load() - - def _load(self) -> xr.Dataset: - """ - Internal method that opens the grib file. Returns a grib message - iterator. - """ - - return xr.open_dataset( - self.filename, - engine="cfgrib", - lock=False, - backend_kwargs=( - { - "filter_by_keys": self.cfgrib_config, - "read_keys": ["orientationOfTheGridInDegrees"], - } - ), - ) - - class GribFiles: """ Class for loading in a set of grib files and combining them over @@ -88,6 +57,11 @@ def _load(self, filenames: list[Path] | None = None): class WholeGribFile: + """ + Class for loading a whole gribfile into a dictionary for different categories of data, mostly + separated by vertical coordinate and bucket type (avg, max, etc.). + """ + def __init__( self, filename: Path, @@ -103,8 +77,6 @@ def _load(self, filename: Path): all_fields: dict = {} for ds in datasets: for var in ds.data_vars: - # var_name = var if var != "unknown" else ds[var].attrs.get("GRIB_cfName", - # ds[var].attrs.get("GRIB_shortName")) var_id = _var_id(ds, str(var)) if all_fields.get(var_id) is None: all_fields[var_id] = ds diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 8c3db06..dcf1e35 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -876,7 +876,7 @@ ctop: # Cloud top height title: Cloud Top Height transform: conversions.m_to_kft unit: kft asl -dewp: # Dew point temperaeure +dewp: # Dew point temperature 2m: cfgrib: shortName: 2d diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 9f01fc5..768bb2b 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -112,7 +112,8 @@ def parallel_maps( # noqa: PLR0915, PLR0912 if index in (top_left, lower_left): current_ax.axis("off") - # Shenanigans to match ensemble member to panel index + # Shenanigans to match ensemble member to panel index. Here's where the ensemble members + # should go: # ---------------- # | | 1 | 2 | 3 | # ---------------- diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 976cdba..35a2bed 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -146,6 +146,9 @@ def __init__( @property def shaded(self): + """ + The main field to be shaded on the map. + """ args = { "fhr": self.fhr, "level": self.level, @@ -479,7 +482,6 @@ def draw(self, show: bool = False): # Create a pop-up to display the figure, if show=True if show: plt.tight_layout() - # plt.show() self.add_logo(self.map.ax) @@ -744,7 +746,7 @@ def _set_overlay_string(self) -> str: contoured.append(f"{user_title}") contoured_units.append(f"{cf.units}") - title = "\n".join(contoured) # Make 'contoured' a multioline string + title = "\n".join(contoured) # Make 'contoured' a multiline string if contoured_units: title = f"{title} ({', '.join(contoured_units)}, contoured)" @@ -846,9 +848,6 @@ def _xy_mesh(self, field: gribdata.FieldData): """Helper function to create mesh for various plot.""" lat, lon = field.latlons() - # if self.map.model == "obs": - # lat, lon = np.meshgrid(lat, lon, sparse=False, indexing="ij") - adjust = 360 if np.any(lon < 0) else 0 return self.map.m(adjust + lon, lat) @@ -968,7 +967,6 @@ def draw(self, show: bool = False): # Create a pop-up to display the figure, if show=True if show: plt.tight_layout() - # plt.show() return cf diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 87266b2..0dfcf82 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -116,8 +116,8 @@ def _add_hydrometeors(self, hydro_subplot: Axes): }, } - pres = self.atmo_profiles.get("pres").get("data") - temp = self.atmo_profiles.get("temp").get("data") + pres = self.atmo_profiles["pres"]["data"] + temp = self.atmo_profiles["temp"]["data"] handles = [] gravity = 9.81 # m/s^2 @@ -312,7 +312,6 @@ def create_diagram(self): skew, hydro_subplot = self._setup_diagram() self._title() - # breakpoint() self._plot_profile(skew) self._plot_wind_barbs(skew) self._plot_labels(skew) @@ -337,12 +336,12 @@ def _plot_hodograph(self, skew: SkewT): # Where the values above 10 km are unchanged, and there are three levels # in each of the 3 layers of interest. # - data_copy: units = np.copy(self.atmo_profiles.get("gh", {}).get("data")) + data_copy: units = np.copy(self.atmo_profiles["gh"]["data"]) agl = data_copy.to("km") # Retrieve the wind data profiles - u_wind = self.atmo_profiles.get("u", {}).get("data") - v_wind = self.atmo_profiles.get("v", {}).get("data") + u_wind = self.atmo_profiles["u"]["data"] + v_wind = self.atmo_profiles["v"]["data"] # Create an inset axes object that is 28% width and height of the # figure and put it in the upper left hand corner. @@ -389,11 +388,11 @@ def _plot_labels(skew: SkewT): def _write_profile(self, csv_path: str | Path): profiles = self.atmo_profiles # dictionary - pres = profiles.get("pres").get("data") - u = profiles.get("u").get("data") - v = profiles.get("v").get("data") - temp = profiles.get("temp").get("data").to("degC") - sphum = profiles.get("sphum").get("data") + pres = profiles["pres"]["data"] + u = profiles["u"]["data"] + v = profiles["v"]["data"] + temp = profiles["temp"]["data"].to("degC") + sphum = profiles["sphum"]["data"] dewpt = mpcalc.dewpoint_from_specific_humidity(pressure=pres, specific_humidity=sphum).to( "degC" @@ -442,9 +441,9 @@ def _plot_profile(self, skew: SkewT): def _plot_wind_barbs(self, skew: SkewT): # Pressure vs wind skew.plot_barbs( - self.atmo_profiles.get("pres", {}).get("data"), - self.atmo_profiles.get("u", {}).get("data"), - self.atmo_profiles.get("v", {}).get("data"), + self.atmo_profiles["pres"]["data"], + self.atmo_profiles["u"]["data"], + self.atmo_profiles["v"]["data"], color="blue", linewidth=0.2, y_clip_radius=0, diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index eaee0b8..3d4e1f1 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -23,6 +23,9 @@ def cfgrib_spec(config: dict, model: str) -> dict: + """ + Given a cfgrib block and a model, return the appropriate sub-block, if it exists. + """ spec: dict = config.get(model, {}) if spec and isinstance(spec, dict): return spec @@ -238,6 +241,9 @@ def path_exists(path: Path | str): def set_level(level: str, model: str, spec: dict): + """ + Given the default_specs level string, extract and set a numeric level in the cfgrib block. + """ nlevel, _ = numeric_level(level=level) level_info = any( key diff --git a/create_graphics.py b/create_graphics.py index 6d0b504..2bc617d 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -68,11 +68,7 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): """ ds = gribfile.WholeGribFile(grib_path).contents args = [(cla, fhr, ds, site, workdir) for site in cla.sites] - # Load global variable here with the full dataset? Process pool only. Try passing pickled object - # as preferred method. - # Concurrent futures library -- map a function over a pool and wait for it. print(f"Queueing {len(args)} Skew Ts") - # parallel_skewt(*args[0]) with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_skewt, args) @@ -123,7 +119,6 @@ def create_maps( ) ) - # parallel_maps(*args[-1]) print(f"Queueing {len(args)} maps") with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_maps, args) diff --git a/image_lists/rap.yml b/image_lists/rap.yml deleted file mode 100644 index cb54815..0000000 --- a/image_lists/rap.yml +++ /dev/null @@ -1,114 +0,0 @@ -hourly: - model: rap - variables: - 1hsnw: - - sfc - acsnod: - - sfc - acsnw: - - sfc - acfrozr: - - sfc - acfrzr: - - sfc - acpcp: - - sfc - cape: - - mu - - mul - - mx90mb - - sfc - ceil: - - ua - cin: - - sfc - cloudcover: - - high - - low - - mid - - total - cref: - - sfc - ctop: - - ua - dewp: - - 2m - echotop: - - sfc - flru: - - sfc - gust: - - 10m - hpbl: - - sfc - lhtfl: - - sfc - ltng: - - sfc - pchg: - - sfc - ptmp: - - 2m - ptyp: - - sfc - pwtr: - - sfc - rh: - - 2m - - 850mb - - mean - rhpw: - - sfc - rvil: - - sfc - shtfl: - - sfc - snod: - - sfc - soilt: - - 1cm - - 4cm - - 10cm - soilw: - - 0cm - - 1cm - - 4cm - - 10cm - solar: - - sfc - temp: - - 2ds - - 2m - - 500mb - - 700mb - - 850mb - - 925mb - - sfc - totp: - - sfc - ulwrf: - - sfc - - nta - uswrf: - - sfc - - nta - vil: - - sfc - vis: - - sfc - vort: - - 500mb - vvel: - - 700mb - wchg: - - 80m - wind: - - 10m - - 80m - - 850mb - - 250mb - wmag: - - 250mb - - 850mb - weasd: - - sfc diff --git a/image_lists/rap_subset.yml b/image_lists/rap_subset.yml deleted file mode 100644 index 17b178b..0000000 --- a/image_lists/rap_subset.yml +++ /dev/null @@ -1,106 +0,0 @@ -hourly: - model: rap - variables: -# 1hsnw: -# - sfc -# acsnod: -# - sfc - acfrozr: - - sfc - acfrzr: - - sfc - acpcp: - - sfc - acsnw: - - sfc - cape: - - mu - - mul - - mx90mb - - sfc - ceil: - - ua - cin: - - sfc - cloudcover: - - high - - low - - mid - - total - cref: - - sfc - ctop: - - ua - dewp: - - 2m - echotop: - - sfc - flru: - - sfc - gust: - - 10m - hpbl: - - sfc - lhtfl: - - sfc - ptmp: - - 2m - ptyp: - - sfc - pwtr: - - sfc - rh: - - 2m - - 850mb - - mean - - pw - rvil: - - sfc - shtfl: - - sfc - snod: - - sfc - soilt: &soilt_levs - - 0cm - - 1cm - - 4cm - - 10cm - - 30cm - - 60cm - - 1m - - 1.6m - - 3m - soilw: *soilt_levs - solar: - - sfc - temp: - - 2ds - - 2m - - 500mb - - 700mb - - 850mb - - 925mb - - sfc - totp: - - sfc - ulwrf: - - sfc - - top - uswrf: - - sfc - - top - vil: - - sfc - vis: - - sfc - vort: - - 500mb - vvel: - - 700mb - weasd: - - sfc - wspeed: - - 10m - - 80m - - 250mb - - 850mb diff --git a/image_lists/rrfs_subset.yml b/image_lists/rrfs_subset.yml index 6c752c8..0bed058 100644 --- a/image_lists/rrfs_subset.yml +++ b/image_lists/rrfs_subset.yml @@ -64,11 +64,11 @@ hourly: - mx25 - sr01 - sr03 -# hlcytot: -# - mn03 -# - mn25 -# - mx03 -# - mx25 + hlcytot: + - mn03 + - mn25 + - mx03 + - mx25 hpbl: - sfc lcl: diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index 7a6a659..4143727 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -395,7 +395,7 @@ def test_fielddata_fire_weather_index(prs_ds, spec): ds=prs_ds, level="sfc", model="hrrr", - short_name="firewxtransform", + short_name="firewx-pygraf", spec=spec, ) diff --git a/tests/datahandler/test_gribfile.py b/tests/datahandler/test_gribfile.py index 3fb3bf7..d799913 100644 --- a/tests/datahandler/test_gribfile.py +++ b/tests/datahandler/test_gribfile.py @@ -6,19 +6,6 @@ from adb_graphics.datahandler import gribfile -def test_gribfile(prsfile): - gf = gribfile.GribFile( - filename=Path(prsfile), - cfgrib_config={ - "shortName": "sp", - "typeOfLevel": "surface", - }, - ) - assert isinstance(gf.contents, Dataset) - assert len(gf.contents.data_vars) == 1 - assert len(gf.contents.data_vars["sp"].shape) == 2 - - @mark.skip(reason="This test requires test data that is not yet available.") def test_gribfiles(): paths = [ From fdef24d438264df017566c806bf07323b4430219 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 2 Dec 2025 12:55:53 -0700 Subject: [PATCH 51/98] Need to install the dev packages. --- .github/workflows/graphics_tests.yml | 6 +++--- .github/workflows/hrrr_maps_tests.yml | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/graphics_tests.yml b/.github/workflows/graphics_tests.yml index b61efdf..06135f7 100644 --- a/.github/workflows/graphics_tests.yml +++ b/.github/workflows/graphics_tests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} steps: - name: Checkout repo uses: actions/checkout@v2 @@ -24,9 +24,9 @@ jobs: environment-file: environment.yml cache-downloads: true cache-environment: true + - name: Install dev pkgs + run: make devenv - name: Lint code run: make lint - shell: bash -el {0} - name: Test code run: make test - shell: bash -el {0} diff --git a/.github/workflows/hrrr_maps_tests.yml b/.github/workflows/hrrr_maps_tests.yml index 7bc8bb9..cddd219 100644 --- a/.github/workflows/hrrr_maps_tests.yml +++ b/.github/workflows/hrrr_maps_tests.yml @@ -47,6 +47,8 @@ jobs: environment-file: environment.yml cache-downloads: true cache-env: true + - name: Install dev pkgs + run: make devenv - name: Test code run: | export GITHUB_WORKSPACE=$(pwd) From 1c9b054ea31deec9989ac4eee3f5f7a65a32e8cc Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 2 Dec 2025 13:01:12 -0700 Subject: [PATCH 52/98] Activate mamba? --- .github/workflows/graphics_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graphics_tests.yml b/.github/workflows/graphics_tests.yml index 06135f7..464c782 100644 --- a/.github/workflows/graphics_tests.yml +++ b/.github/workflows/graphics_tests.yml @@ -25,7 +25,7 @@ jobs: cache-downloads: true cache-environment: true - name: Install dev pkgs - run: make devenv + run: mamba activate && make devenv - name: Lint code run: make lint - name: Test code From 0eda1c836e2b28b2332d238fc4a90162f038283b Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 2 Dec 2025 13:07:45 -0700 Subject: [PATCH 53/98] Try again. --- .github/workflows/graphics_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/graphics_tests.yml b/.github/workflows/graphics_tests.yml index 464c782..ef3699e 100644 --- a/.github/workflows/graphics_tests.yml +++ b/.github/workflows/graphics_tests.yml @@ -25,7 +25,8 @@ jobs: cache-downloads: true cache-environment: true - name: Install dev pkgs - run: mamba activate && make devenv + run: make devenv + shell: bash -el {0} - name: Lint code run: make lint - name: Test code From cf13c58c5b276aac8c3ba8744c39a13f4c8b664a Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 2 Dec 2025 13:13:13 -0700 Subject: [PATCH 54/98] Try again --- .github/workflows/graphics_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graphics_tests.yml b/.github/workflows/graphics_tests.yml index ef3699e..ae84aae 100644 --- a/.github/workflows/graphics_tests.yml +++ b/.github/workflows/graphics_tests.yml @@ -26,7 +26,7 @@ jobs: cache-environment: true - name: Install dev pkgs run: make devenv - shell: bash -el {0} + shell: micromamba-shell {0} - name: Lint code run: make lint - name: Test code From c6243c5a2c427311ea5332996f723651633115d9 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 2 Dec 2025 13:25:17 -0700 Subject: [PATCH 55/98] Try again --- .github/workflows/graphics_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graphics_tests.yml b/.github/workflows/graphics_tests.yml index ae84aae..879cee5 100644 --- a/.github/workflows/graphics_tests.yml +++ b/.github/workflows/graphics_tests.yml @@ -26,7 +26,7 @@ jobs: cache-environment: true - name: Install dev pkgs run: make devenv - shell: micromamba-shell {0} + shell: bash -leo pipefail {0} - name: Lint code run: make lint - name: Test code From 3be841f0acdb29b8c42a242c352a4ed81b20557d Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 2 Dec 2025 13:34:09 -0700 Subject: [PATCH 56/98] Let's try miniforge --- .github/workflows/graphics_tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/graphics_tests.yml b/.github/workflows/graphics_tests.yml index 879cee5..9b1d019 100644 --- a/.github/workflows/graphics_tests.yml +++ b/.github/workflows/graphics_tests.yml @@ -15,18 +15,18 @@ jobs: shell: bash -el {0} steps: - name: Checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@v5 with: lfs: true - - name: Install Micromamba with pygraf environment - uses: mamba-org/setup-micromamba@v1 + - name: Install Miniforge with pygraf environment + uses: conda-incubator/setup-miniconda@v3 with: environment-file: environment.yml + miniforge-version: latest cache-downloads: true cache-environment: true - name: Install dev pkgs run: make devenv - shell: bash -leo pipefail {0} - name: Lint code run: make lint - name: Test code From 590fc826fdff2792863758de8cac2210628548bf Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 2 Dec 2025 13:39:26 -0700 Subject: [PATCH 57/98] Miniforge worked! Activate env and use for hrrr tests --- .github/workflows/graphics_tests.yml | 8 +++----- .github/workflows/hrrr_maps_tests.yml | 9 ++++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/graphics_tests.yml b/.github/workflows/graphics_tests.yml index 9b1d019..6c251a0 100644 --- a/.github/workflows/graphics_tests.yml +++ b/.github/workflows/graphics_tests.yml @@ -23,11 +23,9 @@ jobs: with: environment-file: environment.yml miniforge-version: latest - cache-downloads: true - cache-environment: true - name: Install dev pkgs run: make devenv - - name: Lint code - run: make lint - name: Test code - run: make test + run: | + conda activate pygraf + make test diff --git a/.github/workflows/hrrr_maps_tests.yml b/.github/workflows/hrrr_maps_tests.yml index cddd219..24998fa 100644 --- a/.github/workflows/hrrr_maps_tests.yml +++ b/.github/workflows/hrrr_maps_tests.yml @@ -18,7 +18,7 @@ jobs: shell: bash -l {0} steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Create data and output folders run: | mkdir -p $OUTPUT_LOC @@ -41,16 +41,15 @@ jobs: https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf12.grib2 EOF ls - - name: Install Micromamba with pygraf environment - uses: mamba-org/setup-micromamba@v1 + - name: Install Miniforge with pygraf environment + uses: conda-incubator/setup-miniconda@v3 with: environment-file: environment.yml - cache-downloads: true - cache-env: true - name: Install dev pkgs run: make devenv - name: Test code run: | export GITHUB_WORKSPACE=$(pwd) + conda activate pygraf pytest tests/test_hrrr_maps.py shell: bash -el {0} From 2825512b89099392ea8b127774ee33b5b08437fd Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 2 Dec 2025 14:02:52 -0700 Subject: [PATCH 58/98] Refine test filtering. --- .github/workflows/hrrr_maps_tests.yml | 3 ++- Makefile | 4 ++-- pyproject.toml | 2 +- tests/test_hrrr_maps.py | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/hrrr_maps_tests.yml b/.github/workflows/hrrr_maps_tests.yml index 24998fa..c9f168e 100644 --- a/.github/workflows/hrrr_maps_tests.yml +++ b/.github/workflows/hrrr_maps_tests.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - shell: bash -l {0} + shell: bash -el {0} steps: - name: Checkout repo uses: actions/checkout@v5 @@ -45,6 +45,7 @@ jobs: uses: conda-incubator/setup-miniconda@v3 with: environment-file: environment.yml + miniforge-version: latest - name: Install dev pkgs run: make devenv - name: Test code diff --git a/Makefile b/Makefile index 1f8fd02..52e99e0 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,8 @@ typecheck: mypy --install-types --non-interactive . unittest: - pytest --cov -k "not hrrr" -n 4 . + pytest --cov -k "not hrrr_maps" -n 4 . memtest: - pytest --memray -k"not hrrr" . + pytest --memray -k "not hrrr" . diff --git a/pyproject.toml b/pyproject.toml index 14a142c..9544081 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.coverage.report] exclude_also = ["if TYPE_CHECKING:"] -fail_under = 75 +fail_under = 70 show_missing = true skip_covered = true omit = ["conftest.py", "tests/*"] diff --git a/tests/test_hrrr_maps.py b/tests/test_hrrr_maps.py index f1f1c4c..8b84c31 100644 --- a/tests/test_hrrr_maps.py +++ b/tests/test_hrrr_maps.py @@ -34,7 +34,7 @@ def maps_args(tmp_path) -> list: ] -def test_hrrr_parse_args(tmp_path): +def test_hrrr_maps_parse_args(tmp_path): """ Test parse_args for basic parsing success. Checks if parse_args returns 'maps' in the graphic_type field. @@ -63,7 +63,7 @@ def test_hrrr_parse_args(tmp_path): assert test_args.graphic_type == "maps" -def test_hrrr_file_count(maps_args, tmp_path): +def test_hrrr_maps_file_count(maps_args, tmp_path): """ Test for file count in directory. Can be extended to cover multiple folders. From e45777839d3a98dd2a30182cac1f2cf39c15984b Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 2 Dec 2025 15:21:21 -0700 Subject: [PATCH 59/98] Run this test with more procs and fewer lead times. --- .github/workflows/hrrr_maps_tests.yml | 6 ------ image_lists/hrrr_test.yml | 12 ++++++------ tests/test_hrrr_maps.py | 8 ++++---- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/hrrr_maps_tests.yml b/.github/workflows/hrrr_maps_tests.yml index c9f168e..dcdb5e4 100644 --- a/.github/workflows/hrrr_maps_tests.yml +++ b/.github/workflows/hrrr_maps_tests.yml @@ -33,12 +33,6 @@ jobs: https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf04.grib2 https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf05.grib2 https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf06.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf07.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf08.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf09.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf10.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf11.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf12.grib2 EOF ls - name: Install Miniforge with pygraf environment diff --git a/image_lists/hrrr_test.yml b/image_lists/hrrr_test.yml index 9646336..70c2186 100644 --- a/image_lists/hrrr_test.yml +++ b/image_lists/hrrr_test.yml @@ -2,9 +2,9 @@ hourly: model: hrrr variables: hlcytot: - - mn02 - - mn03 - - mn25 - - mx02 - - mx03 - - mx25 \ No newline at end of file + - mn02 + - mn03 + - mn25 + - mx02 + - mx03 + - mx25 diff --git a/tests/test_hrrr_maps.py b/tests/test_hrrr_maps.py index 8b84c31..2065a67 100644 --- a/tests/test_hrrr_maps.py +++ b/tests/test_hrrr_maps.py @@ -18,10 +18,12 @@ def maps_args(tmp_path) -> list: DATA_LOC, "-f", "0", - "12", + "6", "1", "-o", str(tmp_path / "output"), + "-n", + "3", "-s", "2023031500", "--file_tmpl", @@ -72,9 +74,7 @@ def test_hrrr_maps_file_count(maps_args, tmp_path): create_graphics(maps_args) map_count = 6 count = 0 - folder = "/202303150000/" - output = tmp_path / "output" / folder - assert output.isdir() + output = tmp_path / "output" / "202303150003" for file_name in output.iterdir(): if (output / file_name).is_file(): count += 1 From b95a7b7c28312c3d0b41a235b871af70cd9b9bbe Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 3 Dec 2025 09:09:26 -0700 Subject: [PATCH 60/98] Verbose --- .github/workflows/hrrr_maps_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hrrr_maps_tests.yml b/.github/workflows/hrrr_maps_tests.yml index dcdb5e4..1e1d729 100644 --- a/.github/workflows/hrrr_maps_tests.yml +++ b/.github/workflows/hrrr_maps_tests.yml @@ -46,5 +46,5 @@ jobs: run: | export GITHUB_WORKSPACE=$(pwd) conda activate pygraf - pytest tests/test_hrrr_maps.py + pytest -vv tests/test_hrrr_maps.py shell: bash -el {0} From b6e6bb7a555e0e34b7537cce2f187689043adc3a Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 3 Dec 2025 09:16:51 -0700 Subject: [PATCH 61/98] MORE verbose --- .github/workflows/hrrr_maps_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hrrr_maps_tests.yml b/.github/workflows/hrrr_maps_tests.yml index 1e1d729..a3adf73 100644 --- a/.github/workflows/hrrr_maps_tests.yml +++ b/.github/workflows/hrrr_maps_tests.yml @@ -46,5 +46,5 @@ jobs: run: | export GITHUB_WORKSPACE=$(pwd) conda activate pygraf - pytest -vv tests/test_hrrr_maps.py + pytest -vv -s tests/test_hrrr_maps.py shell: bash -el {0} From 7689e6e9333228b7014e6abaec4a45b92bcaac5e Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 3 Dec 2025 09:32:42 -0700 Subject: [PATCH 62/98] Try again. --- tests/test_hrrr_maps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_hrrr_maps.py b/tests/test_hrrr_maps.py index 2065a67..cd6e878 100644 --- a/tests/test_hrrr_maps.py +++ b/tests/test_hrrr_maps.py @@ -14,6 +14,8 @@ def maps_args(tmp_path) -> list: """Builds HRRR 12-hour accumulated maps.""" return [ "maps", + "-a", + "1", "-d", DATA_LOC, "-f", @@ -22,8 +24,6 @@ def maps_args(tmp_path) -> list: "1", "-o", str(tmp_path / "output"), - "-n", - "3", "-s", "2023031500", "--file_tmpl", From f12506a1e1d75c9979110d2bc84854c822128a4f Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 3 Dec 2025 09:33:28 -0700 Subject: [PATCH 63/98] Only one map. --- image_lists/hrrr_test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/image_lists/hrrr_test.yml b/image_lists/hrrr_test.yml index 70c2186..a0225ac 100644 --- a/image_lists/hrrr_test.yml +++ b/image_lists/hrrr_test.yml @@ -3,8 +3,8 @@ hourly: variables: hlcytot: - mn02 - - mn03 - - mn25 - - mx02 - - mx03 - - mx25 +# - mn03 +# - mn25 +# - mx02 +# - mx03 +# - mx25 From e97c5a3677ca4df083de8b0302d0b7496ee09e7f Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 3 Dec 2025 13:08:18 -0700 Subject: [PATCH 64/98] Changes discussed in person at review. --- .github/workflows/graphics_tests.yml | 2 +- .github/workflows/hrrr_maps_tests.yml | 50 ------------------ README.md | 74 ++++++++++++++++++++++++--- pre.sh | 2 +- tests/test_hrrr_maps.py | 2 +- 5 files changed, 70 insertions(+), 60 deletions(-) delete mode 100644 .github/workflows/hrrr_maps_tests.yml diff --git a/.github/workflows/graphics_tests.yml b/.github/workflows/graphics_tests.yml index 6c251a0..9923ace 100644 --- a/.github/workflows/graphics_tests.yml +++ b/.github/workflows/graphics_tests.yml @@ -15,7 +15,7 @@ jobs: shell: bash -el {0} steps: - name: Checkout repo - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: lfs: true - name: Install Miniforge with pygraf environment diff --git a/.github/workflows/hrrr_maps_tests.yml b/.github/workflows/hrrr_maps_tests.yml deleted file mode 100644 index a3adf73..0000000 --- a/.github/workflows/hrrr_maps_tests.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: hrrr_maps_tests -env: - DATA_LOC: ${{ github.workspace }}/input_data - OUTPUT_LOC: ${{ github.workspace }}/output -on: - push: - branches: - - main - pull_request: - branches: - - main - workflow_dispatch: -jobs: - test_hrrr_maps: - runs-on: ubuntu-latest - defaults: - run: - shell: bash -el {0} - steps: - - name: Checkout repo - uses: actions/checkout@v5 - - name: Create data and output folders - run: | - mkdir -p $OUTPUT_LOC - mkdir -p $DATA_LOC - - name: Fetch Grib Files - run: | - wget -N -P $DATA_LOC -i - << EOF - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf00.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf01.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf02.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf03.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf04.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf05.grib2 - https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20230315/conus/hrrr.t00z.wrfprsf06.grib2 - EOF - ls - - name: Install Miniforge with pygraf environment - uses: conda-incubator/setup-miniconda@v3 - with: - environment-file: environment.yml - miniforge-version: latest - - name: Install dev pkgs - run: make devenv - - name: Test code - run: | - export GITHUB_WORKSPACE=$(pwd) - conda activate pygraf - pytest -vv -s tests/test_hrrr_maps.py - shell: bash -el {0} diff --git a/README.md b/README.md index ab20ba9..8c6200a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # ADB Graphics Creation for UPP Model Output -> Note: This repository is under heavy development. Use at your own risk! - This repository houses a Python-based implementation of the graphics package that is responsible for generating maps for the RAP/HRRR/FV3/RRFS data. It has replaced NCL as the real-time graphics creation package at NOAA GSL for maps and @@ -44,6 +42,26 @@ module load miniconda3/25.11.0 conda activate pygraf ``` +This environment contains the necessary develepment packages. + +## Installing with conda + +Pygraf comes with an environment.yml file for use with any conda installation. Ensure the conda base +environment is activated, and run the following command to create a `pygraf` envirionment suitable +for creating graphics: + +``` +cd pygraf +make env +``` + +For developers who want to run the test suite before contributing new changes to the repository, +additional development packages are required. To install those, run: + +``` +cd pygraf +make devenv +``` ## Stage data @@ -143,7 +161,7 @@ python create_graphics.py \ --tiles full,ATL,CA-NV,CentralCA ``` -NOTE: The graphics already run as a workflow step in the RRFS Retros! They may be +> NOTE: The graphics already run as a workflow step in the RRFS Retros! They may be zipped by default, so you can unzip those files to see your images on disk. ### Creating Skew-T Diagrams @@ -274,16 +292,58 @@ guidelines: - All code must pass tests, and tests must be updated to accommodate new code. - Style beyond linting: - Alphabetize lists (anywhere another order is not more obvious to everyone) - - A single white space line before and after comments. - - A single white space after each method/function. Two after classes. - - Lists are maintained with each item on a single line followed by a comma, - even the last item. This repository is using a minor variation on GitLab flow, requiring new work be contributed via Pull Request from a branch with reviewers (required). Releases will be handled with tags (as opposed to branches, in the original GitLab flow), and will be marked as versions with v[major].[minor].[update]. + +# Running Tests + +GitHub Actions is configured to run several code quality checks, including linting, formatting, and +sorting with `ruff`, type checking with `mypy`, and unit tests with `pytest`. To perform the same +checks locally, developers can run: + +``` +(pygraf) $ make format && make test +``` + +# Working with ecCodes for grib2 + +Two command line utilities are available in the conda environment that will help navigate the +ecCodes interpretation of a grib2 file. `grib_ls` gives a single-line record listing of the entire file by +default, while `grib_dump` provides all the metadata for each record. + +Documentation is available from ECMWF: +* https://confluence.ecmwf.int/display/ECC/grib_ls +* https://confluence.ecmwf.int/display/ECC/grib_dump + + +There are many examples in the documentation, but here are a couple that could help with pygraf +specifically. + +Show the common `pygraf` parameters for the 6th record: +``` +$ grib_ls -p shortName,parameterNumber,typeOfLevel,stepType,level -w count=6 hrrr.t05z.wrfnatf08.grib2 +hrrr.t05z.wrfnatf08.grib2 +shortName parameterNumber typeOfLevel stepType level +grle 32 hybrid instant 1 +``` + +The `count` parameter is nice to use in conjunction with `wgrib2` output where that tool may show +the information needed to identify a variable where the `shortName` from ecCodes may be "unknown". + +To see all available information from that record: + +``` +grib_dump -w count=6 hrrr.t05z.wrfnatf08.grib2 +``` + +> NOTE: When using the `-w` flag with items other than `count`, multiple records may be included in +the output. + + # Contact | Name | Email | diff --git a/pre.sh b/pre.sh index aac7f80..f604856 100644 --- a/pre.sh +++ b/pre.sh @@ -3,7 +3,7 @@ module purge module use -a /contrib/miniconda3/modulefiles -module load miniconda3/4.12.0 +module load miniconda3/25.11.0 conda activate pygraf module list diff --git a/tests/test_hrrr_maps.py b/tests/test_hrrr_maps.py index cd6e878..6709900 100644 --- a/tests/test_hrrr_maps.py +++ b/tests/test_hrrr_maps.py @@ -72,7 +72,7 @@ def test_hrrr_maps_file_count(maps_args, tmp_path): """ # Based on the hrrr_test.yml file, only 6 maps will be created create_graphics(maps_args) - map_count = 6 + map_count = 1 count = 0 output = tmp_path / "output" / "202303150003" for file_name in output.iterdir(): From dc75054dfef6a6a3ced7719206d867440fa77580 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 4 Dec 2025 09:55:59 -0700 Subject: [PATCH 65/98] Updating to newest python solvable. --- devpkgs | 6 +++--- environment.yml | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/devpkgs b/devpkgs index a9fecde..8f2c350 100644 --- a/devpkgs +++ b/devpkgs @@ -1,5 +1,5 @@ -mypy==1.18.* +mypy==1.19.* pytest-cov==7.0.* pytest-xdist==3.8.* -pytest==8.4.* -ruff=0.14.* +pytest==9.0.* +ruff==0.14.* diff --git a/environment.yml b/environment.yml index 2a082fe..1c55564 100644 --- a/environment.yml +++ b/environment.yml @@ -4,15 +4,15 @@ channels: - ufs-community - nodefaults dependencies: - - python=3.10.* + - python=3.13.* - basemap=2.0.* - basemap-data-hires=2.0.* - cfgrib=0.9.* - - dask=2025.10.* - - matplotlib=3.10* + - dask=2025.11.* + - matplotlib=3.10.* - metpy=1.7.* - - notebook=7.4.* - - numpy=2.2.* - - pint=0.24.* - - uwtools=2.10.* - - xarray=2025.6.* + - notebook=7.5.* + - numpy=2.3.* + - pint=0.25.* + - uwtools=2.12.* + - xarray=2025.11.* From c5eb8c333a9c85ebe3c5ca555dc392dbf8987871 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 4 Dec 2025 10:17:46 -0700 Subject: [PATCH 66/98] Pin miniforge version. --- .github/workflows/graphics_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/graphics_tests.yml b/.github/workflows/graphics_tests.yml index 9923ace..b5f37e1 100644 --- a/.github/workflows/graphics_tests.yml +++ b/.github/workflows/graphics_tests.yml @@ -22,7 +22,7 @@ jobs: uses: conda-incubator/setup-miniconda@v3 with: environment-file: environment.yml - miniforge-version: latest + miniforge-version: 25.11.0-0 - name: Install dev pkgs run: make devenv - name: Test code From 6dc04216c009aebbe65a3812e7ac2e7c6b903e40 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 4 Dec 2025 10:40:01 -0700 Subject: [PATCH 67/98] Ignore future warnings from cfgrib. --- adb_graphics/datahandler/gribfile.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 7a879e2..c30ed9b 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -2,11 +2,14 @@ Classes that load grib files. """ +import warnings from pathlib import Path import cfgrib import xarray as xr +warnings.filterwarnings("ignore", category=FutureWarning, module="cfgrib") + class GribFiles: """ @@ -44,6 +47,8 @@ def _load(self, filenames: list[Path] | None = None): engine="cfgrib", concat_dim="time", combine="nested", + compat="override", + coords="minimal", backend_kwargs=( { "filter_by_keys": self.cfgrib_config, @@ -71,7 +76,8 @@ def __init__( def _load(self, filename: Path): datasets = cfgrib.open_datasets( - str(filename), read_keys=["orientationOfTheGridInDegrees", "parameterNumber"] + str(filename), + read_keys=["orientationOfTheGridInDegrees", "parameterNumber"], ) all_fields: dict = {} From 1f5a94b39d15b98324527b556d3b03d958953745 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 4 Dec 2025 10:40:42 -0700 Subject: [PATCH 68/98] Update the pattern for str | None = "default". --- adb_graphics/datahandler/gribdata.py | 8 ++++---- adb_graphics/figure_builders.py | 1 + adb_graphics/figures/maps.py | 24 +++++++++++++----------- adb_graphics/figures/skewt.py | 4 ++-- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index a99b0d9..9d5fe90 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -35,12 +35,12 @@ def __init__( model: str, short_name: str, spec: dict | YAMLConfig, - level: str | None = "ua", + level: str | None = None, ): self.model = model self.spec = spec self.short_name = short_name - self.level = level + self.level = level or "ua" self.fhr = fhr cf = deepcopy(self.vspec) @@ -694,12 +694,12 @@ def __init__( loc: str, short_name: str, spec: dict | YAMLConfig, - level: str | None = "ua", + level: str | None = None, ): super().__init__( fhr=fhr, ds=ds, - level=level, + level=level or "ua", model=model, short_name=short_name, spec=spec, diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 768bb2b..ab94d64 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -41,6 +41,7 @@ def add_obs_panel( level="obs", model="obs", name=short_name, + map_type="maps", ) m = Map( airport_fn=AIRPORTS, diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 35a2bed..55a364f 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -118,28 +118,30 @@ class MapFields: def __init__( self, + ds: dict[str, Dataset], fhr: int, fields_spec: dict, - ds: dict[str, Dataset], level: str, + map_type: str, name: str, - map_type: str | None = None, - **kwargs, + ds2: Path | str | None = None, + model: str | None = None, + tile: str | None = None, ): self.fhr = fhr self.fields_spec = deepcopy(fields_spec) self.ds = ds self.level = level self.map_type = map_type - self.model = kwargs.get("model", "") + self.model = "" if model is None else model self.name = name - self.tile = kwargs.get("tile", "full") + self.tile = tile or "full" self.map_spec = deepcopy(self.fields_spec[self.name][self.level]) set_level(self.level, self.model, self.map_spec) # Required if map_type is "diff" if map_type == "diff": - self.ds2: Path | str = kwargs.get("ds2", "") + self.ds2 = ds2 if not self.ds2: msg = "Diff map requires a second grib path. Provide ds2 argument!" raise ValueError(msg) @@ -157,10 +159,10 @@ def shaded(self): "spec": self.fields_spec, "ds": self.ds, } - field = gribdata.FieldData(**args) + field = gribdata.FieldData(**args) # type: ignore[arg-type] if self.map_type == "diff": args["ds"] = self.ds2 - field2 = gribdata.FieldData(**args) + field2 = gribdata.FieldData(**args) # type: ignore[arg-type] field.data = field.data - field2.data return field @@ -193,6 +195,7 @@ def wind_fields(self, level: str | None = None): for var in ("u", "v"): wind_spec = self.fields_spec[var][lev] set_level(lev, self.model, wind_spec) + args = { "fhr": self.fhr, "level": lev, @@ -201,7 +204,7 @@ def wind_fields(self, level: str | None = None): "spec": self.fields_spec, "ds": self.ds, } - winds.append(gribdata.FieldData(**args)) + winds.append(gribdata.FieldData(**args)) # type: ignore[arg-type] return winds def _overlay_fields(self, spec_sect: str) -> list: @@ -538,13 +541,12 @@ def _draw_panel(self, wind_barbs: bool = True): def _draw_contours(self, ax: Axes, not_labeled: list[str]): """Draw the contour fields requested.""" - model_name = self.model_name main_field = self.field.short_name for contour_field in self.contour_fields: levels = contour_field.contour_kwargs.pop("levels", contour_field.clevs) - if model_name in ["RAP-NCEP", "RRFS-NCEP", "RRFS NA 3km"] and ( + if self.model_name in ["RAP-NCEP", "RRFS-NCEP", "RRFS NA 3km"] and ( main_field == "totp" and contour_field.short_name == "pres" and self.map.tile == "full" diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 0dfcf82..d36b9b2 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -67,7 +67,7 @@ def __init__( model: str, spec: dict | YAMLConfig, max_plev: int | None = 0, - model_name: str | None = "Analysis", + model_name: str | None = None, ): # Initialize on the temperature field since we need to gather # field-specific data from this object, e.g. dates, lat, lon, etc. @@ -75,7 +75,7 @@ def __init__( super().__init__(fhr=fhr, ds=ds, loc=loc, model=model, short_name="temp", spec=spec) self.max_plev = max_plev - self.model_name = model_name + self.model_name = model_name or "Analysis" def _add_hydrometeors(self, hydro_subplot: Axes): mixing_ratios: dict[str, HydroPlotSettings] = { From 8352b466145ace3884968922d6cdd8a9fd391fba Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 4 Dec 2025 10:46:38 -0700 Subject: [PATCH 69/98] _kwargs instead of noqa. --- adb_graphics/conversions.py | 34 ++++++++++++++-------------- adb_graphics/datahandler/gribdata.py | 20 ++++++++-------- adb_graphics/figures/maps.py | 2 +- pyproject.toml | 3 --- 4 files changed, 28 insertions(+), 31 deletions(-) diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index fa303f5..4b75145 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -9,101 +9,101 @@ from xarray.ufuncs import sqrt, square -def k_to_c(field: ndarray, **kwargs): +def k_to_c(field: ndarray, **_kwargs): """Conversion from Kelvin to Celsius.""" return field - 273.15 -def k_to_f(field: ndarray, **kwargs): +def k_to_f(field: ndarray, **_kwargs): """Conversion from Kelvin to Farenheit.""" return (field - 273.15) * 9 / 5 + 32 -def kgm2_to_in(field: ndarray, **kwargs): +def kgm2_to_in(field: ndarray, **_kwargs): """Conversion from kg per m^2 to inches.""" return field * 0.03937 -def magnitude(a: DataArray, b: DataArray, **kwargs) -> DataArray: +def magnitude(a: DataArray, b: DataArray, **_kwargs) -> DataArray: """Return the magnitude of vector components.""" return DataArray(sqrt(square(a) + square(b))) -def m_to_dm(field: ndarray, **kwargs): +def m_to_dm(field: ndarray, **_kwargs): """Conversion from meters to decameters.""" return field / 10.0 -def m_to_in(field: ndarray, **kwargs): +def m_to_in(field: ndarray, **_kwargs): """Conversion from meters to inches.""" return field * 39.3701 -def m_to_kft(field: ndarray, **kwargs): +def m_to_kft(field: ndarray, **_kwargs): """Conversion from meters to kilofeet.""" return field / 304.8 -def m_to_mi(field: ndarray, **kwargs): +def m_to_mi(field: ndarray, **_kwargs): """Conversion from meters to miles.""" return field / 1609.344 -def ms_to_kt(field: ndarray, **kwargs): +def ms_to_kt(field: ndarray, **_kwargs): """Conversion from m s-1 to knots.""" return field * 1.9438 -def pa_to_hpa(field: ndarray, **kwargs): +def pa_to_hpa(field: ndarray, **_kwargs): """Conversion from Pascals to hectopascals.""" return field / 100.0 -def percent(field: ndarray, **kwargs): +def percent(field: ndarray, **_kwargs): """Conversion from values between 0 - 1 to percent.""" return field * 100.0 -def sden_to_slr(field: ndarray, **kwargs): +def sden_to_slr(field: ndarray, **_kwargs): """Convert snow density (kg m-3) to snow-liquid ratio.""" return 1000.0 / field -def to_micro(field: ndarray, **kwargs): +def to_micro(field: ndarray, **_kwargs): """Convert field to micro.""" return field * 1e6 -def to_micrograms_per_m3(field: ndarray, **kwargs): +def to_micrograms_per_m3(field: ndarray, **_kwargs): """Convert field to micrograms per cubic meter.""" return field * 1e9 -def vvel_scale(field: ndarray, **kwargs): +def vvel_scale(field: ndarray, **_kwargs): """Scale vertical velocity for plotting.""" return field * -10 -def vort_scale(field: ndarray, **kwargs): +def vort_scale(field: ndarray, **_kwargs): """Scale vorticity for plotting.""" return field / 1e-05 -def weasd_to_1hsnw(field: ndarray, **kwargs): +def weasd_to_1hsnw(field: ndarray, **_kwargs): """Conversion from snow water equiv to snow (10:1 ratio).""" return field * 10.0 diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 9d5fe90..581c7d9 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -219,7 +219,7 @@ def latlons(self) -> list[np.ndarray]: return [lat, lon] @staticmethod - def opposite(values: DataArray, **kwargs) -> DataArray: # noqa: ARG004 + def opposite(values: DataArray, **_kwargs) -> DataArray: """Returns the opposite of input values.""" return -values @@ -244,7 +244,7 @@ def vector_magnitude( self, field1: DataArray, field2_id: str, - **kwargs, # noqa: ARG002 + **_kwargs, ): """ Returns the vector magnitude of two component vector fields. @@ -311,7 +311,7 @@ def __init__( self.contour_kwargs = {} if contour_kwargs is None else contour_kwargs self.mem = member - def aviation_flight_rules(self, values: DataArray, **kwargs): # noqa: ARG002 + def aviation_flight_rules(self, values: DataArray, **_kwargs): """ Generates a field of Aviation Flight Rules from Ceil and Vis. """ @@ -388,7 +388,7 @@ def data(self) -> DataArray: def data(self, value: DataArray): self._data = value - def field_column_max(self, values: DataArray, **kwargs): # noqa: ARG002 + def field_column_max(self, values: DataArray, **_kwargs): """Returns the column max of the values.""" return values.max(dim=self.vertical_coord) @@ -427,7 +427,7 @@ def field_sum(self, values: DataArray, variable2: str, level2: str, **kwargs): return sum2 - def fire_weather_index(self, values: DataArray, **kwargs): # noqa: ARG002 + def fire_weather_index(self, values: DataArray, **_kwargs): """ Generates a field of Fire Weather Index. @@ -546,30 +546,30 @@ def grid_info(self) -> dict: return grid_info @staticmethod - def icing_adjust_trace(values: DataArray, **kwargs): # noqa: ARG004 + def icing_adjust_trace(values: DataArray, **_kwargs): """Changes the value of ICSEV trace from 4.0 to 0.5, to maintain ascending order.""" return where(values == 4.0, 0.5, values) @staticmethod - def run_max(values: DataArray, **kwargs): # noqa: ARG004 + def run_max(values: DataArray, **_kwargs): """Finds the max hourly value over all the forecast lead times available.""" return values.max(dim="time") # pragma: no cover @staticmethod - def run_min(values: DataArray, **kwargs): # noqa: ARG004 + def run_min(values: DataArray, **_kwargs): """Finds the min hourly value over all the forecast lead times available.""" return values.min(dim="time") # pragma: no cover @staticmethod - def run_total(values: DataArray, **kwargs): # noqa: ARG004 + def run_total(values: DataArray, **_kwargs): """Sums over all the forecast lead times available.""" return values.sum(dim="time") # pragma: no cover - def supercooled_liquid_water(self, **kwargs): # noqa: ARG002 + def supercooled_liquid_water(self, **_kwargs): """ Generates a field of Supercooled Liquid Water. diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 55a364f..dab7652 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -401,7 +401,7 @@ class DataMap: """ - def __init__(self, map_fields: MapFields, map_: Map, model_name: str | None = None, **kwargs): # noqa: ARG002 + def __init__(self, map_fields: MapFields, map_: Map, model_name: str | None = None, **_kwargs): self.field = map_fields.shaded self.contour_fields = map_fields.contours self.hatch_fields = map_fields.hatches diff --git a/pyproject.toml b/pyproject.toml index 9544081..12391e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,9 +80,6 @@ ignore = [ "PT013", # pytest-incorrect-pytest-import "SLF001", # private-member-access ] -"adb_graphics/conversions.py" = [ - "ARG001", # unused-function-argument -] "adb_graphics/datahandler/gribdata.py" = [ "PLR2004", # magic-value-comparison ] From 81d809d8c7d4f3531c9ab1aa0cf7d76327e333e1 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 4 Dec 2025 11:19:11 -0700 Subject: [PATCH 70/98] Suggested changes from pr. --- adb_graphics/datahandler/gribfile.py | 3 +-- adb_graphics/errors.py | 4 ---- adb_graphics/figure_builders.py | 2 +- adb_graphics/figures/maps.py | 3 ++- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index c30ed9b..dba561a 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -52,8 +52,7 @@ def _load(self, filenames: list[Path] | None = None): backend_kwargs=( { "filter_by_keys": self.cfgrib_config, - "indexpath": "", # create a temp file here or pyfakefs to hold it in mem. check - # unittests in wxvx + "indexpath": "", "read_keys": ["orientationOfTheGridInDegrees"], } ), diff --git a/adb_graphics/errors.py b/adb_graphics/errors.py index a498655..62a0e24 100644 --- a/adb_graphics/errors.py +++ b/adb_graphics/errors.py @@ -1,10 +1,6 @@ """Errors specific to the ADB Graphics package.""" -class ArgumentError(ValueError): - """The right arguments are not provided.""" - - class FieldNotUniqueError(Exception): """Exception raised when multiple Grib fields are found with input parameters.""" diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index ab94d64..1ac03be 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -16,7 +16,7 @@ from adb_graphics.figures import skewt from adb_graphics.figures.maps import DataMap, DiffMap, Map, MapFields, MultiPanelDataMap -AIRPORTS = Path("static/Airports_locs.txt") +AIRPORTS = Path(__file__).resolve().parent.parent / "static" / "Airports_locs.txt" def add_obs_panel( diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index dab7652..dff5f0e 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -417,7 +417,8 @@ def wind_fields(self, level: str): def add_logo(ax: Axes): """Puts the NOAA logo at the bottom left of the matplotlib axes.""" - logo = mpimg.imread("static/noaa-logo-50x50.png") + logo_path = Path(__file__).resolve().parent.parent / "static" / "noaa-logo-50x50.png" + logo = mpimg.imread(logo_path) imagebox = mpob.OffsetImage(logo) ab = mpob.AnnotationBbox( From 5baea8b8d8c943b3a4ab6560cc31c64ed2771c19 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 4 Dec 2025 11:26:55 -0700 Subject: [PATCH 71/98] One more parent. --- adb_graphics/figures/maps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index dff5f0e..f460168 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -417,7 +417,7 @@ def wind_fields(self, level: str): def add_logo(ax: Axes): """Puts the NOAA logo at the bottom left of the matplotlib axes.""" - logo_path = Path(__file__).resolve().parent.parent / "static" / "noaa-logo-50x50.png" + logo_path = Path(__file__).resolve().parent.parent.parent / "static" / "noaa-logo-50x50.png" logo = mpimg.imread(logo_path) imagebox = mpob.OffsetImage(logo) From bf0a2a52780eb8388443e0ba566b1055dd72a165 Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:42:50 -0700 Subject: [PATCH 72/98] Apply suggestions from code review Co-authored-by: Paul Madden <136389411+maddenp-cu@users.noreply.github.com> --- adb_graphics/datahandler/gribdata.py | 11 ++++++----- adb_graphics/datahandler/gribfile.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 581c7d9..d582158 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -46,18 +46,19 @@ def __init__( cf = deepcopy(self.vspec) utils.set_level(level=str(level), model=self.model, spec=cf) cf = utils.cfgrib_spec(cf["cfgrib"], self.model) + key = "typeOfLevel" try: - self.vertical_coord = cf["typeOfLevel"] + self.vertical_coord = cf[key] except KeyError: - msg = f"typOfLevel is not a key for {short_name} at {level}. cf: {cf}" + msg = f"{key} is not a key for {short_name} at {level}. cf: {cf}" raise KeyError(msg) from None self.ds = ds @property def anl_dt(self) -> datetime: """ - Returns the initial time of the grib file as a datetime object from - the grib file. + Returns the initial time of the GRIB file as a datetime object from + the GRIB file. """ ret: datetime = to_datetime(self.field.time.values) return ret @@ -361,7 +362,7 @@ def colors(self) -> np.ndarray: def corners(self) -> list: """ - Returns lat and lon of lower left (ll) and upper right(ur) corners. + Returns lat and lon of lower left (ll) and upper right (ur) corners. Order: ll_lat, ur_lat, ll_lon, ur_lon diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index dba561a..ec3d9e2 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -40,7 +40,7 @@ def __init__( self.contents = self._load() def _load(self, filenames: list[Path] | None = None): - """Load the set of files into a single XArray structure.""" + """Load the set of files into a single Xarray structure.""" filenames = self.filenames if filenames is None else filenames ds = xr.open_mfdataset( filenames, From bb9dc754270729928a5541b05f732ea7f16df49e Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 4 Dec 2025 11:49:08 -0700 Subject: [PATCH 73/98] Apply suggested reduction of suppressors. --- pyproject.toml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 12391e6..49ea9da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,9 +24,7 @@ ignore = [ "ANN202", # missing-return-type-private-function "ANN204", # missing-return-type-special-method "ANN205", # missing-return-type-static-method - "ANN206", # missing-return-type-class-method "ANN401", # any-type - "B010", # set-attr-with-constant "C408", # unnecessary-collection-call "C901", # complex-structure "COM812", # missing-trailing-comma @@ -42,7 +40,6 @@ ignore = [ "D203", # incorrect-blank-line-before-class "D205", # missing-blank-line-after-summary "D212", # multi-line-summary-first-line - "D400", # missing-trailing-period "D401", # non-imperative-mood "D404", # docstring-starts-with-this "DTZ001", # call-datetime-without-tzinfo @@ -50,32 +47,21 @@ ignore = [ "DTZ006", # call-datetime-fromtimestamp "DTZ007", # call-datetime-strptime-without-zone "E731", # lambda-assignment - "ERA001", # commented-out-code "FBT001", # boolean-type-hint-positional-argument "FBT002", # boolean-default-value-positional-argument - "FBT003", # boolean-positional-value-in-call - "FLY002", # static-join-to-f-string - "N813", # camelcase-imported-as-lowercase "PLR0913", # too-many-arguments - "PT019", # pytest-fixture-param-without-value "PTH207", # glob "RUF015", # unnecessary-iterable-allocation-for-first-element "S101", # assert - "S103", # bad-file-permissions "S311", # suspicious-non-cryptographic-random-usage "S506", # unsafe-yaml-load "S602", # subprocess-popen-with-shell-equals-true - "S701", # jinja2-autoescape-false "T201", # print - "TC006", # runtime-cast-value - "UP031", # printf-string-formatting - "UP032", # f-string ] [tool.ruff.lint.per-file-ignores] "conftest.py" = [ "ANN001", # missing-type-function-argument - "N802", # invalid-function-name "PLR2004", # magic-value-comparison "PT013", # pytest-incorrect-pytest-import "SLF001", # private-member-access @@ -85,7 +71,6 @@ ignore = [ ] "tests/*" = [ "ANN001", # missing-type-function-argument - "N802", # invalid-function-name "PLR2004", # magic-value-comparison "PT013", # pytest-incorrect-pytest-import "SLF001", # private-member-access From 233f9467ea2cc974d94274fb468d89785ade7d08 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 5 Dec 2025 14:53:38 -0700 Subject: [PATCH 74/98] Updating numeric_level to use regex. --- adb_graphics/figures/maps.py | 9 +++------ adb_graphics/utils.py | 32 ++++++++++++-------------------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index f460168..ef2ea01 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -770,14 +770,13 @@ def _title(self): loc="left", ) - level, lev_unit = numeric_level(f.level) units = f"({f.units}, shaded)" if f.vspec.get("print_units", True) else "" # Title or Atmospheric level and unit in the high center if f.vspec.get("title"): title = f"{f.vspec.get('title')} {units}" else: - level = level if not isinstance(level, list) else level[0] + level, lev_unit = numeric_level(f.level) title = f"{level} {lev_unit} {f.field.long_name} {units}" plt.title(f"{title}", loc="center", y=1.10, fontsize=18) @@ -921,14 +920,13 @@ def _title(self): loc="left", ) - level, lev_unit = f.numeric_level() units = f"({f.units}, shaded)" if f.vspec.get("print_units", True) else "" # Title or Atmospheric level and unit in the high center if f.vspec.get("title"): title = f"Diff: {f.vspec.get('title')} {units}" else: - level = level if not isinstance(level, list) else level[0] + level, lev_unit = numeric_level(f.level) title = f"Diff: {level} {lev_unit} {f.field.long_name} {units}" plt.title(f"{title}", position=(0.5, 1.08), fontsize=18) @@ -1007,14 +1005,13 @@ def title(self): transform=ax.transAxes, ) - level, lev_unit = f.numeric_level() units = f"({f.units}, shaded)" if f.vspec.get("print_units", True) else "" # Title or Atmospheric level and unit in the high center if f.vspec.get("title"): title = f"{f.vspec.get('title')} {units}" else: - level = level if not isinstance(level, list) else level[0] + level, lev_unit = numeric_level(f.level) title = f"{level} {lev_unit} {f.field.long_name} {units}" ax.text( 0, diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 3d4e1f1..0d86e6f 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -4,6 +4,7 @@ import functools import glob +import re import subprocess import sys import time @@ -13,7 +14,6 @@ from importlib.util import find_spec from multiprocessing import Process from pathlib import Path -from string import ascii_letters, digits from typing import Any import numpy as np @@ -158,12 +158,8 @@ def load_yaml(config: Path | str) -> YAMLConfig: def load_sites(arg: str | Path) -> list[str]: - """Check that the sites file exists, and return its contents.""" - - # Check that the file exists + """Return the contents of the sites file, if it exists.""" path = Path(arg) - path.exists() - with path.open() as sites_file: sites: list[str] = sites_file.readlines() return sites @@ -191,20 +187,16 @@ def numeric_level(level: str | None = None) -> tuple[float | int | str, str]: """ level = level if level is not None else "" - - # Gather all the numbers in the string - numbers = "".join([c for c in level if (c in digits or c == ".")]) - - # Convert the numbers to a list, and make integers or floats - if numbers: - lev_val = float(numbers) if "." in numbers else int(numbers) - else: - return "", "" - - # Gather all the letters - lev_unit = "".join([c for c in level if c in ascii_letters]) - - return lev_val, lev_unit + if m := re.match(r"^([0-9.]+)?([a-z]+)?([0-9.]+)?$", level): + groups = m.groups() + units = groups[1] + value = groups[0] or groups[2] + for convert in (int, float): + try: + return convert(value), units + except (TypeError, ValueError): # noqa: PERF203 + pass + return "", "" def old_enough(age: int, file_path: Path | str): From 776ba87a91dd7160895eb91f77903213c5f128fb Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 8 Dec 2025 10:20:17 -0700 Subject: [PATCH 75/98] Use native zipfile instead of subprocess. Replace mock tests with real tests. --- adb_graphics/utils.py | 51 +++++++++++--------- tests/test_utils.py | 105 +++++++++++++++++++++++++++--------------- 2 files changed, 97 insertions(+), 59 deletions(-) diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 0d86e6f..4e38181 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -15,6 +15,7 @@ from multiprocessing import Process from pathlib import Path from typing import Any +from zipfile import ZipFile import numpy as np import yaml @@ -32,7 +33,7 @@ def cfgrib_spec(config: dict, model: str) -> dict: return config -def create_zip(files_to_zip: list[str], zipf: Path | str): +def create_zip(files_to_zip: list[Path], zipf: Path | str): """Create a zip file. Use a locking mechanism -- write a lock file to disk.""" lock_file = Path(f"{zipf}._lock") @@ -42,29 +43,43 @@ def create_zip(files_to_zip: list[str], zipf: Path | str): if not lock_file.exists(): # Create the lock lock_file.touch() - print(f"Writing to zip file {zipf} for files like: {files_to_zip[0][-10:]}") - - cmd = f"zip -uj {zipf} {' '.join(files_to_zip)}" - print(f"Running command: {cmd}") + print(f"Writing to zip file {zipf} for files like: {files_to_zip[0].name}") + overwrite = {} try: - subprocess.run( - cmd, - check=True, - shell=True, - ) + with ZipFile(zipf, "a") as zf: + arcfiles = zf.namelist() + for file in files_to_zip: + if file.name in arcfiles: + arcinfo = zf.getinfo(file.name) + arc_mod_time = datetime(*arcinfo.date_time) + file_mod_time = datetime.fromtimestamp(file.stat().st_mtime) + if file_mod_time > arc_mod_time: + overwrite[file.name] = file + else: + zf.write(file, arcname=Path(file).name) + if overwrite: + tmp_path = Path(f"{zipf}.tmp") + with ZipFile(tmp_path, "w") as tmp: + for item in zf.namelist(): + if not (file := zf.getinfo(item).filename) in overwrite: + tmp.write(file, zf.read(item)) + for arcname, file in overwrite.items(): + tmp.write(file, arcname=arcname) + + tmp_path.rename(zipf) except Exception as e: count += 1 if count >= retry: - msg = "Error on writing zip file!" + msg = "Error writing zip file!" raise RuntimeError(msg) from e else: # Zipping was successful. Remove files that were zipped for file_to_zip in files_to_zip: - Path(file_to_zip).unlink(missing_ok=True) + file_to_zip.unlink(missing_ok=True) break finally: - # Remove the lock lock_file.unlink(missing_ok=True) + # Wait before trying to obtain the lock on the file time.sleep(1) @@ -313,12 +328,6 @@ def zip_products(fhr: int, workdir: Path, zipfiles: dict) -> None: file_tmpl = f"*.skewt.*_f{fhr:03d}.csv" else: file_tmpl = f"*_{tile}_*{fhr:02d}.png" - product_files = glob.glob(str(workdir / file_tmpl)) + product_files = [Path(f) for f in glob.glob(str(workdir / file_tmpl))] if product_files: - zip_proc = Process( - group=None, - target=create_zip, - args=(product_files, zipf), - ) - zip_proc.start() - zip_proc.join() + create_zip(product_files, zipf) diff --git a/tests/test_utils.py b/tests/test_utils.py index 6ef87df..ffbd971 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,9 +2,11 @@ import time from contextlib import contextmanager from copy import deepcopy -from datetime import datetime +from datetime import datetime, timedelta +from os import utime from pathlib import Path from unittest.mock import MagicMock, patch +from zipfile import ZipFile import numpy as np import yaml @@ -44,8 +46,67 @@ def test_create_zip(tmp_path): afile.touch() bfile.touch() zipf = tmp_path / "file.zip" - utils.create_zip([str(f) for f in [afile, bfile]], zipf) - assert zipf.is_file() + utils.create_zip([afile, bfile], zipf) + with ZipFile(zipf, "r") as zf: + assert zf.namelist() == ["a.txt", "b.txt"] + assert not afile.is_file() + assert not bfile.is_file() + +def test_create_zip_existing_empty(tmp_path): + afile = tmp_path / "a.txt" + bfile = tmp_path / "b.txt" + afile.write_text("foo") + bfile.write_text("bar") + zipf = tmp_path / "file.zip" + zipf.touch() + assert zipf.stat().st_size == 0 + utils.create_zip([afile, bfile], zipf) + assert zipf.stat().st_size > 0 + with ZipFile(zipf, "r") as zf: + assert zf.namelist() == ["a.txt", "b.txt"] + assert not afile.is_file() + assert not bfile.is_file() + +def test_create_zip_existing_nonempty(tmp_path): + afile = tmp_path / "a.txt" + bfile = tmp_path / "b.txt" + afile.write_text("foo") + a_mod_time = datetime(2025, 1, 1, 1, 0, 0).timestamp() + utime(afile, (a_mod_time, a_mod_time)) + bfile.write_text("bar") + zipf = tmp_path / "file.zip" + with ZipFile(zipf, "w") as zf: + zf.write(afile, arcname=afile.name) + utils.create_zip([afile, bfile], zipf) + with ZipFile(zipf, "r") as zf: + assert zf.namelist() == ["a.txt", "b.txt"] + # Make sure the file has the older modify time. + assert datetime(*zf.getinfo("a.txt").date_time) == datetime.fromtimestamp(a_mod_time) + assert not afile.is_file() + assert not bfile.is_file() + # Call again and make sure that the "overwrite" branch is not executed. + with patch.object(utils, "ZipFile") as zf: + utils.create_zip([afile, bfile], zipf) + zf.assert_called_once_with(zipf, "a") + + + +def test_create_zip_existing_nonempty_overwrite(tmp_path): + afile = tmp_path / "a.txt" + bfile = tmp_path / "b.txt" + afile.write_text("foo") + bfile.write_text("bar") + zipf = tmp_path / "file.zip" + with ZipFile(zipf, "w") as zf: + zf.write(afile, arcname=afile.name) + # A newer archive file (mod time > previously archived file) will overwrite an older one. + a_mod_time = (datetime.now().replace(second=0, microsecond=0) + timedelta(minutes=5)).timestamp() + utime(afile, (a_mod_time, a_mod_time)) + utils.create_zip([afile, bfile], zipf) + with ZipFile(zipf, "r") as zf: + assert zf.namelist() == ["a.txt", "b.txt"] + # Make sure the archived file has the newer time. + assert datetime(*zf.getinfo("a.txt").date_time) == datetime.fromtimestamp(a_mod_time) assert not afile.is_file() assert not bfile.is_file() @@ -55,10 +116,10 @@ def test_create_zip_error(tmp_path): # Using a different error here (not Exception or RuntimeError) to make sure anything gets # caught in code under test. with ( - patch.object(utils.subprocess, "run", side_effect=ValueError) as run, - raises(RuntimeError, match="Error on writing zip file!"), + patch.object(utils.ZipFile, "write", side_effect=ValueError) as run, + raises(RuntimeError, match="Error writing zip file!"), ): - utils.create_zip(["afile", "bfile"], zipf) + utils.create_zip([Path(f) for f in ("afile", "bfile")], zipf) assert run.call_count == 2 @@ -345,35 +406,3 @@ def test_uniq_wgrib2_list(): assert len(uniq_list) < len(fields_list) assert len(uniq_list) == 1711 - -def test_zip_products(capsys, tmp_path): - (tmp_path / "1_full_XXXX.12.png").touch() - (tmp_path / "2.skewt.XXXX_f012.csv").touch() - mock_proc = MagicMock() - mock_proc.start = MagicMock() - mock_proc.join = MagicMock() - - with ( - patch.object(utils, "Process", return_value=mock_proc) as proc, - patch("time.perf_counter", side_effect=[1.0, 2.0]), - ): - utils.zip_products( - 12, tmp_path, {"full": tmp_path / "full.zip", "skewt_csv": tmp_path / "skewt_csv.zip"} - ) - captured = capsys.readouterr() - assert proc.call_count == 2 - assert mock_proc.start.call_count == 2 - assert mock_proc.join.call_count == 2 - assert "zip_products Elapsed time: 1.0000 seconds" in captured.out - - -def test_zip_products_skips_when_no_files_found(): - with ( - patch("glob.glob", return_value=[]) as mock_glob, - patch("multiprocessing.Process") as mock_process, - ): - workdir = Path("/fake/path") - utils.zip_products(5, workdir, {"tile1": "zip1.zip"}) - - mock_glob.assert_called_once() - mock_process.assert_not_called() From 85fc4a35ee0f8dae2cbf2c30b97e4444a289d226 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 8 Dec 2025 13:33:31 -0700 Subject: [PATCH 76/98] More suggested changes. --- adb_graphics/specs.py | 18 +++------ adb_graphics/utils.py | 55 ++++++++++++------------- image_lists/hrrr_test.yml | 10 ++--- tests/test_common.py | 84 +++++++++++++++++++-------------------- tests/test_specs.py | 1 - tests/test_utils.py | 12 +++--- 6 files changed, 86 insertions(+), 94 deletions(-) diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index fca30c6..e132a0e 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -154,29 +154,21 @@ def hail_colors(self) -> np.ndarray: ncar = get_cmap(self.vspec.get("cmap"), 128)([100, 15, 18, 20, 25, 60, 80, 85, 90]) return np.concatenate((grays, ncar)) - @property - def heat_flux_colors(self) -> np.ndarray: - """Default color map for Latent/Sensible Heat Flux.""" - - grays = get_cmap("Greys", 8)([6, 5, 4, 3, 2]) - ctable = ctables.colortables.get_colortable(self.vspec.get("cmap"))(range(0, 33, 2)) - return np.concatenate((grays, ctable)) - @property def heat_flux_colors_g(self) -> np.ndarray: - """Default color map for Latent/Sensible Heat Flux.""" + """Default color map for ground heat flux.""" return get_cmap(self.vspec.get("cmap"), 128)(range(15, 112, 8)) @property def heat_flux_colors_l(self) -> np.ndarray: - """Default color map for Latent/Sensible Heat Flux.""" + """Default color map for net latent heat flux.""" return get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) @property def heat_flux_colors_s(self) -> np.ndarray: - """Default color map for Latent/Sensible Heat Flux.""" + """Default color map for sensible heat flux.""" return get_cmap(self.vspec.get("cmap"), 128)(range(32, 129, 6)) @@ -264,7 +256,7 @@ def pcp_colors_high(self) -> np.ndarray: @property def pmsl_colors(self) -> np.ndarray: - """Default color map for Surface Pressure.""" + """Default color map for mean sea level pressure.""" ncolors = len(self.vspec.get("clevs", self.clevs)) incr = 128 // ncolors @@ -273,7 +265,7 @@ def pmsl_colors(self) -> np.ndarray: @property def ps_colors(self) -> np.ndarray: - """Default color map for Surface Pressure.""" + """Default color map for surface pressure.""" grays = get_cmap("Greys", 13)(range(13)) segments = [[16, 53], [86, 105], [110, 151, 2], [172, 202, 2]] diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 4e38181..5859702 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -5,14 +5,12 @@ import functools import glob import re -import subprocess import sys import time from collections.abc import Callable from datetime import datetime, timedelta from importlib import import_module from importlib.util import find_spec -from multiprocessing import Process from pathlib import Path from typing import Any from zipfile import ZipFile @@ -33,6 +31,33 @@ def cfgrib_spec(config: dict, model: str) -> dict: return config +def _write_zip(files_to_zip: list[Path], zipf: Path | str): + """Write the zip file, overwriting existing files that have a newer modification timestamp.""" + print(f"Writing to zip file {zipf} for files like: {files_to_zip[0].name}") + overwrite = {} + with ZipFile(zipf, "a") as zf: + arcfiles = zf.namelist() + for file in files_to_zip: + if file.name in arcfiles: + arcinfo = zf.getinfo(file.name) + arc_mod_time = datetime(*arcinfo.date_time) + file_mod_time = datetime.fromtimestamp(file.stat().st_mtime) + if file_mod_time > arc_mod_time: + overwrite[file.name] = file + else: + zf.write(file, arcname=Path(file).name) + if overwrite: + tmp_path = Path(f"{zipf}.tmp") + with ZipFile(tmp_path, "w") as tmp: + for item in zf.namelist(): + if (arcfile := zf.getinfo(item).filename) not in overwrite: + tmp.write(arcfile, str(zf.read(item))) + for arcname, file in overwrite.items(): + tmp.write(file, arcname=arcname) + + tmp_path.rename(zipf) + + def create_zip(files_to_zip: list[Path], zipf: Path | str): """Create a zip file. Use a locking mechanism -- write a lock file to disk.""" @@ -43,30 +68,8 @@ def create_zip(files_to_zip: list[Path], zipf: Path | str): if not lock_file.exists(): # Create the lock lock_file.touch() - print(f"Writing to zip file {zipf} for files like: {files_to_zip[0].name}") - overwrite = {} try: - with ZipFile(zipf, "a") as zf: - arcfiles = zf.namelist() - for file in files_to_zip: - if file.name in arcfiles: - arcinfo = zf.getinfo(file.name) - arc_mod_time = datetime(*arcinfo.date_time) - file_mod_time = datetime.fromtimestamp(file.stat().st_mtime) - if file_mod_time > arc_mod_time: - overwrite[file.name] = file - else: - zf.write(file, arcname=Path(file).name) - if overwrite: - tmp_path = Path(f"{zipf}.tmp") - with ZipFile(tmp_path, "w") as tmp: - for item in zf.namelist(): - if not (file := zf.getinfo(item).filename) in overwrite: - tmp.write(file, zf.read(item)) - for arcname, file in overwrite.items(): - tmp.write(file, arcname=arcname) - - tmp_path.rename(zipf) + _write_zip(files_to_zip, zipf) except Exception as e: count += 1 if count >= retry: @@ -97,8 +100,6 @@ def fhr_list(args: list[int]) -> list[int]: Length > 3: List as is argparse should provide a list of at least one item (nargs='+'). - - Must ensure that the list contains integers. """ args = args if isinstance(args, list) else [args] diff --git a/image_lists/hrrr_test.yml b/image_lists/hrrr_test.yml index a0225ac..70c2186 100644 --- a/image_lists/hrrr_test.yml +++ b/image_lists/hrrr_test.yml @@ -3,8 +3,8 @@ hourly: variables: hlcytot: - mn02 -# - mn03 -# - mn25 -# - mx02 -# - mx03 -# - mx25 + - mn03 + - mn25 + - mx02 + - mx03 + - mx25 diff --git a/tests/test_common.py b/tests/test_common.py index e825c44..37b3d67 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,5 +1,3 @@ -# pylint: disable=invalid-name - """ Pytests for the common utilities included in this package. @@ -202,7 +200,6 @@ def check_transform(self, entry): return True - # pylint: disable=inconsistent-return-statements def get_callable(self, func): """Return the callable function given a function name.""" @@ -213,7 +210,6 @@ def get_callable(self, func): if len(func.split(".")) == 1: # Check all the classes in the gribdata module for attr in dir(gribdata): - # pylint: disable=no-member # Check the methods in each class if func in dir(gribdata.__getattribute__(attr)): method = gribdata.__getattribute__(attr).__dict__.get(func) @@ -345,55 +341,57 @@ def is_a_level(key): """ + # fmt: off allowed_levels = [ - "agl", # above ground level - "best", # Best + "agl", # above ground level + "best", # Best "bndylay", # boundary layer cld cover - "esbl", # ??? - "esblmn", # ??? - "high", # high clouds - "int", # vertical integral - "low", # low clouds - "max", # maximum in column - "maxsfc", # max surface value - "mdn", # maximum downward - "mid", # mid-level clouds - "mnsfc", # min surface value - "msl", # mean sea level - "mu", # most unstable - "mul", # most unstable layer - "mup", # maximum upward - "mu", # most unstable - "obs", # observations - "pw", # wrt precipitable water - "sat", # satellite - "sfc", # surface - "sfclt", # surface (less than) - "top", # nominal top of atmosphere - "total", # total clouds - "ua", # upper air - "uanat", # upper air native file + "esbl", # ??? + "esblmn", # ensemble mean + "high", # high clouds + "int", # vertical integral + "low", # low clouds + "max", # maximum in column + "maxsfc", # max surface value + "mdn", # maximum downward + "mid", # mid-level clouds + "mnsfc", # min surface value + "msl", # mean sea level + "mu", # most unstable + "mul", # most unstable layer + "mup", # maximum upward + "mu", # most unstable + "obs", # observations + "pw", # wrt precipitable water + "sat", # satellite + "sfc", # surface + "sfclt", # surface (less than) + "top", # nominal top of atmosphere + "total", # total clouds + "ua", # upper air + "uanat", # upper air native file ] allowed_lev_type = [ - "cm", # centimeters - "ds", # difference - "ft", # feet - "km", # kilometers - "m", # meters - "mm", # millimeters - "mb", # milibars - "sr", # storm relative + "cm", # centimeters + "ds", # difference + "ft", # feet + "km", # kilometers + "m", # meters + "mm", # millimeters + "mb", # milibars + "sr", # storm relative ] allowed_stat = [ - "in", # ??? + "in", # ??? "ens", # ensemble - "m", # ??? - "maxm", # ??? - "mn", # minimum - "mx", # maximum + "m", # ??? + "maxm", # ??? + "mn", # minimum + "mx", # maximum ] + # fmt: on # Easy check first -- it is in the allowed_levels list if key in allowed_levels: diff --git a/tests/test_specs.py b/tests/test_specs.py index e9c5c50..e278eaa 100644 --- a/tests/test_specs.py +++ b/tests/test_specs.py @@ -52,7 +52,6 @@ def test_centered_diff(levels, expected, spec): ("goes_colors", 151), ("graupel_colors", 19), ("hail_colors", 10), - ("heat_flux_colors", 22), ("heat_flux_colors_g", 13), ("heat_flux_colors_l", 17), ("heat_flux_colors_s", 17), diff --git a/tests/test_utils.py b/tests/test_utils.py index ffbd971..f527a48 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta from os import utime from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import patch from zipfile import ZipFile import numpy as np @@ -52,6 +52,7 @@ def test_create_zip(tmp_path): assert not afile.is_file() assert not bfile.is_file() + def test_create_zip_existing_empty(tmp_path): afile = tmp_path / "a.txt" bfile = tmp_path / "b.txt" @@ -67,6 +68,7 @@ def test_create_zip_existing_empty(tmp_path): assert not afile.is_file() assert not bfile.is_file() + def test_create_zip_existing_nonempty(tmp_path): afile = tmp_path / "a.txt" bfile = tmp_path / "b.txt" @@ -90,7 +92,6 @@ def test_create_zip_existing_nonempty(tmp_path): zf.assert_called_once_with(zipf, "a") - def test_create_zip_existing_nonempty_overwrite(tmp_path): afile = tmp_path / "a.txt" bfile = tmp_path / "b.txt" @@ -100,7 +101,9 @@ def test_create_zip_existing_nonempty_overwrite(tmp_path): with ZipFile(zipf, "w") as zf: zf.write(afile, arcname=afile.name) # A newer archive file (mod time > previously archived file) will overwrite an older one. - a_mod_time = (datetime.now().replace(second=0, microsecond=0) + timedelta(minutes=5)).timestamp() + a_mod_time = ( + datetime.now().replace(second=0, microsecond=0) + timedelta(minutes=5) + ).timestamp() utime(afile, (a_mod_time, a_mod_time)) utils.create_zip([afile, bfile], zipf) with ZipFile(zipf, "r") as zf: @@ -132,7 +135,7 @@ def test_create_zip_locked(tmp_path): zipf_lock = tmp_path / "file.zip._lock" zipf_lock.touch() with raises(TimeoutError), timeout(2): - utils.create_zip([str(f) for f in [afile, bfile]], zipf) + utils.create_zip([afile, bfile], zipf) assert not zipf.is_file() assert afile.is_file() assert bfile.is_file() @@ -405,4 +408,3 @@ def test_uniq_wgrib2_list(): uniq_list = utils.uniq_wgrib2_list(fields_list) assert len(uniq_list) < len(fields_list) assert len(uniq_list) == 1711 - From 7466dfd5427fc6679ae91888bb51d1d534cf6be4 Mon Sep 17 00:00:00 2001 From: Christina Holt <56881914+christinaholtNOAA@users.noreply.github.com> Date: Mon, 8 Dec 2025 14:18:45 -0700 Subject: [PATCH 77/98] Apply suggestions from code review Co-authored-by: Paul Madden <136389411+maddenp-cu@users.noreply.github.com> --- README.md | 8 ++++---- adb_graphics/conversions.py | 2 +- adb_graphics/datahandler/gribdata.py | 8 ++++---- adb_graphics/figure_builders.py | 2 +- adb_graphics/figures/maps.py | 2 +- adb_graphics/figures/skewt.py | 8 ++++---- adb_graphics/specs.py | 2 +- adb_graphics/utils.py | 8 ++++---- conftest.py | 6 +++--- create_graphics.py | 13 +++++-------- 10 files changed, 28 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 8c6200a..02dc0dd 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,12 @@ module load miniconda3/25.11.0 conda activate pygraf ``` -This environment contains the necessary develepment packages. +This environment contains the necessary development packages. ## Installing with conda -Pygraf comes with an environment.yml file for use with any conda installation. Ensure the conda base -environment is activated, and run the following command to create a `pygraf` envirionment suitable +Pygraf comes with an `environment.yml` file for use with any conda installation. Ensure the conda base +environment is activated, and run the following command to create a `pygraf` environment suitable for creating graphics: ``` @@ -320,7 +320,7 @@ Documentation is available from ECMWF: * https://confluence.ecmwf.int/display/ECC/grib_dump -There are many examples in the documentation, but here are a couple that could help with pygraf +There are many examples in the documentation, but here are a couple that could help with `pygraf` specifically. Show the common `pygraf` parameters for the 6th record: diff --git a/adb_graphics/conversions.py b/adb_graphics/conversions.py index 4b75145..e7f0a21 100644 --- a/adb_graphics/conversions.py +++ b/adb_graphics/conversions.py @@ -16,7 +16,7 @@ def k_to_c(field: ndarray, **_kwargs): def k_to_f(field: ndarray, **_kwargs): - """Conversion from Kelvin to Farenheit.""" + """Conversion from Kelvin to Fahrenheit.""" return (field - 273.15) * 9 / 5 + 32 diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index d582158..36c9217 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -418,7 +418,7 @@ def field_mean( return values.sel(isobaricInhPa=levs).mean("isobaricInhPa") def field_sum(self, values: DataArray, variable2: str, level2: str, **kwargs): - """Return the sum of the values.""" + """Returns the sum of the values.""" value2 = self.values( name=variable2, level=level2, do_transform=kwargs.get("do_transform", True) @@ -583,7 +583,7 @@ def supercooled_liquid_water(self, **_kwargs): columns, and (3) uses the layer depth to find the pressure at the next sigma level. - The process is iterative to the topof the atmosphere. + The process is iterative to the top of the atmosphere. """ pres_sfc = self.values(name="pres", level="sfc") * 100.0 # convert back to Pa pres_nat_lev = self.values(name="pres", level="ua") @@ -707,10 +707,10 @@ def __init__( ) self.loc = loc - # The first 31 columns are space delimted + # The first 31 columns are space-delimited self.site_code, _, self.site_num, lat, lon = loc[:31].split() - # The variable lenght site name is included past column 37 + # The variable length site name is included past column 37 self.site_name = loc[37:].rstrip() # Convert the string to a number. Longitude should be positive for all diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 1ac03be..8ca0ee0 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -1,6 +1,6 @@ """ This module is where pieces of the figures are put together. Data is -compbined with maps and skewts to provide the final product. +combined with maps and skewts to provide the final product. """ import gc diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index ef2ea01..22ff457 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -110,7 +110,7 @@ class MapFields: """ - Class that packages all the field objects need for producing + Class that packages all the field objects needed for producing desired map content, i.e. an object that contains all filled contours, hatched spaces, and overlayed contours needed for a full product. diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index d36b9b2..d4a7a8c 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -257,7 +257,7 @@ def atmo_profiles(self): Return a dictionary of atmospheric data profiles for each variable needed by the skewT. - Each of these variables must be have units set appropriately for use + Each of these variables must have units set appropriately for use with MetPy SkewT. Handle those units and conversions here since it differs from the requirements of other graphics units/transforms. """ @@ -466,18 +466,18 @@ def _setup_diagram(self): # Fahrenheit tick labels that will display labels_f = list(range(-20, 125, 20)) * units.degF - # Celcius VALUES for those tick marks. These put the ticks in the right + # Celsius VALUES for those tick marks. These put the ticks in the right # spot. labels = labels_f.to("degC").magnitude - # Set the MINOR tick values to the CELCIUS values. + # Set the MINOR tick values to the CELSIUS values. skew.ax.xaxis.set_minor_locator(FixedLocator(labels)) # Set the MINOR tick labels to the FAHRENHEIT values. skew.ax.set_xticklabels(labels_f.magnitude, minor=True) skew.ax.tick_params(which="minor", length=8) - # Turn off the MAJOR (celcius) tick marks, label the grid lines inside + # Turn off the MAJOR (celsius) tick marks, label the grid lines inside # the axes. skew.ax.tick_params( axis="x", diff --git a/adb_graphics/specs.py b/adb_graphics/specs.py index fca30c6..53d62f2 100644 --- a/adb_graphics/specs.py +++ b/adb_graphics/specs.py @@ -1,6 +1,6 @@ """ This module sets the specifications for certain atmospheric variables. Typically -this is related to a spec that needs some level of computation, i.e. a set of +this is related to a spec that needs some level of computation, e.g. a set of colors from a color map. """ diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 4e38181..4655f93 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -138,7 +138,7 @@ def get_func(val: str): return getattr(module, fun_name) -def join_ranges(loader: yaml.SafeLoader, node: yaml.Node) -> Any: # noqa: ARG001 +def join_ranges(_loader: yaml.SafeLoader, node: yaml.Node) -> Any: """ Merge two or more different ranges into a single array for color bar clevs. @@ -162,7 +162,7 @@ def join_ranges(loader: yaml.SafeLoader, node: yaml.Node) -> Any: # noqa: ARG00 return np.concatenate(list_, axis=0) -def arange_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> Any: # noqa: ARG001 +def arange_constructor(_loader: yaml.SafeLoader, node: yaml.Node) -> Any: return np.arange(*[float(n.value) for n in node.value]) @@ -281,14 +281,14 @@ def wrapper_timer(*args, **kwargs): def to_datetime(string: str): - """Return a datetime object give a string like YYYYMMDDHH.""" + """Return a datetime object given a string like YYYYMMDDHH.""" return datetime.strptime(string, "%Y%m%d%H") def uniq_wgrib2_list(inlist: list[str]): """ - Given a list of wgrib2 output fields, returns a uniq list of fields for + Given a list of wgrib2 output fields, returns a unique list of fields for simplifying a grib2 dataset. Uniqueness is defined by the wgrib output from field 3 (colon delimted) onward, although the original full grib record must be included in the wgrib2 command below. diff --git a/conftest.py b/conftest.py index 3cd24c5..772516b 100644 --- a/conftest.py +++ b/conftest.py @@ -39,20 +39,20 @@ def pytest_addoption(parser): @fixture(scope="session") def natfile(): - """Interface to pass a grib file to pytest.""" + """Interface to pass a grib file to pytest.""" return Path("tests", "data", "wrfnat_hrconus_16.grib2") @fixture(scope="session") def prsfile(): - """Interface to pass a grib file to pytest.""" + """Interface to pass a grib file to pytest.""" return Path("tests", "data", "wrfprs_hrconus_16.grib2") @fixture(scope="session") def spec_file(): - """Interface to pass a grib file to pytest.""" + """Interface to pass a grib file to pytest.""" return Path("adb_graphics", "default_specs.yml") diff --git a/create_graphics.py b/create_graphics.py index 2bc617d..6376d27 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -153,7 +153,7 @@ def load_images(arg: list[Path | str]): file path and dictionary of images to be created. """ - # Agument is expected to be a 2-list of file name and internal + # Argument is expected to be a 2-list of file name and internal # section name. image_file = Path(arg[0]) @@ -175,10 +175,7 @@ def parse_args(argv: list) -> Namespace: containing the settings. """ - parser = ArgumentParser( - description="Script to drive the \ - creation of graphices files." - ) + parser = ArgumentParser(description="Script to drive the creation of graphics files.") # Positional argument parser.add_argument( @@ -230,7 +227,7 @@ def parse_args(argv: list) -> Namespace: "-m", default="Unnamed Experiment", dest="model_name", - help="string to use in title of graphic.", + help="String to use in title of graphic.", type=str, ) parser.add_argument( @@ -514,7 +511,7 @@ def stage_zip_files(tiles: list, zip_dir: Path) -> dict: def graphics_driver(cla: Namespace): # ruff: noqa: PLR0915, PLR0912 # This whole script has likely reached the point of neededing refactoring - # into an object oriented design....each graphics type is it's own object + # into an object oriented design....each graphics type is its own object # sharing a base class. """ Function that interprets the command line arguments to locate the input grib @@ -606,7 +603,7 @@ def graphics_driver(cla: Namespace): break # It's safe to continue on processing the next forecast hour print( - "Cannot find specified file(s), continuing to check on \n \ + "Cannot find specified file(s), continuing to check on \ next forecast hour." ) continue From 1f308291aceb782578a8e8a1c15602e8932d7c8f Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 8 Dec 2025 14:19:32 -0700 Subject: [PATCH 78/98] pragma: no cover with 100% coverage requirement. --- adb_graphics/datahandler/gribdata.py | 19 ++++---- adb_graphics/datahandler/gribfile.py | 4 +- adb_graphics/errors.py | 2 +- adb_graphics/figures/maps.py | 66 +++++++++++++++------------- adb_graphics/figures/skewt.py | 8 ++-- adb_graphics/utils.py | 2 +- create_graphics.py | 26 +++++------ pyproject.toml | 2 +- 8 files changed, 68 insertions(+), 61 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index d582158..487c6a2 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -49,7 +49,7 @@ def __init__( key = "typeOfLevel" try: self.vertical_coord = cf[key] - except KeyError: + except KeyError: # pragma: no cover msg = f"{key} is not a key for {short_name} at {level}. cf: {cf}" raise KeyError(msg) from None self.ds = ds @@ -115,7 +115,7 @@ def _find_var(): for var in ds: if ds[var].attrs["GRIB_shortName"] == short_name: return var - return None + return None # pragma: no cover short_name = cfgribspec.get("shortName", "unknown") vertical_coord = cfgribspec["typeOfLevel"] @@ -140,11 +140,11 @@ def _find_var(): leveled = level is not None and vertical_coord != "hybrid" if len(field.coords[vertical_coord].shape) > 0 and (layered or leveled): if vertical_coord == "depthBelowLandLayer" and level: - level = level / 100.0 + level = level / 100.0 # pragma: no cover field = field.sel(**{vertical_coord: level}) return DataArray(field) - msg = f"Variable {short_name} not found in dataset." - raise ValueError(msg) + msg = f"Variable {short_name} not found in dataset." # pragma: no cover + raise ValueError(msg) # pragma: no cover def get_transform(self, transforms: dict | list | str, val: DataArray) -> DataArray: """ @@ -355,7 +355,7 @@ def colors(self) -> np.ndarray: msg = f"There is no color definition named {color_spec}" raise AttributeError(msg) from e if callable(ret): - return np.asarray(ret()) + return np.asarray(ret()) # pragma: no cover return np.asarray(ret) @property @@ -499,7 +499,7 @@ def grid_info(self) -> dict: grid_info: dict[str, str | float | int | list] = {} var_info = self.field grid_def = var_info.attrs["GRIB_gridDefinitionDescription"].lower() - match grid_def: + match grid_def: # pragma: no cover case x if "lambert" in x: attrs = [ "GRIB_Latin1InDegrees", @@ -728,7 +728,7 @@ def values( self, level: str | None = None, name: str | None = None, - do_transform: bool = True, # noqa: ARG002 + do_transform: bool = False, ) -> DataArray: """ Returns the numpy array of values at the object's x, y location for the @@ -740,6 +740,9 @@ def values( upper air """ + + assert do_transform is False # not supported by this class + # Set the defaults here since this is an instance of an abstract method # level refers to the level key in the specs file. level = level if level is not None else "ua" diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index ec3d9e2..7d53df3 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -11,7 +11,7 @@ warnings.filterwarnings("ignore", category=FutureWarning, module="cfgrib") -class GribFiles: +class GribFiles: # pragma: no cover """ Class for loading in a set of grib files and combining them over forecast hours. @@ -85,7 +85,7 @@ def _load(self, filename: Path): var_id = _var_id(ds, str(var)) if all_fields.get(var_id) is None: all_fields[var_id] = ds - else: + else: # pragma: no cover msg = f"Multiple entries for {var_id} when opening {filename}" raise ValueError(msg) return all_fields diff --git a/adb_graphics/errors.py b/adb_graphics/errors.py index 62a0e24..48d7f6b 100644 --- a/adb_graphics/errors.py +++ b/adb_graphics/errors.py @@ -5,7 +5,7 @@ class FieldNotUniqueError(Exception): """Exception raised when multiple Grib fields are found with input parameters.""" -class GribReadError(Exception): +class GribReadError(Exception): # pragma: no cover """Exception raised when there is an error reading the grib file.""" def __init__(self, name: str, message: str = "was not found"): diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index ef2ea01..c3e9d75 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -267,7 +267,9 @@ def __init__(self, airport_fn: Path, ax: Axes, **kwargs): self.tile = kwargs.get("tile", "full") if self.model == "hrrr" and "WFIP3" in self.tile: - self.grid_info.update({"lat_1": 40.6, "lat_2": 40.6, "lon_0": 289.2}) + self.grid_info.update( + {"lat_1": 40.6, "lat_2": 40.6, "lon_0": 289.2} + ) # pragma: no cover if self.model != "hrrrhi": if self.tile in FULL_TILES: self.corners = self.grid_info.pop("corners") @@ -307,7 +309,7 @@ def boundaries(self): try: self.m.drawcoastlines(linewidth=0.5) - except ValueError: + except ValueError: # pragma: no cover self.m.drawcounties( color="black", linewidth=0.4, @@ -374,7 +376,7 @@ def _get_basemap(self, **get_basemap_kwargs): width=self.width, height=self.height, ) - ) + ) # pragma: no cover basemap_args.update(get_basemap_kwargs) @@ -410,7 +412,7 @@ def __init__(self, map_fields: MapFields, map_: Map, model_name: str | None = No self.model_name = model_name self.plot_scatter = map_fields.fields_spec.get("plot_scatter", False) - def wind_fields(self, level: str): + def wind_fields(self, level: str): # pragma: no cover return self.map_fields.wind_fields(level) @staticmethod @@ -444,9 +446,9 @@ def _colorbar(self, cc: QuadContourSet, ax: Axes): np.amax(self.field.clevs + 1), self.field.ticks, ) - elif self.field.ticks == 0: + elif self.field.ticks == 0: # pragma: no cover ticks = self.field.clevs - else: + else: # pragma: no cover ticks = self.field.clevs[0 : len(self.field.clevs) : -self.field.ticks] ticks = np.around(ticks, 4) @@ -460,11 +462,11 @@ def _colorbar(self, cc: QuadContourSet, ax: Axes): ) tick_labels = [str(t) for t in ticks] - if self.field.short_name == "flru": + if self.field.short_name == "flru": # pragma: no cover tick_labels = [label.rjust(30) for label in ["VFR", "MVFR", "IFR", "LIFR", ""]] # this step is done to allow proper order of icing severity levels (trace before light) - if self.field.short_name == "icsev": + if self.field.short_name == "icsev": # pragma: no cover tick_labels = [label.rjust(30) for label in ["TRACE", "LIGHT", "MODERATE", "HEAVY", ""]] cbar.ax.set_xticklabels(tick_labels, fontsize=12) @@ -504,20 +506,20 @@ def _draw_panel(self, wind_barbs: bool = True): ) not_labeled = [self.field.short_name] - if self.hatch_fields: + if self.hatch_fields: # pragma: no cover not_labeled.extend([h.short_name for h in self.hatch_fields]) # Contour secondary fields, if requested - if self.contour_fields: + if self.contour_fields: # pragma: no cover self._draw_contours(ax, not_labeled) # Add hatched fields, if requested - if self.hatch_fields: + if self.hatch_fields: # pragma: no cover self._draw_hatches(ax) # Add wind barbs, if requested add_wind = self.field.vspec.get("wind", False) - if add_wind and wind_barbs: + if add_wind and wind_barbs: # pragma: no cover self._wind_barbs(add_wind) # Add field values at airports @@ -530,16 +532,16 @@ def _draw_panel(self, wind_barbs: bool = True): and model_name not in ["RRFS NA 3km"] and model_name == "RAP-NCEP" and self.map.tile not in ["full"] - ): + ): # pragma: no cover self._draw_field_values(ax) # Add scatter plot, if requested - if self.plot_scatter: + if self.plot_scatter: # pragma: no cover self._draw_scatter(ax) return cf - def _draw_contours(self, ax: Axes, not_labeled: list[str]): + def _draw_contours(self, ax: Axes, not_labeled: list[str]): # pragma: no cover """Draw the contour fields requested.""" main_field = self.field.short_name @@ -577,7 +579,7 @@ def _draw_contours(self, ax: Axes, not_labeled: list[str]): {self.field.level}" ) - def _draw_scatter(self, ax: Axes): + def _draw_scatter(self, ax: Axes): # pragma: no cover """Plot dots at locations on the map that meet a threshold.""" field = self.field @@ -612,7 +614,9 @@ def _draw_scatter(self, ax: Axes): **field.contour_kwargs, ) - def _draw_field(self, ax: Axes, field: gribdata.FieldData, func: Callable, **kwargs): + def _draw_field( + self, ax: Axes, field: gribdata.FieldData, func: Callable, **kwargs + ): # pragma: no cover """ Internal implementation that calls a matplotlib function. @@ -658,7 +662,7 @@ def _draw_field(self, ax: Axes, field: gribdata.FieldData, func: Callable, **kwa print(f"CLOSE ERROR: {field.short_name} {field.level}") return ret - def _draw_field_values(self, ax: Axes): + def _draw_field_values(self, ax: Axes): # pragma: no cover """Add the text value of the field at airport locations.""" annotate_decimal = self.field.vspec.get("annotate_decimal", 0) airports = self.map.load_airports() @@ -685,7 +689,7 @@ def _draw_field_values(self, ax: Axes): ) data_values.close() - def _draw_hatches(self, ax: Axes): + def _draw_hatches(self, ax: Axes): # pragma: no cover """Draw the hatched regions requested.""" # Levels should be included in the settings dict here since they don't @@ -733,7 +737,7 @@ def _set_overlay_string(self) -> str: contoured = [] contoured_units = [] not_labeled = [f.short_name] - if self.hatch_fields: + if self.hatch_fields: # pragma: no cover cf = self.hatch_fields[0] not_labeled.extend([h.short_name for h in self.hatch_fields]) if not any(list(set(cf.short_name).intersection(["pres"]))): @@ -741,7 +745,7 @@ def _set_overlay_string(self) -> str: contoured.append(f"{user_title} ({cf.units}, hatched)") # Add descriptor string for the important contoured fields - if self.contour_fields: + if self.contour_fields: # pragma: no cover for cf in self.contour_fields: if cf.short_name not in not_labeled: user_title = cf.vspec.get("title", cf.field.long_name) @@ -750,7 +754,7 @@ def _set_overlay_string(self) -> str: contoured_units.append(f"{cf.units}") title = "\n".join(contoured) # Make 'contoured' a multiline string - if contoured_units: + if contoured_units: # pragma: no cover title = f"{title} ({', '.join(contoured_units)}, contoured)" return title @@ -775,7 +779,7 @@ def _title(self): # Title or Atmospheric level and unit in the high center if f.vspec.get("title"): title = f"{f.vspec.get('title')} {units}" - else: + else: # pragma: no cover level, lev_unit = numeric_level(f.level) title = f"{level} {lev_unit} {f.field.long_name} {units}" plt.title(f"{title}", loc="center", y=1.10, fontsize=18) @@ -788,7 +792,7 @@ def _title(self): fontsize=14, ) - def _wind_barbs(self, level: bool | str): + def _wind_barbs(self, level: bool | str): # pragma: no cover """ Draws the wind barbs. A decent stride can be found if you divide the number of grid points on the shorter side by 35. Subdomains are defined @@ -860,7 +864,7 @@ class DiffMap(DataMap): and will not plot overlays and such. """ - def _colorbar(self, cc: QuadContourSet, ax: Axes): + def _colorbar(self, cc: QuadContourSet, ax: Axes): # pragma: no cover """Set the colorbar for a difference field.""" plt.colorbar( @@ -871,7 +875,7 @@ def _colorbar(self, cc: QuadContourSet, ax: Axes): shrink=1.0, ) - def _draw_panel(self, wind_barbs: bool = False): + def _draw_panel(self, wind_barbs: bool = False): # pragma: no cover """Draw a map of the difference field.""" ax = self.map.ax @@ -895,7 +899,7 @@ def _draw_panel(self, wind_barbs: bool = False): levels=self._eq_contours(), ) - def _eq_contours(self): + def _eq_contours(self): # pragma: no cover """Center the contours based on the data min/max.""" minval = np.amin(self.field.data) @@ -905,7 +909,7 @@ def _eq_contours(self): maxval = max(abs(minval), abs(maxval)) return np.linspace(-maxval, maxval, 21) - def _title(self): + def _title(self): # pragma: no cover """Draw the title for a map.""" f = self.field @@ -960,13 +964,13 @@ def draw(self, show: bool = False): # Finish with the colorbar on the last panel only # Plot it on the full figure scale. - if self.last_panel: + if self.last_panel: # pragma: no cover cax = plt.axes((0.0, 0.0, 1.0, 0.2)) self._colorbar(ax=cax, cc=cf) cax.axis("off") # Create a pop-up to display the figure, if show=True - if show: + if show: # pragma: no cover plt.tight_layout() return cf @@ -985,7 +989,7 @@ def _label_member(self): transform=ax.transAxes, ) - def title(self): + def title(self): # pragma: no cover """Draw the title for a map.""" f = self.field diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index d36b9b2..58bc2b8 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -135,7 +135,7 @@ def _add_hydrometeors(self, hydro_subplot: Axes): except (errors.NoGraphicsDefinitionForVariableError, IndexError, ValueError): try: profile = self.values(name=mixr, level="uanat") * 1000.0 * scale - except errors.NoGraphicsDefinitionForVariableError: + except errors.NoGraphicsDefinitionForVariableError: # pragma: no cover print(f"missing {mixr} for hydrometeor plot, skipping that field.") continue mixr_total: units = 0.0 @@ -632,7 +632,7 @@ def thermo_variables(self): lev = items.get("level", "ua") spec = self.spec.get(varname, {}).get(lev) - if not spec: + if not spec: # pragma: no cover raise errors.NoGraphicsDefinitionForVariableError(varname, lev) try: @@ -641,7 +641,7 @@ def thermo_variables(self): if transforms: vals = self.get_transform(transforms, vals) - except errors.GribReadError: + except errors.GribReadError: # pragma: no cover vals = DataArray([]) thermo[var]["data"] = vals thermo[var]["units"] = spec.get("unit") @@ -730,7 +730,7 @@ def label_line(ax: Axes, label: str, segment: np.ndarray, **kwargs): if end == "top": trans_angle -= 180 - else: + else: # pragma: no cover trans_angle = 0 # Set a bunch of keyword arguments diff --git a/adb_graphics/utils.py b/adb_graphics/utils.py index 5859702..1bb4309 100644 --- a/adb_graphics/utils.py +++ b/adb_graphics/utils.py @@ -310,7 +310,7 @@ def uniq_wgrib2_list(inlist: list[str]): @timer -def zip_products(fhr: int, workdir: Path, zipfiles: dict) -> None: +def zip_products(fhr: int, workdir: Path, zipfiles: dict) -> None: # pragma: no cover """ Spin up a subprocess to zip all the product files into the staged zip files. diff --git a/create_graphics.py b/create_graphics.py index 2bc617d..e9b2ae4 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -38,7 +38,7 @@ def check_file( data_root: Path | None = None, file_tmpl: str | None = None, mem: int | None = None, -) -> tuple[Path, bool]: +) -> tuple[Path, bool]: # pragma: no cover """ Given the command line arguments, the forecast hour, and a potential ensemble member, build a full path to the file and ensure it exists. @@ -61,7 +61,7 @@ def check_file( return grib_path, old_enough -def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): +def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): # pragma: no cover """ Generate arguments for parallel processing of Skew T graphics, and generate a pool of workers to complete the tasks. @@ -79,7 +79,7 @@ def create_maps( grib_paths: list[Path], workdir: Path, grib_path2: Path | None = None, -): +): # pragma: no cover """ Generate arguments for parallel processing of plan-view maps and generate a pool of workers to complete the task. @@ -124,7 +124,7 @@ def create_maps( pool.starmap(parallel_maps, args) -def generate_tile_list(arg_list: list) -> list[str]: +def generate_tile_list(arg_list: list) -> list[str]: # pragma: no cover """ Given the input arguments -- a list if the argument is provided, return the list. If no arg is provided, defaults to the full domain, and if 'all' @@ -146,7 +146,7 @@ def generate_tile_list(arg_list: list) -> list[str]: return arg_list -def load_images(arg: list[Path | str]): +def load_images(arg: list[Path | str]): # pragma: no cover """ Check that input image file exists, and that it contains the requested section. Return a 2-list (required by argparse) of the @@ -169,7 +169,7 @@ def load_images(arg: list[Path | str]): return [images.get("model"), images.get("variables")] -def parse_args(argv: list) -> Namespace: +def parse_args(argv: list) -> Namespace: # pragma: no cover """ Set up argparse command line arguments, and return the Namespace containing the settings. @@ -363,7 +363,7 @@ def parse_args(argv: list) -> Namespace: return parser.parse_args(argv) -def pre_proc_grib_files(cla: Namespace, fhr: int) -> tuple[Path, bool]: +def pre_proc_grib_files(cla: Namespace, fhr: int) -> tuple[Path, bool]: # pragma: no cover """ Use the command line argument object (cla) to determine the grib file location at a given forecast hour. If multiple data input paths and file @@ -447,7 +447,7 @@ def pre_proc_grib_files(cla: Namespace, fhr: int) -> tuple[Path, bool]: return combined_fp, True -def remove_accumulated_images(cla: Namespace): +def remove_accumulated_images(cla: Namespace): # pragma: no cover """ Searches for all images that correspond with specs that have the accumulate entry set to True and removes them from the list of images to @@ -469,7 +469,7 @@ def remove_accumulated_images(cla: Namespace): del cla.images[1][variable] -def remove_proc_grib_files(cla: Namespace) -> None: +def remove_proc_grib_files(cla: Namespace) -> None: # pragma: no cover """Find all processed grib files produced by this script and remove them.""" # Prepare template with all viable forecast hours -- glob accepts * @@ -485,7 +485,7 @@ def remove_proc_grib_files(cla: Namespace) -> None: Path(file_path).unlink() -def stage_zip_files(tiles: list, zip_dir: Path) -> dict: +def stage_zip_files(tiles: list, zip_dir: Path) -> dict: # pragma: no cover """ Stage the zip files in the appropriate directory for each tile to be plotted. Return the dictionary of zipfile paths. @@ -511,7 +511,7 @@ def stage_zip_files(tiles: list, zip_dir: Path) -> dict: @utils.timer -def graphics_driver(cla: Namespace): +def graphics_driver(cla: Namespace): # pragma: no cover # ruff: noqa: PLR0915, PLR0912 # This whole script has likely reached the point of neededing refactoring # into an object oriented design....each graphics type is it's own object @@ -678,7 +678,7 @@ def graphics_driver(cla: Namespace): remove_proc_grib_files(cla) -def create_graphics(argv: list): +def create_graphics(argv: list): # pragma: no cover """ Function to perform a series of checks on command line arguments. """ @@ -738,4 +738,4 @@ def create_graphics(argv: list): if __name__ == "__main__": - create_graphics(sys.argv[1:]) + create_graphics(sys.argv[1:]) # pragma: no cover diff --git a/pyproject.toml b/pyproject.toml index 49ea9da..43e1f55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.coverage.report] exclude_also = ["if TYPE_CHECKING:"] -fail_under = 70 +fail_under = 100 show_missing = true skip_covered = true omit = ["conftest.py", "tests/*"] From f6dd1f5fdfb6a0bb235c6930cd4e72af23c1de82 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 8 Dec 2025 14:38:32 -0700 Subject: [PATCH 79/98] A few changes I missed on GH. --- adb_graphics/datahandler/gribfile.py | 11 +++++------ create_graphics.py | 3 +-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 7d53df3..e3a7ec0 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -39,11 +39,10 @@ def __init__( self.cfgrib_config = cfgrib_config self.contents = self._load() - def _load(self, filenames: list[Path] | None = None): + def _load(self): """Load the set of files into a single Xarray structure.""" - filenames = self.filenames if filenames is None else filenames ds = xr.open_mfdataset( - filenames, + self.filenames, engine="cfgrib", concat_dim="time", combine="nested", @@ -71,11 +70,11 @@ def __init__( filename: Path, ): self.filename = filename - self.contents = self._load(filename) + self.contents = self._load() - def _load(self, filename: Path): + def _load(self): datasets = cfgrib.open_datasets( - str(filename), + str(self.filename), read_keys=["orientationOfTheGridInDegrees", "parameterNumber"], ) diff --git a/create_graphics.py b/create_graphics.py index c1f3504..1bdbff0 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -603,8 +603,7 @@ def graphics_driver(cla: Namespace): # pragma: no cover break # It's safe to continue on processing the next forecast hour print( - "Cannot find specified file(s), continuing to check on \ - next forecast hour." + "Cannot find specified file(s), continuing to check on next forecast hour." ) continue From 29460447b6605e948fe6829394826818fe9f9428 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 8 Dec 2025 15:10:28 -0700 Subject: [PATCH 80/98] Renaming. --- adb_graphics/datahandler/gribdata.py | 36 ++++++++++++++-------------- adb_graphics/datahandler/gribfile.py | 11 +++++---- adb_graphics/figure_builders.py | 2 +- adb_graphics/figures/skewt.py | 8 +++---- conftest.py | 4 ++-- create_graphics.py | 10 ++++---- tests/datahandler/test_gribdata.py | 28 +++++++++++----------- tests/datahandler/test_gribfile.py | 10 ++++---- 8 files changed, 54 insertions(+), 55 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 7ea31ec..b0006a3 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -236,7 +236,7 @@ def valid_dt(self) -> datetime: return self.anl_dt + fh @abc.abstractmethod - def values( + def get_values( self, level: str | None = None, name: str | None = None, do_transform: bool = True ) -> DataArray: """Returns the values of a given variable.""" @@ -255,7 +255,7 @@ def vector_magnitude( """ var, lev = field2_id.split("_") if "_" in field2_id else (field2_id, self.level) - field2 = self.values(level=lev, name=var, do_transform=False) + field2 = self.get_values(level=lev, name=var, do_transform=False) mag = conversions.magnitude(field1, field2) field1.close() field2.close() @@ -318,7 +318,7 @@ def aviation_flight_rules(self, values: DataArray, **_kwargs): """ ceil = values - vis = self.values(name="vis", level="sfc") + vis = self.get_values(name="vis", level="sfc") flru = where((ceil > 1.0) & (ceil < 3.0), 1.01, 0.0) flru = where((vis > 3.0) & (vis < 5.0), 1.01, flru) @@ -382,7 +382,7 @@ def data(self) -> DataArray: the values associated with a given object -- helpful for differences. """ if not hasattr(self, "_data"): - self._data = self.values() + self._data = self.get_values() return self._data @data.setter @@ -397,7 +397,7 @@ def field_column_max(self, values: DataArray, **_kwargs): def field_diff(self, values: DataArray, variable2: str, level2: str, **kwargs): """Subtracts the values from variable2 from self.field.""" - value2 = self.values( + value2 = self.get_values( name=variable2, level=level2, do_transform=kwargs.get("do_transform", True) ) diff = values - value2 @@ -420,7 +420,7 @@ def field_mean( def field_sum(self, values: DataArray, variable2: str, level2: str, **kwargs): """Returns the sum of the values.""" - value2 = self.values( + value2 = self.get_values( name=variable2, level=level2, do_transform=kwargs.get("do_transform", True) ) sum2 = values + value2 @@ -440,11 +440,11 @@ def fire_weather_index(self, values: DataArray, **_kwargs): # Gather fields from the input veg = values - temp = self.values(level="2m", name="temp", do_transform=False) - dewpt = self.values(level="2m", name="dewp", do_transform=False) - weasd = self.values(level="sfc", name="weasd", do_transform=False) - gust = self.values(level="10m", name="gust", do_transform=False) - soilm = self.values(level="sfc", name="soilm", do_transform=False) + temp = self.get_values(level="2m", name="temp", do_transform=False) + dewpt = self.get_values(level="2m", name="dewp", do_transform=False) + weasd = self.get_values(level="sfc", name="weasd", do_transform=False) + gust = self.get_values(level="10m", name="gust", do_transform=False) + soilm = self.get_values(level="sfc", name="soilm", do_transform=False) # A few derived fields dewpt_depression = temp - dewpt @@ -585,11 +585,11 @@ def supercooled_liquid_water(self, **_kwargs): The process is iterative to the top of the atmosphere. """ - pres_sfc = self.values(name="pres", level="sfc") * 100.0 # convert back to Pa - pres_nat_lev = self.values(name="pres", level="ua") - temp = self.values(name="temp", level="ua") - cloud_mixing_ratio = self.values(name="clwmr", level="ua") - rain_mixing_ratio = self.values(name="rwmr", level="ua") + pres_sfc = self.get_values(name="pres", level="sfc") * 100.0 # convert back to Pa + pres_nat_lev = self.get_values(name="pres", level="ua") + temp = self.get_values(name="temp", level="ua") + cloud_mixing_ratio = self.get_values(name="clwmr", level="ua") + rain_mixing_ratio = self.get_values(name="rwmr", level="ua") gravity = 9.81 slw = pres_sfc * 0.0 # start with array of zero values @@ -635,7 +635,7 @@ def units(self) -> str: return str(self.vspec.get("unit", self.field.units)) - def values( + def get_values( self, level: str | None = None, name: str | None = None, do_transform: bool = True ) -> DataArray: """ @@ -724,7 +724,7 @@ def __init__( if self.site_lon < 0: self.site_lon = self.site_lon + 360.0 - def values( + def get_values( self, level: str | None = None, name: str | None = None, diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index e3a7ec0..e8ae0a2 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -3,6 +3,7 @@ """ import warnings +from functools import cached_property from pathlib import Path import cfgrib @@ -37,9 +38,9 @@ def __init__( self.filenames = filenames self.cfgrib_config = cfgrib_config - self.contents = self._load() - def _load(self): + @cached_property + def datasets(self): """Load the set of files into a single Xarray structure.""" ds = xr.open_mfdataset( self.filenames, @@ -70,9 +71,9 @@ def __init__( filename: Path, ): self.filename = filename - self.contents = self._load() - def _load(self): + @cached_property + def datasets(self): datasets = cfgrib.open_datasets( str(self.filename), read_keys=["orientationOfTheGridInDegrees", "parameterNumber"], @@ -85,7 +86,7 @@ def _load(self): if all_fields.get(var_id) is None: all_fields[var_id] = ds else: # pragma: no cover - msg = f"Multiple entries for {var_id} when opening {filename}" + msg = f"Multiple entries for {var_id} when opening {self.filename}" raise ValueError(msg) return all_fields diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 8ca0ee0..e4ae1dc 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -172,7 +172,7 @@ def parallel_maps( # noqa: PLR0915, PLR0912 if spec.get("include_obs", False) and cla.obs_file_path: # Add observation panel to lower left. Currently only # supported for composite reflectivity. - obs_ds = gribfile.WholeGribFile(cla.obs_file_path).contents + obs_ds = gribfile.WholeGribFile(cla.obs_file_path).datasets add_obs_panel( ax=axes[8], model_name=cla.model_name, diff --git a/adb_graphics/figures/skewt.py b/adb_graphics/figures/skewt.py index 5a28c20..4781ed7 100644 --- a/adb_graphics/figures/skewt.py +++ b/adb_graphics/figures/skewt.py @@ -131,10 +131,10 @@ def _add_hydrometeors(self, hydro_subplot: Axes): # Get the profile values scale = settings.get("scale", 1.0) try: - profile = self.values(name=mixr) * 1000.0 * scale + profile = self.get_values(name=mixr) * 1000.0 * scale except (errors.NoGraphicsDefinitionForVariableError, IndexError, ValueError): try: - profile = self.values(name=mixr, level="uanat") * 1000.0 * scale + profile = self.get_values(name=mixr, level="uanat") * 1000.0 * scale except errors.NoGraphicsDefinitionForVariableError: # pragma: no cover print(f"missing {mixr} for hydrometeor plot, skipping that field.") continue @@ -296,7 +296,7 @@ def atmo_profiles(self): for var, items in atmo_vars.items(): # Get the profile values and attach MetPy units - vals = self.values(name=var).to_numpy() * items["units"] + vals = self.get_values(name=var).to_numpy() * items["units"] # Apply any needed transformations transform = items.get("transform") @@ -636,7 +636,7 @@ def thermo_variables(self): raise errors.NoGraphicsDefinitionForVariableError(varname, lev) try: - vals = self.values(level=lev, name=varname) + vals = self.get_values(level=lev, name=varname) transforms = spec.get("transform") if transforms: vals = self.get_transform(transforms, vals) diff --git a/conftest.py b/conftest.py index 772516b..d01ce8f 100644 --- a/conftest.py +++ b/conftest.py @@ -58,9 +58,9 @@ def spec_file(): @fixture(scope="session") def prs_ds(prsfile): - return gribfile.WholeGribFile(prsfile).contents + return gribfile.WholeGribFile(prsfile).datasets @fixture(scope="session") def nat_ds(natfile): - return gribfile.WholeGribFile(natfile).contents + return gribfile.WholeGribFile(natfile).datasets diff --git a/create_graphics.py b/create_graphics.py index 1bdbff0..c140df6 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -66,7 +66,7 @@ def create_skewt(cla: Namespace, fhr: int, grib_path: Path, workdir: Path): # p Generate arguments for parallel processing of Skew T graphics, and generate a pool of workers to complete the tasks. """ - ds = gribfile.WholeGribFile(grib_path).contents + ds = gribfile.WholeGribFile(grib_path).datasets args = [(cla, fhr, ds, site, workdir) for site in cla.sites] print(f"Queueing {len(args)} Skew Ts") with Pool(processes=cla.nprocs) as pool: @@ -85,7 +85,7 @@ def create_maps( generate a pool of workers to complete the task. """ - ds = gribfile.WholeGribFile(grib_paths[-1]).contents + ds = gribfile.WholeGribFile(grib_paths[-1]).datasets for tile in cla.tiles: args = [] for variable, levels in cla.images[1].items(): @@ -104,7 +104,7 @@ def create_maps( if (accumulate or grib_acc) and fhr == 0: continue if accumulate: - ads = gribfile.GribFiles(grib_paths, vspec).contents + ads = gribfile.GribFiles(grib_paths, vspec).datasets args.append( ( @@ -602,9 +602,7 @@ def graphics_driver(cla: Namespace): # pragma: no cover print(f"Waiting for {grib_path} to be available.") break # It's safe to continue on processing the next forecast hour - print( - "Cannot find specified file(s), continuing to check on next forecast hour." - ) + print("Cannot find specified file(s), continuing to check on next forecast hour.") continue # Create the working directory diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index 4143727..5d10e10 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -11,7 +11,7 @@ class ConcreteUPPData(gribdata.UPPData): - def values( + def get_values( self, level: str | None = None, # noqa: ARG002 name: str | None = None, # noqa: ARG002 @@ -383,7 +383,7 @@ def test_fielddata_corners_single_dim(fielddata_obj): def test_fielddata_data_getter_and_setter(fielddata_obj): - assert np.array_equal(fielddata_obj.data, fielddata_obj.values()) + assert np.array_equal(fielddata_obj.data, fielddata_obj.get_values()) new_data = ones_like(fielddata_obj.field) fielddata_obj.data = new_data assert np.array_equal(fielddata_obj.data, new_data) @@ -473,52 +473,52 @@ def test_fielddata_units_in_vspec(fielddata_obj): def test_fielddata_values_args_no_transform(fielddata_obj, lev, var): fielddata_obj.vspec["transform"] = None fielddata_obj.model = "hrrr" - assert not np.array_equal(fielddata_obj.values(level=lev, name=var), fielddata_obj.field) + assert not np.array_equal(fielddata_obj.get_values(level=lev, name=var), fielddata_obj.field) def test_fielddata_values_args_transform(fielddata_obj): fielddata_obj.vspec["transform"] = "opposite" fielddata_obj.model = "hrrr" - assert np.array_equal(fielddata_obj.values(level="sfc", name="temp"), -fielddata_obj.field) + assert np.array_equal(fielddata_obj.get_values(level="sfc", name="temp"), -fielddata_obj.field) def test_fielddata_values_no_args_no_transform(fielddata_obj): field = ones_like(fielddata_obj.ds["t_surface_instant"]) fielddata_obj.ds = {"t_surface_instant": field} fielddata_obj.vspec["transform"] = None - assert np.array_equal(fielddata_obj.values(), field.t) + assert np.array_equal(fielddata_obj.get_values(), field.t) def test_fielddata_values_no_args_transform(fielddata_obj): field = ones_like(fielddata_obj.ds["t_surface_instant"]) fielddata_obj.ds = {"t_surface_instant": field} fielddata_obj.vspec["transform"] = "opposite" - assert np.array_equal(fielddata_obj.values(), -field.t.squeeze()) + assert np.array_equal(fielddata_obj.get_values(), -field.t.squeeze()) def test_fielddata_values_bad_name_level(fielddata_obj_ro): with raises(errors.NoGraphicsDefinitionForVariableError): - fielddata_obj_ro.values(level="foo", name="temp") + fielddata_obj_ro.get_values(level="foo", name="temp") with raises(errors.NoGraphicsDefinitionForVariableError): - fielddata_obj_ro.values(level="sfc", name="foo") + fielddata_obj_ro.get_values(level="sfc", name="foo") with raises(errors.NoGraphicsDefinitionForVariableError): - fielddata_obj_ro.values(level="bar", name="foo") + fielddata_obj_ro.get_values(level="bar", name="foo") def test_profiledata_values(profiledata_obj): - assert profiledata_obj.values().shape == (50,) + assert profiledata_obj.get_values().shape == (50,) def test_profiledata_values_bad_name_level(profiledata_obj): with raises(errors.NoGraphicsDefinitionForVariableError): - profiledata_obj.values(level="foo", name="temp") + profiledata_obj.get_values(level="foo", name="temp") with raises(errors.NoGraphicsDefinitionForVariableError): - profiledata_obj.values(level="sfc", name="foo") + profiledata_obj.get_values(level="sfc", name="foo") with raises(errors.NoGraphicsDefinitionForVariableError): - profiledata_obj.values(level="bar", name="foo") + profiledata_obj.get_values(level="bar", name="foo") def test_profiledata_values_one_level(profiledata_obj): - value = profiledata_obj.values(name="hlcy", level="sr01") + value = profiledata_obj.get_values(name="hlcy", level="sr01") assert value.shape == () # A single number assert value == 47.7 diff --git a/tests/datahandler/test_gribfile.py b/tests/datahandler/test_gribfile.py index d799913..433fb1e 100644 --- a/tests/datahandler/test_gribfile.py +++ b/tests/datahandler/test_gribfile.py @@ -20,8 +20,8 @@ def test_gribfiles(): "typeOfLevel": "surface", }, ) - assert isinstance(gf.contents, dict) - assert isinstance(gf.contents["sp_surface_instant"], Dataset) - assert len(gf.contents) == 1 - assert len(gf.contents["sp_surface_instant"].data_vars) == 1 - assert len(gf.contents["sp_surface_instant"].data_vars["sp"].shape) == 3 + assert isinstance(gf.datasets, dict) + assert isinstance(gf.datasets["sp_surface_instant"], Dataset) + assert len(gf.datasets) == 1 + assert len(gf.datasets["sp_surface_instant"].data_vars) == 1 + assert len(gf.datasets["sp_surface_instant"].data_vars["sp"].shape) == 3 From 0e9cf2724e950826a4744ada9cda12cdfceface2 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 9 Dec 2025 09:06:06 -0700 Subject: [PATCH 81/98] A few more coverage statements. --- adb_graphics/datahandler/gribdata.py | 2 +- adb_graphics/figures/maps.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index b0006a3..638db6f 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -538,7 +538,7 @@ def grid_info(self) -> dict: grid_info[bm_arg] = val del val - else: + else: # pragma: no cover grid_info["lat_0"] = 20.44 grid_info["lon_0"] = 202.54 grid_info["width"] = 2000000 diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 00ceafd..4a9b741 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -140,7 +140,7 @@ def __init__( self.map_spec = deepcopy(self.fields_spec[self.name][self.level]) set_level(self.level, self.model, self.map_spec) # Required if map_type is "diff" - if map_type == "diff": + if map_type == "diff": # pragma: no cover self.ds2 = ds2 if not self.ds2: msg = "Diff map requires a second grib path. Provide ds2 argument!" @@ -160,7 +160,7 @@ def shaded(self): "ds": self.ds, } field = gribdata.FieldData(**args) # type: ignore[arg-type] - if self.map_type == "diff": + if self.map_type == "diff": # pragma: no cover args["ds"] = self.ds2 field2 = gribdata.FieldData(**args) # type: ignore[arg-type] field.data = field.data - field2.data @@ -173,10 +173,10 @@ def contours(self): # We won't plot contours on multipanel plots, or full global # plots. - if self.map_type == "enspanel": + if self.map_type == "enspanel": # pragma: no cover return [] - if "global" in self.model and self.tile in ["full"]: + if "global" in self.model and self.tile in ["full"]: # pragma: no cover return [] return self._overlay_fields("contours") @@ -187,7 +187,7 @@ def hatches(self): return self._overlay_fields("hatches") - def wind_fields(self, level: str | None = None): + def wind_fields(self, level: str | None = None): # pragma: no cover """Return u, v tuple of wind fields.""" lev = level or self.level @@ -207,7 +207,7 @@ def wind_fields(self, level: str | None = None): winds.append(gribdata.FieldData(**args)) # type: ignore[arg-type] return winds - def _overlay_fields(self, spec_sect: str) -> list: + def _overlay_fields(self, spec_sect: str) -> list: # pragma: no cover """ Create FieldData objects for the specified overlay type - hatches or contours. """ @@ -273,7 +273,7 @@ def __init__(self, airport_fn: Path, ax: Axes, **kwargs): if self.model != "hrrrhi": if self.tile in FULL_TILES: self.corners = self.grid_info.pop("corners") - else: + else: # pragma: no cover self.corners = TILE_DEFS[self.tile]["corners"] self.grid_info.pop("corners") else: @@ -293,12 +293,12 @@ def __init__(self, airport_fn: Path, ax: Axes, **kwargs): if self.tile in ["HI", "Florida", "PuertoRico"] or self.model in [ "hrrrhi", "hrrrcar", - ]: + ]: # pragma: no cover area_thresh = 100 self.m = self._get_basemap(area_thresh=area_thresh, **self.grid_info) - if self.model == "hrrrhi": + if self.model == "hrrrhi": # pragma: no cover parallels = np.arange(0.0, 81, 5.0) self.m.drawparallels(parallels, labels=[False, True, True, False]) meridians = np.arange(10.0, 351.0, 5.0) @@ -316,7 +316,9 @@ def boundaries(self): zorder=2, ) else: - if self.model not in ["global", "hfip"] and self.tile not in FULL_TILES: + if ( + self.model not in ["global", "hfip"] and self.tile not in FULL_TILES + ): # pragma: no cover self.m.drawcounties( antialiased=False, color="black", From 593ab177c8364159da6c55bd9aedb60055e32afa Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 9 Dec 2025 09:32:49 -0700 Subject: [PATCH 82/98] Missed one more block. --- adb_graphics/figures/maps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adb_graphics/figures/maps.py b/adb_graphics/figures/maps.py index 4a9b741..a3ac783 100644 --- a/adb_graphics/figures/maps.py +++ b/adb_graphics/figures/maps.py @@ -276,7 +276,7 @@ def __init__(self, airport_fn: Path, ax: Axes, **kwargs): else: # pragma: no cover self.corners = TILE_DEFS[self.tile]["corners"] self.grid_info.pop("corners") - else: + else: # pragma: no cover self.corners = None if self.tile in FULL_TILES: self.width = self.grid_info.pop("width") From 2083ce8c803bbb6210ff002274b34a822e9eaf02 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 7 Jan 2026 14:07:38 -0700 Subject: [PATCH 83/98] Add backend args for open_datasets. --- adb_graphics/datahandler/gribfile.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index e8ae0a2..2afe5e3 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -77,6 +77,12 @@ def datasets(self): datasets = cfgrib.open_datasets( str(self.filename), read_keys=["orientationOfTheGridInDegrees", "parameterNumber"], + backend_kwargs=( + { + "indexpath": "", + "read_keys": ["orientationOfTheGridInDegrees"], + } + ), ) all_fields: dict = {} From 5f8c2870d940b093d319592a0133a44978511099 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 14 Jan 2026 21:12:56 +0000 Subject: [PATCH 84/98] WIP Debugging CAR and HI runs. --- adb_graphics/datahandler/gribdata.py | 18 +- adb_graphics/datahandler/gribfile.py | 2 +- adb_graphics/default_specs.yml | 798 +++++++++++++++++++++------ check_fields.py | 48 ++ conftest.py | 22 +- image_lists/hrrrcar_subset.yml | 16 - 6 files changed, 701 insertions(+), 203 deletions(-) create mode 100644 check_fields.py diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 638db6f..0b92d46 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -121,6 +121,7 @@ def _find_var(): vertical_coord = cfgribspec["typeOfLevel"] step_type = cfgribspec.get("stepType", "instant") var_id = f"{short_name}_{vertical_coord}_{step_type}" + vertical_coord = "level" if vertical_coord == "unknown" else vertical_coord ds: Dataset | dict = self.ds.get(var_id, {}) if ds == {}: msg = f"{var_id} is not a valid key for the dataset" @@ -128,14 +129,17 @@ def _find_var(): var = _find_var() if var is not None: field = ds[var] - top = cfgribspec.get("topLevel", cfgribspec.get("scaledValueOfFirstFixedSurface")) - bottom = cfgribspec.get( - "bottomLevel", cfgribspec.get("scaledValueOfSecondFixedSurface") - ) - layered = top is not None or bottom is not None - level = top if top in field.coords[vertical_coord] else bottom + level = cfgribspec.get("level") + layered = False + if level is None: + top = cfgribspec.get("topLevel", cfgribspec.get("scaledValueOfFirstFixedSurface")) + bottom = cfgribspec.get( + "bottomLevel", cfgribspec.get("scaledValueOfSecondFixedSurface") + ) + layered = top is not None or bottom is not None + level = top if top in field.coords[vertical_coord] else bottom if level is None: - level = cfgribspec.get("level", utils.numeric_level(self.level)[0]) + level = utils.numeric_level(self.level)[0] level = None if level == "" else level leveled = level is not None and vertical_coord != "hybrid" if len(field.coords[vertical_coord].shape) > 0 and (layered or leveled): diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index 2afe5e3..a70d2a3 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -98,7 +98,7 @@ def datasets(self): def _var_id(ds: xr.Dataset, var: str): - vertical_dim = ds[list(ds.data_vars)[0]].attrs.get("GRIB_typeOfLevel") + vertical_dim = ds[list(ds.data_vars)[0]].attrs.get("GRIB_typeOfLevel", "unknown") var_name = ds[var].attrs.get("GRIB_shortName") step_type = ds[var].attrs.get("GRIB_stepType", "nostepType") return f"{var_name}_{vertical_dim}_{step_type}" diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index dcf1e35..b0d9e66 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -80,12 +80,16 @@ 1hsnw: # 1 hr Accumulated Snow Using 10:1 Ratio sfc: cfgrib: - hrrr: + hrrr: &1hrsnw_hrrr shortName: sdwe level: 0 typeOfLevel: surface stepType: accum stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' + hrrrcar: + <<: *1hrsnw_hrrr + hrrrhi: + <<: *1hrsnw_hrrr rrfs: parameterNumber: 50 level: 0 @@ -108,10 +112,16 @@ 1ref: # Reflectivity at 1 km AGL 1000m: &refl cfgrib: - hrrr: + hrrr: &hrrr_1ref shortName: refd typeOfLevel: heightAboveGround level: 1000 + hrrrcar: + <<: *hrrr_1ref + shortName: unknown + parameterNumber: 195 + hrrrhi: + <<: *hrrr_1ref global: shortName: refd typeOfLevel: heightAboveGround @@ -146,20 +156,38 @@ acfrzr: # Run Total Freezing Rain sfc: <<: *graupel cfgrib: - shortName: frzr - level: 0 - typeOfLevel: surface - stepType: accum + hrrr: &named_frzr + shortName: frzr + level: 0 + typeOfLevel: surface + stepType: accum + hrrrcar: + <<: *named_frzr + shortName: unknown + parameterCategory: 1 + parameterNumber: 225 + rrfs: + <<: *named_frzr ncl_name: FRZR_P8_L1_GLC0_acc title: Run Total Freezing Rain acpcp: # Accumulated run total precipitation sfc: cfgrib: - shortName: tp - level: 0 - typeOfLevel: surface - stepRange: '{{ "%d-%d" % (0, fhr) }}' - stepType: accum + hrrr: &named_acctp + shortName: tp + level: 0 + typeOfLevel: surface + stepRange: '{{ "%d-%d" % (0, fhr) }}' + stepType: accum + hrrrcar: + <<: *named_acctp + shortName: unknown + parameterCategory: 1 + parameterNumber: 8 + hrrrhi: + <<: *named_acctp + rrfs: + <<: *named_acctp clevs: [0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 15, 20, 40] cmap: gist_ncar colors: rainbow12_colors @@ -414,9 +442,19 @@ bc2: # Black Carbon 2 cape: mu: &cape # Most Unstable CAPE cfgrib: - shortName: cape - topLevel: 25500 - typeOfLevel: pressureFromGroundLayer + hrrr: &named_cape + shortName: cape + topLevel: 25500 + typeOfLevel: pressureFromGroundLayer + hrrrcar: + <<: *named_cape + shortName: unknown + parameterCategory: 7 + parameterNumber: 6 + hrrrhi: + <<: *named_cape + rrfs: + <<: *named_cape clevs: [1, 100, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000] cmap: gist_ncar colors: vort_colors @@ -442,9 +480,19 @@ cape: mul: # Most Unstable Layer CAPE <<: *cape cfgrib: - shortName: cape - topLevel: 18000 - typeOfLevel: pressureFromGroundLayer + hrrr: &named_mulcape + shortName: cape + topLevel: 18000 + typeOfLevel: pressureFromGroundLayer + hrrrcar: + <<: *named_mulcape + shortName: unknown + parameterCategory: 7 + parameterNumber: 6 + hrrrhi: + <<: *named_mulcape + rrfs: + <<: *named_mulcape contours: cape: colors: white @@ -465,9 +513,19 @@ cape: mx90mb: # Lowest 90 mb Mixed Layer CAPE <<: *cape cfgrib: - shortName: cape - topLevel: 9000 - typeOfLevel: pressureFromGroundLayer + hrrr: &named_mixcape + shortName: cape + topLevel: 9000 + typeOfLevel: pressureFromGroundLayer + hrrrcar: + <<: *named_mixcape + shortName: unknown + parameterCategory: 7 + parameterNumber: 6 + hrrrhi: + <<: *named_mixcape + rrfs: + <<: *named_mixcape contours: cape: colors: white @@ -488,9 +546,19 @@ cape: sfc: <<: *cape cfgrib: - shortName: cape - level: 0 - typeOfLevel: surface + hrrr: &named_sfccape + shortName: cape + level: 0 + typeOfLevel: surface + hrrrcar: + <<: *named_sfccape + shortName: unknown + parameterCategory: 7 + parameterNumber: 6 + hrrrhi: + <<: *named_sfccape + rrfs: + <<: *named_sfccape contours: cin: colors: 'k' @@ -596,9 +664,19 @@ cicep: # Categorical Ice Pellets cin: # Surface Convective Inhibition mu: cfgrib: - shortName: cin - level: 25500 - typeOfLevel: pressureFromGroundLayer + hrrr: &named_cin + shortName: cin + level: 25500 + typeOfLevel: pressureFromGroundLayer + hrrrcar: + <<: *named_cin + shortName: unknown + parameterCategory: 7 + parameterNumber: 7 + hrrrhi: + <<: *named_cin + rrfs: + <<: *named_cin clevs: [-300, -200, -150, -100, -75, -50, -40, -30, -20, -10, -1] cmap: gist_ncar colors: cin_colors @@ -607,17 +685,38 @@ cin: # Surface Convective Inhibition vertical_index: 2 mx90mb: # Lowest 90 mb Mixed Layer CIN cfgrib: - shortName: cin - level: 9000 - typeOfLevel: pressureFromGroundLayer + hrrr: &named_mixcin + shortName: cin + level: 9000 + typeOfLevel: pressureFromGroundLayer + hrrrcar: + <<: *named_mixcin + shortName: unknown + parameterCategory: 7 + parameterNumber: 7 + hrrrhi: + <<: *named_mixcin + rrfs: + <<: *named_mixcin + clevs: [-300, -200, -150, -100, -75, -50, -40, -30, -20, -10, -1] ncl_name: CIN_P0_2L108_{grid} title: 'ML CIN < -50' vertical_index: 0 sfc: &sfc_cin cfgrib: - shortName: cin - level: 0 - typeOfLevel: surface + hrrr: &named_sfccin + shortName: cin + level: 0 + typeOfLevel: surface + hrrrcar: + <<: *named_sfccin + shortName: unknown + parameterCategory: 7 + parameterNumber: 7 + hrrrhi: + <<: *named_sfccin + rrfs: + <<: *named_sfccin clevs: [-300, -200, -150, -100, -75, -50, -40, -30, -20, -10, -1] cmap: gist_ncar colors: cin_colors @@ -645,9 +744,18 @@ cloudcover: high: <<: *cld_cover cfgrib: - shortName: hcc - typeOfLevel: highCloudLayer - level: 0 + hrrr: &high_cld_cover + shortName: hcc + typeOfLevel: highCloudLayer + level: 0 + hrrrcar: + <<: *high_cld_cover + typeOfLevel: unknown + hrrrhi: + <<: *high_cld_cover + rrfs: + <<: *high_cld_cover + clevs: [2, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95] cmap: gist_ncar colors: cldcov_colors @@ -658,27 +766,53 @@ cloudcover: low: <<: *cld_cover cfgrib: - shortName: lcc - typeOfLevel: lowCloudLayer - level: 0 + hrrr: &low_cld_cover + shortName: lcc + typeOfLevel: lowCloudLayer + level: 0 + hrrrcar: + <<: *low_cld_cover + typeOfLevel: unknown + hrrrhi: + <<: *low_cld_cover + rrfs: + <<: *low_cld_cover ncl_name: LCDC_P0_L214_{grid} title: Low-Level Cloud Cover mid: <<: *cld_cover cfgrib: - shortName: mcc - typeOfLevel: middleCloudLayer - level: 0 + hrrr: &mid_cld_cover + shortName: mcc + typeOfLevel: middleCloudLayer + level: 0 + hrrrcar: + <<: *mid_cld_cover + typeOfLevel: unknown + hrrrhi: + <<: *mid_cld_cover + rrfs: + <<: *mid_cld_cover ncl_name: MCDC_P0_L224_{grid} title: Mid-Level Cloud Cover total: <<: *cld_cover cfgrib: - hrrr: + hrrr: &tcc shortName: tcc typeOfLevel: atmosphere stepType: instant level: 0 + hrrrcar: + <<: *tcc + shortName: unknown + parameterNumber: 1 + parameterCategory: 6 + hrrrhi: + <<: *tcc + shortName: unknown + parameterNumber: 1 + parameterCategory: 6 rrfs: shortName: tcc typeOfLevel: atmosphereSingleLayer @@ -813,8 +947,17 @@ cpofp: # Frozen Precipitation Percentage crain: # Categorical Rain sfc: cfgrib: - shortName: crain - typeOfLevel: surface + hrrr: &named_crain + shortName: crain + typeOfLevel: surface + hrrrcar: + parameterCategory: 1 + parameterName: 192 + typeOfLevel: surface + hrrrhi: + <<: *named_crain + rrfs: + <<: *named_crain ncl_name: CRAIN_P0_L1_{grid} cref: # Composite reflectivity esbl: @@ -832,6 +975,13 @@ cref: # Composite reflectivity shortName: refc typeOfLevel: atmosphere level: 0 + hrrrcar: &refc_noname + parameterCategory: 16 + parameterNumber: 196 + level: 0 + typeOfLevel: atmosphere + hrrrhi: + <<: *refc_noname global: shortName: refc typeOfLevel: atmosphereSingleLayer @@ -992,11 +1142,15 @@ fullintdust: # Full vertically integrated dust (Fine dust + Coarse dust) echotop: # Echo Top sfc: cfgrib: - hrrr: + hrrr: &hrrr_echotop parameterNumber: 3 parameterCategory: 16 typeOfLevel: cloudTop level: 0 + hrrrcar: + <<: *hrrr_echotop + hrrrhi: + <<: *hrrr_echotop rrfs: parameterNumber: 3 parameterCategory: 16 @@ -1076,9 +1230,18 @@ firewx-pygraf: flru: # Aviation Flight Rules sfc: cfgrib: - shortName: gh - typeOfLevel: cloudCeiling - level: 0 + hrrr: &cloud_base + shortName: gh + typeOfLevel: cloudCeiling + level: 0 + hrrrcar: + <<: *cloud_base + typeOfLevel: cloudBase + hrrrhi: + <<: *cloud_base + rrfs: + <<: *cloud_base + clevs: [0.0, 1.0, 2.0, 3.0, 4.0] cmap: gist_ncar colors: flru_colors @@ -1098,6 +1261,16 @@ G113bt: # GOES-W Brightness temperature, water vapor (Ch 3) shortName: SBT113 typeOfLevel: nominalTop level: 0 + hrrrcar: + parameterCategory: 192 + parameterNumber: 7 + typeOfLevel: nominalTop + level: 0 + hrrrhi: + parameterCategory: 192 + typeOfLevel: nominalTop + parameterNumber: 7 + level: 0 rrfs: parameterNumber: 242 typeOfLevel: atmosphere @@ -1114,9 +1287,22 @@ G114bt: # GOES-W Brightness temperature, infrared (Ch 4) sat: <<: *goes_sat cfgrib: - shortName: SBT114 - typeOfLevel: nominalTop - level: 0 + hrrr: &g114_bt + shortName: SBT114 + typeOfLevel: nominalTop + level: 0 + hrrrcar: + parameterCategory: 192 + parameterNumber: 7 + typeOfLevel: nominalTop + level: 0 + hrrrhi: + parameterCategory: 192 + parameterNumber: 7 + typeOfLevel: nominalTop + level: 0 + rrfs: + <<: *g114_bt ncl_name: SBT114_P0_L8_{grid} title: GOES-W Brightness Temperature, Infrared unit: ch 4 @@ -1124,18 +1310,45 @@ G123bt: # GOES-E Brightness temperature, water vapor (Ch 3) sat: <<: *goes_sat cfgrib: - shortName: SBT123 - typeOfLevel: nominalTop - level: 0 + hrrr: &g123_bt + shortName: SBT123 + typeOfLevel: nominalTop + level: 0 + hrrrcar: + parameterCategory: 192 + parameterNumber: 1 + level: 0 + typeOfLevel: nominalTop + hrrrhi: + parameterCategory: 192 + parameterNumber: 1 + typeOfLevel: nominalTop + level: 0 + rrfs: + <<: *g123_bt + ncl_name: SBT123_P0_L8_{grid} title: GOES-E Brightness Temperature, Water Vapor G124bt: # GOES-E Brightness temperature, infrared (Ch 4) sat: <<: *goes_sat cfgrib: - shortName: SBT124 - typeOfLevel: nominalTop - level: 0 + hrrr: &g124_bt + shortName: SBT124 + typeOfLevel: nominalTop + level: 0 + hrrrcar: + typeOfLevel: nominalTop + parameterCategory: 192 + parameterNumber: 2 + level: 0 + hrrrhi: + typeOfLevel: nominalTop + parameterCategory: 192 + parameterNumber: 2 + level: 0 + rrfs: + <<: *g124_bt ncl_name: SBT124_P0_L8_{grid} title: GOES-E Brightness Temperature, Infrared @@ -1242,11 +1455,15 @@ gust: hail: # Max 1h Hail diameter maxsfc: &hail # surface cfgrib: - hrrr: + hrrr: &hrrr_hail shortName: hail typeOfLevel: sigma level: 0 stepType: max + hrrrcar: + <<: *hrrr_hail + hrrrhi: + <<: *hrrr_hail rrfs: shortName: hail typeOfLevel: surface @@ -1262,11 +1479,15 @@ hail: # Max 1h Hail diameter max: # total atmosphere <<: *hail cfgrib: - hrrr: + hrrr: &hrrr_maxhail shortName: hail typeOfLevel: atmosphere level: 0 stepType: max + hrrrcar: + <<: *hrrr_maxhail + hrrrhi: + <<: *hrrr_maxhail rrfs: shortName: hail typeOfLevel: surface @@ -1455,16 +1676,19 @@ hlcytot: hpbl: # Height of Planetary Boundary Layer sfc: cfgrib: - hrrr: + hrrr: &hpbl shortName: blh typeOfLevel: surface - global: + hrrrcar: &unnamed_blh parameterNumber: 196 parameterCategory: 3 typeOfLevel: surface + hrrrhi: + <<: *hpbl + global: + <<: *unnamed_blh rrfs: - shortName: blh - typeOfLevel: surface + <<: *hpbl clevs: [25, 50, 100, 200, 300, 500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000] cmap: ir_rgbv colors: pbl_colors @@ -1591,9 +1815,22 @@ lhtflavg: li: # Lifted Index best: &lifted_index cfgrib: - shortName: 4lftx - typeOfLevel: pressureFromGroundLayer - topLevel: 18000 + hrrr: &named_bestli + shortName: 4lftx + typeOfLevel: pressureFromGroundLayer + topLevel: 18000 + hrrrcar: + <<: *named_bestli + shortName: unknown + parameterCategory: 7 + parameterNumber: 193 + hrrrhi: + <<: *named_bestli + shortName: unknown + parameterCategory: 7 + parameterNumber: 193 + rrfs: + <<: *named_bestli clevs: !arange [-15, 16] cmap: Spectral colors: lifted_index_colors @@ -1604,17 +1841,35 @@ li: # Lifted Index sfc: <<: *lifted_index cfgrib: - shortName: lftx - typeOfLevel: isobaricLayer - level: 500 + hrrr: &named_lftx + shortName: lftx + typeOfLevel: isobaricLayer + level: 500 + hrrrcar: + <<: *named_lftx + shortName: unknown + parameterCategory: 7 + parameterNumber: 192 + hrrrhi: + <<: *named_lftx + rrfs: + <<: *named_lftx ncl_name: LFTX_P0_2L100_{grid} title: Surface Lifted Index lpl: # Lifted parcel level agl: - cfgrib: - shortName: plpl - typeOfLevel: pressureFromGroundLayer - level: 25500 + cfgrib: &cfgrib_lpl + hrrr: &named_plpl + shortName: plpl + typeOfLevel: pressureFromGroundLayer + level: 25500 + hrrrcar: + <<: *named_plpl + shortName: unknown + parameterCategory: 3 + parameterNumber: 200 + rrfs: + <<: *named_plpl ncl_name: PLPL_P0_2L108_{grid} title: Lifted Parcel Level AGL >50 transform: @@ -1625,18 +1880,26 @@ lpl: # Lifted parcel level unit: hPa ua: cfgrib: - shortName: plpl - typeOfLevel: pressureFromGroundLayer - level: 25500 + <<: *cfgrib_lpl ncl_name: PLPL_P0_2L108_{grid} transform: conversions.pa_to_hpa unit: hPa ltg3: # Lightning Threat (LTG1 ... LTG2) sfc: cfgrib: - shortName: ltng - typeOfLevel: atmosphere - level: 0 + hrrr: &named_ltng + shortName: ltng + typeOfLevel: atmosphere + level: 0 + hrrrcar: &unnamed_ltng + <<: *named_ltng + shortName: unknown + parameterCategory: 17 + parameterNumber: 192 + hrrrhi: + <<: *unnamed_ltng + rrfs: + <<: *named_ltng clevs: [0.02, 0.5, 1.0, 1.5, 2.0, 2.5, 3, 4, 5, 6, 7, 8, 10, 12] cmap: NWSReflectivity colors: cref_colors @@ -1782,10 +2045,17 @@ pres: unit: hPa msl: cfgrib: - hrrr: + hrrr: &hrrr_mslp shortName: mslma typeOfLevel: meanSea level: 0 + hrrrcar: + <<: *hrrr_mslp + shortName: unknown + parameterCategory: 3 + parameterNumber: 198 + hrrrhi: + <<: *hrrr_mslp global: shortName: prmsl typeOfLevel: meanSea @@ -1850,10 +2120,14 @@ presmin: ptmp: # Potential temperature 2m: cfgrib: - hrrr: + hrrr: &hrrr_pt shortName: pt typeOfLevel: heightAboveGround level: 2 + hrrrcar: + <<: *hrrr_pt + hrrrhi: + <<: *hrrr_pt rrfs: shortName: pt typeOfLevel: surface @@ -1867,12 +2141,22 @@ ptmp: # Potential temperature wind: 10m ptyp: # Hourly total precipitation sfc: - cfgrib: - shortName: tp - typeOfLevel: surface - level: 0 - stepType: accum - stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' + cfgrib: &cfgrib_precip + hrrr: &named_tp + shortName: tp + typeOfLevel: surface + level: 0 + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' + stepType: accum + hrrrcar: + <<: *named_tp + shortName: unknown + parameterCategory: 1 + parameterNumber: 8 + hrrrhi: + <<: *named_tp + rrfs: + <<: *named_tp clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar colors: pcp_colors @@ -1929,9 +2213,17 @@ ptyp: # Hourly total precipitation pwtr: # Precipitable water sfc: &pwtr cfgrib: - shortName: pwat - typeOfLevel: atmosphereSingleLayer - level: 0 + hrrr: &pwat + shortName: pwat + typeOfLevel: atmosphereSingleLayer + level: 0 + hrrrcar: + <<: *pwat + typeOfLevel: unknown + hrrrhi: + <<: *pwat + rrfs: + <<: *pwat clevs: !arange [4, 81, 4] cmap: gist_ncar colors: pw_colors @@ -1943,25 +2235,39 @@ ref: # Maximum reflectivity for past hour at 1 km AGL m10: <<: *refl cfgrib: - hrrr: + hrrr: &named_refd shortName: refd typeOfLevel: isothermal level: 263 stepType: instant + hrrrcar: + <<: *named_refd + parameterCategory: 16 + parameterNumber: 195 + shortName: unknown + hrrrhi: + <<: *named_refd + parameterCategory: 16 + parameterNumber: 195 + shortName: unknown rrfs: + <<: *named_refd shortName: rare - typeOfLevel: isothermal - level: 263 ncl_name: REFD_P0_L20_{grid} title: -10C Isothermal Reflectivity maxm10: <<: *refl cfgrib: - hrrr: + hrrr: &named_maxm10 shortName: refd typeOfLevel: isothermal level: 263 stepType: max + hrrrcar: + <<: *named_maxm10 + shortName: unknown + parameterCategory: 16 + parameterNumber: 195 ncl_name: REFD_P8_L20_{grid}_max1h title: Max 1h -10C Isothermal Reflectivity rh: # Relative Humidity @@ -2071,10 +2377,14 @@ rh: # Relative Humidity rvil: # Radar-derived Vertically Integrated Liquid sfc: &vert_int_liq cfgrib: - hrrr: + hrrr: &hrrr_rvil shortName: veril typeOfLevel: atmosphere level: 0 + hrrrcar: + <<: *hrrr_rvil + hrrrhi: + <<: *hrrr_rvil rrfs: shortName: veril typeOfLevel: atmosphereSingleLayer @@ -2295,21 +2605,20 @@ soilt: # Soil Temperature 10cm: <<: *soilt_levs cfgrib: - hrrr: + hrrr: &st_10cm shortName: st typeOfLevel: depthBelowLandLayer scaledValueOfFirstFixedSurface: 10 scaledValueOfSecondFixedSurface: 10 + hrrrcar: + <<: *st_10cm + hrrrhi: + <<: *st_10cm global: - shortName: st - typeOfLevel: depthBelowLandLayer + <<: *st_10cm scaledValueOfFirstFixedSurface: 0 - scaledValueOfSecondFixedSurface: 10 rrfs: - shortName: st - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 10 - scaledValueOfSecondFixedSurface: 10 + <<: *st_10cm title: Soil Temperature at 10cm 30cm: <<: *soilt_levs @@ -2333,21 +2642,20 @@ soilt: # Soil Temperature 1m: <<: *soilt_levs cfgrib: - hrrr: + hrrr: &st_1m shortName: st typeOfLevel: depthBelowLandLayer scaledValueOfFirstFixedSurface: 100 scaledValueOfSecondFixedSurface: 100 + hrrrcar: + <<: *st_1m + hrrrhi: + <<: *st_1m global: - shortName: st - typeOfLevel: depthBelowLandLayer + <<: *st_1m scaledValueOfFirstFixedSurface: 40 - scaledValueOfSecondFixedSurface: 100 rrfs: - shortName: st - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 100 - scaledValueOfSecondFixedSurface: 100 + <<: *st_1m title: Soil Temperature at 1m 1.6m: <<: *soilt_levs @@ -2368,10 +2676,20 @@ soilt: # Soil Temperature soilw: # Soil Moisture 0cm: &soilw_levs cfgrib: - shortName: soilw - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 0 - scaledValueOfSecondFixedSurface: 0 + hrrr: &soilw_0cm + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 0 + scaledValueOfSecondFixedSurface: 0 + hrrrcar: + <<: *soilw_0cm + shortName: unknown + parameterCategory: 0 + parameterNumber: 2 + hrrrhi: + <<: *soilw_0cm + rrfs: + <<: *soilw_0cm clevs: [0, 0.01, 0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50] cmap: jet_r colors: soilw_colors @@ -2382,98 +2700,185 @@ soilw: # Soil Moisture 1cm: <<: *soilw_levs cfgrib: - shortName: soilw - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 1 - scaledValueOfSecondFixedSurface: 1 + hrrr: &soilw_1cm + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 1 + scaledValueOfSecondFixedSurface: 1 + hrrrcar: + <<: *soilw_1cm + shortName: unknown + parameterCategory: 0 + parameterNumber: 2 + hrrrhi: + <<: *soilw_1cm + rrfs: + <<: *soilw_1cm title: Soil Moisture at 1cm 4cm: <<: *soilw_levs cfgrib: - shortName: soilw - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 4 - scaledValueOfSecondFixedSurface: 4 + hrrr: &soilw_4cm + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 4 + scaledValueOfSecondFixedSurface: 4 + hrrrcar: + <<: *soilw_4cm + shortName: unknown + parameterCategory: 0 + parameterNumber: 2 + hrrrhi: + <<: *soilw_4cm + rrfs: + <<: *soilw_4cm title: Soil Moisture at 4cm 10cm: <<: *soilw_levs cfgrib: - hrrr: &soilw_cfgrib_10 + hrrr: &soilw_cfgrib_10cm shortName: soilw typeOfLevel: depthBelowLandLayer scaledValueOfFirstFixedSurface: 10 scaledValueOfSecondFixedSurface: 10 + hrrrcar: + <<: *soilw_cfgrib_10cm + shortName: unknown + parameterCategory: 0 + parameterNumber: 2 + hrrrhi: + <<: *soilw_cfgrib_10cm global: - <<: *soilw_cfgrib_10 + <<: *soilw_cfgrib_10cm scaledValueOfFirstFixedSurface: 0 rrfs: - <<: *soilw_cfgrib_10 + <<: *soilw_cfgrib_10cm title: Soil Moisture at 10cm 30cm: <<: *soilw_levs cfgrib: - shortName: soilw - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 30 - scaledValueOfSecondFixedSurface: 30 + hrrr: &soilw_30cm + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 30 + scaledValueOfSecondFixedSurface: 30 + hrrrcar: + <<: *soilw_30cm + shortName: unknown + parameterCategory: 0 + parameterNumber: 2 + hrrrhi: + <<: *soilw_30cm + rrfs: + <<: *soilw_30cm title: Soil Moisture at 30cm 40cm: <<: *soilw_levs cfgrib: - shortName: soilw - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 40 - scaledValueOfSecondFixedSurface: 40 + hrrr: &soilw_40cm + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 40 + scaledValueOfSecondFixedSurface: 40 + hrrrcar: + <<: *soilw_40cm + shortName: unknown + parameterCategory: 0 + parameterNumber: 2 + hrrrhi: + <<: *soilw_40cm + rrfs: + <<: *soilw_40cm title: Soil Moisture at 40cm 60cm: <<: *soilw_levs cfgrib: - shortName: soilw - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 60 - scaledValueOfSecondFixedSurface: 60 + hrrr: &soilw_60cm + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 60 + scaledValueOfSecondFixedSurface: 60 + hrrrcar: + <<: *soilw_60cm + shortName: unknown + parameterCategory: 0 + parameterNumber: 2 + hrrrhi: + <<: *soilw_60cm + rrfs: + <<: *soilw_60cm title: Soil Moisture at 60cm 1m: <<: *soilw_levs cfgrib: - hrrr: &soilw_cfgrib_100 + hrrr: &soilw_cfgrib_100cm shortName: soilw typeOfLevel: depthBelowLandLayer scaledValueOfFirstFixedSurface: 100 scaledValueOfSecondFixedSurface: 100 + hrrrcar: + <<: *soilw_cfgrib_100cm + shortName: unknown + parameterCategory: 0 + parameterNumber: 2 + hrrrhi: + <<: *soilw_cfgrib_100cm global: - <<: *soilw_cfgrib_100 + <<: *soilw_cfgrib_100cm scaledValueOfFirstFixedSurface: 40 rrfs: - <<: *soilw_cfgrib_100 + <<: *soilw_cfgrib_100cm title: Soil Moisture at 1m 1.6m: <<: *soilw_levs cfgrib: - shortName: soilw - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 160 - scaledValueOfSecondFixedSurface: 160 + hrrr: &soilw_cfgrib_160cm + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 160 + scaledValueOfSecondFixedSurface: 160 + hrrrcar: + <<: *soilw_cfgrib_160cm + shortName: unknown + parameterCategory: 0 + parameterNumber: 2 + hrrrhi: + <<: *soilw_cfgrib_160cm + rrfs: + <<: *soilw_cfgrib_160cm title: Soil Moisture at 1.6m 3m: <<: *soilw_levs cfgrib: - shortName: soilw - typeOfLevel: depthBelowLandLayer - scaledValueOfFirstFixedSurface: 300 - scaledValueOfSecondFixedSurface: 300 + hrrr: &soilw_cfgrib_300cm + shortName: soilw + typeOfLevel: depthBelowLandLayer + scaledValueOfFirstFixedSurface: 300 + scaledValueOfSecondFixedSurface: 300 + hrrrcar: + <<: *soilw_cfgrib_300cm + shortName: unknown + parameterCategory: 0 + parameterNumber: 2 + hrrrhi: + <<: *soilw_cfgrib_300cm + rrfs: + <<: *soilw_cfgrib_300cm title: Soil Moisture at 3m solar: # Incoming Solar Radiation sfc: &incoming_radiation cfgrib: - hrrr: + hrrr: &sdswrf shortName: sdswrf level: 0 typeOfLevel: surface + hrrrcar: + <<: *sdswrf + hrrrhi: + <<: *sdswrf rrfs: + <<: *sdswrf shortName: csdsf - level: 0 - typeOfLevel: surface clevs: [50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100] cmap: gist_ncar colors: vort_colors @@ -2643,10 +3048,7 @@ totp: # Hourly total precipitation sfc: <<: *precip cfgrib: - shortName: tp - typeOfLevel: surface - stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' - stepType: accum + <<: *cfgrib_precip contours: pres_msl: colors: red @@ -2809,14 +3211,16 @@ ulwrf: # Upward Longwave Radiation Flux unit: W/m$^{2}$ top: # Nominal top of atmosphere cfgrib: - hrrr: + hrrr: &ulwrf parameterNumber: 4 typeOfLevel: nominalTop level: 0 + hrrrcar: + <<: *ulwrf + hrrrhi: + <<: *ulwrf rrfs: - parameterNumber: 4 - typeOfLevel: nominalTop - level: 0 + <<: *ulwrf stepType: avg clevs: !arange [80, 341, 2] cmap: ir_rgbv_r @@ -2957,14 +3361,40 @@ vbdsf: # Incoming Direct Radiation sfc: <<: *incoming_radiation cfgrib: - shortName: vbdsf - typeOfLevel: surface + hrrr: + shortName: vbdsf + typeOfLevel: surface + hrrrcar: + parameterCategory: 4 + parameterNumber: 200 + typeOfLevel: unknown + hrrrhi: + parameterCategory: 4 + parameterNumber: 200 + typeOfLevel: unknown + rrfs: + shortName: vbdsf + typeOfLevel: surface ncl_name: VBDSF_P0_L1_{grid} title: Incoming Direct Radiation vddsf: # Incoming Diffuse Radiation sfc: <<: *incoming_radiation cfgrib: + hrrr: + shortName: vddsf + typeOfLevel: surface + hrrrcar: + parameterCategory: 4 + parameterNumber: 201 + typeOfLevel: unknown + hrrrhi: + parameterCategory: 4 + parameterNumber: 201 + typeOfLevel: unknown + rrfs: + shortName: vddsf + typeOfLevel: surface shortName: vddsf typeOfLevel: surface ncl_name: VDDSF_P0_L1_{grid} @@ -2972,9 +3402,16 @@ vddsf: # Incoming Diffuse Radiation vig: # Vertically-integrated graupel sfc: cfgrib: - parameterNumber: 74 - typeOfLevel: atmosphereSingleLayer - level: 0 + hrrr: &hrrr_vig + parameterCategory: 1 + parameterNumber: 74 + typeOfLevel: atmosphereSingleLayer + level: 0 + hrrrcar: + <<: *hrrr_vig + typeOfLevel: unknown + rrfs: + <<: *hrrr_vig clevs: !arange [5, 91, 5] cmap: jet colors: graupel_colors @@ -3024,15 +3461,20 @@ vort: # Absolute vorticity vvel: # Vertical velocity 700mb: cfgrib: - global: + global: &named_vvel shortName: w typeOfLevel: isobaricInhPa hrrr: - shortName: w - typeOfLevel: isobaricInhPa + <<: *named_vvel + hrrrcar: + parameterCategory: 1 + parameterNumber: 74 + typeOfLevel: unknown + hrrrhi: + <<: *named_vvel rrfs: + <<: *named_vvel shortName: wz - typeOfLevel: isobaricInhPa clevs: !arange [-17, 34, 5] cmap: gist_ncar colors: vvel_colors @@ -3210,18 +3652,22 @@ wspeed: # Wind Speed wind: 10m mdn: # Hourly Maximum Downdraft Velocity cfgrib: - hrrr: + hrrr: &wspeed_mdn + parameterCategory: 2 parameterNumber: 221 typeOfLevel: pressureFromGroundLayer topLevel: 10000 bottomLevel: 100000 stepType: max + hrrrcar: + <<: *wspeed_mdn + hrrrhi: + <<: *wspeed_mdn rrfs: - parameterNumber: 221 + <<: *wspeed_mdn typeOfLevel: isobaricLayer topLevel: 100 bottomLevel: 1000 - stepType: max clevs: [-40, -35, -30, -25, -22.5, -20, -17.5, -15, -12.5, -10, -7.5, -5, -2.5, -2, -1.5, -1, -0.5] cmap: jet colors: mdn_colors @@ -3231,18 +3677,20 @@ wspeed: # Wind Speed unit: m/s mup: # Hourly Maximum Updraft Velocity cfgrib: - hrrr: + hrrr: &wspeed_mup + parameterCategory: 2 parameterNumber: 220 typeOfLevel: pressureFromGroundLayer topLevel: 10000 bottomLevel: 100000 stepType: max + hrrrcar: + <<: *wspeed_mup rrfs: - parameterNumber: 220 + <<: *wspeed_mup typeOfLevel: isobaricLayer topLevel: 100 bottomLevel: 1000 - stepType: max clevs: [0.5, 1, 1.5, 2, 2.5, 5, 7.5, 10, 12.5, 15, 17.5, 20, 22.5, 25, 30, 35, 40] cmap: jet colors: mup_colors diff --git a/check_fields.py b/check_fields.py new file mode 100644 index 0000000..9638f60 --- /dev/null +++ b/check_fields.py @@ -0,0 +1,48 @@ +import glob +import sys + +from adb_graphics import utils +from adb_graphics.datahandler import gribfile, gribdata +from create_graphics import load_images + + +def main(args): + + grib_file, image_file = args + + model, var_levels = load_images([image_file, "hourly"]) + + ds = gribfile.WholeGribFile(grib_file).datasets + first_item = next(iter(ds.keys())) + fhr = int(ds[first_item].step.dt.total_seconds() // 3600) + + specs = utils.load_yaml("adb_graphics/default_specs.yml") + specs.dereference(context={"fhr": int(fhr), "file_type": "prs"}) + + + for variable, levels in var_levels.items(): + for level in levels: + spec = specs.get(variable, {}).get(level) + if spec is None: + print(f"No spec for {variable} at {level}") + continue + vspec = utils.cfgrib_spec(spec["cfgrib"], model) + args = { + "fhr": fhr, + "level": level, + "model": model, + "short_name": variable, + "spec": specs, + "ds": ds, + } + try: + field = gribdata.FieldData(**args).data + except Exception as e: + print(str(e)) + continue + +if __name__ == "__main__": + main(sys.argv[1:]) + + + diff --git a/conftest.py b/conftest.py index d01ce8f..972e44b 100644 --- a/conftest.py +++ b/conftest.py @@ -8,11 +8,12 @@ import glob from pathlib import Path +import pytest + from pytest import fixture from adb_graphics.datahandler import gribfile - @fixture(scope="session", autouse=True) def cleanup_data_idx(): yield # Nothing to be done before tests @@ -36,17 +37,29 @@ def pytest_addoption(parser): help="Path to prs-file.", ) + parser.addoption( + "--image-file", + action="store", + help="Path to image list file.", + type=Path, + ) + +def pytest_configure(config): + pytest.image_file = config.getoption("--image-file") @fixture(scope="session") -def natfile(): +def natfile(pytestconfig): """Interface to pass a grib file to pytest.""" - + if path := pytestconfig.getoption("--nat-file"): + return Path(path) return Path("tests", "data", "wrfnat_hrconus_16.grib2") @fixture(scope="session") -def prsfile(): +def prsfile(pytestconfig): """Interface to pass a grib file to pytest.""" + if path := pytestconfig.getoption("--prs-file"): + return Path(path) return Path("tests", "data", "wrfprs_hrconus_16.grib2") @@ -61,6 +74,7 @@ def prs_ds(prsfile): return gribfile.WholeGribFile(prsfile).datasets + @fixture(scope="session") def nat_ds(natfile): return gribfile.WholeGribFile(natfile).datasets diff --git a/image_lists/hrrrcar_subset.yml b/image_lists/hrrrcar_subset.yml index 5f08167..7f1783f 100644 --- a/image_lists/hrrrcar_subset.yml +++ b/image_lists/hrrrcar_subset.yml @@ -16,16 +16,9 @@ hourly: - mul - mx90mb - sfc - ceil: - - ua - ceilexp: - - ua - ceilexp2: - - ua cin: - sfc cloudcover: - - bndylay - high - low - mid @@ -40,14 +33,10 @@ hourly: - 2m echotop: - sfc - firewx-pygraf: - - sfc flru: - sfc G113bt: - sat - G114bt: - - sat G123bt: - sat G124bt: @@ -85,8 +74,6 @@ hourly: - sfc ltg3: - sfc - mfrp: - - sfc mref: - sfc pres: @@ -142,9 +129,6 @@ hourly: - sfc totp: - sfc - trc1: - - sfc - - int ulwrf: - sfc - top From 6335b16a6ca51e9d72dcf3f18fac45ed848988aa Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 15 Jan 2026 12:09:13 -0700 Subject: [PATCH 85/98] Testing with all. --- adb_graphics/default_specs.yml | 168 +++++++++++++++++++++++++-------- 1 file changed, 128 insertions(+), 40 deletions(-) diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index b0d9e66..40d94a7 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -119,9 +119,13 @@ hrrrcar: <<: *hrrr_1ref shortName: unknown + parameterCategory: 16 parameterNumber: 195 hrrrhi: <<: *hrrr_1ref + shortName: unknown + parameterCategory: 16 + parameterNumber: 195 global: shortName: refd typeOfLevel: heightAboveGround @@ -179,12 +183,14 @@ acpcp: # Accumulated run total precipitation typeOfLevel: surface stepRange: '{{ "%d-%d" % (0, fhr) }}' stepType: accum - hrrrcar: + hrrrcar: &unnamed_acctp <<: *named_acctp shortName: unknown parameterCategory: 1 parameterNumber: 8 hrrrhi: + <<: *unnamed_acctp + global: <<: *named_acctp rrfs: <<: *named_acctp @@ -306,17 +312,19 @@ acsnod: # Accumulated snow acsnw: # Run Total Accumulated Snow Using 10:1 Ratio sfc: cfgrib: - hrrr: + hrrr: &named_weasd shortName: sdwe level: 0 typeOfLevel: surface stepType: accum stepRange: '{{ "%d-%d" % (0, fhr) }}' + hrrrhi: &unnamed_weasd + <<: *named_weasd + shortName: unknown + parameterCategory: 1 + parameterNumber: 13 rrfs: - parameterNumber: 50 - level: 0 - typeOfLevel: surface - stepRange: '{{ "%d-%d" % (0, fhr) }}' + <<: *unnamed_weasd clevs: [0.01, 0.1, 1, 2, 3, 4, 6, 8, 10, 12, 18, 24] cmap: gist_ncar colors: snow_colors @@ -446,12 +454,14 @@ cape: shortName: cape topLevel: 25500 typeOfLevel: pressureFromGroundLayer - hrrrcar: + hrrrcar: &unnamed_cape <<: *named_cape shortName: unknown parameterCategory: 7 parameterNumber: 6 hrrrhi: + <<: *unnamed_cape + global: <<: *named_cape rrfs: <<: *named_cape @@ -491,6 +501,8 @@ cape: parameterNumber: 6 hrrrhi: <<: *named_mulcape + global: + <<: *named_mulcape rrfs: <<: *named_mulcape contours: @@ -524,6 +536,8 @@ cape: parameterNumber: 6 hrrrhi: <<: *named_mixcape + global: + <<: *named_mixcape rrfs: <<: *named_mixcape contours: @@ -550,12 +564,14 @@ cape: shortName: cape level: 0 typeOfLevel: surface - hrrrcar: + hrrrcar: &unnamed_sfccape <<: *named_sfccape shortName: unknown parameterCategory: 7 parameterNumber: 6 hrrrhi: + <<: *unnamed_sfccape + global: <<: *named_sfccape rrfs: <<: *named_sfccape @@ -592,9 +608,17 @@ cellv: ceil: # Ceiling ua: &ceil cfgrib: - shortName: gh - typeOfLevel: cloudCeiling - level: 0 + hrrr: &named_ceil + shortName: gh + typeOfLevel: cloudCeiling + level: 0 + hrrrhi: + <<: *named_ceil + typeOfLevel: cloudBase + global: + <<: *named_ceil + rrfs: + <<: *named_ceil clevs: [0, 0.1, 0.3, 0.5, 1, 2, 3, 5, 10, 15, 20, 30, 52] cmap: gist_ncar colors: ceil_colors @@ -650,16 +674,34 @@ cloudbase: # Cloud-base height cfrzr: # Categorical Freezing Rain sfc: cfgrib: - shortName: cfrzr - typeOfLevel: surface + hrrr: &named_cfrzr + shortName: cfrzr + typeOfLevel: surface + hrrrcar: &unnamed_cfrzr + parameterCategory: 1 + parameterNumber: 193 + typeOfLevel: unknown + hrrrhi: + <<: *unnamed_cfrzr + rrfs: + <<: *named_cfrzr ncl_name: CFRZR_P0_L1_{grid} cicep: # Categorical Ice Pellets sfc: cfgrib: - shortName: cicep - typeOfLevel: surface - level: 0 - stepType: instant + hrrr: &named_cicep + shortName: cicep + typeOfLevel: surface + level: 0 + stepType: instant + hrrrcar: &unnamed_cicep + parameterCategory: 1 + parameterNumber: 193 + typeOfLevel: unknown + hrrrhi: + <<: *unnamed_cicep + rrfs: + <<: *named_cicep ncl_name: CICEP_P0_L1_{grid} cin: # Surface Convective Inhibition mu: @@ -668,12 +710,14 @@ cin: # Surface Convective Inhibition shortName: cin level: 25500 typeOfLevel: pressureFromGroundLayer - hrrrcar: + hrrrcar: &unnamed_cin <<: *named_cin shortName: unknown parameterCategory: 7 parameterNumber: 7 hrrrhi: + <<: *unnamed_cin + global: <<: *named_cin rrfs: <<: *named_cin @@ -696,6 +740,8 @@ cin: # Surface Convective Inhibition parameterNumber: 7 hrrrhi: <<: *named_mixcin + global: + <<: *named_mixcin rrfs: <<: *named_mixcin clevs: [-300, -200, -150, -100, -75, -50, -40, -30, -20, -10, -1] @@ -708,12 +754,14 @@ cin: # Surface Convective Inhibition shortName: cin level: 0 typeOfLevel: surface - hrrrcar: + hrrrcar: &unnamed_sfccin <<: *named_sfccin shortName: unknown parameterCategory: 7 parameterNumber: 7 hrrrhi: + <<: *unnamed_sfccin + global: <<: *named_sfccin rrfs: <<: *named_sfccin @@ -753,9 +801,11 @@ cloudcover: typeOfLevel: unknown hrrrhi: <<: *high_cld_cover + typeOfLevel: unknown + global: + <<: *high_cld_cover rrfs: <<: *high_cld_cover - clevs: [2, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95] cmap: gist_ncar colors: cldcov_colors @@ -775,6 +825,9 @@ cloudcover: typeOfLevel: unknown hrrrhi: <<: *low_cld_cover + typeOfLevel: unknown + global: + <<: *low_cld_cover rrfs: <<: *low_cld_cover ncl_name: LCDC_P0_L214_{grid} @@ -791,6 +844,9 @@ cloudcover: typeOfLevel: unknown hrrrhi: <<: *mid_cld_cover + typeOfLevel: unknown + global: + <<: *mid_cld_cover rrfs: <<: *mid_cld_cover ncl_name: MCDC_P0_L224_{grid} @@ -950,12 +1006,12 @@ crain: # Categorical Rain hrrr: &named_crain shortName: crain typeOfLevel: surface - hrrrcar: + hrrrcar: &unnamed_crain parameterCategory: 1 parameterName: 192 typeOfLevel: surface hrrrhi: - <<: *named_crain + <<: *unnamed_crain rrfs: <<: *named_crain ncl_name: CRAIN_P0_L1_{grid} @@ -1067,6 +1123,7 @@ dlwrfavg: # Downward Longwave Radiation Flux Average cfgrib: shortName: sdlwrf typeOfLevel: surface + stepType: avg clevs: !arange [200, 501, 12] cmap: gist_ncar colors: radiation_colors @@ -1099,6 +1156,7 @@ dswrfavg: # Downward Shortwave Radiation Flux Average cfgrib: shortName: sdswrf typeOfLevel: surface + stepType: avg clevs: [0, 50, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] colors: rainbow12_colors ncl_name: DSWRF_P8_L1_{grid}_avg6h @@ -1222,9 +1280,17 @@ firewx-pygraf: sfc: <<: *firewx cfgrib: - shortName: vgtyp - typeOfLevel: surface - level: 0 + hrrr: &named_vgtyp + shortName: vgtyp + typeOfLevel: surface + level: 0 + hrrrhi: + <<: *named_vgtyp + shortName: gppbfas + parameterCategory: 0 + parameterNumber: 198 + rrfs: + <<: *named_vgtyp ncl_name: VGTYP_P0_L1_{grid} transform: fire_weather_index flru: # Aviation Flight Rules @@ -1239,6 +1305,7 @@ flru: # Aviation Flight Rules typeOfLevel: cloudBase hrrrhi: <<: *cloud_base + typeOfLevel: cloudBase rrfs: <<: *cloud_base @@ -1684,7 +1751,7 @@ hpbl: # Height of Planetary Boundary Layer parameterCategory: 3 typeOfLevel: surface hrrrhi: - <<: *hpbl + <<: *unnamed_blh global: <<: *unnamed_blh rrfs: @@ -1863,11 +1930,15 @@ lpl: # Lifted parcel level shortName: plpl typeOfLevel: pressureFromGroundLayer level: 25500 - hrrrcar: + hrrrcar: &unnamed_plpl <<: *named_plpl shortName: unknown parameterCategory: 3 parameterNumber: 200 + hrrrhi: + <<: *unnamed_plpl + global: + <<: *named_plpl rrfs: <<: *named_plpl ncl_name: PLPL_P0_2L108_{grid} @@ -2045,26 +2116,23 @@ pres: unit: hPa msl: cfgrib: - hrrr: &hrrr_mslp + hrrr: &named_mslp shortName: mslma typeOfLevel: meanSea level: 0 - hrrrcar: - <<: *hrrr_mslp + hrrrcar: &unnamed_mslp + <<: *named_mslp shortName: unknown parameterCategory: 3 parameterNumber: 198 hrrrhi: - <<: *hrrr_mslp + <<: *unnamed_mslp global: + <<: *named_mslp shortName: prmsl - typeOfLevel: meanSea - level: 0 rrfs: + <<: *named_mslp shortName: mslet - typeOfLevel: meanSea - level: 0 - parameterNumber: 192 clevs: !arange [976, 1051, 4] cmap: Spectral_r colors: pmsl_colors @@ -2148,13 +2216,13 @@ ptyp: # Hourly total precipitation level: 0 stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' stepType: accum - hrrrcar: + hrrrcar: &unnamed_tp <<: *named_tp shortName: unknown parameterCategory: 1 parameterNumber: 8 hrrrhi: - <<: *named_tp + <<: *unnamed_tp rrfs: <<: *named_tp clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 1.0, 2.0] @@ -2222,6 +2290,9 @@ pwtr: # Precipitable water typeOfLevel: unknown hrrrhi: <<: *pwat + typeOfLevel: unknown + global: + <<: *pwat rrfs: <<: *pwat clevs: !arange [4, 81, 4] @@ -2556,8 +2627,18 @@ snod: # Snow Depth soilm: # Soil Moisture Availability sfc: cfgrib: - shortName: mstav - typeOfLevel: depthBelowLand + hrrr: &named_mstav + shortName: mstav + typeOfLevel: depthBelowLand + hrrrcar: + <<: *named_mstav + hrrrhi: + <<: *named_mstav + shortName: unknown + parameterNumber: 194 + parameterCategory: 0 + rrfs: + <<: *named_mstav level: 0 clevs: [0, 5, 15, 25, 35, 45, 55, 65, 75, 85, 95] cmap: Spectral @@ -2876,6 +2957,8 @@ solar: # Incoming Solar Radiation <<: *sdswrf hrrrhi: <<: *sdswrf + global: + <<: *sdswrf rrfs: <<: *sdswrf shortName: csdsf @@ -3410,6 +3493,9 @@ vig: # Vertically-integrated graupel hrrrcar: <<: *hrrr_vig typeOfLevel: unknown + hrrrhi: + <<: *hrrr_vig + typeOfLevel: unknown rrfs: <<: *hrrr_vig clevs: !arange [5, 91, 5] @@ -3686,6 +3772,8 @@ wspeed: # Wind Speed stepType: max hrrrcar: <<: *wspeed_mup + hrrrhi: + <<: *wspeed_mup rrfs: <<: *wspeed_mup typeOfLevel: isobaricLayer From ccdd977c4457181b8e3e264b66af4f9552d4bc6b Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 15 Jan 2026 12:21:35 -0700 Subject: [PATCH 86/98] Lint and test. --- adb_graphics/datahandler/gribdata.py | 2 +- check_fields.py | 48 ---------------------------- conftest.py | 13 +------- 3 files changed, 2 insertions(+), 61 deletions(-) delete mode 100644 check_fields.py diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 0b92d46..63949b7 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -144,7 +144,7 @@ def _find_var(): leveled = level is not None and vertical_coord != "hybrid" if len(field.coords[vertical_coord].shape) > 0 and (layered or leveled): if vertical_coord == "depthBelowLandLayer" and level: - level = level / 100.0 # pragma: no cover + level = float(level) / 100.0 # pragma: no cover field = field.sel(**{vertical_coord: level}) return DataArray(field) msg = f"Variable {short_name} not found in dataset." # pragma: no cover diff --git a/check_fields.py b/check_fields.py deleted file mode 100644 index 9638f60..0000000 --- a/check_fields.py +++ /dev/null @@ -1,48 +0,0 @@ -import glob -import sys - -from adb_graphics import utils -from adb_graphics.datahandler import gribfile, gribdata -from create_graphics import load_images - - -def main(args): - - grib_file, image_file = args - - model, var_levels = load_images([image_file, "hourly"]) - - ds = gribfile.WholeGribFile(grib_file).datasets - first_item = next(iter(ds.keys())) - fhr = int(ds[first_item].step.dt.total_seconds() // 3600) - - specs = utils.load_yaml("adb_graphics/default_specs.yml") - specs.dereference(context={"fhr": int(fhr), "file_type": "prs"}) - - - for variable, levels in var_levels.items(): - for level in levels: - spec = specs.get(variable, {}).get(level) - if spec is None: - print(f"No spec for {variable} at {level}") - continue - vspec = utils.cfgrib_spec(spec["cfgrib"], model) - args = { - "fhr": fhr, - "level": level, - "model": model, - "short_name": variable, - "spec": specs, - "ds": ds, - } - try: - field = gribdata.FieldData(**args).data - except Exception as e: - print(str(e)) - continue - -if __name__ == "__main__": - main(sys.argv[1:]) - - - diff --git a/conftest.py b/conftest.py index 972e44b..4739554 100644 --- a/conftest.py +++ b/conftest.py @@ -8,12 +8,11 @@ import glob from pathlib import Path -import pytest - from pytest import fixture from adb_graphics.datahandler import gribfile + @fixture(scope="session", autouse=True) def cleanup_data_idx(): yield # Nothing to be done before tests @@ -37,15 +36,6 @@ def pytest_addoption(parser): help="Path to prs-file.", ) - parser.addoption( - "--image-file", - action="store", - help="Path to image list file.", - type=Path, - ) - -def pytest_configure(config): - pytest.image_file = config.getoption("--image-file") @fixture(scope="session") def natfile(pytestconfig): @@ -74,7 +64,6 @@ def prs_ds(prsfile): return gribfile.WholeGribFile(prsfile).datasets - @fixture(scope="session") def nat_ds(natfile): return gribfile.WholeGribFile(natfile).datasets From fed4606b357d15fde6adc7d3f923b698504082da Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 23 Jan 2026 22:33:40 +0000 Subject: [PATCH 87/98] Adding support for regional mpas. --- adb_graphics/default_specs.yml | 174 ++++++++++++++++++++++----- image_lists/regional_mpas_subset.yml | 9 +- 2 files changed, 144 insertions(+), 39 deletions(-) diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 40d94a7..5e0449d 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -127,9 +127,9 @@ parameterCategory: 16 parameterNumber: 195 global: - shortName: refd - typeOfLevel: heightAboveGround - level: 1000 + <<: *hrrr_1ref + mpas: + <<: *hrrr_1ref rrfs: shortName: rare typeOfLevel: heightAboveGround @@ -170,6 +170,8 @@ acfrzr: # Run Total Freezing Rain shortName: unknown parameterCategory: 1 parameterNumber: 225 + mpas: + <<: *named_frzr rrfs: <<: *named_frzr ncl_name: FRZR_P8_L1_GLC0_acc @@ -192,6 +194,8 @@ acpcp: # Accumulated run total precipitation <<: *unnamed_acctp global: <<: *named_acctp + mpas: + <<: *named_acctp rrfs: <<: *named_acctp clevs: [0.01, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 15, 20, 40] @@ -463,6 +467,8 @@ cape: <<: *unnamed_cape global: <<: *named_cape + mpas: + <<: *named_cape rrfs: <<: *named_cape clevs: [1, 100, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000] @@ -503,6 +509,8 @@ cape: <<: *named_mulcape global: <<: *named_mulcape + mpas: + <<: *named_mulcape rrfs: <<: *named_mulcape contours: @@ -538,6 +546,8 @@ cape: <<: *named_mixcape global: <<: *named_mixcape + mpas: + <<: *named_mixcape rrfs: <<: *named_mixcape contours: @@ -573,6 +583,8 @@ cape: <<: *unnamed_sfccape global: <<: *named_sfccape + mpas: + <<: *named_sfccape rrfs: <<: *named_sfccape contours: @@ -617,6 +629,8 @@ ceil: # Ceiling typeOfLevel: cloudBase global: <<: *named_ceil + mpas: + <<: *named_ceil rrfs: <<: *named_ceil clevs: [0, 0.1, 0.3, 0.5, 1, 2, 3, 5, 10, 15, 20, 30, 52] @@ -683,6 +697,8 @@ cfrzr: # Categorical Freezing Rain typeOfLevel: unknown hrrrhi: <<: *unnamed_cfrzr + mpas: + <<: *named_cfrzr rrfs: <<: *named_cfrzr ncl_name: CFRZR_P0_L1_{grid} @@ -700,6 +716,8 @@ cicep: # Categorical Ice Pellets typeOfLevel: unknown hrrrhi: <<: *unnamed_cicep + mpas: + <<: *named_cicep rrfs: <<: *named_cicep ncl_name: CICEP_P0_L1_{grid} @@ -719,6 +737,8 @@ cin: # Surface Convective Inhibition <<: *unnamed_cin global: <<: *named_cin + mpas: + <<: *named_cin rrfs: <<: *named_cin clevs: [-300, -200, -150, -100, -75, -50, -40, -30, -20, -10, -1] @@ -742,6 +762,8 @@ cin: # Surface Convective Inhibition <<: *named_mixcin global: <<: *named_mixcin + mpas: + <<: *named_mixcin rrfs: <<: *named_mixcin clevs: [-300, -200, -150, -100, -75, -50, -40, -30, -20, -10, -1] @@ -763,6 +785,8 @@ cin: # Surface Convective Inhibition <<: *unnamed_sfccin global: <<: *named_sfccin + mpas: + <<: *named_sfccin rrfs: <<: *named_sfccin clevs: [-300, -200, -150, -100, -75, -50, -40, -30, -20, -10, -1] @@ -804,6 +828,8 @@ cloudcover: typeOfLevel: unknown global: <<: *high_cld_cover + mpas: + <<: *high_cld_cover rrfs: <<: *high_cld_cover clevs: [2, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95] @@ -828,6 +854,8 @@ cloudcover: typeOfLevel: unknown global: <<: *low_cld_cover + mpas: + <<: *low_cld_cover rrfs: <<: *low_cld_cover ncl_name: LCDC_P0_L214_{grid} @@ -847,6 +875,8 @@ cloudcover: typeOfLevel: unknown global: <<: *mid_cld_cover + mpas: + <<: *mid_cld_cover rrfs: <<: *mid_cld_cover ncl_name: MCDC_P0_L224_{grid} @@ -869,11 +899,12 @@ cloudcover: shortName: unknown parameterNumber: 1 parameterCategory: 6 + mpas: + <<: *tcc + typeOfLevel: atmosphereSingleLayer rrfs: - shortName: tcc + <<: *tcc typeOfLevel: atmosphereSingleLayer - stepType: instant - level: 0 ncl_name: TCDC_P0_L{level_type}_{grid} title: Total Cloud Cover clwmr: # Cloud water Mixing Ratio @@ -1012,6 +1043,8 @@ crain: # Categorical Rain typeOfLevel: surface hrrrhi: <<: *unnamed_crain + mpas: + <<: *named_crain rrfs: <<: *named_crain ncl_name: CRAIN_P0_L1_{grid} @@ -1042,6 +1075,8 @@ cref: # Composite reflectivity shortName: refc typeOfLevel: atmosphereSingleLayer level: 0 + mpas: + <<: *refc_noname rrfs: parameterNumber: 5 parameterCategory: 16 @@ -1200,20 +1235,22 @@ fullintdust: # Full vertically integrated dust (Fine dust + Coarse dust) echotop: # Echo Top sfc: cfgrib: - hrrr: &hrrr_echotop + hrrr: &named_echotop parameterNumber: 3 parameterCategory: 16 typeOfLevel: cloudTop level: 0 hrrrcar: - <<: *hrrr_echotop + <<: *named_echotop hrrrhi: - <<: *hrrr_echotop - rrfs: + <<: *named_echotop + mpas: &unnamed_echotop parameterNumber: 3 parameterCategory: 16 typeOfLevel: atmosphereSingleLayer level: 0 + rrfs: + <<: *unnamed_echotop clevs: !arange [4, 50, 3] cmap: NWSReflectivity colors: cref_colors @@ -1306,6 +1343,8 @@ flru: # Aviation Flight Rules hrrrhi: <<: *cloud_base typeOfLevel: cloudBase + mpas: + <<: *cloud_base rrfs: <<: *cloud_base @@ -1338,6 +1377,10 @@ G113bt: # GOES-W Brightness temperature, water vapor (Ch 3) typeOfLevel: nominalTop parameterNumber: 7 level: 0 + mpas: + parameterNumber: 242 + typeOfLevel: atmosphere + level: 0 rrfs: parameterNumber: 242 typeOfLevel: atmosphere @@ -1368,6 +1411,8 @@ G114bt: # GOES-W Brightness temperature, infrared (Ch 4) parameterNumber: 7 typeOfLevel: nominalTop level: 0 + mpas: + <<: *g114_bt rrfs: <<: *g114_bt ncl_name: SBT114_P0_L8_{grid} @@ -1381,19 +1426,17 @@ G123bt: # GOES-E Brightness temperature, water vapor (Ch 3) shortName: SBT123 typeOfLevel: nominalTop level: 0 - hrrrcar: + hrrrcar: &unnamed_g123_bt parameterCategory: 192 parameterNumber: 1 level: 0 typeOfLevel: nominalTop hrrrhi: - parameterCategory: 192 - parameterNumber: 1 - typeOfLevel: nominalTop - level: 0 + <<: *unnamed_g123_bt + mpas: + <<: *g123_bt rrfs: <<: *g123_bt - ncl_name: SBT123_P0_L8_{grid} title: GOES-E Brightness Temperature, Water Vapor G124bt: # GOES-E Brightness temperature, infrared (Ch 4) @@ -1404,16 +1447,15 @@ G124bt: # GOES-E Brightness temperature, infrared (Ch 4) shortName: SBT124 typeOfLevel: nominalTop level: 0 - hrrrcar: + hrrrcar: &unnamed_g124_bt typeOfLevel: nominalTop parameterCategory: 192 parameterNumber: 2 level: 0 hrrrhi: - typeOfLevel: nominalTop - parameterCategory: 192 - parameterNumber: 2 - level: 0 + <<: *unnamed_g124_bt + mpas: + <<: *g124_bt rrfs: <<: *g124_bt @@ -1555,10 +1597,11 @@ hail: # Max 1h Hail diameter <<: *hrrr_maxhail hrrrhi: <<: *hrrr_maxhail + mpas: + <<: *hrrr_maxhail rrfs: - shortName: hail + <<: *hrrr_maxhail typeOfLevel: surface - stepType: max ncl_name: HAIL_P8_L10_{grid}_max1h title: Max 1h Hail/Graupel Diameter, Entire Column hailcast: # Max 1h Hail diameter @@ -1754,6 +1797,8 @@ hpbl: # Height of Planetary Boundary Layer <<: *unnamed_blh global: <<: *unnamed_blh + mpas: + <<: *hpbl rrfs: <<: *hpbl clevs: [25, 50, 100, 200, 300, 500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000] @@ -1896,6 +1941,8 @@ li: # Lifted Index shortName: unknown parameterCategory: 7 parameterNumber: 193 + mpas: + <<: *named_bestli rrfs: <<: *named_bestli clevs: !arange [-15, 16] @@ -1919,6 +1966,8 @@ li: # Lifted Index parameterNumber: 192 hrrrhi: <<: *named_lftx + mpas: + <<: *named_lftx rrfs: <<: *named_lftx ncl_name: LFTX_P0_2L100_{grid} @@ -1939,6 +1988,8 @@ lpl: # Lifted parcel level <<: *unnamed_plpl global: <<: *named_plpl + mpas: + <<: *named_plpl rrfs: <<: *named_plpl ncl_name: PLPL_P0_2L108_{grid} @@ -2130,6 +2181,8 @@ pres: global: <<: *named_mslp shortName: prmsl + mpas: + <<: *named_mslp rrfs: <<: *named_mslp shortName: mslet @@ -2196,6 +2249,8 @@ ptmp: # Potential temperature <<: *hrrr_pt hrrrhi: <<: *hrrr_pt + mpas: + <<: *hrrr_pt rrfs: shortName: pt typeOfLevel: surface @@ -2223,6 +2278,8 @@ ptyp: # Hourly total precipitation parameterNumber: 8 hrrrhi: <<: *unnamed_tp + mpas: + <<: *named_tp rrfs: <<: *named_tp clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 1.0, 2.0] @@ -2293,6 +2350,8 @@ pwtr: # Precipitable water typeOfLevel: unknown global: <<: *pwat + mpas: + <<: *pwat rrfs: <<: *pwat clevs: !arange [4, 81, 4] @@ -2321,6 +2380,8 @@ ref: # Maximum reflectivity for past hour at 1 km AGL parameterCategory: 16 parameterNumber: 195 shortName: unknown + mpas: + <<: *named_refd rrfs: <<: *named_refd shortName: rare @@ -2339,6 +2400,8 @@ ref: # Maximum reflectivity for past hour at 1 km AGL shortName: unknown parameterCategory: 16 parameterNumber: 195 + mpas: + <<: *named_maxm10 ncl_name: REFD_P8_L20_{grid}_max1h title: Max 1h -10C Isothermal Reflectivity rh: # Relative Humidity @@ -2456,10 +2519,12 @@ rvil: # Radar-derived Vertically Integrated Liquid <<: *hrrr_rvil hrrrhi: <<: *hrrr_rvil + mpas: + <<: *hrrr_rvil + typeOfLevel: atmosphereSingleLayer rrfs: - shortName: veril + <<: *hrrr_rvil typeOfLevel: atmosphereSingleLayer - level: 0 clevs: [0.05, 0.15, 0.76, 3.47, 6.92, 12, 31.6, 35, 40, 45, 50, 55, 60, 65, 70] cmap: NWSReflectivity colors: cref_colors @@ -2632,7 +2697,7 @@ soilm: # Soil Moisture Availability typeOfLevel: depthBelowLand hrrrcar: <<: *named_mstav - hrrrhi: + hrrrhi: &unnamed_mstav <<: *named_mstav shortName: unknown parameterNumber: 194 @@ -2698,6 +2763,8 @@ soilt: # Soil Temperature global: <<: *st_10cm scaledValueOfFirstFixedSurface: 0 + mpas: + <<: *st_10cm rrfs: <<: *st_10cm title: Soil Temperature at 10cm @@ -2735,6 +2802,8 @@ soilt: # Soil Temperature global: <<: *st_1m scaledValueOfFirstFixedSurface: 40 + mpas: + <<: *st_1m rrfs: <<: *st_1m title: Soil Temperature at 1m @@ -2769,6 +2838,8 @@ soilw: # Soil Moisture parameterNumber: 2 hrrrhi: <<: *soilw_0cm + mpas: + <<: *soilw_0cm rrfs: <<: *soilw_0cm clevs: [0, 0.01, 0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50] @@ -2793,6 +2864,8 @@ soilw: # Soil Moisture parameterNumber: 2 hrrrhi: <<: *soilw_1cm + mpas: + <<: *soilw_1cm rrfs: <<: *soilw_1cm title: Soil Moisture at 1cm @@ -2811,6 +2884,8 @@ soilw: # Soil Moisture parameterNumber: 2 hrrrhi: <<: *soilw_4cm + mpas: + <<: *soilw_4cm rrfs: <<: *soilw_4cm title: Soil Moisture at 4cm @@ -2832,6 +2907,8 @@ soilw: # Soil Moisture global: <<: *soilw_cfgrib_10cm scaledValueOfFirstFixedSurface: 0 + mpas: + <<: *soilw_cfgrib_10cm rrfs: <<: *soilw_cfgrib_10cm title: Soil Moisture at 10cm @@ -2850,6 +2927,8 @@ soilw: # Soil Moisture parameterNumber: 2 hrrrhi: <<: *soilw_30cm + mpas: + <<: *soilw_30cm rrfs: <<: *soilw_30cm title: Soil Moisture at 30cm @@ -2886,6 +2965,8 @@ soilw: # Soil Moisture parameterNumber: 2 hrrrhi: <<: *soilw_60cm + mpas: + <<: *soilw_60cm rrfs: <<: *soilw_60cm title: Soil Moisture at 60cm @@ -2907,6 +2988,8 @@ soilw: # Soil Moisture global: <<: *soilw_cfgrib_100cm scaledValueOfFirstFixedSurface: 40 + mpas: + <<: *soilw_cfgrib_100cm rrfs: <<: *soilw_cfgrib_100cm title: Soil Moisture at 1m @@ -2925,6 +3008,8 @@ soilw: # Soil Moisture parameterNumber: 2 hrrrhi: <<: *soilw_cfgrib_160cm + mpas: + <<: *soilw_cfgrib_160cm rrfs: <<: *soilw_cfgrib_160cm title: Soil Moisture at 1.6m @@ -2943,6 +3028,8 @@ soilw: # Soil Moisture parameterNumber: 2 hrrrhi: <<: *soilw_cfgrib_300cm + mpas: + <<: *soilw_cfgrib_300cm rrfs: <<: *soilw_cfgrib_300cm title: Soil Moisture at 3m @@ -2959,6 +3046,8 @@ solar: # Incoming Solar Radiation <<: *sdswrf global: <<: *sdswrf + mpas: + <<: *sdswrf rrfs: <<: *sdswrf shortName: csdsf @@ -3302,6 +3391,8 @@ ulwrf: # Upward Longwave Radiation Flux <<: *ulwrf hrrrhi: <<: *ulwrf + mpas: + <<: *ulwrf rrfs: <<: *ulwrf stepType: avg @@ -3328,7 +3419,8 @@ ulwrfavg: # Upward Longwave Radiation Flux unit: W/m$^{2}$ top: # Nominal top of atmosphere cfgrib: - parameterName: Upward long-wave radiation flux + parameterCategory: 5 + parameterNumber: 4 typeOfLevel: nominalTop stepType: avg clevs: !arange [80, 341, 2] @@ -3444,20 +3536,22 @@ vbdsf: # Incoming Direct Radiation sfc: <<: *incoming_radiation cfgrib: - hrrr: + hrrr: &named_vbdsf shortName: vbdsf typeOfLevel: surface - hrrrcar: + hrrrcar: &unnamed_vbdsf parameterCategory: 4 parameterNumber: 200 typeOfLevel: unknown hrrrhi: + <<: *unnamed_vbdsf parameterCategory: 4 parameterNumber: 200 typeOfLevel: unknown + mpas: + <<: *named_vbdsf rrfs: - shortName: vbdsf - typeOfLevel: surface + <<: *named_vbdsf ncl_name: VBDSF_P0_L1_{grid} title: Incoming Direct Radiation vddsf: # Incoming Diffuse Radiation @@ -3496,6 +3590,8 @@ vig: # Vertically-integrated graupel hrrrhi: <<: *hrrr_vig typeOfLevel: unknown + mpas: + <<: *hrrr_vig rrfs: <<: *hrrr_vig clevs: !arange [5, 91, 5] @@ -3558,6 +3654,9 @@ vvel: # Vertical velocity typeOfLevel: unknown hrrrhi: <<: *named_vvel + mpas: + <<: *named_vvel + shortName: wz rrfs: <<: *named_vvel shortName: wz @@ -3622,6 +3721,9 @@ weasd: # Water equivalent of accumulated snow depth unit: in snoliqr: # Snow-liquid ratio (from U. Utah diagnostic in UPP) sfc: + cfgrib: + shortName: rsn + typeOfLevel: surface clevs: [6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28] cmap: gist_ncar colors: snow_colors @@ -3749,6 +3851,11 @@ wspeed: # Wind Speed <<: *wspeed_mdn hrrrhi: <<: *wspeed_mdn + mpas: + <<: *wspeed_mdn + typeOfLevel: isobaricLayer + topLevel: 100 + bottomLevel: 1000 rrfs: <<: *wspeed_mdn typeOfLevel: isobaricLayer @@ -3774,6 +3881,11 @@ wspeed: # Wind Speed <<: *wspeed_mup hrrrhi: <<: *wspeed_mup + mpas: + <<: *wspeed_mup + typeOfLevel: isobaricLayer + topLevel: 100 + bottomLevel: 1000 rrfs: <<: *wspeed_mup typeOfLevel: isobaricLayer diff --git a/image_lists/regional_mpas_subset.yml b/image_lists/regional_mpas_subset.yml index 6641ccc..86d2d52 100644 --- a/image_lists/regional_mpas_subset.yml +++ b/image_lists/regional_mpas_subset.yml @@ -1,5 +1,5 @@ hourly: - model: regional_mpas + model: mpas variables: 1ref: - 1000m @@ -56,11 +56,8 @@ hourly: - sat gust: - 10m - 1hachail: - - sfc hail: - max - - maxsfc hlcy: - in25 - mn02 @@ -87,8 +84,6 @@ hourly: li: - best - sfc - ltg3: - - sfc mfrp: - sfc mref: @@ -119,8 +114,6 @@ hourly: - sfc snod: - sfc - soilm: - - sfc soilt: &soilt_levs - 0cm - 1cm From cb1b490febb44b21c259e7898faa7e5b11944364 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 3 Feb 2026 17:39:44 +0000 Subject: [PATCH 88/98] Updates for accumulated RRFS A fields. --- adb_graphics/datahandler/gribdata.py | 2 +- adb_graphics/datahandler/gribfile.py | 14 +++++--------- adb_graphics/default_specs.yml | 22 ++++++++++++++++++---- create_graphics.py | 12 ++++++++---- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 63949b7..8fb7d40 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -124,7 +124,7 @@ def _find_var(): vertical_coord = "level" if vertical_coord == "unknown" else vertical_coord ds: Dataset | dict = self.ds.get(var_id, {}) if ds == {}: - msg = f"{var_id} is not a valid key for the dataset" + msg = f"{var_id} is not a valid key for dataset while plotting {self.short_name} at {self.level}" raise ValueError(msg) var = _find_var() if var is not None: diff --git a/adb_graphics/datahandler/gribfile.py b/adb_graphics/datahandler/gribfile.py index a70d2a3..306727a 100644 --- a/adb_graphics/datahandler/gribfile.py +++ b/adb_graphics/datahandler/gribfile.py @@ -26,14 +26,10 @@ def __init__( """ Initialize GribFiles object. - coord_dims dict containing the name of the dimension to - concat (key), and a list of its values (value). - Ex: {'fhr': [2, 3, 4]} - filenames dict containing list of files names for the 0h and 1h - forecast lead times ('01fcst'), and all the free forecast - hours after that ('free_fcst'). - filetype key to use for dict when setting variable_names - model string describing the model type + filenames dict containing list of files names for the 0h and 1h + forecast lead times ('01fcst'), and all the free forecast + hours after that ('free_fcst'). + cfgrib_config config for the variable to load. """ self.filenames = filenames @@ -45,9 +41,9 @@ def datasets(self): ds = xr.open_mfdataset( self.filenames, engine="cfgrib", - concat_dim="time", combine="nested", compat="override", + concat_dim="time", coords="minimal", backend_kwargs=( { diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 5e0449d..2c195ec 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -165,6 +165,7 @@ acfrzr: # Run Total Freezing Rain level: 0 typeOfLevel: surface stepType: accum + stepRange: '{{ "%d-%d" % (0, fhr) }}' hrrrcar: <<: *named_frzr shortName: unknown @@ -304,6 +305,7 @@ acsnod: # Accumulated snow level: 0 typeOfLevel: surface stepType: accum + stepRange: '{{ "%d-%d" % (0, fhr) }}' parameterName: Total snowfall clevs: [0.01, 0.1, 1, 2, 3, 4, 6, 8, 10, 12, 18, 24] cmap: gist_ncar @@ -1569,14 +1571,14 @@ hail: # Max 1h Hail diameter typeOfLevel: sigma level: 0 stepType: max + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' hrrrcar: <<: *hrrr_hail hrrrhi: <<: *hrrr_hail rrfs: - shortName: hail + <<: *hrrr_hail typeOfLevel: surface - stepType: max clevs: [0.10, 0.25, 0.50, 0.75, 1.0, 1.5, 2.0, 2.5, 3.0] cmap: gist_ncar colors: hail_colors @@ -1593,6 +1595,7 @@ hail: # Max 1h Hail diameter typeOfLevel: atmosphere level: 0 stepType: max + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' hrrrcar: <<: *hrrr_maxhail hrrrhi: @@ -1637,6 +1640,7 @@ hlcy: # Helicity topLevel: 2000 bottomLevel: 0 stepType: min + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' clevs: !arange [-300, -24, 25] cmap: gist_ncar colors: rainbow12_reverse @@ -2036,6 +2040,7 @@ ltng: # Lightning typeOfLevel: atmosphere level: 0 stepType: max + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' clevs: !arange [5, 91, 5] cmap: jet colors: graupel_colors @@ -2081,6 +2086,7 @@ mref: # Maximum reflectivity for past hour at 1 km AGL parameterNumber: 198 typeOfLevel: heightAboveGround stepType: max + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' level: 1000 ncl_name: MAXREF_P8_L103_{grid}_max1h title: Max 1km agl Reflectivity (over prev hr) @@ -2269,8 +2275,8 @@ ptyp: # Hourly total precipitation shortName: tp typeOfLevel: surface level: 0 - stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' stepType: accum + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' hrrrcar: &unnamed_tp <<: *named_tp shortName: unknown @@ -2395,6 +2401,7 @@ ref: # Maximum reflectivity for past hour at 1 km AGL typeOfLevel: isothermal level: 263 stepType: max + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' hrrrcar: <<: *named_maxm10 shortName: unknown @@ -3076,6 +3083,7 @@ ssrun: # Storm Surface Runoff typeOfLevel: surface level: 0 stepType: accum + stepRange: '{{ "%d-%d" % (0, fhr) }}' clevs: [0.002, 0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 1.0, 2.0] cmap: gist_ncar colors: pcp_colors @@ -3396,6 +3404,7 @@ ulwrf: # Upward Longwave Radiation Flux rrfs: <<: *ulwrf stepType: avg + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' clevs: !arange [80, 341, 2] cmap: ir_rgbv_r colors: radiation_mix_colors @@ -3423,6 +3432,7 @@ ulwrfavg: # Upward Longwave Radiation Flux parameterNumber: 4 typeOfLevel: nominalTop stepType: avg + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' clevs: !arange [80, 341, 2] cmap: ir_rgbv_r colors: radiation_mix_colors @@ -3444,7 +3454,6 @@ uswrf: # Upward Shortwave Radiation Flux title: Upward SW Radiation Flux, Surface top: # Nominal top of atmosphere <<: *radiation_flux - cfgrib: cfgrib: parameterNumber: 8 parameterCategory: 4 @@ -3690,6 +3699,7 @@ vvort: # Vertical vorticity typeOfLevel: heightAboveGroundLayer topLevel: 1000 stepType: max + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' clevs: !arange [0.0025, 0.0301, 0.0025] cmap: gist_ncar colors: vort_colors @@ -3704,6 +3714,7 @@ vvort: # Vertical vorticity typeOfLevel: heightAboveGroundLayer topLevel: 2000 stepType: max + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' title: 0-2km Max Vertical Vorticity (over prev hour) weasd: # Water equivalent of accumulated snow depth sfc: @@ -3834,6 +3845,7 @@ wspeed: # Wind Speed typeOfLevel: heightAboveGround level: 10 stepType: max + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' ncl_name: WIND_P8_L103_{grid}_max1h title: Max 10m Wind (over prev hour) transform: conversions.ms_to_kt @@ -3847,6 +3859,7 @@ wspeed: # Wind Speed topLevel: 10000 bottomLevel: 100000 stepType: max + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' hrrrcar: <<: *wspeed_mdn hrrrhi: @@ -3877,6 +3890,7 @@ wspeed: # Wind Speed topLevel: 10000 bottomLevel: 100000 stepType: max + stepRange: '{{ "%d-%d" % (fhr-1, fhr) }}' hrrrcar: <<: *wspeed_mup hrrrhi: diff --git a/create_graphics.py b/create_graphics.py index c140df6..36c09f4 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -85,7 +85,7 @@ def create_maps( generate a pool of workers to complete the task. """ - ds = gribfile.WholeGribFile(grib_paths[-1]).datasets + ds = None for tile in cla.tiles: args = [] for variable, levels in cla.images[1].items(): @@ -99,12 +99,17 @@ def create_maps( accumulate = spec.get("accumulate", False) vspec = utils.cfgrib_spec(spec["cfgrib"], cla.images[0]) grib_acc = ( - vspec.get("stepRange", "").startswith("-1") or vspec.get("stepRange") == "0-0" + vspec.get("stepRange", "").startswith("-1") or + vspec.get("stepRange") == "0-0" or + vspec.get("stepType") in ["max", "min", "accum"], ) if (accumulate or grib_acc) and fhr == 0: + print(f"Skipping accumulated {variable} at {level} at fhr=0") continue if accumulate: - ads = gribfile.GribFiles(grib_paths, vspec).datasets + ads = gribfile.GribFiles(grib_paths[1:], vspec).datasets + elif ds is None: + ds = gribfile.WholeGribFile(grib_paths[-1]).datasets args.append( ( @@ -118,7 +123,6 @@ def create_maps( grib_path2, ) ) - print(f"Queueing {len(args)} maps") with Pool(processes=cla.nprocs) as pool: pool.starmap(parallel_maps, args) From f99ae1f6e3f5eb38e86fa8614e90d81d7e070528 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 3 Feb 2026 18:22:19 +0000 Subject: [PATCH 89/98] Global working. --- adb_graphics/default_specs.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 2c195ec..3c441f8 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -1062,28 +1062,25 @@ cref: # Composite reflectivity sfc: <<: *refl cfgrib: - hrrr: + hrrr: &named_refc shortName: refc typeOfLevel: atmosphere level: 0 - hrrrcar: &refc_noname + hrrrcar: &unnamed_refc parameterCategory: 16 parameterNumber: 196 level: 0 typeOfLevel: atmosphere hrrrhi: - <<: *refc_noname + <<: *unnamed_refc global: - shortName: refc - typeOfLevel: atmosphereSingleLayer - level: 0 + <<: *named_refc mpas: - <<: *refc_noname + <<: *unnamed_refc rrfs: + <<: *unnamed_refc parameterNumber: 5 - parameterCategory: 16 typeOfLevel: atmosphereSingleLayer - level: 0 ncl_name: REFC_P0_L{level_type}_{grid} title: Composite Reflectivity include_obs: True From 8c9c5ed542f138ee9f3216f8cb0fbecbe368bec9 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Mon, 9 Feb 2026 16:04:54 +0000 Subject: [PATCH 90/98] Linting. --- adb_graphics/datahandler/gribdata.py | 2 +- create_graphics.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 8fb7d40..5557d2d 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -124,7 +124,7 @@ def _find_var(): vertical_coord = "level" if vertical_coord == "unknown" else vertical_coord ds: Dataset | dict = self.ds.get(var_id, {}) if ds == {}: - msg = f"{var_id} is not a valid key for dataset while plotting {self.short_name} at {self.level}" + msg = f"{var_id} is not a key for dataset plotting {self.short_name} at {self.level}" raise ValueError(msg) var = _find_var() if var is not None: diff --git a/create_graphics.py b/create_graphics.py index 36c09f4..805d55c 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -99,9 +99,9 @@ def create_maps( accumulate = spec.get("accumulate", False) vspec = utils.cfgrib_spec(spec["cfgrib"], cla.images[0]) grib_acc = ( - vspec.get("stepRange", "").startswith("-1") or - vspec.get("stepRange") == "0-0" or - vspec.get("stepType") in ["max", "min", "accum"], + vspec.get("stepRange", "").startswith("-1") + or vspec.get("stepRange") == "0-0" + or vspec.get("stepType") in ["max", "min", "accum"], ) if (accumulate or grib_acc) and fhr == 0: print(f"Skipping accumulated {variable} at {level} at fhr=0") From 244b34e1b1f3fb903d065c33f34089d3964e11ae Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Tue, 17 Feb 2026 23:19:47 +0000 Subject: [PATCH 91/98] Fix for global cref. --- adb_graphics/default_specs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 3c441f8..7ce7154 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -1075,6 +1075,7 @@ cref: # Composite reflectivity <<: *unnamed_refc global: <<: *named_refc + typeOfLevel: atmosphereSingleLayer mpas: <<: *unnamed_refc rrfs: From e319a529daf434f9edac3eef6dc17af1f29f3e08 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 18 Feb 2026 18:14:21 +0000 Subject: [PATCH 92/98] Fixes for RTMA --- create_graphics.py | 2 +- image_lists/rtma.yml | 34 ---------------------------------- 2 files changed, 1 insertion(+), 35 deletions(-) diff --git a/create_graphics.py b/create_graphics.py index 805d55c..5edce6d 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -101,7 +101,7 @@ def create_maps( grib_acc = ( vspec.get("stepRange", "").startswith("-1") or vspec.get("stepRange") == "0-0" - or vspec.get("stepType") in ["max", "min", "accum"], + or vspec.get("stepType") in ["max", "min", "accum"] ) if (accumulate or grib_acc) and fhr == 0: print(f"Skipping accumulated {variable} at {level} at fhr=0") diff --git a/image_lists/rtma.yml b/image_lists/rtma.yml index 6091e34..ce9bc32 100644 --- a/image_lists/rtma.yml +++ b/image_lists/rtma.yml @@ -1,34 +1,21 @@ hourly: model: hrrr variables: -# 1hsm: -# - sfc 1ref: - 1000m -# 3hsm: -# - sfc cape: - mu - mul - mx90mb - sfc - ceil: - - ua - ceilexp: - - ua - ceilexp2: - - ua cin: - sfc cloudcover: - high - low - mid - - total cpofp: - sfc - cref: - - sfc ctop: - ua dewp: @@ -45,13 +32,6 @@ hourly: - sat G124bt: - sat - gust: - - 10m - hlcy: - - in16 - - in25 - - sr01 - - sr03 hpbl: - sfc lcl: @@ -61,19 +41,12 @@ hourly: li: - best - sfc - ltg3: - - sfc -# mnvv: -# - sfc mref: - sfc pres: - - msl - sfc ptmp: - 2m -# ptyp: -# - sfc pwtr: - sfc ref: @@ -82,9 +55,6 @@ hourly: - 2m - 850mb - mean - - pw - rvil: - - sfc shear: - 01km - 06km @@ -92,8 +62,6 @@ hourly: - sfc snod: - sfc - soilm: - - sfc solar: - sfc temp: @@ -114,8 +82,6 @@ hourly: - sfc vddsf: - sfc - vil: - - sfc vis: - sfc vort: From 6a653c6001f100c09a3e952bce714dde5dbfc060 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Mon, 16 Mar 2026 22:14:22 +0000 Subject: [PATCH 93/98] Fixing global run. --- create_graphics.py | 2 +- image_lists/global.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/create_graphics.py b/create_graphics.py index 5edce6d..62a1ad4 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -101,7 +101,7 @@ def create_maps( grib_acc = ( vspec.get("stepRange", "").startswith("-1") or vspec.get("stepRange") == "0-0" - or vspec.get("stepType") in ["max", "min", "accum"] + or vspec.get("stepType") in ["max", "min", "accum", "avg"] ) if (accumulate or grib_acc) and fhr == 0: print(f"Skipping accumulated {variable} at {level} at fhr=0") diff --git a/image_lists/global.yml b/image_lists/global.yml index 8105b8c..f2f32d4 100644 --- a/image_lists/global.yml +++ b/image_lists/global.yml @@ -24,8 +24,6 @@ hourly: - sfc dewp: - 2m - dlwrfavg: - - sfc dswrfavg: - sfc gh: From 4fe275760f50c47dddab700990c7e83d82dd29ad Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 18 Mar 2026 21:25:39 +0000 Subject: [PATCH 94/98] Fix HRRR NAT graphics. --- adb_graphics/datahandler/gribdata.py | 6 +++--- adb_graphics/default_specs.yml | 24 ++++++++++++++++++++++++ adb_graphics/figure_builders.py | 9 ++++++--- create_graphics.py | 6 +++++- pre.sh | 9 +++++++-- 5 files changed, 45 insertions(+), 9 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index 5557d2d..f606b51 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -141,9 +141,9 @@ def _find_var(): if level is None: level = utils.numeric_level(self.level)[0] level = None if level == "" else level - leveled = level is not None and vertical_coord != "hybrid" + leveled = level is not None if len(field.coords[vertical_coord].shape) > 0 and (layered or leveled): - if vertical_coord == "depthBelowLandLayer" and level: + if vertical_coord == "depthBelowLandLayer" and leveled: level = float(level) / 100.0 # pragma: no cover field = field.sel(**{vertical_coord: level}) return DataArray(field) @@ -574,7 +574,7 @@ def run_total(values: DataArray, **_kwargs): return values.sum(dim="time") # pragma: no cover - def supercooled_liquid_water(self, **_kwargs): + def supercooled_liquid_water(self, val, **_kwargs): """ Generates a field of Supercooled Liquid Water. diff --git a/adb_graphics/default_specs.yml b/adb_graphics/default_specs.yml index 7ce7154..c887a55 100644 --- a/adb_graphics/default_specs.yml +++ b/adb_graphics/default_specs.yml @@ -3280,6 +3280,11 @@ trc1: unit: $mg/m^2$ 1000ft: &tracer1 # requires nat data + cfgrib: &trc1_cfgrib + parameterCategory: 20 + parameterNumber: 0 + typeOfLevel: hybrid + level: 5 clevs: [1, 2, 4, 6, 8, 12, 16, 20, 25, 30, 40, 60, 100, 200] colors: smoke_colors ncl_name: MASSDEN_P0_L105_{grid} @@ -3293,6 +3298,9 @@ trc1: 6000ft: # requires nat data <<: *tracer1 + cfgrib: + <<: *trc1_cfgrib + level: 12 title: 6000ft AGL Smoke vertical_index: 11 sfc: @@ -3356,11 +3364,19 @@ u: 850mb: *ua_uwind 925mb: *ua_uwind 1000ft: &nat_uwind + cfgrib: + shortName: u + typeOfLevel: hybrid + level: 5 ncl_name: UGRD_P0_L105_{grid} transform: conversions.ms_to_kt vertical_index: 4 6000ft: <<: *nat_uwind + cfgrib: + shortName: u + typeOfLevel: hybrid + level: 12 vertical_index: 11 max: cfgrib: @@ -3522,11 +3538,19 @@ v: 850mb: *ua_vwind 925mb: *ua_vwind 1000ft: &nat_vwind + cfgrib: + shortName: v + typeOfLevel: hybrid + level: 5 ncl_name: VGRD_P0_L105_{grid} transform: conversions.ms_to_kt vertical_index: 4 6000ft: <<: *nat_vwind + cfgrib: + shortName: v + typeOfLevel: hybrid + level: 12 vertical_index: 11 max: cfgrib: diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index e4ae1dc..0d0466f 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -168,6 +168,7 @@ def parallel_maps( # noqa: PLR0915, PLR0912 if index == 0: dm.title() dm.add_logo(current_ax) + continue elif index == lower_left: if spec.get("include_obs", False) and cla.obs_file_path: # Add observation panel to lower left. Currently only @@ -182,10 +183,12 @@ def parallel_maps( # noqa: PLR0915, PLR0912 spec=cla.specs, tile=tile, ) - else: - dm.draw(show=True) - else: + continue + try: dm.draw(show=True) + except: + print(f"Error occurred while creating map for {variable} at {level}.") + raise # Build the output path png_file = f"{variable}_{tile}_{level}_f{fhr:03d}.png" diff --git a/create_graphics.py b/create_graphics.py index 62a1ad4..397c656 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -97,7 +97,11 @@ def create_maps( msg = f"graphics: {variable} {level}" raise errors.NoGraphicsDefinitionForVariableError(msg) accumulate = spec.get("accumulate", False) - vspec = utils.cfgrib_spec(spec["cfgrib"], cla.images[0]) + try: + vspec = utils.cfgrib_spec(spec["cfgrib"], cla.images[0]) + except KeyError: + print(f"ERROR: {variable} at {level} has no cfgrib entry") + raise grib_acc = ( vspec.get("stepRange", "").startswith("-1") or vspec.get("stepRange") == "0-0" diff --git a/pre.sh b/pre.sh index f604856..fcb3828 100644 --- a/pre.sh +++ b/pre.sh @@ -2,8 +2,13 @@ module purge -module use -a /contrib/miniconda3/modulefiles -module load miniconda3/25.11.0 +if [[ $(hostname) == u* ]] ; then + module use -a /contrib/miniconda/modulefiles + module load miniconda/25.3.1 +else + module use -a /contrib/miniconda3/modulefiles + module load miniconda3/25.11.0 +fi conda activate pygraf module list From 57881de82d1b6b2c3a08bc50acb13f127584d9eb Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 19 Mar 2026 14:58:53 +0000 Subject: [PATCH 95/98] Lint and pass tests. --- adb_graphics/datahandler/gribdata.py | 3 ++- adb_graphics/figure_builders.py | 4 ++-- tests/datahandler/test_gribdata.py | 2 +- tests/test_figure_builders.py | 12 +++++++++++- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/adb_graphics/datahandler/gribdata.py b/adb_graphics/datahandler/gribdata.py index f606b51..c6def0f 100644 --- a/adb_graphics/datahandler/gribdata.py +++ b/adb_graphics/datahandler/gribdata.py @@ -144,6 +144,7 @@ def _find_var(): leveled = level is not None if len(field.coords[vertical_coord].shape) > 0 and (layered or leveled): if vertical_coord == "depthBelowLandLayer" and leveled: + assert isinstance(level, (int, float, str)) # pragma: no cover level = float(level) / 100.0 # pragma: no cover field = field.sel(**{vertical_coord: level}) return DataArray(field) @@ -574,7 +575,7 @@ def run_total(values: DataArray, **_kwargs): return values.sum(dim="time") # pragma: no cover - def supercooled_liquid_water(self, val, **_kwargs): + def supercooled_liquid_water(self, values: DataArray, **_kwargs): # noqa: ARG002 """ Generates a field of Supercooled Liquid Water. diff --git a/adb_graphics/figure_builders.py b/adb_graphics/figure_builders.py index 0d0466f..1da327e 100644 --- a/adb_graphics/figure_builders.py +++ b/adb_graphics/figure_builders.py @@ -169,7 +169,7 @@ def parallel_maps( # noqa: PLR0915, PLR0912 dm.title() dm.add_logo(current_ax) continue - elif index == lower_left: + if index == lower_left: if spec.get("include_obs", False) and cla.obs_file_path: # Add observation panel to lower left. Currently only # supported for composite reflectivity. @@ -186,7 +186,7 @@ def parallel_maps( # noqa: PLR0915, PLR0912 continue try: dm.draw(show=True) - except: + except: # pragma: no cover print(f"Error occurred while creating map for {variable} at {level}.") raise diff --git a/tests/datahandler/test_gribdata.py b/tests/datahandler/test_gribdata.py index 5d10e10..91e9fae 100644 --- a/tests/datahandler/test_gribdata.py +++ b/tests/datahandler/test_gribdata.py @@ -443,7 +443,7 @@ def test_fielddata_supercooled_liquid_water(nat_ds, spec): short_name="slw", spec=spec, ) - slw = fd.supercooled_liquid_water() + slw = fd.supercooled_liquid_water(fd.field) assert not np.array_equal(slw, fd.field) diff --git a/tests/test_figure_builders.py b/tests/test_figure_builders.py index ac9096c..3d00dbe 100644 --- a/tests/test_figure_builders.py +++ b/tests/test_figure_builders.py @@ -5,7 +5,7 @@ from unittest.mock import call, patch import numpy as np -from pytest import fixture +from pytest import fixture, raises from uwtools.api.config import get_yaml_config from adb_graphics import figure_builders, utils @@ -108,6 +108,16 @@ def test_parallel_maps(parallel_maps_args, tmp_path): assert (tmp_path / "temp_full_sfc_f016.png").is_file() +def test_parallel_maps_bad_draw(capsys, parallel_maps_args): + with ( + patch.object(figure_builders.DataMap, "draw", side_effect=ValueError("foo")), + raises(ValueError, match="foo"), + ): + figure_builders.parallel_maps(**parallel_maps_args) + captured = capsys.readouterr() + assert "temp at sfc" in captured.out + + def test_parallel_maps_enspanel(parallel_maps_args, prsfile, tmp_path): parallel_maps_args["cla"].ens_size = 9 parallel_maps_args["cla"].graphic_type = "enspanel" From 2a4673fc85a670b627842d9d967ee8ed223549a7 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 19 Mar 2026 18:53:26 +0000 Subject: [PATCH 96/98] Fix accumulated variables with missing files. --- create_graphics.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/create_graphics.py b/create_graphics.py index 397c656..23fcd1f 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -110,7 +110,10 @@ def create_maps( if (accumulate or grib_acc) and fhr == 0: print(f"Skipping accumulated {variable} at {level} at fhr=0") continue - if accumulate: + if accumulate and not cla.all_leads: + print(f"Skipping accumulated {variable} at {level}. Not enough data.") + continue + if accumulate and cla.all_leads: ads = gribfile.GribFiles(grib_paths[1:], vspec).datasets elif ds is None: ds = gribfile.WholeGribFile(grib_paths[-1]).datasets @@ -119,7 +122,7 @@ def create_maps( ( cla, fhr, - ads if accumulate else ds, + ads if accumulate and cla.all_leads else ds, level, variable, workdir, From b8712d0040fcfba968d8311ae3c6e577fe00e7cd Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 20 Mar 2026 19:58:14 +0000 Subject: [PATCH 97/98] Fix for accumulations beyond fh=2. --- create_graphics.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/create_graphics.py b/create_graphics.py index 23fcd1f..dbd871c 100644 --- a/create_graphics.py +++ b/create_graphics.py @@ -113,8 +113,13 @@ def create_maps( if accumulate and not cla.all_leads: print(f"Skipping accumulated {variable} at {level}. Not enough data.") continue - if accumulate and cla.all_leads: - ads = gribfile.GribFiles(grib_paths[1:], vspec).datasets + if accumulate and cla.all_leads: # pragma: no cover + try: + vspec = {k: v for k, v in vspec.items() if k != "stepRange"} + ads = gribfile.GribFiles(grib_paths[1:], vspec).datasets + except ValueError: + print(f"Error loading all times for {variable} at {level}.") + raise elif ds is None: ds = gribfile.WholeGribFile(grib_paths[-1]).datasets From 0f1ccf7c4d5ddeb7c20972da608fc85618ff313f Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 21 Apr 2026 16:00:38 +0000 Subject: [PATCH 98/98] Remove 80m wind for RTMA. --- image_lists/rtma_ncep.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/image_lists/rtma_ncep.yml b/image_lists/rtma_ncep.yml index a1dbe25..6299c56 100644 --- a/image_lists/rtma_ncep.yml +++ b/image_lists/rtma_ncep.yml @@ -11,4 +11,3 @@ hourly: - sfc wspeed: - 10m - - 80m