Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add e3sm_to_cmip task #650

Merged
merged 16 commits into from
Dec 18, 2024
Merged

Add e3sm_to_cmip task #650

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
e99178b
Add e3sm_to_cmip task
forsyth2 Dec 7, 2024
7fc7e2b
Add dependency handling
forsyth2 Dec 10, 2024
3a8dc06
Update defaults
forsyth2 Dec 10, 2024
e7d43e1
Fix cmortables_dir
forsyth2 Dec 10, 2024
f92ad22
Currently testing on v3
forsyth2 Dec 11, 2024
2cbc136
Image check tests passing
forsyth2 Dec 12, 2024
e93a577
Clean up code
forsyth2 Dec 12, 2024
5812010
Address comments
forsyth2 Dec 12, 2024
44c93e0
Update parameters
forsyth2 Dec 12, 2024
6e37e19
Handle ts dependency and output
forsyth2 Dec 13, 2024
c1116e9
Unit tests passing
forsyth2 Dec 13, 2024
b3948aa
v3 image check test passing
forsyth2 Dec 13, 2024
98a3cd3
Min cases with diff years
forsyth2 Dec 17, 2024
0bd5d90
Simplified tests passing
forsyth2 Dec 17, 2024
ec85cbf
New expected results for weekly tests
forsyth2 Dec 18, 2024
2d3bec6
Reset utils.py
forsyth2 Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions tests/integration/generated/test_min_case_add_dependencies_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ years = "1985:1995:10",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:50:00"
# ts is in 5 year increments
years = "1985:1995:5"
Expand All @@ -35,7 +34,6 @@ years = "1985:1995:5"
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
ts_fmt = "cmip"
vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"

[[ land_monthly ]]
Expand All @@ -45,7 +43,6 @@ years = "1985:1995:5"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
ts_fmt = "cmip"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[[ rof_monthly ]]
Expand Down Expand Up @@ -79,6 +76,27 @@ years = "1985:1995:5"
mapping_file = "glb"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[e3sm_to_cmip]
active = True
walltime = "00:50:00"
# ts is in 5 year increments
years = "1985:1995:5"

[[ atm_monthly_180x360_aave ]]
# for e3sm_diags (depend_on_ts), ilamb
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

e3sm_diags currently doesn't depend on e3sm_to_cmip, pmp will

frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"

[[ land_monthly ]]
# for ilamb
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[e3sm_diags]
active = True
environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
Expand Down
19 changes: 16 additions & 3 deletions tests/integration/generated/test_min_case_carryover_dependencies_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,12 @@ walltime = "00:30:00"

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
ts_fmt = "cmip"

[[ rof_monthly ]]
extra_vars = 'areatotal2'
Expand Down Expand Up @@ -99,7 +97,22 @@ walltime = "00:30:00"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
ts_fmt = "cmip"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[e3sm_to_cmip]
active = True
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

# TODO: Add "tc_analysis" back in after empty dat is resolved.
Expand Down
2 changes: 0 additions & 2 deletions tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,12 @@ years = "1985:1989:4",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
#ts_fmt = "cmip"

[e3sm_diags]
active = True
Expand Down
2 changes: 0 additions & 2 deletions tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_1_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,9 @@ years = "1985:1989:4",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
#ts_fmt = "cmip"
2 changes: 0 additions & 2 deletions tests/integration/generated/test_min_case_e3sm_diags_depend_on_ts_mvm_2_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,12 @@ years = "1995:1999:4",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
#ts_fmt = "cmip"

[e3sm_diags]
active = True
Expand Down
1 change: 0 additions & 1 deletion tests/integration/generated/test_min_case_e3sm_diags_streamflow_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ years = "1985:1989:4",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ rof_monthly ]]
Expand Down
1 change: 0 additions & 1 deletion tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_1_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ years = "1985:1989:4",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ rof_monthly ]]
Expand Down
1 change: 0 additions & 1 deletion tests/integration/generated/test_min_case_e3sm_diags_streamflow_mvm_2_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ years = "1995:1999:4",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ rof_monthly ]]
Expand Down
1 change: 0 additions & 1 deletion tests/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ years = "1985:1989:4",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_daily_180x360_aave ]]
Expand Down
1 change: 0 additions & 1 deletion ...s/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_1_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ years = "1985:1989:4",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_daily_180x360_aave ]]
Expand Down
1 change: 0 additions & 1 deletion ...s/integration/generated/test_min_case_e3sm_diags_tropical_subseasonal_mvm_2_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ years = "1995:1999:4",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_daily_180x360_aave ]]
Expand Down
12 changes: 0 additions & 12 deletions ...tion/generated/test_min_case_global_time_series_comprehensive_v3_setup_only_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ years = "1985:1989:2",

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_monthly_glb ]]
Expand Down Expand Up @@ -51,14 +50,3 @@ walltime = "00:30:00"

# (This cfg is the setup portion only)
# [global_time_series]
# active = True
# climo_years = "1985-1989", "1990-1995",
# environment_commands = "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>"
# experiment_name = "v3.LR.historical_0051"
# figstr = "v3.LR.historical_0051"
# moc_file=mocTimeSeries_1985-1995.nc
# plots_lnd = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILWATER_10CM,TSA,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
# ts_num_years = 5
# ts_years = "1985-1989", "1985-1995",
# walltime = "00:30:00"
# years = "1985-1995",
1 change: 0 additions & 1 deletion tests/integration/generated/test_min_case_global_time_series_custom_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"
years = "1985:1995:5",

Expand Down
1 change: 0 additions & 1 deletion tests/integration/generated/test_min_case_global_time_series_original_8_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_monthly_glb ]]
Expand Down
1 change: 0 additions & 1 deletion tests/integration/generated/test_min_case_global_time_series_original_8_no_ocn_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_monthly_glb ]]
Expand Down
20 changes: 18 additions & 2 deletions tests/integration/generated/test_min_case_ilamb_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:50:00"
ts_fmt = "cmip"
years = "1985:2014:30"

[[ atm_monthly_180x360_aave ]]
Expand All @@ -32,6 +30,24 @@ years = "1985:2014:30"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[e3sm_to_cmip]
active = True
walltime = "00:50:00"
years = "1985:2014:30"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
vars = "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,TAUX,TAUY,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT,CLDTOT,CLDHGH,CLDMED,CLDLOW,U,ICEFRAC,LANDFRAC,OCNFRAC,PS,CLDICE,CLDLIQ,T,AODDUST"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[ilamb]
active = True
nodes = 8
Expand Down
14 changes: 12 additions & 2 deletions tests/integration/generated/test_min_case_ilamb_land_only_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@ www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_min_case_

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:50:00"
ts_fmt = "cmip"
years = "1985:2014:30"

[[ land_monthly ]]
Expand All @@ -26,6 +24,18 @@ years = "1985:2014:30"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[e3sm_to_cmip]
active = True
walltime = "00:50:00"
years = "1985:2014:30"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"

[ilamb]
active = True
land_only = True
Expand Down
3 changes: 0 additions & 3 deletions tests/integration/generated/test_min_case_nco_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,12 @@ walltime = "00:30:00"

[ts]
active = True
e3sm_to_cmip_environment_commands = ""
walltime = "00:30:00"

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
ts_fmt = "cmip"

[[ atm_daily_180x360_aave ]]
frequency = "daily"
Expand Down Expand Up @@ -85,5 +83,4 @@ walltime = "00:30:00"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
ts_fmt = "cmip"
vars = "FSH,RH2M,LAISHA,LAISUN,QINTR,QOVER,QRUNOFF,QSOIL,QVEGE,QVEGT,SOILICE,SOILLIQ,SOILWATER_10CM,TSA,TSOI,H2OSNO,TOTLITC,CWDC,SOIL1C,SOIL2C,SOIL3C,SOIL4C,WOOD_HARVESTC,TOTVEGC,NBP,GPP,AR,HR"
23 changes: 19 additions & 4 deletions tests/integration/generated/test_weekly_bundles_chrysalis.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
# > atm_monthly_diurnal_8xdaily_180x360_aave
# ts > atm_monthly_180x360_aave
# > land_monthly
# e3sm_to_cmip > atm_monthly_180x360_aave
# > land_monthly
# e3sm_diags > atm_monthly_180x360_aave
# bundle 2:
# ts > atm_monthly_glb
Expand Down Expand Up @@ -59,14 +61,12 @@ years = "1985:1989:2",
[ts]
active = True
bundle = "bundle1"
e3sm_to_cmip_environment_commands = ""
years = "1985:1989:2",

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
ts_fmt = "cmip"

[[ atm_monthly_glb ]]
bundle = "bundle2" # Override bundle1
Expand All @@ -77,14 +77,12 @@ years = "1985:1989:2",
years = "1985:1995:5",

[[ land_monthly ]]
e3sm_to_cmip_environment_commands = ""
extra_vars = "landfrac"
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "LAISHA,LAISUN"
ts_fmt = "cmip"

[[ rof_monthly ]]
bundle = "bundle3" # Override bundle1, let bundle1 finish first because "e3sm_diags: atm_monthly_180x360_aave_mvm" requires "ts: atm_monthly_180x360_aave"
Expand All @@ -95,6 +93,23 @@ years = "1985:1989:2",
mapping_file = ""
vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"

[e3sm_to_cmip]
active = True
bundle = "bundle1"
years = "1985:1989:2",

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"

[[ land_monthly ]]
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
mapping_file = "map_r05_to_cmip6_180x360_aave.20231110.nc"
vars = "LAISHA,LAISUN"

# TODO: Add "tc_analysis" back in after empty dat is resolved.
# [tc_analysis]
# active = True
Expand Down
Loading
Loading