From 4c315ae16446dfd918d26c81dd10dc3673db2fbd Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 11 May 2026 14:29:04 -0600 Subject: [PATCH 1/6] Initial commit --- .github/workflows/cmp_fortran_to_metadata.yml | 27 +++++ ccpp/config/ccpp_prebuild_config.py | 7 ++ test/test_fortran_to_metadata.py | 111 ++++++++++++++++++ 3 files changed, 145 insertions(+) create mode 100644 .github/workflows/cmp_fortran_to_metadata.yml create mode 100755 test/test_fortran_to_metadata.py diff --git a/.github/workflows/cmp_fortran_to_metadata.yml b/.github/workflows/cmp_fortran_to_metadata.yml new file mode 100644 index 000000000..d947eb17a --- /dev/null +++ b/.github/workflows/cmp_fortran_to_metadata.yml @@ -0,0 +1,27 @@ +name: Run Capgen fortran to metadata comparision script + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + env: + SCM_ROOT: ${{ github.workspace }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Compare Fortran source files to CCPP metadata files + run: | + cd ${SCM_ROOT} + cp test/test_fortran_to_metadata.py . + ./test_fortran_to_metadata.py \ No newline at end of file diff --git a/ccpp/config/ccpp_prebuild_config.py b/ccpp/config/ccpp_prebuild_config.py index 3f693f8ff..7128acd27 100755 --- a/ccpp/config/ccpp_prebuild_config.py +++ b/ccpp/config/ccpp_prebuild_config.py @@ -43,9 +43,16 @@ }, 'module_radsw_parameters' : { 'module_radsw_parameters' : '', + 'cmpfsw_type' : '', + 'sfcfsw_type' : '', + 'profsw_type' : '', + 'topfsw_type' : '', }, 'module_radlw_parameters' : { 'module_radlw_parameters' : '', + 'sfcflw_type' : '', + 'proflw_type' : '', + 'topflw_type' : '', }, 'module_ozphys' : { 'module_ozphys' : '', diff --git a/test/test_fortran_to_metadata.py b/test/test_fortran_to_metadata.py new file mode 100755 index 000000000..eded5060a --- /dev/null +++ b/test/test_fortran_to_metadata.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python + +############################################################################### +# Dependencies +############################################################################### +import argparse +import os +import sys +# Using Prebuild +sys.path.append('ccpp/config') +from ccpp_prebuild_config import SCHEME_FILES, TYPEDEFS_NEW_METADATA, VARIABLE_DEFINITION_FILES +# Using Capgen +sys.path.append('ccpp/framework/scripts') +from metadata_table import register_ddts + +############################################################################### +# Argument list +############################################################################### +parser = argparse.ArgumentParser() + +############################################################################### +# Main program +############################################################################### +def main(): + # Get command line arguments (N/A) + args = parser.parse_args() + + # + script_dir = 'ccpp/framework/scripts/' + f2m_script = script_dir+'/ccpp_fortran_to_metadata.py' + + # Create list of known DDTs (using prebuild config script) + ddts_prebuild = [] + for ct, typedefs in enumerate(TYPEDEFS_NEW_METADATA): + for typedef in TYPEDEFS_NEW_METADATA[typedefs]: + ddts_prebuild.append(typedef) + # end for + # end for + #print(ddts_prebuild) + + # Create list of known DDTs (using capgen register_ddts) + # ######################################################################## + # ######################################################################## + # DJS: It's hard to know how Capgen is implemented, but to find the DDTs, + # we need names for all the source files (scheme and host). Here I + # just used the files lists used by the CCPP prebuild configuration. + # ######################################################################## + # ######################################################################## + # First, convert source file suffixes from fortran to .meta + # Scheme files + scheme_metadata = [] + for scheme_file in SCHEME_FILES: + scheme_metadata.append(os.path.splitext(scheme_file)[0]+'.meta') + # end for + # Host files + host_metadata = [] + # DJS: The last file in VARIABLE_DEFINITION_FILES is not a valid CCPP metadata file. Omit. + nhost = len(VARIABLE_DEFINITION_FILES) + for host_file in VARIABLE_DEFINITION_FILES[0:nhost-1]: + host_metadata.append(os.path.splitext(host_file)[0]+'.meta') + # end for + + # Finally, register the DDTs + # NOTE: This returns a list of (lowercase) ddt names. + ddts_capgen = register_ddts(scheme_metadata+host_metadata) + #print(ddts_capgen) + + # Create string from DDT list. + init = True + known_ddts = '' + for ddt in ddts_prebuild: + if (init): + known_ddts = ddt + init = False + else: + known_ddts = known_ddts+','+ddt + # end if + # end for + + # + # Run Fortran-to-metadata comparision script + # + # DJS: This fails if you use the known_ddts from Capgen, which are all lowercase. + error_count = 0 + error_file = [] + com = f2m_script+' --ddt-names '+known_ddts+' ' + for scheme_file in SCHEME_FILES: + result = os.system(com+scheme_file) + if (result != 0): + error_count = error_count + 1 + error_file.append(scheme_file) + # end if + # end for + + print('------------------------------------------------------------------------------------') + if (error_count == 0): + print("SUCCESS! All metadata and Fortran files are consistent") + else: + print("ERROR! There are differences between the metadata and Fortran source files") + print(" ",str(error_count)+' files have issues:') + for ierror in error_file: + print(" ",ierror) + # end for + print(" See messages above for more details") + # end if + print('------------------------------------------------------------------------------------') + +############################################################################### +############################################################################### +if __name__ == '__main__': + main() From 363846e2e29d3314d40ef11a4027257d4af0b408 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 11 May 2026 14:31:41 -0600 Subject: [PATCH 2/6] Update CI script --- .github/workflows/cmp_fortran_to_metadata.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/cmp_fortran_to_metadata.yml b/.github/workflows/cmp_fortran_to_metadata.yml index d947eb17a..588e3d28c 100644 --- a/.github/workflows/cmp_fortran_to_metadata.yml +++ b/.github/workflows/cmp_fortran_to_metadata.yml @@ -15,6 +15,12 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Initialize Submodules + run: | + git config --global --add safe.directory ${SCM_ROOT} + cd ${SCM_ROOT} + git submodule update --init --recursive + - name: Set up Python uses: actions/setup-python@v5 with: From 6f98810fa87385469ca28ad716f6279d0db1a6a8 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 11 May 2026 14:33:32 -0600 Subject: [PATCH 3/6] Update CI script --- .github/workflows/cmp_fortran_to_metadata.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cmp_fortran_to_metadata.yml b/.github/workflows/cmp_fortran_to_metadata.yml index 588e3d28c..405c9582a 100644 --- a/.github/workflows/cmp_fortran_to_metadata.yml +++ b/.github/workflows/cmp_fortran_to_metadata.yml @@ -28,6 +28,5 @@ jobs: - name: Compare Fortran source files to CCPP metadata files run: | - cd ${SCM_ROOT} cp test/test_fortran_to_metadata.py . ./test_fortran_to_metadata.py \ No newline at end of file From 1b1c4ac2e5242e1fae9b2c2cdbe75619f5503cae Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 11 May 2026 16:10:55 -0600 Subject: [PATCH 4/6] Housekeeping --- test/test_fortran_to_metadata.py | 72 ++++++++++---------------------- 1 file changed, 22 insertions(+), 50 deletions(-) diff --git a/test/test_fortran_to_metadata.py b/test/test_fortran_to_metadata.py index eded5060a..6beb78934 100755 --- a/test/test_fortran_to_metadata.py +++ b/test/test_fortran_to_metadata.py @@ -6,69 +6,37 @@ import argparse import os import sys -# Using Prebuild sys.path.append('ccpp/config') from ccpp_prebuild_config import SCHEME_FILES, TYPEDEFS_NEW_METADATA, VARIABLE_DEFINITION_FILES -# Using Capgen -sys.path.append('ccpp/framework/scripts') -from metadata_table import register_ddts - -############################################################################### -# Argument list -############################################################################### -parser = argparse.ArgumentParser() ############################################################################### # Main program ############################################################################### def main(): - # Get command line arguments (N/A) - args = parser.parse_args() + # This list of files is not compared since they are known to fail. + files_known_to_not_pass_test = [ + 'ccpp/physics/physics/MP/GFDL/fv_sat_adj.F90', + 'ccpp/physics/physics/Radiation/RRTMG/rrtmg_lw_post.F90', + 'ccpp/physics/physics/SFC_Layer/UFS/sfc_diff.f', + 'ccpp/physics/physics/SFC_Models/Lake/CLM/clm_lake.f90', + 'ccpp/physics/physics/smoke_dust/rrfs_smoke_wrapper.F90'] - # + # Comparison script from framework script_dir = 'ccpp/framework/scripts/' f2m_script = script_dir+'/ccpp_fortran_to_metadata.py' # Create list of known DDTs (using prebuild config script) - ddts_prebuild = [] + ddts = [] for ct, typedefs in enumerate(TYPEDEFS_NEW_METADATA): for typedef in TYPEDEFS_NEW_METADATA[typedefs]: - ddts_prebuild.append(typedef) + ddts.append(typedef) # end for # end for - #print(ddts_prebuild) - - # Create list of known DDTs (using capgen register_ddts) - # ######################################################################## - # ######################################################################## - # DJS: It's hard to know how Capgen is implemented, but to find the DDTs, - # we need names for all the source files (scheme and host). Here I - # just used the files lists used by the CCPP prebuild configuration. - # ######################################################################## - # ######################################################################## - # First, convert source file suffixes from fortran to .meta - # Scheme files - scheme_metadata = [] - for scheme_file in SCHEME_FILES: - scheme_metadata.append(os.path.splitext(scheme_file)[0]+'.meta') - # end for - # Host files - host_metadata = [] - # DJS: The last file in VARIABLE_DEFINITION_FILES is not a valid CCPP metadata file. Omit. - nhost = len(VARIABLE_DEFINITION_FILES) - for host_file in VARIABLE_DEFINITION_FILES[0:nhost-1]: - host_metadata.append(os.path.splitext(host_file)[0]+'.meta') - # end for - - # Finally, register the DDTs - # NOTE: This returns a list of (lowercase) ddt names. - ddts_capgen = register_ddts(scheme_metadata+host_metadata) - #print(ddts_capgen) # Create string from DDT list. init = True known_ddts = '' - for ddt in ddts_prebuild: + for ddt in ddts: if (init): known_ddts = ddt init = False @@ -77,24 +45,27 @@ def main(): # end if # end for - # # Run Fortran-to-metadata comparision script - # - # DJS: This fails if you use the known_ddts from Capgen, which are all lowercase. error_count = 0 error_file = [] com = f2m_script+' --ddt-names '+known_ddts+' ' for scheme_file in SCHEME_FILES: - result = os.system(com+scheme_file) - if (result != 0): - error_count = error_count + 1 - error_file.append(scheme_file) + if (scheme_file not in files_known_to_not_pass_test): + result = os.system(com+scheme_file) + if (result != 0): + error_count = error_count + 1 + error_file.append(scheme_file) + # end if + else: + print("skipping ",scheme_file) # end if # end for + # Display results print('------------------------------------------------------------------------------------') if (error_count == 0): print("SUCCESS! All metadata and Fortran files are consistent") + sys.exit(0) else: print("ERROR! There are differences between the metadata and Fortran source files") print(" ",str(error_count)+' files have issues:') @@ -102,6 +73,7 @@ def main(): print(" ",ierror) # end for print(" See messages above for more details") + sys.exit(1) # end if print('------------------------------------------------------------------------------------') From 3a2a78031103617d6cd4edec12f7f8ca0ead02aa Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 11 May 2026 16:17:28 -0600 Subject: [PATCH 5/6] Fix fortran to metadata inconsistency --- .gitmodules | 4 ++-- ccpp/physics | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index b9de8d317..a76057a8d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,8 @@ branch = develop [submodule "ccpp-physics"] path = ccpp/physics - url = https://github.com/NCAR/ccpp-physics - branch = scm/dev + url = https://github.com/dustinswales/ccpp-physics + branch = bug/flake_metadata [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 135a5dda8..fb235bb21 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 135a5dda80f383c29c9a1ac937a392c2e74c5683 +Subproject commit fb235bb21572a05cb2d7a9cd3765e95b763b0b79 From 809cf301359392f8c92fff165c1df292100d9ed2 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 11 May 2026 17:01:40 -0600 Subject: [PATCH 6/6] Omission from previous commit --- ccpp/physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp/physics b/ccpp/physics index fb235bb21..47af3a34d 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit fb235bb21572a05cb2d7a9cd3765e95b763b0b79 +Subproject commit 47af3a34d5e71c1876a7c95f8cd76ded3e5aca7b