Add refinediag preanalysis tests

.github/workflows/test_cloud_runner.yml

@@ -74,10 +74,6 @@ jobs:
 
           singularity exec --writable-tmpfs ${bind_paths} ${container_path} ${runscript_path}
 
-          # Can potentially do this to get list of tasks
-          #task_list=$(cylc list test_pp__ptest__ttest)
-          #task1=$(echo $task_list | cut -d ' ' -f1) ....
-
       - name: Upload workflow-run log files
         uses: actions/upload-artifact@v4
         with:
@@ -166,6 +162,106 @@ jobs:
               exit 2
           fi
 
+      - name: Print stage-history-refined successes or failures
+        run: |
+          set +e
+          # Successes
+          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history-refined/0[1-3]/job.status)
+          exit_status_1=$?
+          # Failures
+          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history-refined/0[1-3]/job.status)
+          exit_status_2=$?
+          if [ $exit_status_1 -eq 0 ]; then
+              echo $success | sed 's/ /\n/g' #split string by spaces
+          elif [ $exit_status_1 -eq 1 ]; then
+              echo "No succeeded tasks found"
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+          if [ $exit_status_2 -eq 0 ]; then
+              echo "Failures found"
+              echo $failure | sed 's/ /\n/g' #split string by spaces
+              exit 1   #exit with error
+          elif [ $exit_status_2 -eq 1 ]; then
+              echo "No task failures found."
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+
+      - name: Print refine-diag successes or failures
+        run: |
+          set +e
+          # Successes 
+          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/refineDiag-*/0[1-3]/job.status)
+          exit_status_1=$?
+
+          # Failures
+          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/refineDiag-*/0[1-3]/job.status)
+          exit_status_2=$?
+
+          if [ $exit_status_1 -eq 0 ]; then
+              echo $success | sed 's/ /\n/g' #split string by spaces
+
+              # Check if refine-diag task executed correctly
+              # history files copied into another history file 
+              # ........
+
+          elif [ $exit_status_1 -eq 1 ]; then
+              echo "No succeeded tasks found"
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+
+          if [ $exit_status_2 -eq 0 ]; then
+              echo "Failures found"
+              echo $failure | sed 's/ /\n/g' #split string by spaces
+              exit 1   #exit with error
+          elif [ $exit_status_2 -eq 1 ]; then
+              echo "No task failures found."
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+
+      - name: Print pre-analysis successes or failures
+        run: |
+          set +e
+          # Successes 
+          success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/preAnalysis-*/0[1-3]/job.status)
+          exit_status_1=$?
+          # Failures
+          failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/preAnalysis-*/0[1-3]/job.status)
+          exit_status_2=$?
+          if [ $exit_status_1 -eq 0 ]; then
+              echo $success | sed 's/ /\n/g' #split string by spaces
+              # Check if preanalysis task executed correctly: Check database exists
+              if [ -s "/contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/db" ]; then
+                  echo "Pre-analysis task produced non-empty db file"
+                  exit 0
+              else
+                  echo "Pre-analysis task did not produce expected, non-empty db file"
+                  exit 1
+              fi
+          elif [ $exit_status_1 -eq 1 ]; then
+              echo "No succeeded tasks found"
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+          if [ $exit_status_2 -eq 0 ]; then
+              echo "Failures found"
+              echo $failure | sed 's/ /\n/g' #split string by spaces
+              exit 1   #exit with error
+          elif [ $exit_status_2 -eq 1 ]; then
+              echo "No task failures found."
+          else
+              echo "WARNING: error with grep execution"
+              exit 2
+          fi
+
       - name: Print regrid-xy successes or failures
         run: |
           set +e
@@ -446,6 +542,13 @@ jobs:
               exit 2
           fi
 
+        ## I think this will work (without cylc clean) because the
+        ## OUTPUT_DIR is bind mounted from the host
+        ## This is to retain clean workspace on cloud and to not waste
+        ## allocation
+      - name: Clean up output folder
+        run: rm -rf /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest
+
 ## Keep in here for now as a just in case reference ##
 #      - name: Print any succeeded tasks
 #        run: |

flow.cylc

@@ -465,6 +465,7 @@
             env
             set -x
             mkdir -p $work $tempCache $refineDiagDir
+            hsmget -v -t -a $histDir -p {{ PTMP_DIR }}/$histDir -w $work $hsmdate/\*
             cd $work/$hsmdate
             ls
         """
@@ -500,19 +501,23 @@
             fi
             if [[ $refinedCount > 0 ]]; then
                 for file in $(ls -1 *nc); do
-                    list_ncvars.csh -st01234 $file |& tee $CYLC_WORKFLOW_SHARE_DIR/refineDiag.log
+                    # list the variables names in the netcdf file
+                    ncdump -h $file |
+                    grep -E "float|double" |
+                    awk '{print $2}' |
+                    cut -d '(' -f1 |& tee $CYLC_WORKFLOW_SHARE_DIR/refineDiag.log
                 done
             else
                 echo ERROR: RefineDiag script did not create any NetCDF files as it was expected to do
                 exit 1
             fi
             if [[ -f {{ REFINED_HISTORY_DIR }}/$oname.nc.tar ]]; then
+                hsmget -v -t -a {{ REFINED_HISTORY_DIR }} -p {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} \
+                    -w $TMPDIR/modify_refineDiag $hsmdate/\*
                 echo "the contents of {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} is..."
                 ls {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}
                 echo "the contents of {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}/$oname.nc is..."
                 ls {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}/$oname.nc
-                hsmget -v -t -a {{ REFINED_HISTORY_DIR }} -p {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} \
-                    -w $TMPDIR/modify_refineDiag $hsmdate/\*
                 mv -f * $TMPDIR/modify_refineDiag
                 mv -f $TMPDIR/modify_refineDiag/* .
                 rm -rf $TMPDIR/modify_refineDiag

for_gh_runner/dummy_preanalysis.csh

@@ -0,0 +1,3 @@
+#!/bin/csh
+
+echo "Pre-analysis script is running!"

for_gh_runner/dummy_refinediag.csh

@@ -0,0 +1,26 @@
+#!/bin/csh
+
+## RefineDiag scripts reads raw history files and generates new, refined history files
+# Script copies test history file to another history file
+set input_dir = `pwd`
+
+if ( $?refineDiagDir ) then
+    set output_dir = $refineDiagDir
+else
+    echo "ERROR: refineDiagDir environment variable is not set."
+    exit 1
+endif
+
+if ( -d "$input_dir" ) then
+    #ls -aF
+    echo "Input directory found: $input_dir"
+    echo "good to go"
+    foreach INFILE (*atmos_month*.nc)
+        set OUTFILE = ${INFILE:r}_refined.${INFILE:e}
+        echo "OUTFILE $OUTFILE"
+        cp "$INFILE" "$output_dir/$OUTFILE"
+    end
+else
+    echo "Error: Input directory $input_dir does not exist."
+    exit 1
+endif

for_gh_runner/run_pp_locally.sh

@@ -58,6 +58,13 @@ if [ -d $cylc_src_dir_name ]; then
     sleep 10s
 fi
 
+## More cleaning neede for refineDiag output
+if [ -d /work/$USER/fre-wf-local-testing/refined_history ]; then
+    echo -e "Refine Diag scripts previously run, removing ..."
+    rm -rf "/work/$USER/fre-wf-local-testing/refined_history"
+fi
+
+
 # if you want this, comment on https://github.com/NOAA-GFDL/fre-cli/issues/673
 #echo ""
 #echo ""

for_gh_runner/runscript.sh

@@ -75,13 +75,20 @@ check_exit_status () {
 fre_pp_steps () {
     set -x
 
+    ## Clean previous experiment
     echo "experiment cleaning, if it was previously installed"
     if [ -d /mnt/cylc-run/${name} ]; then
         echo -e "\n${name} previously installed"
         echo "   Removing ${name}..."
         cylc clean ${name}
     fi
 
+    ## More cleaning needed for refineDiag output
+    if [ -d /mnt/$USER/refined_history ]; then
+        echo -e "Refine Diag scripts previously run, removing ..."
+        rm -rf /mnt/$USER/refined_history
+    fi 
+
     ## Checkout
     echo -e "\nCreating $name directory in ${HOME}/cylc-src/${name} ..."
     rm -rf /mnt/cylc-src/${name}
@@ -94,13 +101,13 @@ fre_pp_steps () {
     #Not sure if needed because if no global.cylc found, cylc uses default, which utilizes background jobs anyway ...
     #export CYLC_CONF_PATH=/mnt/cylc-src/${name}/generic-global-config/
 
-    ## Configure the rose-suite and rose-app files for the workflow
-    echo -e "\nRunning fre pp configure-yaml, combining separate yaml configs into one, then writing rose-suite/app config files ..."
+    ## Configure the rose-suite file for the workflow
+    echo -e "\nRunning fre pp configure-yaml, combining separate yaml configs into one, then writing rose-suite config file ..."
     fre -vv pp configure-yaml -e ${expname} -p ${plat} -t ${targ} -y ${yamlfile}
     check_exit_status "CONFIGURE-YAML"
 
     ## Validate the configuration files
-    echo -e "\nRunning fre pp validate, validating rose-suite/app config files ..."
+    echo -e "\nRunning fre pp validate, validating rose-suite config file ..."
     fre -vv pp validate -e ${expname} -p ${plat} -t ${targ}
     check_exit_status "VALIDATE"
 

for_gh_runner/yaml_workflow/local_settings.yaml

@@ -1,21 +1,36 @@
 directories:
 ## to test staging data from archive, use the below
 #  history_dir:        "/archive/inl/test_cloud_runner_history_files"
+#
 # to have a faster turn-around (does not test archive data staging), use the below
-  history_dir:        "/work/inl/test_cloud_runner_history_files"
+  history_dir:         "/work/inl/test_cloud_runner_history_files"
+  refined_history_dir: "/work/$USER/fre-wf-local-testing/refined_history"
   pp_dir:             !join [/work/$USER/test_cloud_runner_scratch_dir/, *FRE_STEM, /, pp]
 #  analysis_dir:       !join [/nbhome/$USER/, *FRE_STEM, /, *name]
   ptmp_dir:           "/xtmp/$USER/ptmp"
 
 postprocess:
   settings:
     history_segment:         "P1Y"
-#    site:                    "ppan"
-    site:                    "ppan_test"
+    site:                    "ppan"
+#    site:                    "ppan_test"
     pp_chunks:               [*PP_CHUNK96]
     pp_start:                *EXP_AMIP_START
     pp_stop:                 *EXP_AMIP_END
     pp_grid_spec:            *GRID_SPEC96_LOCAL
   switches:
     clean_work:              True
     do_atmos_plevel_masking: True
+
+  # to be first-step-user-script
+  preanalysis:
+    vitals:
+      script: "/work/d4s/fre-wf-local-testing/dummy_preanalysis.csh"
+      inputs: ['atmos_month', 'aerosol_month'] # not used currently
+      do_preanalysis: True
+  refinediag:
+    ocean:
+      script: "/work/d4s/fre-wf-local-testing/dummy_refinediag.csh"
+      inputs: ['ocean_monthly', 'ocean_bgc'] # not used currently
+      outputs: ['ocean_refined']             # not used currently
+      do_refinediag: True

for_gh_runner/yaml_workflow/settings.yaml

@@ -1,9 +1,10 @@
 # directories shared across tools
 directories:
-  history_dir:        "/mnt/history"
-  pp_dir:             "/mnt/pp"
+  history_dir:         "/mnt/history"
+  refined_history_dir: "/mnt/$USER/refined_history" 
+  pp_dir:              "/mnt/pp"
 #  analysis_dir:       !join [/nbhome/$USER/, *FRE_STEM, /, *name]
-  ptmp_dir:           "/mnt/ptmp"
+  ptmp_dir:            "/mnt/ptmp"
 
 postprocess:
   settings:
@@ -16,3 +17,20 @@ postprocess:
   switches:
     clean_work:              True
     do_atmos_plevel_masking: True
+
+  # to be first-step-user-scripts
+  preanalysis:
+    vitals:
+      script: "/contrib-efs/fre-workflows-runners/actions-runner-1/_work/fre-workflows/fre-workflows/for_gh_runner/dummy_preanalysis.csh"
+      inputs: ['atmos_month', 'aerosol_month'] #not used currently
+      # to be user-script_on
+      do_preanalysis: True
+
+  # to be refinediag-user-scripts
+  refinediag:
+    ocean:
+      script: "/contrib-efs/fre-workflows-runners/actions-runner-1/_work/fre-workflows/fre-workflows/for_gh_runner/dummy_refinediag.csh"
+      inputs: ['ocean_monthly', 'ocean_bgc'] # not used currently
+      outputs: ['ocean_refined']             # not used currently
+      # to be user_script_on
+      do_refinediag: True

site/gaea.cylc

@@ -22,7 +22,7 @@
             COPY_TOOL=gcp
 
     [[STAGE-HISTORY]]
-        pre-script = module load hsm/1.3.0
+        pre-script = module load hsm/1.4.0
 
     [[MAKE-TIMESERIES]]
         pre-script = module load cdo

site/ppan.cylc

@@ -8,7 +8,7 @@
             #module load conda
             #conda activate user-fre-cli
 
-            which fre
+            #which fre
             module load gcp/2.3
             module list
         """
@@ -24,15 +24,15 @@
 
     [[STAGE-HISTORY]]
         pre-script = """
-            module load gcp/2.3 hsm/1.3.0
+            module load gcp/2.3 hsm/1.4.0
         """
 
 {% if DO_REFINEDIAG or DO_PREANALYSIS %}
     [[PRE-ANALYSIS]]
         pre-script = """
             env
             set -x
-            module load hsm/1.3.0
+            module load hsm/1.4.0
             module load gcp
             # needed only for list_ncvars.
             module load fre-nctools/2024.05.01

site/ppan_test.cylc

@@ -30,11 +30,11 @@
             epmt check
 
             #### if you want to use your own conda environment, comment out 'module load fre/...' below
-            module load fre/{{ FRE_VERSION }}
-            module load gcp/2.3 hsm/1.3.0
+            #module load fre/{{ FRE_VERSION }}
+            module load gcp/2.3 hsm/1.4.0
 
             #### if you want to use your own conda environment, edit the PATH and uncomment below like so:
-            #export PATH=/home/$USER/conda/envs/fre-cli/bin:$PATH
+            export PATH=/home/$USER/conda/envs/fre-cli/bin:$PATH
         """
         env-script = """
             {{ EPMT_PREAMBLE }}