diff --git a/.github/workflows/test_cloud_runner.yml b/.github/workflows/test_cloud_runner.yml index 9452d4cd..a52a7329 100644 --- a/.github/workflows/test_cloud_runner.yml +++ b/.github/workflows/test_cloud_runner.yml @@ -74,10 +74,6 @@ jobs: singularity exec --writable-tmpfs ${bind_paths} ${container_path} ${runscript_path} - # Can potentially do this to get list of tasks - #task_list=$(cylc list test_pp__ptest__ttest) - #task1=$(echo $task_list | cut -d ' ' -f1) .... - - name: Upload workflow-run log files uses: actions/upload-artifact@v4 with: @@ -166,6 +162,106 @@ jobs: exit 2 fi + - name: Print stage-history-refined successes or failures + run: | + set +e + # Successes + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history-refined/0[1-3]/job.status) + exit_status_1=$? + # Failures + failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history-refined/0[1-3]/job.status) + exit_status_2=$? + if [ $exit_status_1 -eq 0 ]; then + echo $success | sed 's/ /\n/g' #split string by spaces + elif [ $exit_status_1 -eq 1 ]; then + echo "No succeeded tasks found" + else + echo "WARNING: error with grep execution" + exit 2 + fi + if [ $exit_status_2 -eq 0 ]; then + echo "Failures found" + echo $failure | sed 's/ /\n/g' #split string by spaces + exit 1 #exit with error + elif [ $exit_status_2 -eq 1 ]; then + echo "No task failures found." + else + echo "WARNING: error with grep execution" + exit 2 + fi + + - name: Print refine-diag successes or failures + run: | + set +e + # Successes + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/refineDiag-*/0[1-3]/job.status) + exit_status_1=$? + + # Failures + failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/refineDiag-*/0[1-3]/job.status) + exit_status_2=$? + + if [ $exit_status_1 -eq 0 ]; then + echo $success | sed 's/ /\n/g' #split string by spaces + + # Check if refine-diag task executed correctly + # history files copied into another history file + # ........ + + elif [ $exit_status_1 -eq 1 ]; then + echo "No succeeded tasks found" + else + echo "WARNING: error with grep execution" + exit 2 + fi + + if [ $exit_status_2 -eq 0 ]; then + echo "Failures found" + echo $failure | sed 's/ /\n/g' #split string by spaces + exit 1 #exit with error + elif [ $exit_status_2 -eq 1 ]; then + echo "No task failures found." + else + echo "WARNING: error with grep execution" + exit 2 + fi + + - name: Print pre-analysis successes or failures + run: | + set +e + # Successes + success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/preAnalysis-*/0[1-3]/job.status) + exit_status_1=$? + # Failures + failure=$(grep -E "CYLC_JOB_EXIT=ERR" /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/job/*/preAnalysis-*/0[1-3]/job.status) + exit_status_2=$? + if [ $exit_status_1 -eq 0 ]; then + echo $success | sed 's/ /\n/g' #split string by spaces + # Check if preanalysis task executed correctly: Check database exists + if [ -s "/contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest/log/db" ]; then + echo "Pre-analysis task produced non-empty db file" + exit 0 + else + echo "Pre-analysis task did not produce expected, non-empty db file" + exit 1 + fi + elif [ $exit_status_1 -eq 1 ]; then + echo "No succeeded tasks found" + else + echo "WARNING: error with grep execution" + exit 2 + fi + if [ $exit_status_2 -eq 0 ]; then + echo "Failures found" + echo $failure | sed 's/ /\n/g' #split string by spaces + exit 1 #exit with error + elif [ $exit_status_2 -eq 1 ]; then + echo "No task failures found." + else + echo "WARNING: error with grep execution" + exit 2 + fi + - name: Print regrid-xy successes or failures run: | set +e @@ -446,6 +542,13 @@ jobs: exit 2 fi + ## I think this will work (without cylc clean) because the + ## OUTPUT_DIR is bind mounted from the host + ## This is to retain clean workspace on cloud and to not waste + ## allocation + - name: Clean up output folder + run: rm -rf /contrib-efs/container-test/ppp-setup/cylc-run/test_pp__ptest__ttest + ## Keep in here for now as a just in case reference ## # - name: Print any succeeded tasks # run: | diff --git a/flow.cylc b/flow.cylc index 0cd99c79..7f9be19a 100644 --- a/flow.cylc +++ b/flow.cylc @@ -465,6 +465,7 @@ env set -x mkdir -p $work $tempCache $refineDiagDir + hsmget -v -t -a $histDir -p {{ PTMP_DIR }}/$histDir -w $work $hsmdate/\* cd $work/$hsmdate ls """ @@ -500,19 +501,23 @@ fi if [[ $refinedCount > 0 ]]; then for file in $(ls -1 *nc); do - list_ncvars.csh -st01234 $file |& tee $CYLC_WORKFLOW_SHARE_DIR/refineDiag.log + # list the variables names in the netcdf file + ncdump -h $file | + grep -E "float|double" | + awk '{print $2}' | + cut -d '(' -f1 |& tee $CYLC_WORKFLOW_SHARE_DIR/refineDiag.log done else echo ERROR: RefineDiag script did not create any NetCDF files as it was expected to do exit 1 fi if [[ -f {{ REFINED_HISTORY_DIR }}/$oname.nc.tar ]]; then + hsmget -v -t -a {{ REFINED_HISTORY_DIR }} -p {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} \ + -w $TMPDIR/modify_refineDiag $hsmdate/\* echo "the contents of {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} is..." ls {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} echo "the contents of {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}/$oname.nc is..." ls {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}/$oname.nc - hsmget -v -t -a {{ REFINED_HISTORY_DIR }} -p {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} \ - -w $TMPDIR/modify_refineDiag $hsmdate/\* mv -f * $TMPDIR/modify_refineDiag mv -f $TMPDIR/modify_refineDiag/* . rm -rf $TMPDIR/modify_refineDiag diff --git a/for_gh_runner/dummy_preanalysis.csh b/for_gh_runner/dummy_preanalysis.csh new file mode 100644 index 00000000..97a10923 --- /dev/null +++ b/for_gh_runner/dummy_preanalysis.csh @@ -0,0 +1,3 @@ +#!/bin/csh + +echo "Pre-analysis script is running!" diff --git a/for_gh_runner/dummy_refinediag.csh b/for_gh_runner/dummy_refinediag.csh new file mode 100644 index 00000000..f1ceb1d9 --- /dev/null +++ b/for_gh_runner/dummy_refinediag.csh @@ -0,0 +1,26 @@ +#!/bin/csh + +## RefineDiag scripts reads raw history files and generates new, refined history files +# Script copies test history file to another history file +set input_dir = `pwd` + +if ( $?refineDiagDir ) then + set output_dir = $refineDiagDir +else + echo "ERROR: refineDiagDir environment variable is not set." + exit 1 +endif + +if ( -d "$input_dir" ) then + #ls -aF + echo "Input directory found: $input_dir" + echo "good to go" + foreach INFILE (*atmos_month*.nc) + set OUTFILE = ${INFILE:r}_refined.${INFILE:e} + echo "OUTFILE $OUTFILE" + cp "$INFILE" "$output_dir/$OUTFILE" + end +else + echo "Error: Input directory $input_dir does not exist." + exit 1 +endif diff --git a/for_gh_runner/run_pp_locally.sh b/for_gh_runner/run_pp_locally.sh index fa20a487..5af4bd20 100755 --- a/for_gh_runner/run_pp_locally.sh +++ b/for_gh_runner/run_pp_locally.sh @@ -58,6 +58,13 @@ if [ -d $cylc_src_dir_name ]; then sleep 10s fi +## More cleaning neede for refineDiag output +if [ -d /work/$USER/fre-wf-local-testing/refined_history ]; then + echo -e "Refine Diag scripts previously run, removing ..." + rm -rf "/work/$USER/fre-wf-local-testing/refined_history" +fi + + # if you want this, comment on https://github.com/NOAA-GFDL/fre-cli/issues/673 #echo "" #echo "" diff --git a/for_gh_runner/runscript.sh b/for_gh_runner/runscript.sh index d3f7a934..744fd6a7 100755 --- a/for_gh_runner/runscript.sh +++ b/for_gh_runner/runscript.sh @@ -75,6 +75,7 @@ check_exit_status () { fre_pp_steps () { set -x + ## Clean previous experiment echo "experiment cleaning, if it was previously installed" if [ -d /mnt/cylc-run/${name} ]; then echo -e "\n${name} previously installed" @@ -82,6 +83,12 @@ fre_pp_steps () { cylc clean ${name} fi + ## More cleaning needed for refineDiag output + if [ -d /mnt/$USER/refined_history ]; then + echo -e "Refine Diag scripts previously run, removing ..." + rm -rf /mnt/$USER/refined_history + fi + ## Checkout echo -e "\nCreating $name directory in ${HOME}/cylc-src/${name} ..." rm -rf /mnt/cylc-src/${name} @@ -94,13 +101,13 @@ fre_pp_steps () { #Not sure if needed because if no global.cylc found, cylc uses default, which utilizes background jobs anyway ... #export CYLC_CONF_PATH=/mnt/cylc-src/${name}/generic-global-config/ - ## Configure the rose-suite and rose-app files for the workflow - echo -e "\nRunning fre pp configure-yaml, combining separate yaml configs into one, then writing rose-suite/app config files ..." + ## Configure the rose-suite file for the workflow + echo -e "\nRunning fre pp configure-yaml, combining separate yaml configs into one, then writing rose-suite config file ..." fre -vv pp configure-yaml -e ${expname} -p ${plat} -t ${targ} -y ${yamlfile} check_exit_status "CONFIGURE-YAML" ## Validate the configuration files - echo -e "\nRunning fre pp validate, validating rose-suite/app config files ..." + echo -e "\nRunning fre pp validate, validating rose-suite config file ..." fre -vv pp validate -e ${expname} -p ${plat} -t ${targ} check_exit_status "VALIDATE" diff --git a/for_gh_runner/yaml_workflow/local_settings.yaml b/for_gh_runner/yaml_workflow/local_settings.yaml index d24224ba..339d6e9a 100644 --- a/for_gh_runner/yaml_workflow/local_settings.yaml +++ b/for_gh_runner/yaml_workflow/local_settings.yaml @@ -1,8 +1,10 @@ directories: ## to test staging data from archive, use the below # history_dir: "/archive/inl/test_cloud_runner_history_files" +# # to have a faster turn-around (does not test archive data staging), use the below - history_dir: "/work/inl/test_cloud_runner_history_files" + history_dir: "/work/inl/test_cloud_runner_history_files" + refined_history_dir: "/work/$USER/fre-wf-local-testing/refined_history" pp_dir: !join [/work/$USER/test_cloud_runner_scratch_dir/, *FRE_STEM, /, pp] # analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] ptmp_dir: "/xtmp/$USER/ptmp" @@ -10,8 +12,8 @@ directories: postprocess: settings: history_segment: "P1Y" -# site: "ppan" - site: "ppan_test" + site: "ppan" +# site: "ppan_test" pp_chunks: [*PP_CHUNK96] pp_start: *EXP_AMIP_START pp_stop: *EXP_AMIP_END @@ -19,3 +21,16 @@ postprocess: switches: clean_work: True do_atmos_plevel_masking: True + + # to be first-step-user-script + preanalysis: + vitals: + script: "/work/d4s/fre-wf-local-testing/dummy_preanalysis.csh" + inputs: ['atmos_month', 'aerosol_month'] # not used currently + do_preanalysis: True + refinediag: + ocean: + script: "/work/d4s/fre-wf-local-testing/dummy_refinediag.csh" + inputs: ['ocean_monthly', 'ocean_bgc'] # not used currently + outputs: ['ocean_refined'] # not used currently + do_refinediag: True diff --git a/for_gh_runner/yaml_workflow/settings.yaml b/for_gh_runner/yaml_workflow/settings.yaml index 2e48c873..7f7f34ad 100644 --- a/for_gh_runner/yaml_workflow/settings.yaml +++ b/for_gh_runner/yaml_workflow/settings.yaml @@ -1,9 +1,10 @@ # directories shared across tools directories: - history_dir: "/mnt/history" - pp_dir: "/mnt/pp" + history_dir: "/mnt/history" + refined_history_dir: "/mnt/$USER/refined_history" + pp_dir: "/mnt/pp" # analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] - ptmp_dir: "/mnt/ptmp" + ptmp_dir: "/mnt/ptmp" postprocess: settings: @@ -16,3 +17,20 @@ postprocess: switches: clean_work: True do_atmos_plevel_masking: True + + # to be first-step-user-scripts + preanalysis: + vitals: + script: "/contrib-efs/fre-workflows-runners/actions-runner-1/_work/fre-workflows/fre-workflows/for_gh_runner/dummy_preanalysis.csh" + inputs: ['atmos_month', 'aerosol_month'] #not used currently + # to be user-script_on + do_preanalysis: True + + # to be refinediag-user-scripts + refinediag: + ocean: + script: "/contrib-efs/fre-workflows-runners/actions-runner-1/_work/fre-workflows/fre-workflows/for_gh_runner/dummy_refinediag.csh" + inputs: ['ocean_monthly', 'ocean_bgc'] # not used currently + outputs: ['ocean_refined'] # not used currently + # to be user_script_on + do_refinediag: True diff --git a/site/gaea.cylc b/site/gaea.cylc index 06da0e5c..9939d198 100644 --- a/site/gaea.cylc +++ b/site/gaea.cylc @@ -22,7 +22,7 @@ COPY_TOOL=gcp [[STAGE-HISTORY]] - pre-script = module load hsm/1.3.0 + pre-script = module load hsm/1.4.0 [[MAKE-TIMESERIES]] pre-script = module load cdo diff --git a/site/ppan.cylc b/site/ppan.cylc index b28a4e0b..79a030b3 100644 --- a/site/ppan.cylc +++ b/site/ppan.cylc @@ -8,7 +8,7 @@ #module load conda #conda activate user-fre-cli - which fre + #which fre module load gcp/2.3 module list """ @@ -24,7 +24,7 @@ [[STAGE-HISTORY]] pre-script = """ - module load gcp/2.3 hsm/1.3.0 + module load gcp/2.3 hsm/1.4.0 """ {% if DO_REFINEDIAG or DO_PREANALYSIS %} @@ -32,7 +32,7 @@ pre-script = """ env set -x - module load hsm/1.3.0 + module load hsm/1.4.0 module load gcp # needed only for list_ncvars. module load fre-nctools/2024.05.01 diff --git a/site/ppan_test.cylc b/site/ppan_test.cylc index d3674068..d6ef2eb6 100644 --- a/site/ppan_test.cylc +++ b/site/ppan_test.cylc @@ -30,11 +30,11 @@ epmt check #### if you want to use your own conda environment, comment out 'module load fre/...' below - module load fre/{{ FRE_VERSION }} - module load gcp/2.3 hsm/1.3.0 + #module load fre/{{ FRE_VERSION }} + module load gcp/2.3 hsm/1.4.0 #### if you want to use your own conda environment, edit the PATH and uncomment below like so: - #export PATH=/home/$USER/conda/envs/fre-cli/bin:$PATH + export PATH=/home/$USER/conda/envs/fre-cli/bin:$PATH """ env-script = """ {{ EPMT_PREAMBLE }}