diff --git a/config.sh b/config.sh index 000d821..5b17e5f 100755 --- a/config.sh +++ b/config.sh @@ -80,14 +80,17 @@ export controlanal="false" # hybrid-cov high-res control analysis as in ops source $MODULESHOME/init/sh if [ "$machine" == 'hera' ]; then export basedir=/scratch2/BMC/gsienkf/${USER} + export codedir=/scratch2/BMC/gsienkf/Jeffrey.S.Whitaker/ export datadir=$basedir export hsidir="/ESRL/BMC/gsienkf/2year/whitaker/${exptname}" export obs_datapath=/scratch1/NCEPDEV/global/glopara/dump module purge module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack - module load hpc/1.2.0 - module load hpc-intel/2022.1.2 - module load hpc-impi/2022.1.2 + module load hpc/1.1.0 + module load hpc-intel/18.0.5.274 + module load hpc-impi/2018.0.4 + module load prod_util/1.2.2 + module load hdf5/1.10.6 module load netcdf/4.7.4 module load pio/2.5.2 @@ -97,6 +100,7 @@ if [ "$machine" == 'hera' ]; then export WGRIB=`which wgrib` elif [ "$machine" == 'orion' ]; then export basedir=/work/noaa/gsienkf/${USER} + export codedir=/work/noaa/gsienkf/whitaker/ export datadir=$basedir export hsidir="/ESRL/BMC/gsienkf/2year/whitaker/${exptname}" #export obs_datapath=/work/noaa/rstprod/dump @@ -117,6 +121,7 @@ elif [ "$machine" == 'orion' ]; then export WGRIB=`which wgrib` elif [ "$machine" == 'gaea' ]; then export basedir=/lustre/f2/dev/${USER} + export codedir=/lustre/f2/dev/Jeffrey.S.Whitaker export datadir=/lustre/f2/scratch/${USER} export hsidir="/ESRL/BMC/gsienkf/2year/whitaker/${exptname}" #export hsidir="/3year/NCEPDEV/GEFSRR/${exptname}" @@ -387,7 +392,7 @@ elif [ $LEVS -eq 127 ]; then export s_ens_v=7.7 # 20 levels fi # use pre-generated bias files. -#export biascorrdir=${datadir}/biascor +#export biascorrdir=${codedir}/biascor export nanals=80 # if nanals2>0, extend nanals2 members out to FHMAX + ANALINC (one extra assim window) @@ -404,7 +409,7 @@ if [ "$machine" == 'hera' ]; then export FIXDIR_gcyc=$FIXDIR #export FIXDIR_gcyc=/scratch1/NCEPDEV/global/glopara/fix # for GFSv16 export python=/contrib/anaconda/2.3.0/bin/python - export gsipath=${basedir}/gsi/GSI + export gsipath=${codedir}/gsi/GSI export fixgsi=${gsipath}/fix export fixcrtm=/scratch2/NCEPDEV/nwprod/NCEPLIBS/fix/crtm_v2.3.0 export execdir=${enkfscripts}/exec_${machine} @@ -417,7 +422,7 @@ elif [ "$machine" == 'orion' ]; then #export FIXDIR_gcyc=/work/noaa/global/glopara/fix_NEW # for GFSv16 export python=`which python` export fv3gfspath=/work/noaa/global/glopara - export gsipath=${basedir}/GSI + export gsipath=${codedir}/GSI export fixgsi=${gsipath}/fix export fixcrtm=$fv3gfspath/crtm/crtm_v2.3.0 export execdir=${enkfscripts}/exec_${machine} @@ -431,7 +436,7 @@ elif [ "$machine" == 'gaea' ]; then export fv3gfspath=/lustre/f2/dev/Jeffrey.S.Whitaker/fv3_reanl/fv3gfs/global_shared.v15.0.0 export FIXFV3=${fv3gfspath}/fix/fix_fv3_gmted2010 export FIXGLOBAL=${fv3gfspath}/fix/fix_am - export gsipath=/lustre/f2/dev/Jeffrey.S.Whitaker/GSI-github-jswhit + export gsipath=${codedir}/GSI-github-jswhit export fixgsi=${gsipath}/fix export fixcrtm=/lustre/f2/pdata/ncep_shared/NCEPLIBS/lib/crtm/v2.2.6/fix #export fixcrtm=${fixgsi}/crtm_v2.2.3 @@ -461,7 +466,7 @@ export HYBENSINFO=${fixgsi}/global_hybens_info.l${LEVS}.txt # only used if readi # in stratosphere/mesosphere #export HYBENSMOOTHINFO=${fixgsi}/global_hybens_smoothinfo.l${LEVS}.txt export OZINFO=${fixgsi}/global_ozinfo.txt -export CONVINFO=${fixgsi}/global_convinfo.txt +#export CONVINFO=${fixgsi}/global_convinfo.txt export SATINFO=${fixgsi}/global_satinfo.txt export NLAT=$((${LATA}+2)) # default is to use berror file in gsi fix dir. diff --git a/exec_hera/global_enkf b/exec_hera/global_enkf deleted file mode 100755 index ebb95a4..0000000 Binary files a/exec_hera/global_enkf and /dev/null differ diff --git a/exec_hera/global_enkf b/exec_hera/global_enkf new file mode 120000 index 0000000..2fb8d30 --- /dev/null +++ b/exec_hera/global_enkf @@ -0,0 +1 @@ +../../../gerrit-hera/gsi/GSI_mine/install/bin/enkf.x \ No newline at end of file diff --git a/exec_hera/global_gsi b/exec_hera/global_gsi deleted file mode 100755 index 2a09979..0000000 Binary files a/exec_hera/global_gsi and /dev/null differ diff --git a/exec_hera/global_gsi b/exec_hera/global_gsi new file mode 120000 index 0000000..27579fd --- /dev/null +++ b/exec_hera/global_gsi @@ -0,0 +1 @@ +../../../gerrit-hera/gsi/GSI_mine/install/bin/gsi.x \ No newline at end of file diff --git a/hera_preamble b/hera_preamble index 7e47181..164a54c 100644 --- a/hera_preamble +++ b/hera_preamble @@ -1,7 +1,8 @@ #!/bin/sh -#SBATCH -q debug -#SBATCH -t 00:30:00 -##SBATCH -t 01:30:00 +##SBATCH -q debug +##SBATCH -t 00:30:00 +#SBATCH -q batch +#SBATCH -t 01:30:00 #SBATCH -A gsienkf #SBATCH -N 20 #SBATCH -J C96_hybcov_p8 @@ -24,8 +25,8 @@ elif [ $NODES -eq 20 ]; then # 20 nodes, 2 threads export control_threads=2 export control_proc=800 - export write_groups_ctl=2 # write groups for control forecast. - export write_tasks_ctl=8 + export write_groups_ctl=4 # write groups for control forecast. + export write_tasks_ctl=4 export layout_ctl="8,8" # layout_x,layout_y (total # mpi tasks = $layout_x*$layout_y*6=($fg_proc/$fg_threads) - $write_tasks*$write_groups) elif [ $NODES -eq 40 ]; then # 40 nodes, 4 threads @@ -69,11 +70,11 @@ fi # C96 ensemble export fg_proc=$corespernode export fg_threads=1 -export enkf_threads=4 -export gsi_control_threads=2 +export enkf_threads=10 +export gsi_control_threads=4 export write_groups=1 # 40 cores per node -export write_tasks=4 +export write_tasks=4 export layout="3, 2" # C384 ensemble, C768 control (run with 80 nodes) diff --git a/main.sh b/main.sh index 3e7fdae..52854cc 100755 --- a/main.sh +++ b/main.sh @@ -50,34 +50,37 @@ if [ "$REALTIME" == "NO" ]; then # Set CONVINFO if [[ "$analdate" -ge 2021052012 ]]; then - export CONVINFO=$fixgsi/gfsv16_historical/global_convinfo.txt.2021052012 + CONVINFO2=$fixgsi/gfsv16_historical/global_convinfo.txt.2021052012 elif [[ "$analdate" -ge 2021032212 ]]; then - export CONVINFO=$fixgsi/gfsv16_historical/global_convinfo.txt.2021032212 + CONVINFO2=$fixgsi/gfsv16_historical/global_convinfo.txt.2021032212 elif [[ "$analdate" -ge 2020091612 ]]; then - export CONVINFO=$fixgsi/gfsv16_historical/global_convinfo.txt.2020091612 + CONVINFO2=$fixgsi/gfsv16_historical/global_convinfo.txt.2020091612 elif [[ "$analdate" -ge 2020052612 ]]; then - export CONVINFO=$fixgsi/gfsv16_historical/global_convinfo.txt.2020052612 + CONVINFO2=$fixgsi/gfsv16_historical/global_convinfo.txt.2020052612 elif [[ "$analdate" -ge 2020040718 ]]; then - export CONVINFO=$fixgsi/gfsv16_historical/global_convinfo.txt.2020040718 + CONVINFO2=$fixgsi/gfsv16_historical/global_convinfo.txt.2020040718 elif [[ "$analdate" -ge 2019110706 ]]; then - export CONVINFO=$fixgsi/gfsv16_historical/global_convinfo.txt.2019110706 + CONVINFO2=$fixgsi/gfsv16_historical/global_convinfo.txt.2019110706 elif [[ "$analdate" -ge 2019021900 ]]; then - export CONVINFO=$fixgsi/gfsv16_historical/global_convinfo.txt.2019021900 + CONVINFO2=$fixgsi/gfsv16_historical/global_convinfo.txt.2019021900 elif [[ "$analdate" -ge "2018022818" ]]; then - export CONVINFO=$fixgsi/fv3_historical/global_convinfo.txt.2018022818 + CONVINFO2=$fixgsi/fv3_historical/global_convinfo.txt.2018022818 elif [[ "$analdate" -ge "2018010512" ]]; then - export CONVINFO=$fixgsi/fv3_historical/global_convinfo.txt.2018010512 + CONVINFO2=$fixgsi/fv3_historical/global_convinfo.txt.2018010512 elif [[ "$analdate" -ge "2017071912" ]]; then - export CONVINFO=$fixgsi/fv3_historical/global_convinfo.txt.2017071912 + CONVINFO2=$fixgsi/fv3_historical/global_convinfo.txt.2017071912 elif [[ "$analdate" -ge "2016031512" ]]; then - export CONVINFO=$fixgsi/fv3_historical/global_convinfo.txt.2016031512 + CONVINFO2=$fixgsi/fv3_historical/global_convinfo.txt.2016031512 elif [[ "$analdate" -ge "2014041400" ]]; then - export CONVINFO=$fixgsi/fv3_historical/global_convinfo.txt.2014041400 + CONVINFO2=$fixgsi/fv3_historical/global_convinfo.txt.2014041400 else echo "no convinfo found" exit 1 fi +export CONVINFO=${CONVINFO:-$CONVINFO2} + + # Set OZINFO if [[ "$analdate" -ge 2020011806 ]]; then export OZINFO=$fixgsi/gfsv16_historical/global_ozinfo.txt.2020011806