source: NEMO/branches/UKMO/AMM15_v3_6_STABLE_new_ersem/NEMOGCM/CONFIG/AMM7_FABM_ERSEM/EXP2020/Archer2-MultiMonthHindcast.pbs

#!/bin/bash
#SBATCH --job-name=AMM7_ERSEM_test
##SBATCH --time=00:20:00
#SBATCH --time=00:45:00
##SBATCH --nodes=2
#SBATCH --nodes=12
#SBATCH --ntasks=1224
##SBATCH --ntasks=100
#SBATCH --account=n01-PML
#SBATCH --partition=standard
#SBATCH --qos=standard
##SBATCH --reservation=shortqos
##SBATCH --qos=short

# NOTE:
# the simulation year and month must be passed to this slurm script at runtime
# by defning externally the variables y0,m0 passing them to the pbs script,
# e.g.:
# "sbatch MYH2020slurm.sh 1981 1"
#

#export OMP_NUM_THREADS=1

module restore /etc/cray-pe.d/PrgEnv-gnu
module load cray-hdf5-parallel
module load cray-netcdf-hdf5parallel
module unload craype-network-ofi
module unload cray-mpich
module load craype-network-ucx
module load cray-mpich-ucx
module load libfabric
module load gcc
module swap gcc/10.1.0 gcc/9.3.0

set -u

#number of month to launch:
nmmax=31

y0=$1
m0=$2

echo "Submitting year" $y0 "month" $m0 "..."

ystart=2000 #1981
yend=2000 #2015

RUNDIR=/work/n01/n01/gle/n2o_2020
INPUTS=/work/n01/n01/gle/n2o_2020  #<put here the path to the folder containing the restart files for the start of the simulation>

cd $RUNDIR

#echo $PBS_JOBID $y0 $m0 "MultiYear chain" >> jobs.log
echo "Submitting year" $y0 "..."

stopflag=0

#general time setting:
dt=300 #time step
nsstart=$(date -d "$ystart-01-01 00:00:00 GMT" +%s) #seconds since EPOCH for total simulation start

#cleanup:
./annualclean.sh $RUNDIR

#link annual forcing files:
./setAnnualLinks.sh $y0 $RUNDIR

nm=1 #initialise month counter

while [ $nm -le $nmmax ]
do
 #Compute previous and next month:
 mp=$(( $m0 + 1 ))
 if [ $mp -eq 13 ]
 then
    mp=1
    yp=$(( y0 + 1 ))
 else
    yp=$y0
 fi
 mm=$(( $m0 - 1 ))
 if [ $mm -eq 0 ]
 then
    mm=12
    ym=$(( y0 - 1 ))
 else
    ym=$y0
 fi
 echo "Processing month $nm of $nmmax..."
 if [ $y0 -le $yend ]
 then
   echo "Preparing $y0 $m0 ..."
   ./monthlyclean.sh $RUNDIR

   m0str=$(printf %02d $m0)
   mpstr=$(printf %02d $mp)
   mmstr=$(printf %02d $mm)


   ARCHIVEDIR=$RUNDIR/$y0/$m0str

   #restarts:
   rm -rf restart.nc restart_trc.nc restart_[0-9]???.nc restart_trc_[0-9]???.nc
   if [ $y0 -eq $ystart ] && [ $m0 -eq 1 ]
   then
      ln -sf restart_hd_1999_12.nc restart.nc
      ln -sf restartX_trc.nc restart_trc.nc
      rst=0
      euler=0
   else
      if [ -s $RUNDIR/$ym/$mmstr/restart_0000.nc ]
      then 
         ln -sf $RUNDIR/$ym/$mmstr/restart_????.nc .
      elif [ -s $RUNDIR/$ym/$mmstr/restart.nc ]
      then
         ln -sf $RUNDIR/$ym/$mmstr/restart.nc .
      else
         stopflag=1
      fi
      if [ -s $RUNDIR/$ym/$mmstr/restart_trc_0000.nc ]
      then 
         ln -sf $RUNDIR/$ym/$mmstr/restart_trc_????.nc .
      elif [ -s $RUNDIR/$ym/$mmstr/restart_trc.nc ]
      then
         ln -sf $RUNDIR/$ym/$mmstr/restart_trc.nc .
      else
         stopflag=0
      fi
      rst=2
      euler=0
   fi

   #compute run-time:
   case $m0 in
      4|6|9|11) nit=$(( 86400*30/dt )) ;;
      2) if [ $(( y0 % 4 )) -ne 0 -o $(( y0 % 100)) -eq 0 -a $(( $y0 % 400 )) -ne 0 ]; then nit=$(( 86400*28/dt )); else nit=$(( 86400*29/dt )); fi ;;
      *) nit=$(( 86400*31/dt )) ;;
   esac

   #compute start iteration and end iteration:
   ns0=$(date -d "$y0-${m0str}-01 00:00:00 GMT" +%s) #seconds since EPOCH for this chunk
   n0=$(( ns0 - nsstart ))
   n0=$(( n0 / dt + 1 ))
   nend=$(( n0 + nit -1 ))
   d0=$y0${m0str}01
   if [ $y0 -lt 1990 -o $y0 -gt 2009 ]
   then
      lclim=.true.
   else
      lclim=.false.
   fi
   cat namelist.template \
                | sed "s,__DATE0__,$d0,g" \
                | sed "s,__RST__,$rst,g" \
                | sed "s,__N0__,$n0,g" \
                | sed "s,__NEND__,$nend,g" \
                | sed "s,__LCLIM__,$lclim,g" \
                | sed "s,__EULER__,$euler,g" \
                > namelist_cfg
   cat namelist_top.template \
                | sed "s,__RST__,$rst,g" \
                > namelist_top_cfg

   if [ $stopflag -ne 0 ]
   then
      echo "Not ready to launch. Forced exit."
      exit
   fi

   echo "Launching $y0 $m0 at $(date +%s) seconds since 1970-01-01 00:00:00"

   ./amm7standard.slurm

   echo "Finished $y0 $m0 at $(date +%s) seconds since 1970-01-01 00:00:00"

   #prepare archive directory
   mkdir -p $ARCHIVEDIR
   #prepare restart files:
   for file in amm7_*${nend}_restart_????.nc
   do
      fn=${file: -7:4} #file number
      mv $file $ARCHIVEDIR/restart_$fn.nc
   done
   for file in amm7_*${nend}_restart_trc_????.nc
   do
      fn=${file: -7:4} #file number
      mv $file $ARCHIVEDIR/restart_trc_$fn.nc
   done

   #move outputs:
   mv amm7_1d_${d0}_[1-2]???????_*.nc $ARCHIVEDIR
   mv amm7_1m_${d0}_[1-2]???????_ptrc_T.nc $ARCHIVEDIR
   mv amm7_1m_${d0}_[1-2]???????_grid_?.nc $ARCHIVEDIR
   bzip2 ocean.output && mv -f ocean.output.bz2 $ARCHIVEDIR
   mv timing.output $ARCHIVEDIR
   mv -f time.step $ARCHIVEDIR

   #archive:
   fname=$y0/$m0str
   echo "Archiving in $fname ..."
   qsub -v folder=$fname,RUNDIR=$RUNDIR archiveFolder.pbs

   #prepare for next launch:
   if [ $yp -gt $y0 ]
   then
      ./annualclean.sh $RUNDIR
      ./setAnnualLinks.sh $yp $RUNDIR
   fi
 fi
 y0=$yp
 m0=$mp
 nm=$(( $nm + 1 ))
done

#resubmit:
if [ $y0 -le $yend ]
then
   echo 'Submitting' $y0 $m0 '...'
   #qsub -v y0=$y0,m0=$m0 MultipleYearHindcast2020.pbs
   sbatch MYH2020slurm.sh $y0 $m0
   echo "Done."
else
   echo "All done."
fi