[15480] | 1 | #!/bin/bash |
---|
| 2 | #SBATCH --job-name=AMM7_ERSEM_test |
---|
| 3 | ##SBATCH --time=00:20:00 |
---|
| 4 | #SBATCH --time=00:45:00 |
---|
| 5 | ##SBATCH --nodes=2 |
---|
| 6 | #SBATCH --nodes=12 |
---|
| 7 | #SBATCH --ntasks=1224 |
---|
| 8 | ##SBATCH --ntasks=100 |
---|
| 9 | #SBATCH --account=n01-PML |
---|
| 10 | #SBATCH --partition=standard |
---|
| 11 | #SBATCH --qos=standard |
---|
| 12 | ##SBATCH --reservation=shortqos |
---|
| 13 | ##SBATCH --qos=short |
---|
| 14 | |
---|
| 15 | # NOTE: |
---|
| 16 | # the simulation year and month must be passed to this slurm script at runtime |
---|
| 17 | # by defning externally the variables y0,m0 passing them to the pbs script, |
---|
| 18 | # e.g.: |
---|
| 19 | # "sbatch MYH2020slurm.sh 1981 1" |
---|
| 20 | # |
---|
| 21 | |
---|
| 22 | #export OMP_NUM_THREADS=1 |
---|
| 23 | |
---|
| 24 | module restore /etc/cray-pe.d/PrgEnv-gnu |
---|
| 25 | module load cray-hdf5-parallel |
---|
| 26 | module load cray-netcdf-hdf5parallel |
---|
| 27 | module unload craype-network-ofi |
---|
| 28 | module unload cray-mpich |
---|
| 29 | module load craype-network-ucx |
---|
| 30 | module load cray-mpich-ucx |
---|
| 31 | module load libfabric |
---|
| 32 | module load gcc |
---|
| 33 | module swap gcc/10.1.0 gcc/9.3.0 |
---|
| 34 | |
---|
| 35 | set -u |
---|
| 36 | |
---|
| 37 | #number of month to launch: |
---|
| 38 | nmmax=31 |
---|
| 39 | |
---|
| 40 | y0=$1 |
---|
| 41 | m0=$2 |
---|
| 42 | |
---|
| 43 | echo "Submitting year" $y0 "month" $m0 "..." |
---|
| 44 | |
---|
| 45 | ystart=2000 #1981 |
---|
| 46 | yend=2000 #2015 |
---|
| 47 | |
---|
| 48 | RUNDIR=/work/n01/n01/gle/n2o_2020 |
---|
| 49 | INPUTS=/work/n01/n01/gle/n2o_2020 #<put here the path to the folder containing the restart files for the start of the simulation> |
---|
| 50 | |
---|
| 51 | cd $RUNDIR |
---|
| 52 | |
---|
| 53 | #echo $PBS_JOBID $y0 $m0 "MultiYear chain" >> jobs.log |
---|
| 54 | echo "Submitting year" $y0 "..." |
---|
| 55 | |
---|
| 56 | stopflag=0 |
---|
| 57 | |
---|
| 58 | #general time setting: |
---|
| 59 | dt=300 #time step |
---|
| 60 | nsstart=$(date -d "$ystart-01-01 00:00:00 GMT" +%s) #seconds since EPOCH for total simulation start |
---|
| 61 | |
---|
| 62 | #cleanup: |
---|
| 63 | ./annualclean.sh $RUNDIR |
---|
| 64 | |
---|
| 65 | #link annual forcing files: |
---|
| 66 | ./setAnnualLinks.sh $y0 $RUNDIR |
---|
| 67 | |
---|
| 68 | nm=1 #initialise month counter |
---|
| 69 | |
---|
| 70 | while [ $nm -le $nmmax ] |
---|
| 71 | do |
---|
| 72 | #Compute previous and next month: |
---|
| 73 | mp=$(( $m0 + 1 )) |
---|
| 74 | if [ $mp -eq 13 ] |
---|
| 75 | then |
---|
| 76 | mp=1 |
---|
| 77 | yp=$(( y0 + 1 )) |
---|
| 78 | else |
---|
| 79 | yp=$y0 |
---|
| 80 | fi |
---|
| 81 | mm=$(( $m0 - 1 )) |
---|
| 82 | if [ $mm -eq 0 ] |
---|
| 83 | then |
---|
| 84 | mm=12 |
---|
| 85 | ym=$(( y0 - 1 )) |
---|
| 86 | else |
---|
| 87 | ym=$y0 |
---|
| 88 | fi |
---|
| 89 | echo "Processing month $nm of $nmmax..." |
---|
| 90 | if [ $y0 -le $yend ] |
---|
| 91 | then |
---|
| 92 | echo "Preparing $y0 $m0 ..." |
---|
| 93 | ./monthlyclean.sh $RUNDIR |
---|
| 94 | |
---|
| 95 | m0str=$(printf %02d $m0) |
---|
| 96 | mpstr=$(printf %02d $mp) |
---|
| 97 | mmstr=$(printf %02d $mm) |
---|
| 98 | |
---|
| 99 | |
---|
| 100 | ARCHIVEDIR=$RUNDIR/$y0/$m0str |
---|
| 101 | |
---|
| 102 | #restarts: |
---|
| 103 | rm -rf restart.nc restart_trc.nc restart_[0-9]???.nc restart_trc_[0-9]???.nc |
---|
| 104 | if [ $y0 -eq $ystart ] && [ $m0 -eq 1 ] |
---|
| 105 | then |
---|
| 106 | ln -sf restart_hd_1999_12.nc restart.nc |
---|
| 107 | ln -sf restartX_trc.nc restart_trc.nc |
---|
| 108 | rst=0 |
---|
| 109 | euler=0 |
---|
| 110 | else |
---|
| 111 | if [ -s $RUNDIR/$ym/$mmstr/restart_0000.nc ] |
---|
| 112 | then |
---|
| 113 | ln -sf $RUNDIR/$ym/$mmstr/restart_????.nc . |
---|
| 114 | elif [ -s $RUNDIR/$ym/$mmstr/restart.nc ] |
---|
| 115 | then |
---|
| 116 | ln -sf $RUNDIR/$ym/$mmstr/restart.nc . |
---|
| 117 | else |
---|
| 118 | stopflag=1 |
---|
| 119 | fi |
---|
| 120 | if [ -s $RUNDIR/$ym/$mmstr/restart_trc_0000.nc ] |
---|
| 121 | then |
---|
| 122 | ln -sf $RUNDIR/$ym/$mmstr/restart_trc_????.nc . |
---|
| 123 | elif [ -s $RUNDIR/$ym/$mmstr/restart_trc.nc ] |
---|
| 124 | then |
---|
| 125 | ln -sf $RUNDIR/$ym/$mmstr/restart_trc.nc . |
---|
| 126 | else |
---|
| 127 | stopflag=0 |
---|
| 128 | fi |
---|
| 129 | rst=2 |
---|
| 130 | euler=0 |
---|
| 131 | fi |
---|
| 132 | |
---|
| 133 | #compute run-time: |
---|
| 134 | case $m0 in |
---|
| 135 | 4|6|9|11) nit=$(( 86400*30/dt )) ;; |
---|
| 136 | 2) if [ $(( y0 % 4 )) -ne 0 -o $(( y0 % 100)) -eq 0 -a $(( $y0 % 400 )) -ne 0 ]; then nit=$(( 86400*28/dt )); else nit=$(( 86400*29/dt )); fi ;; |
---|
| 137 | *) nit=$(( 86400*31/dt )) ;; |
---|
| 138 | esac |
---|
| 139 | |
---|
| 140 | #compute start iteration and end iteration: |
---|
| 141 | ns0=$(date -d "$y0-${m0str}-01 00:00:00 GMT" +%s) #seconds since EPOCH for this chunk |
---|
| 142 | n0=$(( ns0 - nsstart )) |
---|
| 143 | n0=$(( n0 / dt + 1 )) |
---|
| 144 | nend=$(( n0 + nit -1 )) |
---|
| 145 | d0=$y0${m0str}01 |
---|
| 146 | if [ $y0 -lt 1990 -o $y0 -gt 2009 ] |
---|
| 147 | then |
---|
| 148 | lclim=.true. |
---|
| 149 | else |
---|
| 150 | lclim=.false. |
---|
| 151 | fi |
---|
| 152 | cat namelist.template \ |
---|
| 153 | | sed "s,__DATE0__,$d0,g" \ |
---|
| 154 | | sed "s,__RST__,$rst,g" \ |
---|
| 155 | | sed "s,__N0__,$n0,g" \ |
---|
| 156 | | sed "s,__NEND__,$nend,g" \ |
---|
| 157 | | sed "s,__LCLIM__,$lclim,g" \ |
---|
| 158 | | sed "s,__EULER__,$euler,g" \ |
---|
| 159 | > namelist_cfg |
---|
| 160 | cat namelist_top.template \ |
---|
| 161 | | sed "s,__RST__,$rst,g" \ |
---|
| 162 | > namelist_top_cfg |
---|
| 163 | |
---|
| 164 | if [ $stopflag -ne 0 ] |
---|
| 165 | then |
---|
| 166 | echo "Not ready to launch. Forced exit." |
---|
| 167 | exit |
---|
| 168 | fi |
---|
| 169 | |
---|
| 170 | echo "Launching $y0 $m0 at $(date +%s) seconds since 1970-01-01 00:00:00" |
---|
| 171 | |
---|
| 172 | ./amm7standard.slurm |
---|
| 173 | |
---|
| 174 | echo "Finished $y0 $m0 at $(date +%s) seconds since 1970-01-01 00:00:00" |
---|
| 175 | |
---|
| 176 | #prepare archive directory |
---|
| 177 | mkdir -p $ARCHIVEDIR |
---|
| 178 | #prepare restart files: |
---|
| 179 | for file in amm7_*${nend}_restart_????.nc |
---|
| 180 | do |
---|
| 181 | fn=${file: -7:4} #file number |
---|
| 182 | mv $file $ARCHIVEDIR/restart_$fn.nc |
---|
| 183 | done |
---|
| 184 | for file in amm7_*${nend}_restart_trc_????.nc |
---|
| 185 | do |
---|
| 186 | fn=${file: -7:4} #file number |
---|
| 187 | mv $file $ARCHIVEDIR/restart_trc_$fn.nc |
---|
| 188 | done |
---|
| 189 | |
---|
| 190 | #move outputs: |
---|
| 191 | mv amm7_1d_${d0}_[1-2]???????_*.nc $ARCHIVEDIR |
---|
| 192 | mv amm7_1m_${d0}_[1-2]???????_ptrc_T.nc $ARCHIVEDIR |
---|
| 193 | mv amm7_1m_${d0}_[1-2]???????_grid_?.nc $ARCHIVEDIR |
---|
| 194 | bzip2 ocean.output && mv -f ocean.output.bz2 $ARCHIVEDIR |
---|
| 195 | mv timing.output $ARCHIVEDIR |
---|
| 196 | mv -f time.step $ARCHIVEDIR |
---|
| 197 | |
---|
| 198 | #archive: |
---|
| 199 | fname=$y0/$m0str |
---|
| 200 | echo "Archiving in $fname ..." |
---|
| 201 | qsub -v folder=$fname,RUNDIR=$RUNDIR archiveFolder.pbs |
---|
| 202 | |
---|
| 203 | #prepare for next launch: |
---|
| 204 | if [ $yp -gt $y0 ] |
---|
| 205 | then |
---|
| 206 | ./annualclean.sh $RUNDIR |
---|
| 207 | ./setAnnualLinks.sh $yp $RUNDIR |
---|
| 208 | fi |
---|
| 209 | fi |
---|
| 210 | y0=$yp |
---|
| 211 | m0=$mp |
---|
| 212 | nm=$(( $nm + 1 )) |
---|
| 213 | done |
---|
| 214 | |
---|
| 215 | #resubmit: |
---|
| 216 | if [ $y0 -le $yend ] |
---|
| 217 | then |
---|
| 218 | echo 'Submitting' $y0 $m0 '...' |
---|
| 219 | #qsub -v y0=$y0,m0=$m0 MultipleYearHindcast2020.pbs |
---|
| 220 | sbatch MYH2020slurm.sh $y0 $m0 |
---|
| 221 | echo "Done." |
---|
| 222 | else |
---|
| 223 | echo "All done." |
---|
| 224 | fi |
---|