1 | #!/bin/bash |
---|
2 | #SBATCH --job-name=AMM7_ERSEM_test |
---|
3 | ##SBATCH --time=00:20:00 |
---|
4 | #SBATCH --time=00:45:00 |
---|
5 | ##SBATCH --nodes=2 |
---|
6 | #SBATCH --nodes=12 |
---|
7 | #SBATCH --ntasks=1224 |
---|
8 | ##SBATCH --ntasks=100 |
---|
9 | #SBATCH --account=n01-PML |
---|
10 | #SBATCH --partition=standard |
---|
11 | #SBATCH --qos=standard |
---|
12 | ##SBATCH --reservation=shortqos |
---|
13 | ##SBATCH --qos=short |
---|
14 | |
---|
15 | # NOTE: |
---|
16 | # the simulation year and month must be passed to this slurm script at runtime |
---|
17 | # by defning externally the variables y0,m0 passing them to the pbs script, |
---|
18 | # e.g.: |
---|
19 | # "sbatch MYH2020slurm.sh 1981 1" |
---|
20 | # |
---|
21 | |
---|
22 | #export OMP_NUM_THREADS=1 |
---|
23 | |
---|
24 | module restore /etc/cray-pe.d/PrgEnv-gnu |
---|
25 | module load cray-hdf5-parallel |
---|
26 | module load cray-netcdf-hdf5parallel |
---|
27 | module unload craype-network-ofi |
---|
28 | module unload cray-mpich |
---|
29 | module load craype-network-ucx |
---|
30 | module load cray-mpich-ucx |
---|
31 | module load libfabric |
---|
32 | module load gcc |
---|
33 | module swap gcc/10.1.0 gcc/9.3.0 |
---|
34 | |
---|
35 | set -u |
---|
36 | |
---|
37 | #number of month to launch: |
---|
38 | nmmax=31 |
---|
39 | |
---|
40 | y0=$1 |
---|
41 | m0=$2 |
---|
42 | |
---|
43 | echo "Submitting year" $y0 "month" $m0 "..." |
---|
44 | |
---|
45 | ystart=2000 #1981 |
---|
46 | yend=2000 #2015 |
---|
47 | |
---|
48 | RUNDIR=/work/n01/n01/gle/n2o_2020 |
---|
49 | INPUTS=/work/n01/n01/gle/n2o_2020 #<put here the path to the folder containing the restart files for the start of the simulation> |
---|
50 | |
---|
51 | cd $RUNDIR |
---|
52 | |
---|
53 | #echo $PBS_JOBID $y0 $m0 "MultiYear chain" >> jobs.log |
---|
54 | echo "Submitting year" $y0 "..." |
---|
55 | |
---|
56 | stopflag=0 |
---|
57 | |
---|
58 | #general time setting: |
---|
59 | dt=300 #time step |
---|
60 | nsstart=$(date -d "$ystart-01-01 00:00:00 GMT" +%s) #seconds since EPOCH for total simulation start |
---|
61 | |
---|
62 | #cleanup: |
---|
63 | ./annualclean.sh $RUNDIR |
---|
64 | |
---|
65 | #link annual forcing files: |
---|
66 | ./setAnnualLinks.sh $y0 $RUNDIR |
---|
67 | |
---|
68 | nm=1 #initialise month counter |
---|
69 | |
---|
70 | while [ $nm -le $nmmax ] |
---|
71 | do |
---|
72 | #Compute previous and next month: |
---|
73 | mp=$(( $m0 + 1 )) |
---|
74 | if [ $mp -eq 13 ] |
---|
75 | then |
---|
76 | mp=1 |
---|
77 | yp=$(( y0 + 1 )) |
---|
78 | else |
---|
79 | yp=$y0 |
---|
80 | fi |
---|
81 | mm=$(( $m0 - 1 )) |
---|
82 | if [ $mm -eq 0 ] |
---|
83 | then |
---|
84 | mm=12 |
---|
85 | ym=$(( y0 - 1 )) |
---|
86 | else |
---|
87 | ym=$y0 |
---|
88 | fi |
---|
89 | echo "Processing month $nm of $nmmax..." |
---|
90 | if [ $y0 -le $yend ] |
---|
91 | then |
---|
92 | echo "Preparing $y0 $m0 ..." |
---|
93 | ./monthlyclean.sh $RUNDIR |
---|
94 | |
---|
95 | m0str=$(printf %02d $m0) |
---|
96 | mpstr=$(printf %02d $mp) |
---|
97 | mmstr=$(printf %02d $mm) |
---|
98 | |
---|
99 | |
---|
100 | ARCHIVEDIR=$RUNDIR/$y0/$m0str |
---|
101 | |
---|
102 | #restarts: |
---|
103 | rm -rf restart.nc restart_trc.nc restart_[0-9]???.nc restart_trc_[0-9]???.nc |
---|
104 | if [ $y0 -eq $ystart ] && [ $m0 -eq 1 ] |
---|
105 | then |
---|
106 | ln -sf restart_hd_1999_12.nc restart.nc |
---|
107 | ln -sf restartX_trc.nc restart_trc.nc |
---|
108 | rst=0 |
---|
109 | euler=0 |
---|
110 | else |
---|
111 | if [ -s $RUNDIR/$ym/$mmstr/restart_0000.nc ] |
---|
112 | then |
---|
113 | ln -sf $RUNDIR/$ym/$mmstr/restart_????.nc . |
---|
114 | elif [ -s $RUNDIR/$ym/$mmstr/restart.nc ] |
---|
115 | then |
---|
116 | ln -sf $RUNDIR/$ym/$mmstr/restart.nc . |
---|
117 | else |
---|
118 | stopflag=1 |
---|
119 | fi |
---|
120 | if [ -s $RUNDIR/$ym/$mmstr/restart_trc_0000.nc ] |
---|
121 | then |
---|
122 | ln -sf $RUNDIR/$ym/$mmstr/restart_trc_????.nc . |
---|
123 | elif [ -s $RUNDIR/$ym/$mmstr/restart_trc.nc ] |
---|
124 | then |
---|
125 | ln -sf $RUNDIR/$ym/$mmstr/restart_trc.nc . |
---|
126 | else |
---|
127 | stopflag=0 |
---|
128 | fi |
---|
129 | rst=2 |
---|
130 | euler=0 |
---|
131 | fi |
---|
132 | |
---|
133 | #compute run-time: |
---|
134 | case $m0 in |
---|
135 | 4|6|9|11) nit=$(( 86400*30/dt )) ;; |
---|
136 | 2) if [ $(( y0 % 4 )) -ne 0 -o $(( y0 % 100)) -eq 0 -a $(( $y0 % 400 )) -ne 0 ]; then nit=$(( 86400*28/dt )); else nit=$(( 86400*29/dt )); fi ;; |
---|
137 | *) nit=$(( 86400*31/dt )) ;; |
---|
138 | esac |
---|
139 | |
---|
140 | #compute start iteration and end iteration: |
---|
141 | ns0=$(date -d "$y0-${m0str}-01 00:00:00 GMT" +%s) #seconds since EPOCH for this chunk |
---|
142 | n0=$(( ns0 - nsstart )) |
---|
143 | n0=$(( n0 / dt + 1 )) |
---|
144 | nend=$(( n0 + nit -1 )) |
---|
145 | d0=$y0${m0str}01 |
---|
146 | if [ $y0 -lt 1990 -o $y0 -gt 2009 ] |
---|
147 | then |
---|
148 | lclim=.true. |
---|
149 | else |
---|
150 | lclim=.false. |
---|
151 | fi |
---|
152 | cat namelist.template \ |
---|
153 | | sed "s,__DATE0__,$d0,g" \ |
---|
154 | | sed "s,__RST__,$rst,g" \ |
---|
155 | | sed "s,__N0__,$n0,g" \ |
---|
156 | | sed "s,__NEND__,$nend,g" \ |
---|
157 | | sed "s,__LCLIM__,$lclim,g" \ |
---|
158 | | sed "s,__EULER__,$euler,g" \ |
---|
159 | > namelist_cfg |
---|
160 | cat namelist_top.template \ |
---|
161 | | sed "s,__RST__,$rst,g" \ |
---|
162 | > namelist_top_cfg |
---|
163 | |
---|
164 | if [ $stopflag -ne 0 ] |
---|
165 | then |
---|
166 | echo "Not ready to launch. Forced exit." |
---|
167 | exit |
---|
168 | fi |
---|
169 | |
---|
170 | echo "Launching $y0 $m0 at $(date +%s) seconds since 1970-01-01 00:00:00" |
---|
171 | |
---|
172 | ./amm7standard.slurm |
---|
173 | |
---|
174 | echo "Finished $y0 $m0 at $(date +%s) seconds since 1970-01-01 00:00:00" |
---|
175 | |
---|
176 | #prepare archive directory |
---|
177 | mkdir -p $ARCHIVEDIR |
---|
178 | #prepare restart files: |
---|
179 | for file in amm7_*${nend}_restart_????.nc |
---|
180 | do |
---|
181 | fn=${file: -7:4} #file number |
---|
182 | mv $file $ARCHIVEDIR/restart_$fn.nc |
---|
183 | done |
---|
184 | for file in amm7_*${nend}_restart_trc_????.nc |
---|
185 | do |
---|
186 | fn=${file: -7:4} #file number |
---|
187 | mv $file $ARCHIVEDIR/restart_trc_$fn.nc |
---|
188 | done |
---|
189 | |
---|
190 | #move outputs: |
---|
191 | mv amm7_1d_${d0}_[1-2]???????_*.nc $ARCHIVEDIR |
---|
192 | mv amm7_1m_${d0}_[1-2]???????_ptrc_T.nc $ARCHIVEDIR |
---|
193 | mv amm7_1m_${d0}_[1-2]???????_grid_?.nc $ARCHIVEDIR |
---|
194 | bzip2 ocean.output && mv -f ocean.output.bz2 $ARCHIVEDIR |
---|
195 | mv timing.output $ARCHIVEDIR |
---|
196 | mv -f time.step $ARCHIVEDIR |
---|
197 | |
---|
198 | #archive: |
---|
199 | fname=$y0/$m0str |
---|
200 | echo "Archiving in $fname ..." |
---|
201 | qsub -v folder=$fname,RUNDIR=$RUNDIR archiveFolder.pbs |
---|
202 | |
---|
203 | #prepare for next launch: |
---|
204 | if [ $yp -gt $y0 ] |
---|
205 | then |
---|
206 | ./annualclean.sh $RUNDIR |
---|
207 | ./setAnnualLinks.sh $yp $RUNDIR |
---|
208 | fi |
---|
209 | fi |
---|
210 | y0=$yp |
---|
211 | m0=$mp |
---|
212 | nm=$(( $nm + 1 )) |
---|
213 | done |
---|
214 | |
---|
215 | #resubmit: |
---|
216 | if [ $y0 -le $yend ] |
---|
217 | then |
---|
218 | echo 'Submitting' $y0 $m0 '...' |
---|
219 | #qsub -v y0=$y0,m0=$m0 MultipleYearHindcast2020.pbs |
---|
220 | sbatch MYH2020slurm.sh $y0 $m0 |
---|
221 | echo "Done." |
---|
222 | else |
---|
223 | echo "All done." |
---|
224 | fi |
---|