1 | #!/bin/bash |
---|
2 | #SBATCH --time=08:00:00 |
---|
3 | #SBATCH --nodes=4 |
---|
4 | #SBATCH --ntasks-per-node=20 |
---|
5 | #SBATCH --threads-per-core=1 |
---|
6 | #SBATCH --job-name=ORCA2 |
---|
7 | #SBATCH --partition=all |
---|
8 | #SBATCH --mail-type=ALL |
---|
9 | #SBATCH --mail-user=momm |
---|
10 | #SBATCH --exclusive |
---|
11 | |
---|
12 | |
---|
13 | # NOTE: |
---|
14 | # the simulation year and month must be passed to this pbs script at runtime |
---|
15 | # by the command line, e.g.: |
---|
16 | # "sbatch 1980 1 MonthlyChainHindcast.sh" |
---|
17 | # |
---|
18 | |
---|
19 | module load intel |
---|
20 | module load intel-mpi |
---|
21 | module load hdf5-intelmpi |
---|
22 | module load netcdf-intelmpi |
---|
23 | |
---|
24 | export I_MPI_PMI_LIBRARY=/usr/lib64/libpmi.so |
---|
25 | export I_MPI_PIN_PROCS=0-19 |
---|
26 | export I_MPI_EXTRA_FILESYSTEM=on |
---|
27 | export I_MPI_EXTRA_FILESYSTEM_LIST=gpfs |
---|
28 | export OMP_NUM_THREADS=1 |
---|
29 | ulimit -s 10240 |
---|
30 | #ulimit -s unlimited |
---|
31 | |
---|
32 | #NOTE: |
---|
33 | # start and end cycle of perpetual cycle need to be passed by command line argument as first and second argument respectively. |
---|
34 | |
---|
35 | CORES=$(( SLURM_NTASKS )) |
---|
36 | |
---|
37 | ncycle=$1 |
---|
38 | nend=$2 |
---|
39 | nstart=1 |
---|
40 | nm=$(( ncycle - 1 )) |
---|
41 | np=$(( ncycle + 1 )) |
---|
42 | set -u #break on unset variables |
---|
43 | |
---|
44 | RUNDIR=/work/momm/ORCA2 |
---|
45 | ARCHIVEDIR=$RUNDIR/$ncycle |
---|
46 | |
---|
47 | cd $RUNDIR |
---|
48 | |
---|
49 | stopflag=0 |
---|
50 | |
---|
51 | #restarts: |
---|
52 | rm -rf restart.nc restart_trc.nc restart_[0-9]???.nc restart_trc_[0-9]???.nc |
---|
53 | if [ $ncycle -eq $nstart ] |
---|
54 | then |
---|
55 | ln -sf $RUNDIR/0/restart_trc.nc restart_trc.nc |
---|
56 | lrst=.FALSE. |
---|
57 | rst=0 |
---|
58 | euler=0 |
---|
59 | else |
---|
60 | lrst=.TRUE. |
---|
61 | if [ -s $RUNDIR/$nm/restart_0000.nc ] |
---|
62 | then |
---|
63 | ln -sf $RUNDIR/$nm/restart_????.nc . |
---|
64 | elif [ -s $RUNDIR/$nm/restart.nc ] |
---|
65 | then |
---|
66 | ln -sf $RUNDIR/$nm/restart.nc . |
---|
67 | else |
---|
68 | stopflag=1 |
---|
69 | fi |
---|
70 | if [ -s $RUNDIR/$nm/restart_trc_0000.nc ] |
---|
71 | then |
---|
72 | ln -sf $RUNDIR/$nm/restart_trc_????.nc . |
---|
73 | elif [ -s $RUNDIR/$nm/restart_trc.nc ] |
---|
74 | then |
---|
75 | ln -sf $RUNDIR/$nm/restart_trc.nc . |
---|
76 | else |
---|
77 | stopflag=1 |
---|
78 | fi |
---|
79 | rst=2 |
---|
80 | euler=0 |
---|
81 | fi |
---|
82 | |
---|
83 | #compute run-time: |
---|
84 | dt=5760 #time step in seconds |
---|
85 | nit=$(( 86400*365/dt )) |
---|
86 | |
---|
87 | #compute start iteration and end iteration: |
---|
88 | ns0=$(( 86400*365 * (ncycle - 1) )) |
---|
89 | n0=$(( ns0 / dt + 1 )) |
---|
90 | nend=$(( n0 + nit -1 )) |
---|
91 | d0=$(printf %04d $ncycle)0101 |
---|
92 | |
---|
93 | #restart |
---|
94 | cat namelist.template \ |
---|
95 | | sed "s,__DATE0__,$d0,g" \ |
---|
96 | | sed "s,__LRST__,$lrst,g" \ |
---|
97 | | sed "s,__RST__,$rst,g" \ |
---|
98 | | sed "s,__N0__,$n0,g" \ |
---|
99 | | sed "s,__NEND__,$nend,g" \ |
---|
100 | | sed "s,__EULER__,$euler,g" \ |
---|
101 | > namelist_cfg |
---|
102 | cat namelist_top.template \ |
---|
103 | | sed "s,__RST__,$rst,g" \ |
---|
104 | > namelist_top_cfg |
---|
105 | |
---|
106 | #cat > ./xios.conf <<EOL |
---|
107 | #$COMPUTETAG ./nemo.exe |
---|
108 | #$XIOSTAG ./xios_server.exe |
---|
109 | #EOL |
---|
110 | |
---|
111 | echo "Total number of nodes / cores used:" $SLURM_NNODES "/" $CORES |
---|
112 | #echo "Compute cores used: $COMPUTECORES" |
---|
113 | #echo "XIOS cores used:" $XIOSCORES |
---|
114 | #echo "Buffer cores blocked for XIOS:" $XIOSBLOCKEDCORES |
---|
115 | |
---|
116 | #cat > xios.conf <<EOL |
---|
117 | #0-359 ./nemo.exe |
---|
118 | #360 ./xios_server.exe |
---|
119 | #EOL |
---|
120 | |
---|
121 | if [ $stopflag -ne 0 ] |
---|
122 | then |
---|
123 | echo "Not ready to launch. Forced exit." |
---|
124 | exit |
---|
125 | fi |
---|
126 | |
---|
127 | echo "Launching NEMO at $(date +%s) seconds since 1970-01-01 00:00:00" |
---|
128 | |
---|
129 | #srun -K1 -n $(( COMPUTECORES + XIOSCORES )) -m plane=20 --multi-prog ./xios.conf |
---|
130 | srun -K1 -n $CORES ./nemo.exe |
---|
131 | |
---|
132 | echo "Finished NEMO at $(date +%s) seconds since 1970-01-01 00:00:00" |
---|
133 | |
---|
134 | #prepare archive directory |
---|
135 | echo "Archiving to" $ARCHIVEDIR "..." |
---|
136 | mkdir -p $ARCHIVEDIR |
---|
137 | #prepare restart files: |
---|
138 | for file in ORCA2_*${nend}_restart_????.nc |
---|
139 | do |
---|
140 | fn=${file: -7:4} #file number |
---|
141 | mv $file $ARCHIVEDIR/restart_$fn.nc |
---|
142 | done |
---|
143 | for file in ORCA2_*${nend}_restart_trc_????.nc |
---|
144 | do |
---|
145 | fn=${file: -7:4} #file number |
---|
146 | mv $file $ARCHIVEDIR/restart_trc_$fn.nc |
---|
147 | done |
---|
148 | for file in ORCA2_*${nend}_restart_ice_????.nc |
---|
149 | do |
---|
150 | fn=${file: -7:4} #file number |
---|
151 | mv $file $ARCHIVEDIR/restart_ice_$fn.nc |
---|
152 | done |
---|
153 | |
---|
154 | mv ORCA2_5d_*.nc $ARCHIVEDIR |
---|
155 | mv ORCA2_1m_ptrc_T.nc $ARCHIVEDIR |
---|
156 | mv ORCA2_1m_grid_?.nc $ARCHIVEDIR |
---|
157 | bzip2 ocean.output && mv -f ocean.output.bz2 $ARCHIVEDIR |
---|
158 | |
---|
159 | #resubmit: |
---|
160 | if [ $np -le $nendcycle ] |
---|
161 | then |
---|
162 | echo "Submitting cycle" $np 'of' $nendcycle '...' |
---|
163 | sbatch Perpetual.sh $np $nendcycle |
---|
164 | echo "Done." |
---|
165 | else |
---|
166 | echo "All done." |
---|
167 | fi |
---|
168 | |
---|
169 | #archive: |
---|
170 | ./archiveFolder.sh $ncycle >& ./archive.$ncycle.log & |
---|
171 | |
---|