1 | #!/usr/bin/env bash |
---|
2 | |
---|
3 | #SBATCH -J sette |
---|
4 | #SBATCH -o sette.%j.out |
---|
5 | #SBATCH -e sette.%j.err |
---|
6 | #SBATCH --export=ALL |
---|
7 | #SBATCH --parsable |
---|
8 | #SBATCH --exclusive |
---|
9 | #SBATCH --nodes=NODES |
---|
10 | #SBATCH --time=00:15:00 |
---|
11 | |
---|
12 | ##SBATCH -p debug64 |
---|
13 | ##SBATCH -A smer |
---|
14 | ##SBATCH --qos=debug |
---|
15 | |
---|
16 | #SBATCH -p normal64 |
---|
17 | ##SBATCH -A cmems |
---|
18 | ##SBATCH --qos=coper |
---|
19 | |
---|
20 | |
---|
21 | # Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these |
---|
22 | # (via sed operating on this template job file). |
---|
23 | # |
---|
24 | export XIO_HOME="/home/ext/mr/smer/samsong/SRC/XIOS/xios-2.5/MET_INTEL16_r1859" |
---|
25 | export SETTE_DIR="DEF_SETTE_DIR" |
---|
26 | |
---|
27 | # DEFAULT NEMO/XIOS PROC NUMBER PER NODE |
---|
28 | O_PER_NODE=38 |
---|
29 | X_PER_NODE=2 |
---|
30 | |
---|
31 | # REQUESTED TOTAL NEMO/XIOS PROC NUMBER |
---|
32 | OCORES=NPROCS |
---|
33 | XCORES=NXIOPROCS |
---|
34 | |
---|
35 | # AVAILABLE PER NODE & TOTAL PROC NUMBER |
---|
36 | A_PER_NODE=$((SLURM_CPUS_ON_NODE/2)) |
---|
37 | ACORES=$((SLURM_CPUS_ON_NODE/2*SLURM_JOB_NUM_NODES)) |
---|
38 | |
---|
39 | # ADJUST PROC PER NODE |
---|
40 | if [ $OCORES -le $O_PER_NODE ]; then O_PER_NODE=$OCORES; fi |
---|
41 | if [ $XCORES -gt $X_PER_NODE ]; then X_PER_NODE=$((A_PER_NODE-O_PER_NODE)); fi |
---|
42 | |
---|
43 | # TOTAL NEMO/XIOS PROC NUMBER |
---|
44 | OCORES=$(( O_PER_NODE * SLURM_JOB_NUM_NODES )) |
---|
45 | XCORES=$(( X_PER_NODE * SLURM_JOB_NUM_NODES )) |
---|
46 | TCORES=$(( OCORES + XCORES )) |
---|
47 | |
---|
48 | # CHECK PROC PER NODE + CHECK TOTAL PROC |
---|
49 | if [[ $((O_PER_NODE+X_PER_NODE)) -gt $((SLURM_CPUS_ON_NODE/2)) ]] || [[ ${TCORES} -gt ${ACORES} ]]; then |
---|
50 | exit 1 |
---|
51 | else |
---|
52 | echo "OCORES: $OCORES" |
---|
53 | echo "XCORES: $XCORES" |
---|
54 | fi |
---|
55 | |
---|
56 | |
---|
57 | ############################################################### |
---|
58 | # |
---|
59 | # |
---|
60 | # load sette functions (only post_test_tidyup needed) |
---|
61 | # |
---|
62 | . ${SETTE_DIR}/all_functions.sh |
---|
63 | ############################################################### |
---|
64 | # |
---|
65 | # modules to load |
---|
66 | #module purge |
---|
67 | #module load intel/16.1.150 |
---|
68 | #module load intelmpi/5.1.2.150 |
---|
69 | #module load netcdf/4.4.0 |
---|
70 | #module load hdf5/1.8.16 |
---|
71 | #module load zlib/1.2.8 szip/2.1 |
---|
72 | |
---|
73 | # Don't remove neither change the following line |
---|
74 | # BODY |
---|
75 | # |
---|
76 | # Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these |
---|
77 | # (via sed operating on this template job file). Note that the number of compute nodes required |
---|
78 | # is also set by the fcm_job.sh on the PBS select header line above. |
---|
79 | # |
---|
80 | # These variables are needed by post_test_tidyup function in all_functions.sh |
---|
81 | # |
---|
82 | export INPUT_DIR=DEF_INPUT_DIR |
---|
83 | export CONFIG_DIR=DEF_CONFIG_DIR |
---|
84 | export TOOLS_DIR=DEF_TOOLS_DIR |
---|
85 | export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION |
---|
86 | export NEW_CONF=DEF_NEW_CONF |
---|
87 | export CMP_NAM=DEF_CMP_NAM |
---|
88 | export TEST_NAME=DEF_TEST_NAME |
---|
89 | export EXE_DIR=DEF_EXE_DIR |
---|
90 | ulimit -a |
---|
91 | ulimit -s unlimited |
---|
92 | # |
---|
93 | # end of set up |
---|
94 | ############################################################### |
---|
95 | # |
---|
96 | # change to the working directory |
---|
97 | # |
---|
98 | cd $EXE_DIR |
---|
99 | |
---|
100 | echo Running on host `hostname` |
---|
101 | echo Time is `date` |
---|
102 | echo Directory is `pwd` |
---|
103 | # |
---|
104 | # Run the parallel MPI executable |
---|
105 | # |
---|
106 | if [ $XCORES -gt 0 ]; then |
---|
107 | # |
---|
108 | # Run MPMD case |
---|
109 | # |
---|
110 | #XIOS will run on a separate node so will run in parallel queue |
---|
111 | if [ ! -f ./xios_server.exe ] && [ -f ${XIO_HOME}/bin/xios_server.exe ]; then |
---|
112 | cp ${XIO_HOME}/bin/xios_server.exe . |
---|
113 | fi |
---|
114 | if [ ! -f ./xios_server.exe ]; then |
---|
115 | echo "./xios_server.exe not found" |
---|
116 | echo "run aborted" |
---|
117 | exit |
---|
118 | fi |
---|
119 | |
---|
120 | cat > mpmd.conf <<EOF |
---|
121 | 0-$((OCORES-1)) ./nemo |
---|
122 | ${OCORES}-$((TCORES-1)) ./xios_server.exe |
---|
123 | EOF |
---|
124 | echo time srun --cpu_bind=cores --mpi=pmi2 -m cyclic -n ${TCORES} --multi-prog ./mpmd.conf |
---|
125 | time srun --cpu_bind=cores --mpi=pmi2 -m cyclic -n ${TCORES} --multi-prog ./mpmd.conf |
---|
126 | # |
---|
127 | else |
---|
128 | # |
---|
129 | # Run SPMD case |
---|
130 | # |
---|
131 | echo time srun --cpu_bind=cores --mpi=pmi2 -m cyclic -n ${OCORES} ./nemo |
---|
132 | time srun --cpu_bind=cores --mpi=pmi2 -m cyclic -n ${OCORES} ./nemo |
---|
133 | fi |
---|
134 | # |
---|
135 | |
---|
136 | # |
---|
137 | post_test_tidyup |
---|
138 | # END_BODY |
---|
139 | # Don't remove neither change the previous line |
---|
140 | exit |
---|
141 | |
---|