[12798] | 1 | #!/usr/bin/env bash |
---|
| 2 | |
---|
| 3 | #SBATCH -J sette |
---|
| 4 | #SBATCH -o sette.%j.out |
---|
| 5 | #SBATCH -e sette.%j.err |
---|
| 6 | #SBATCH --export=ALL |
---|
| 7 | #SBATCH --parsable |
---|
| 8 | #SBATCH --exclusive |
---|
| 9 | #SBATCH --nodes=NODES |
---|
| 10 | #SBATCH --time=00:15:00 |
---|
| 11 | |
---|
| 12 | ##SBATCH -p debug64 |
---|
| 13 | ##SBATCH -A smer |
---|
| 14 | ##SBATCH --qos=debug |
---|
| 15 | |
---|
| 16 | #SBATCH -p normal64 |
---|
| 17 | ##SBATCH -A cmems |
---|
| 18 | ##SBATCH --qos=coper |
---|
| 19 | |
---|
| 20 | |
---|
| 21 | # Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these |
---|
| 22 | # (via sed operating on this template job file). |
---|
| 23 | # |
---|
| 24 | export XIO_HOME="/home/ext/mr/smer/samsong/SRC/XIOS/xios-2.5/MET_INTEL16_r1859" |
---|
| 25 | export SETTE_DIR="DEF_SETTE_DIR" |
---|
| 26 | |
---|
| 27 | # DEFAULT NEMO/XIOS PROC NUMBER PER NODE |
---|
| 28 | O_PER_NODE=38 |
---|
| 29 | X_PER_NODE=2 |
---|
| 30 | |
---|
| 31 | # REQUESTED TOTAL NEMO/XIOS PROC NUMBER |
---|
| 32 | OCORES=NPROCS |
---|
| 33 | XCORES=NXIOPROCS |
---|
| 34 | |
---|
| 35 | # AVAILABLE PER NODE & TOTAL PROC NUMBER |
---|
| 36 | A_PER_NODE=$((SLURM_CPUS_ON_NODE/2)) |
---|
| 37 | ACORES=$((SLURM_CPUS_ON_NODE/2*SLURM_JOB_NUM_NODES)) |
---|
| 38 | |
---|
| 39 | # ADJUST PROC PER NODE |
---|
| 40 | if [ $OCORES -le $O_PER_NODE ]; then O_PER_NODE=$OCORES; fi |
---|
| 41 | if [ $XCORES -gt $X_PER_NODE ]; then X_PER_NODE=$((A_PER_NODE-O_PER_NODE)); fi |
---|
| 42 | |
---|
| 43 | # TOTAL NEMO/XIOS PROC NUMBER |
---|
| 44 | OCORES=$(( O_PER_NODE * SLURM_JOB_NUM_NODES )) |
---|
| 45 | XCORES=$(( X_PER_NODE * SLURM_JOB_NUM_NODES )) |
---|
| 46 | TCORES=$(( OCORES + XCORES )) |
---|
| 47 | |
---|
| 48 | # CHECK PROC PER NODE + CHECK TOTAL PROC |
---|
| 49 | if [[ $((O_PER_NODE+X_PER_NODE)) -gt $((SLURM_CPUS_ON_NODE/2)) ]] || [[ ${TCORES} -gt ${ACORES} ]]; then |
---|
| 50 | exit 1 |
---|
| 51 | else |
---|
| 52 | echo "OCORES: $OCORES" |
---|
| 53 | echo "XCORES: $XCORES" |
---|
| 54 | fi |
---|
| 55 | |
---|
| 56 | |
---|
| 57 | ############################################################### |
---|
| 58 | # |
---|
| 59 | # |
---|
| 60 | # load sette functions (only post_test_tidyup needed) |
---|
| 61 | # |
---|
| 62 | . ${SETTE_DIR}/all_functions.sh |
---|
| 63 | ############################################################### |
---|
| 64 | # |
---|
| 65 | # modules to load |
---|
| 66 | #module purge |
---|
| 67 | #module load intel/16.1.150 |
---|
| 68 | #module load intelmpi/5.1.2.150 |
---|
| 69 | #module load netcdf/4.4.0 |
---|
| 70 | #module load hdf5/1.8.16 |
---|
| 71 | #module load zlib/1.2.8 szip/2.1 |
---|
| 72 | |
---|
| 73 | # Don't remove neither change the following line |
---|
| 74 | # BODY |
---|
| 75 | # |
---|
| 76 | # Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these |
---|
| 77 | # (via sed operating on this template job file). Note that the number of compute nodes required |
---|
| 78 | # is also set by the fcm_job.sh on the PBS select header line above. |
---|
| 79 | # |
---|
| 80 | # These variables are needed by post_test_tidyup function in all_functions.sh |
---|
| 81 | # |
---|
| 82 | export INPUT_DIR=DEF_INPUT_DIR |
---|
| 83 | export CONFIG_DIR=DEF_CONFIG_DIR |
---|
| 84 | export TOOLS_DIR=DEF_TOOLS_DIR |
---|
| 85 | export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION |
---|
| 86 | export NEW_CONF=DEF_NEW_CONF |
---|
| 87 | export CMP_NAM=DEF_CMP_NAM |
---|
| 88 | export TEST_NAME=DEF_TEST_NAME |
---|
| 89 | export EXE_DIR=DEF_EXE_DIR |
---|
| 90 | ulimit -a |
---|
| 91 | ulimit -s unlimited |
---|
| 92 | # |
---|
| 93 | # end of set up |
---|
| 94 | ############################################################### |
---|
| 95 | # |
---|
| 96 | # change to the working directory |
---|
| 97 | # |
---|
| 98 | cd $EXE_DIR |
---|
| 99 | |
---|
| 100 | echo Running on host `hostname` |
---|
| 101 | echo Time is `date` |
---|
| 102 | echo Directory is `pwd` |
---|
| 103 | # |
---|
| 104 | # Run the parallel MPI executable |
---|
| 105 | # |
---|
| 106 | if [ $XCORES -gt 0 ]; then |
---|
| 107 | # |
---|
| 108 | # Run MPMD case |
---|
| 109 | # |
---|
| 110 | #XIOS will run on a separate node so will run in parallel queue |
---|
| 111 | if [ ! -f ./xios_server.exe ] && [ -f ${XIO_HOME}/bin/xios_server.exe ]; then |
---|
| 112 | cp ${XIO_HOME}/bin/xios_server.exe . |
---|
| 113 | fi |
---|
| 114 | if [ ! -f ./xios_server.exe ]; then |
---|
| 115 | echo "./xios_server.exe not found" |
---|
| 116 | echo "run aborted" |
---|
| 117 | exit |
---|
| 118 | fi |
---|
| 119 | |
---|
| 120 | cat > mpmd.conf <<EOF |
---|
| 121 | 0-$((OCORES-1)) ./nemo |
---|
| 122 | ${OCORES}-$((TCORES-1)) ./xios_server.exe |
---|
| 123 | EOF |
---|
| 124 | echo time srun --cpu_bind=cores --mpi=pmi2 -m cyclic -n ${TCORES} --multi-prog ./mpmd.conf |
---|
| 125 | time srun --cpu_bind=cores --mpi=pmi2 -m cyclic -n ${TCORES} --multi-prog ./mpmd.conf |
---|
| 126 | # |
---|
| 127 | else |
---|
| 128 | # |
---|
| 129 | # Run SPMD case |
---|
| 130 | # |
---|
| 131 | echo time srun --cpu_bind=cores --mpi=pmi2 -m cyclic -n ${OCORES} ./nemo |
---|
| 132 | time srun --cpu_bind=cores --mpi=pmi2 -m cyclic -n ${OCORES} ./nemo |
---|
| 133 | fi |
---|
| 134 | # |
---|
| 135 | |
---|
| 136 | # |
---|
| 137 | post_test_tidyup |
---|
| 138 | # END_BODY |
---|
| 139 | # Don't remove neither change the previous line |
---|
| 140 | exit |
---|
| 141 | |
---|