Changeset 1500 for trunk/libIGCM

Timestamp:

11/19/19 16:35:09 (4 years ago)

Author:

aclsce

Message:

• JeanZay? : added functionality ( use of slurm explicit binding) to run MPMD configuration with MPI-OMP (Execution type 2 in libIGCM classification).

File:

• trunk/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh (modified) (6 diffs)

trunk/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh

@@ -1136 +1136 @@
   elif [ ${executionType} -eq 2 ] ; then
     # MPMD + MPI + OMP
-      sed -e "s/::openMPthreads::/${openMPthreads}/" \
-          -e "s/::JobNumProcTot::/${mpiTasks}/"    \
-          ${file} > ${file}.tmp
-
+      (( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
+      [ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
+      sed -e "/::openMPthreads::/d"                 \
+        -e "s/::JobNumProcTot::/${mpiTasks}/"       \
+        -e "/ntasks/i\#SBATCH --nodes=${nodeNumber}"\
+        -e "/ntasks/i\#SBATCH --exclusive"          \
+         ${file} > ${file}.tmp
   elif [ ${executionType} -eq 3 ] ; then
     # SPMD + MPI/OMP
@@ -1245 +1248 @@
 
     # MPI-OpenMP (MPMD)
-    # example of run_file
-    # 0-70   env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
-    # 71-71  env OMP_NUM_THREADS=1 ./script_xios.x.ksh
-
-      # Loop on the components to build run_file and script_exec files
-      current_core=0
+    # export SLURM_HOSTFILE=./hostlist 
+    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
+    # example of  hostlist file :
+    # r3i3n33
+    # r3i3n33
+    # ...
+    # example of run_file :
+    # 0 taskset -c 0-4 ./script_lmdz.x.ksh
+    # 1 taskset -c 5-9 ./script_lmdz.x.ksh
+    #...
+    # 32 taskset -c 0 ./script_opa.xx.ksh
+    # 33 taskset -c 1 ./script_opa.xx.ksh
+    #...
+    # 62 taskset -c 30 ./script_xios.x.ksh
+
+    #  Hosts treatment
+        IFS=$'\n'; set -f
+        listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
+        unset IFS; set +f
+        rm -f hostlist
+
+     # Loop on the components to build run_file and script_exec files
+        rank=0
+        current_core=0
+
       for comp in ${config_ListOfComponents[*]} ; do
 
@@ -1268 +1290 @@
           eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
           eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
+     
 
           # Build script files
@@ -1274 +1298 @@
           echo ""  >> script_${ExeNameOut}.ksh
           if [ ${comp_proc_omp_loc} -gt 1 ] ; then
+          # Check if the number of threads is correct
+              case ${comp_proc_omp_loc} in
+                  2|4|5|10|20)
+                      IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
+                      ;;
+                  *)
+                      IGCM_debug_Exit "ERROR with OMP parameters !"
+                      IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
+                      IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
+                      IGCM_debug_Verif_Exit
+                      ;;
+              esac
             echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
             echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
+            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
+
           fi
 
@@ -1284 +1322 @@
 
           # Complete run_file
-          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
-          (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
-          echo "${current_core}-${end_core}  env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >> run_file
-          (( current_core = ${end_core} + 1 ))
-
+          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
+              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
+          else
+              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
+          fi
+
+          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
+             (( index_host = current_core / NB_CORE_PER_NODE ))
+             host_value=${listnodes[${index_host}]}
+             (( slot =  current_core % NB_CORE_PER_NODE ))
+               echo "$host_value" >> hostlist
+               echo "$rank taskset -c $slot-$((slot+comp_proc_omp_loc-1)) ./script_${ExeNameOut}.ksh" >> run_file
+             (( rank = rank + 1 ))
+             (( current_core = current_core + offset_comp_proc_loc ))
+          done
         fi
       done
@@ -1295 +1343 @@
       ## This is used when no error comes from executables and when something stopped an executable without notice. 
       export SLURM_WAIT=60
-
-      EXECUTION="${HOST_MPIRUN_COMMAND} --multi-prog ./run_file"
+      export SLURM_HOSTFILE=./hostlist
+
+      EXECUTION="${HOST_MPIRUN_COMMAND} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
 
       IGCM_sys_Chmod u+x run_file