Changeset 1000 for trunk/libIGCM/libIGCM_sys

Timestamp:

03/28/14 16:45:03 (10 years ago)

Author:

aclsce

Message:

Added treatment of MPMD mode with MPI-OpenMP hybrid parallelization for Curie machine.

Example of final command :
mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
with

• script_lmdz.x.ksh : #!/bin/ksh

export KMP_STACKSIZE=3g
export KMP_LIBRARY=turnaround
export MKL_SERIAL=YES
OMP_NUM_THREADS=4
./lmdz.x

-rankfile :
rank 0=curie5296 slot=0,1,2,3
rank 1=curie5296 slot=4,5,6,7
...

File:

• trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh (modified) (1 diff)

trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh

@@ -2083 +2083 @@
   EXECUTION=${HOST_MPIRUN_COMMAND}
 
+# MPMD mode
+
   if ( ${OK_PARA_MPMD} ) ; then
 
-    if [ -f run_file ] ; then
-      IGCM_sys_Rm -f run_file
-    fi
-    touch run_file
-
-# run_file construction
-
-# Then first loop on the components for the coupler ie oasis
-
-## the coupler ie oasis must be the first one
-    for comp in ${config_ListOfComponents[*]} ; do
-
-      eval ExeNameIn=\${config_Executable_${comp}[0]}
-      eval ExeNameOut=\${config_Executable_${comp}[1]}
+# Only MPI (MPMD)
+      if  ( ! ${OK_PARA_OMP} ) ; then
+
+          if [ -f run_file ] ; then
+              IGCM_sys_Rm -f run_file
+          fi
+          touch run_file
+
+# Build run_file
+
+# First loop on the components for the coupler ie oasis (only if oasis3)
+
+## the coupler ie oasis3 must be the first one
+          for comp in ${config_ListOfComponents[*]} ; do
+
+              eval ExeNameIn=\${config_Executable_${comp}[0]}
+              eval ExeNameOut=\${config_Executable_${comp}[1]}
 
       # for CPL component only
-      if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
-
-        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
-        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
-        echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
+              if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
+
+                  eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+                  eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+                  echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
+              fi
+          done
+
+# Then second loop on the components
+
+          for comp in ${config_ListOfComponents[*]} ; do
+
+              eval ExeNameIn=\${config_Executable_${comp}[0]}
+              eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+      # Only if we really have an executable for the component and not the coupler ie oasis:
+              if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
+
+                  eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+                  eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+                  echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
+              fi
+          done
+
+          EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
+
+          IGCM_sys_Chmod u+x run_file
+          if ( $DEBUG_sys ) ; then
+              echo "run_file contains : "
+              cat run_file
+          fi
+
+# MPI-OpenMP (MPMD)   
+      else
+
+# Use of mpirun instead of ccc_mprun
+          EXECUTION="time mpirun"
+
+#  Hosts treatment
+          ${EXECUTION} hostname | sort | uniq > hosts.tmp
+          
+          i=0
+          rm -f hosts rankfile
+          IGCM_debug_Print 1 "sys Curie, Hosts available :"
+          for nodes in `cat hosts.tmp`
+          do
+              host[$i]=$nodes
+              echo "${host[$i]}" >> hosts
+              IGCM_debug_Print 1 ${host[$i]}
+              i=$((i+1))
+          done
+          rm -f hosts.tmp
+
+          listnodes=${host[*]}
+          
+          EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile" 
+
+# Initialisation
+          rank=0
+          current_core=0
+          core_per_node=16
+          init_exec=n
+
+# Loop on the components
+          for comp in ${config_ListOfComponents[*]} ; do
+              
+              eval ExeNameIn=\${config_Executable_${comp}[0]}
+              eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+# Not possible if oasis has an executable (i.e old version of oasis3)   
+              if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
+                  IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
+                  IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
+                  IGCM_debug_Verif_Exit 
+              fi
+
+# Only if we really have an executable for the component :
+              if [ "X${ExeNameOut}" != X\"\" ] ; then
+                  
+                  eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+                  eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+                  echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+                  echo ""  >> script_${ExeNameOut}.ksh
+                  if [ ${comp_proc_omp_loc} -gt 1 ] ; then
+
+# Check if the number of threads is correct
+                      case ${comp_proc_omp_loc} in
+                          2|4|8|16)
+                              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
+                              ;;
+                          *) 
+                              IGCM_debug_Exit "ERROR with OMP parameters !"
+                              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
+                              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
+                              IGCM_debug_Verif_Exit
+                              ;;
+                      esac
+                      echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+                      echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+                      echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+                      echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
+                  fi
+#               echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
+#               echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
+                  echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
+                  IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+                  
+                  if [ ${init_exec} = y ] ; then
+                      EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
+                  else  
+                      EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
+                      init_exec=y
+                  fi
+
+# Build rankfile : method used to assign cores and nodes for the MPI process
+# Ex : 
+#rank 0=curie5296 slot=0,1,2,3 
+#rank 1=curie5296 slot=4,5,6,7
+# Example of final command :
+# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
+# with script_lmdz.x.ksh :
+# #!/bin/ksh
+#export KMP_STACKSIZE=3g
+#export KMP_LIBRARY=turnaround
+#export MKL_SERIAL=YES
+#OMP_NUM_THREADS=4
+#./lmdz.x
+#
+
+                  for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
+                      (( index_host = current_core / core_per_node ))
+                      host_value=${host[${index_host}]}
+                      (( slot =  current_core % core_per_node ))
+                      virg=","
+                      string_final=""
+                      for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
+                          string=$index$virg
+                          string_final=$string_final$string
+                      done
+                      string_final=$( echo $string_final | sed "s/.$//" )
+                      echo "rank $rank=$host_value slot=$string_final" >> rankfile
+                      (( rank = rank + 1 ))
+                      (( current_core = current_core + comp_proc_omp_loc ))
+                  done
+              fi
+
+          done
       fi
-    done
-
-# Then second loop on the components
-
-    for comp in ${config_ListOfComponents[*]} ; do
-
-      eval ExeNameIn=\${config_Executable_${comp}[0]}
-      eval ExeNameOut=\${config_Executable_${comp}[1]}
-
-      # Only if we really have an executable for the component and not the coupler ie oasis:
-      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
-
-        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
-        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
-        echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
-      fi
-    done
-
-    EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
-
-    IGCM_sys_Chmod u+x run_file
-    if ( $DEBUG_sys ) ; then
-      echo "run_file contains : "
-      cat run_file
-    fi
-
-  else # Only one executable. launch it.
+
+# Only one executable (SPMD mode).
+  else 
 
     for comp in ${config_ListOfComponents[*]} ; do