Changeset 1390 for trunk/libIGCM

Timestamp:

05/02/17 10:20:35 (7 years ago)

Author:

mafoipsl

Message:

On curie : for MPMD mode add a parameter in config.card (ExecutionModeOnCurie?) to select ccc_mprun or mpirun method. mpirun is used by default.
These methods are described here : wiki:IntegrationOpenMP/EnteteCommandesCurie?

File:

• trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh (modified) (7 diffs)

trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh

@@ -1111 +1111 @@
 
   elif [ ${executionType} -eq 2 ] ; then
-    # MPMD + MPI + OMP
-    sed -e "/::openMPthreads::/d"                  \
-        -e "s/::JobNumProcTot::/${coreNumber}/"    \
-      ${file} > ${file}.tmp
-
+    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
+    if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
+      sed -e "/::openMPthreads::/d"                  \
+          -e "s/::JobNumProcTot::/${coreNumber}/"    \
+        ${file} > ${file}.tmp
+    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
+      sed -e "/::openMPthreads::/d"                  \
+          -e "s/::JobNumProcTot::/${coreNumber}/"    \
+          -e "/--cpu_bind=none/d"                    \
+        ${file} > ${file}.tmp
+    else
+     IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
+     IGCM_debug_Exit "IGCM_sys_updateHeaders"
+    fi
   elif [ ${executionType} -eq 3 ] ; then
     # SPMD + MPI/OMP
@@ -1184 +1193 @@
   EXECUTION=${HOST_MPIRUN_COMMAND}
 
-  # MPMD mode
   if ( ${OK_PARA_MPMD} ) ; then
+
+    # MPMD mode
+    # 1 MPI only
+    # 2 MPI/OpenMP mpirun method
+    # 3 MPI/OpenMP ccc_mprun method
 
     if [ -f run_file ] ; then
@@ -1192 +1205 @@
     touch run_file
 
-    # Only MPI (MPMD)
+    # case 1 : Only MPI (MPMD)
     if  ( ! ${OK_PARA_OMP} ) ; then
 
@@ -1228 +1241 @@
       done
 
+      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
+
+      IGCM_sys_Chmod u+x run_file
+      if ( $DEBUG_sys ) ; then
+        echo "run_file contains : "
+        cat run_file
+      fi
+
+    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
+
+    # 2 MPI/OpenMP mpirun method
+
+      # Use of mpirun instead of ccc_mprun
+      EXECUTION="time mpirun"
+
+      #  Hosts treatment
+      ${EXECUTION} hostname | sort | uniq > hosts.tmp
+
+      i=0
+      rm -f hosts rankfile
+      IGCM_debug_Print 1 "sys Curie, Hosts available :"
+      for nodes in `cat hosts.tmp`
+      do
+        host[$i]=$nodes
+        echo "${host[$i]}" >> hosts
+        IGCM_debug_Print 1 ${host[$i]}
+        i=$((i+1))
+      done
+      rm -f hosts.tmp
+
+      listnodes=${host[*]}
+
+      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
+
+      # Initialisation
+      rank=0
+      current_core=0
+      core_per_node=16
+      init_exec=n
+
+      # Loop on the components
+      for comp in ${config_ListOfComponents[*]} ; do
+
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        # Not possible if oasis has an executable (i.e old version of oasis3)
+        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
+          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
+          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
+          IGCM_debug_Verif_Exit
+        fi
+
+        # Only if we really have an executable for the component :
+        if [ "X${ExeNameOut}" != X\"\" ] ; then
+
+          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+          echo ""  >> script_${ExeNameOut}.ksh
+          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
+
+            # Check if the number of threads is correct
+            case ${comp_proc_omp_loc} in
+            2|4|8|16)
+              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
+              ;;
+            *)
+              IGCM_debug_Exit "ERROR with OMP parameters !"
+              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
+              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
+              IGCM_debug_Verif_Exit
+              ;;
+            esac
+            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
+          fi
+
+          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
+          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
+          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
+          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
+          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+
+          if [ ${init_exec} = y ] ; then
+            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
+          else
+            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
+            init_exec=y
+          fi
+
+          # Build rankfile : method used to assign cores and nodes for the MPI process
+          # Ex :
+          #rank 0=curie5296 slot=0,1,2,3
+          #rank 1=curie5296 slot=4,5,6,7
+          # Example of final command :
+          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
+          # with script_lmdz.x.ksh :
+          # #!/bin/ksh
+          #export KMP_STACKSIZE=3g
+          #export KMP_LIBRARY=turnaround
+          #export MKL_SERIAL=YES
+          #OMP_NUM_THREADS=4
+          #./lmdz.x
+
+          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
+            (( index_host = current_core / core_per_node ))
+            host_value=${host[${index_host}]}
+            (( slot =  current_core % core_per_node ))
+            virg=","
+            string_final=""
+            for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
+              string=$index$virg
+              string_final=$string_final$string
+            done
+            string_final=$( echo $string_final | sed "s/.$//" )
+            echo "rank $rank=$host_value slot=$string_final" >> rankfile
+            (( rank = rank + 1 ))
+            (( current_core = current_core + comp_proc_omp_loc ))
+          done
+        fi
+      done
+
+    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
+
+    # 3 MPI/OpenMP ccc_mprun method
+
     # MPI-OpenMP (MPMD)
     # example of run_file
@@ -1233 +1376 @@
     # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
     # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
-
-    else
 
       # Loop on the components
@@ -1301 +1442 @@
       module load feature/bridge/heterogenous_mpmd
 
-    fi
-
-    # for both MPMD/MPI and MPMD/MPI/OpenMP mode
-
-    EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
-
-    IGCM_sys_Chmod u+x run_file
-    if ( $DEBUG_sys ) ; then
-      echo "run_file contains : "
-      cat run_file
-    fi
-
+      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method) 
+
+      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
+
+      IGCM_sys_Chmod u+x run_file
+      if ( $DEBUG_sys ) ; then
+        echo "run_file contains : "
+        cat run_file
+      fi
+
+    else
+
+      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
+
+    fi # 1 2 or 3 if ${OK_PARA_MPMD}
+
+  else
   # Only one executable (SPMD mode).
-  else
 
     for comp in ${config_ListOfComponents[*]} ; do
@@ -1359 +1504 @@
     done
 
-  fi
+  fi # ${OK_PARA_MPMD}
 
   IGCM_debug_Print 1 "sys Curie : execution command is "