Changeset 1466 for trunk/libIGCM

Timestamp:

07/13/18 09:38:15 (6 years ago)

Author:

aclsce

Message:

• On curie, allow to run with mpirun command on MPI-MPMD mode by defining ExecutionModeOnCurie?=mpirun in config.card.

File:

• trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh (modified) (2 diffs)

trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh

@@ -1140 +1140 @@
 
   if [ ${executionType} -eq 1 ] ; then
-    # MPMD + MPI
-    sed -e "/::openMPthreads::/d"                  \
-        -e "s/::JobNumProcTot::/${coreNumber}/"    \
-        -e "/#MSUB -x/d"                           \
-        -e "/--cpu_bind=none/d"                    \
-      ${file} > ${file}.tmp
-
+    # MPMD + MPI : mpirun
+    if [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ; then
+        sed -e "/::openMPthreads::/d"                  \
+            -e "s/::JobNumProcTot::/${coreNumber}/"    \
+             ${file} > ${file}.tmp
+    else
+        # MPMD + MPI : ccc_mprun
+        sed -e "/::openMPthreads::/d"                  \
+            -e "s/::JobNumProcTot::/${coreNumber}/"    \
+            -e "/#MSUB -x/d"                           \
+            -e "/--cpu_bind=none/d"                    \
+            ${file} > ${file}.tmp
+    fi
   elif [ ${executionType} -eq 2 ] ; then
     # MPMD + MPI + OMP : mpirun/ccc_mprun/error
@@ -1245 +1251 @@
     if  ( ! ${OK_PARA_OMP} ) ; then
 
+        if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then 
+      # use of ccc_mprun (default mode) 
       # Build run_file
 
       # First loop on the components for the coupler ie oasis (only if oasis3)
       # the coupler ie oasis3 must be the first one
-      for comp in ${config_ListOfComponents[*]} ; do
-
-        eval ExeNameIn=\${config_Executable_${comp}[0]}
-        eval ExeNameOut=\${config_Executable_${comp}[1]}
+            for comp in ${config_ListOfComponents[*]} ; do
+
+                eval ExeNameIn=\${config_Executable_${comp}[0]}
+                eval ExeNameOut=\${config_Executable_${comp}[1]}
 
         # for CPL component only
-        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
-
-          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
-          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
-          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
-        fi
-      done
+                if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
+
+                    eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+                    eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+                    echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
+                fi
+            done
 
       # Then second loop on the components
-      for comp in ${config_ListOfComponents[*]} ; do
-
-        eval ExeNameIn=\${config_Executable_${comp}[0]}
-        eval ExeNameOut=\${config_Executable_${comp}[1]}
+            for comp in ${config_ListOfComponents[*]} ; do
+
+                eval ExeNameIn=\${config_Executable_${comp}[0]}
+                eval ExeNameOut=\${config_Executable_${comp}[1]}
 
         # Only if we really have an executable for the component and not the coupler ie oasis:
-        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
-
-          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
-          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
-          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
-        fi
-      done
-
-      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
-
-      IGCM_sys_Chmod u+x run_file
-      if ( $DEBUG_sys ) ; then
-        echo "run_file contains : "
-        cat run_file
-      fi
+                if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
+
+                    eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+                    eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+                    echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
+                fi
+            done
+
+            EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
+
+            IGCM_sys_Chmod u+x run_file
+            if ( $DEBUG_sys ) ; then
+                echo "run_file contains : "
+                cat run_file
+            fi
+
+        elif [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ; then
+            # use of mpirun 
+
+                  # Use of mpirun instead of ccc_mprun
+            EXECUTION="time mpirun"
+
+      #  Hosts treatment
+            ${EXECUTION} hostname | sort | uniq > hosts.tmp
+
+            i=0
+            rm -f hosts rankfile
+            IGCM_debug_Print 1 "sys Curie, Hosts available :"
+            for nodes in `cat hosts.tmp`
+            do
+                host[$i]=$nodes
+                echo "${host[$i]}" >> hosts
+                IGCM_debug_Print 1 ${host[$i]}
+                i=$((i+1))
+            done
+            rm -f hosts.tmp
+
+            listnodes=${host[*]}
+
+            EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
+
+      # Initialisation
+            rank=0
+            current_core=0
+            init_exec=n
+
+      # Loop on the components
+            for comp in ${config_ListOfComponents[*]} ; do
+
+                eval ExeNameIn=\${config_Executable_${comp}[0]}
+                eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        # Not possible if oasis has an executable (i.e old version of oasis3)
+                if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
+                    IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
+                    IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
+                    IGCM_debug_Verif_Exit
+                fi
+
+        # Only if we really have an executable for the component :
+                if [ "X${ExeNameOut}" != X\"\" ] ; then
+
+                    eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+                    eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+                    eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
+
+                    echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+                    echo ""  >> script_${ExeNameOut}.ksh
+                    if [ ${comp_proc_omp_loc} -gt 1 ] ; then
+
+            # Check if the number of threads is correct
+                        case ${comp_proc_omp_loc} in
+                            2|4|8|16)
+                                IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
+                                ;;
+                            *)
+                                IGCM_debug_Exit "ERROR with OMP parameters !"
+                                IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
+                                IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
+                                IGCM_debug_Verif_Exit
+                                ;;
+                        esac
+                        echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+                        echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+                        echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+                        echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
+                    fi
+
+          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
+                    echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
+                    echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
+                    echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
+                    IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+
+                    if [ ${init_exec} = y ] ; then
+                        EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
+                    else
+                        EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
+                        init_exec=y
+                    fi
+
+          # Build rankfile : method used to assign cores and nodes for the MPI process
+          # Ex :
+          #rank 0=curie5296 slot=0,1,2,3
+          #rank 1=curie5296 slot=4,5,6,7
+          # Example of final command :
+          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
+          # with script_lmdz.x.ksh :
+          # #!/bin/ksh
+          #export KMP_STACKSIZE=3g
+          #export KMP_LIBRARY=turnaround
+          #export MKL_SERIAL=YES
+          #OMP_NUM_THREADS=4
+          #./lmdz.x
+                    if [ ${comp_proc_nod_loc} -gt 1 ] ; then
+                        (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
+                    else
+                        (( offset_comp_proc_loc =  comp_proc_omp_loc ))
+                    fi
+                    for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
+                        (( index_host = current_core / NB_CORE_PER_NODE ))
+                        host_value=${host[${index_host}]}
+                        (( slot =  current_core % NB_CORE_PER_NODE ))
+                        virg=","
+                        string_final=""
+                        for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
+                            string=$index$virg
+                            string_final=$string_final$string
+                        done
+                        string_final=$( echo $string_final | sed "s/.$//" )
+                        echo "rank $rank=$host_value slot=$string_final" >> rankfile
+                        (( rank = rank + 1 ))
+                        (( current_core = current_core + offset_comp_proc_loc ))
+                    done
+                fi
+            done
+
+        fi
 
     elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then