Changeset 490 for branches/libIGCM_MPI_OpenMP

Timestamp:

06/16/11 12:31:47 (13 years ago)

Author:

mmaipsl

Message:

First version for MPI/OpenMP integration in libIGCM.
Work only on titane for instance.

Location:

branches/libIGCM_MPI_OpenMP

Files:

• AA_job (modified) (2 diffs)
• libIGCM_config/libIGCM_config.ksh (modified) (2 diffs)
• libIGCM_sys/libIGCM_sys_titane.ksh (modified) (3 diffs)

branches/libIGCM_MPI_OpenMP/AA_job

@@ -440 +440 @@
 
     echo "========================================================================"
-    if ( [ X${BATCH_NUM_PROC_TOT} != X ] && [ "${BATCH_NUM_PROC_TOT}" -gt 1 ] ) ; then
-        MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND}
-        eval MPIRUN_OPTIONS=${config_UserChoices_JobRunOptions}
-    else
-        MPIRUN_COMMAND=${MPIRUN_COMMAND:="time "}
-        MPIRUN_OPTIONS=${MPIRUN_OPTIONS:=""}
-    fi
-
     if [ ${DRYRUN} -le 1 ] ; then
         REAL_DATE_INIT=$( date )
-        echo "EXECUTION of : ${MPIRUN_COMMAND} ${MPIRUN_OPTIONS} ./${config_Executable_Name}   > ${Exe_Output} 2>&1"
         echo                                                                                   > ${Exe_Output}
         echo "#######################################"                                        >> ${Exe_Output}
-        echo "EXECUTION of : ${MPIRUN_COMMAND} ${MPIRUN_OPTIONS} ./${config_Executable_Name}" >> ${Exe_Output}
+        echo "EXECUTION of : ${EXECUTION}"                                                    >> ${Exe_Output}
         echo                                                                                  >> ${Exe_Output}
         typeset RET
         RUN_DATE_BEGIN=$( date '+%Y-%m-%dT%H:%M:%S' )
-        ${MPIRUN_COMMAND} ${MPIRUN_OPTIONS} ./${config_Executable_Name} >> ${Exe_Output} 2>&1
+        ${EXECUTION}  >> ${Exe_Output} 2>&1
         RET=$?
         RUN_DATE_END=$( date '+%Y-%m-%dT%H:%M:%S' )
@@ -473 +464 @@
         echo                                            >> ${Exe_Output}
     else
-        echo "EXECUTION of : ${MPIRUN_COMMAND} ${MPIRUN_OPTIONS} ./${config_Executable_Name} simulated for DRYRUN = " $DRYRUN
+        echo "EXECUTION of : ${EXECUTION} simulated for DRYRUN = " $DRYRUN
+        echo "EXECUTION of : ${EXECUTION} simulated for DRYRUN = " $DRYRUN > ${Exe_Output}
         if ( $DEBUG_debug ) ; then
             echo "FOR EXECUTION DRYRUN mode = " $DRYRUN >> stack

branches/libIGCM_MPI_OpenMP/libIGCM_config/libIGCM_config.ksh

@@ -108 +108 @@
     eval Script_Output_Prefix=${config_UserChoices_Script_Output_Prefix:='Script_Output'}
     IGCM_debug_Print 3 "Script_Output_Prefix = ${Script_Output_Prefix}"
-    eval Exe_Output=out_${config_Executable_Name}
-    IGCM_debug_Print 3 "Exe_Output           = ${Exe_Output}"
+#MM : obsolete !
+#    eval Exe_Output=out_${config_Executable_Name}
+    eval Exe_Output=out_execution
+#    IGCM_debug_Print 3 "Exe_Output           = ${Exe_Output}"
 
     #==================================
@@ -536 +538 @@
     fi
 
+    typeset ExeNameIn
+    typeset comp i
+    typeset tempvar tempvarMPI tempvarNOD NbElts j
+
+    (( PROCESSUS_NUMBER = 0 ))
+    (( i=0 ))
+
+    OK_PARA_MPI=false
+    OK_PARA_OMP=false
+    OK_PARA_NOD=false
+    for comp in ${config_ListOfComponents[*]} ; do
+
+        IGCM_debug_Print 1 ${comp}
+
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+
+        # NO order in config.card for parallelized values ! 
+        # just use suffix : MPI , OMP and NOD (for number of NODes.)
+
+        # NOD is the number of NODes allocated
+        eval ${comp}_PROC_NOD=0
+
+        # MPI is the number of MPI processus per nodes
+        eval ${comp}_PROC_MPI=0
+
+        # OMP is the number of OpenMP threads per MPI processus
+        eval ${comp}_PROC_OMP=0
+        
+        # Only if we really have an executable for the component :
+        if [ X${ExeNameIn} != X\"\" ] ; then
+            eval NbElts=\${#config_Executable_${comp}[@]}
+            if [ ${NbElts} -gt 2 ] ; then
+                (( j = 2 ))
+                eval ${comp}_PROC_MPI=1
+                eval ${comp}_PROC_OMP=1
+                eval ${comp}_PROC_NOD=1
+
+                while [ $j -lt ${NbElts} ] ; do 
+                    eval tempvar=\${config_Executable_${comp}[${j}]}
+
+                    echo 
+#MM BUG impossible ?!
+                    if [ X${tempvar} == X ] ; then
+                        echo "Error reading MPI/OMP parameters !!!" 
+                        echo "Check your config.card." 
+                    # MM : Ã  enlever :
+                        exit 1
+                        #break
+                    fi
+
+# pb                eval ${comp}_PROC_MPI=1
+# pb                eval ${comp}_PROC_OMP=1
+# pb                eval ${comp}_PROC_NOD=1
+
+                    eval IGCM_debug_Print 2 ${tempvar}
+
+                    case ${tempvar} in
+                        *MPI)
+                            # Read MPI parameter for composante
+                            eval ${comp}_PROC_MPI=$( echo ${tempvar} | sed -e "s/MPI//" )
+                            OK_PARA_MPI=true;;
+                        *OMP)
+                            # Read OMP parameter for composante
+                            eval ${comp}_PROC_OMP=$( echo ${tempvar} | sed -e "s/OMP//" )
+                            OK_PARA_OMP=true;;
+                        *NOD)
+                            # Read NOD (NumBer of Nodes) parameter for composante
+                            eval ${comp}_PROC_NOD=$( echo ${tempvar} | sed -e "s/NOD//" )
+                            OK_PARA_NOD=true
+                            OK_PARA_MPI=true;;
+                    esac
+
+                    (( j = j + 1 ))
+                done
+
+                eval tempvarMPI=\${${comp}_PROC_MPI}
+                eval tempvarNOD=\${${comp}_PROC_NOD}
+                eval tempvarOMP=\${${comp}_PROC_OMP}
+                (( PROCESSUS_NUMBER = PROCESSUS_NUMBER + tempvarMPI * tempvarNOD * tempvarOMP ))
+            fi
+        fi
+
+        (( i=i+1 ))
+    done
+
+    # sequential case !
+    if [ ${PROCESSUS_NUMBER} -eq 0 ] ; then
+        (( PROCESSUS_NUMBER = 1 ))
+        echo "PROCESSUS_NUMBER is all 0 (sequential use of old definition in config->Executable->list)."
+        echo "We set it to 1."
+    fi
+
+    IGCM_debug_Print 1 "MPI/OMP treatment PROCESSUS_NUMBER = ${PROCESSUS_NUMBER}"
+
+    # Verification with PBS parameter
+    if [ X${BATCH_NUM_PROC_TOT} != X ] ; then
+        if [ ${BATCH_NUM_PROC_TOT} -ne ${PROCESSUS_NUMBER} ] ; then
+            echo "Warning with parallelization parameters !"
+            echo "PBS variable BATCH_NUM_PROC_TOT = ${BATCH_NUM_PROC_TOT} "
+            echo " is the total number of _processors_ reserved."
+            echo "It is not equal to the sum of _processus_ in config.card = ${PROCESSUS_NUMBER}."
+        fi
+        if ( ! ${OK_PARA_MPI} ) ; then
+            if [ ${BATCH_NUM_PROC_TOT} -gt 1 ] ; then
+                echo "You have given a reservation of ${BATCH_NUM_PROC_TOT} processors,"
+                echo "but you havn't filled the parallel parameter in config->Executable->list."
+                echo "This is the old method and it is no more supported."
+                echo "Please read documentation or config specific comments."
+                echo "We stop now."
+                exit 1
+            else 
+                PROCESSUS_NUMBER=1
+            fi
+        fi
+    else
+        if ( ${OK_PARA_MPI} ) ; then
+            echo "You have not given a value of ${BATCH_NUM_PROC_TOT} processors,"
+            echo "but you have filled the parallel parameter in config->Executable->list."
+            echo "This is the old method and it is no more supported."
+            echo "Please read documentation or config specific comments."
+            echo "We stop now."
+            exit 1
+        fi
+    fi
+
+#     if ( ${OK_PARA_MPI} || ${OK_PARA_OMP} || ${OK_PARA_NOD} ) ; then
+    IGCM_sys_build_execution_scripts
+#     fi
+
     ExecutionFail=false
 

branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_titane.ksh

@@ -167 +167 @@
 #- Max number of arguments passed to nco operator or demigration command
 UNIX_MAX_LIMIT=360
+
+NUM_COREPERNODE=8
+        
 
 #D-#==================================================
@@ -1266 +1269 @@
         echo "IGCM_sys_activ_variables"
     fi
-    ulimit -s 2097152
+#    ulimit -s 2097152
+
+    typeset max_omp
+
+    ulimit -s unlimited
+
+    (( max_omp = 0 ))
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+        
+    
+        # Only if we really have an executable for the component :
+        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+            if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
+
+                (( max_omp = comp_proc_omp_loc ))
+            fi
+        fi
+    done
+    if [ ${max_omp} -gt 1 ] ; then
+        module load openmp/${max_omp}thds
+    fi
+
     IGCM_debug_PopStack "IGCM_sys_activ_variables"
 }
@@ -1281 +1310 @@
 }
 
+
+# function IGCM_sys_build_run_file {
 ############################################################
 # Build run file
 
-function IGCM_sys_build_run_file {
-    IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
-    if ( $DEBUG_sys ) ; then
-        echo "IGCM_sys_build_run_file " $@
-    fi
-
-    # set Number of processors for OCE here
-    NUM_PROC_OCE=5
-
-    (( NUM_PROC_ATM = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE - 1))
-    (( nb_tot_m1    = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE ))
-    if [ $1 = MPI1 ]; then
-        cat <<EOF > run_file
--np 1 ./oasis
--np ${NUM_PROC_ATM} ./lmdz.x
--np ${NUM_PROC_OCE} ./opa.xx
-EOF
-        config_UserChoices_JobRunOptions='"--app"'
-        IGCM_sys_Chmod u+x run_file
-    fi
-    IGCM_debug_PopStack "IGCM_sys_build_run_file"
+#     IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
+#     if ( $DEBUG_sys ) ; then
+#       echo "IGCM_sys_build_run_file " $@
+#     fi
+
+#     # set Number of processors for OCE here
+#     NUM_PROC_OCE=5
+
+#     (( NUM_PROC_ATM = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE - 1))
+#     (( nb_tot_m1    = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE ))
+#     if [ $1 = MPI1 ]; then
+#       cat <<EOF > run_file
+# -np 1 ./oasis
+# -np ${NUM_PROC_ATM} ./lmdz.x
+# -np ${NUM_PROC_OCE} ./opa.xx
+# EOF
+#       config_UserChoices_JobRunOptions='"--app"'
+#       IGCM_sys_Chmod u+x run_file
+#     fi
+#     IGCM_debug_PopStack "IGCM_sys_build_run_file"
   
-}
+# }
+
+############################################################
+# Build MPI/OMP scripts
+function IGCM_sys_build_execution_scripts 
+{
+    IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
+    if ( $DEBUG_sys ) ; then
+        echo "IGCM_sys_build_execution_scripts " $@
+    fi
+
+    # Verification with PBS parameter
+    if [ X${BATCH_NUM_PROC_TOT} != X ] ; then
+
+        if ( ${OK_PARA_MPI} ) ; then
+            EXECUTION=${HOST_MPIRUN_COMMAND}
+        else
+            if [ ${BATCH_NUM_PROC_TOT} -eq 1 ] ; then
+                EXECUTION=${MPIRUN_COMMAND:="time "}
+            fi
+        fi
+    else
+        if ( ! ${OK_PARA_MPI} ) ; then
+            EXECUTION=${MPIRUN_COMMAND:="time "}
+        fi
+    fi
+
+    if [ ${PROCESSUS_NUMBER} -gt 1 ] ; then
+
+    #  Hosts treatment
+
+        ${EXECUTION} hostname | sort | uniq > hosts.tmp
+        
+        i=0
+        rm -f hosts
+        IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
+        for nodes in `cat hosts.tmp`
+        do
+            host[$i]=$nodes
+            echo "${host[$i]} slots=8 max_slots=8" >> hosts
+            IGCM_debug_Print 1 ${host[$i]}
+            i=$((i+1))
+        done
+        rm -f hosts.tmp
+
+        listnodes=${host[*]}
+    
+        EXECUTION="${EXECUTION} -hostfile hosts" 
+    fi
+
+# Initialisation
+
+    init_node=y
+    node_num_current=0
+    start_num=0
+    init_exec=n
+
+
+# Test : if oasis is there, we put it at the first position
+                
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        if [ "X${comp}" = "XCPL" ]  ; then
+            
+            eval ExeNameIn=\${config_Executable_${comp}[0]}
+            eval ExeNameOut=\${config_Executable_${comp}[1]}
+            
+            echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+            echo ""  >> script_${ExeNameOut}.ksh
+            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+            echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
+            IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+            
+            init_node=n
+            
+            (( nombre_restant_node = NUM_COREPERNODE - 1 ))
+            node_num_current=0
+            node_current=${host[${node_num_current}]}
+            
+            EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh" 
+            
+            init_exec=y
+            start_num=1
+            
+        fi
+        
+    done
+    
+# Then loop on the components (except for oasis)
+
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+        
+    
+        # Only if we really have an executable for the component :
+        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+            
+            if ( ${OK_PARA_MPI} ) ; then 
+                
+                echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+#               echo "set -vx" >> script_${ExeNameOut}.ksh
+                echo ""  >> script_${ExeNameOut}.ksh
+                echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+                echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+                echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+                echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
+                echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
+                echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
+                IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+                
+                node_num=0
+                
+# We define the number of MPI process to be assigned for the component 
+
+                nombre_restant_comp=${comp_proc_mpi_loc}
+                
+# Loop on the allocated nodes
+
+                for node in ${listnodes} ; do
+
+# We go to the current node
+
+                    if [ ${node_num} = ${node_num_current} ] ; then
+
+                        node_current=${host[${node_num_current}]}
+
+# If first time on the node : initialisation 
+
+                        if [ ${init_node} = y ] ; then
+                            nombre_restant_node=${NUM_COREPERNODE}
+                        fi
+        
+# Test on the number of OMP threads
+
+                        if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
+                            (( node_num = node_num + 1 ))
+                            node_num_current=${node_num}
+                            init_node=y
+                            continue
+                        fi
+
+# Number of MPI process to assign
+
+                        (( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
+ 
+                        if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
+                            num_corempi=${nombre_restant_comp}
+                        fi
+
+                        (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
+                        (( nombre_restant_comp = nombre_restant_comp - num_corempi ))
+                    
+                        if [ ${init_exec} = y ] ; then
+                            EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
+                        else    
+                            EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
+                            init_exec=y
+                        fi
+
+                        ((  start_num = num_corempi + start_num ))
+        
+                    else
+                        
+                        (( node_num = node_num + 1 )) 
+                        continue
+                    fi
+                
+# Test on the number of core/process remaining on the node/component 
+                
+                    if [ ${nombre_restant_node} = 0 ] ; then
+                        (( node_num = node_num + 1 ))
+                        node_num_current=${node_num}
+                        init_node=y
+
+                        if [ ${nombre_restant_comp} = 0 ] ; then
+                            break 1
+                        fi
+                    else
+
+                        node_num_current=${node_num}
+                        init_node=n
+                        
+                        if [ ${nombre_restant_comp} = 0 ] ; then
+                            break 1
+                        fi
+                    fi
+                done
+            else
+                EXECUTION="${EXECUTION} ./${ExeNameOut}"
+            fi
+        fi
+    done
+
+    IGCM_debug_Print 1 "sys Titane : La commande d execution est "
+    IGCM_debug_Print 1 $EXECUTION
+
+    IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
+}
+
+
 
 ##############################################################