Changeset 492 for branches

Timestamp:

08/10/11 14:17:48 (13 years ago)

Author:

mmaipsl

Message:

Arnaud, Martial : integrate vargas for mixte parallelization MPI/OpenMP.

File:

• branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_vargas.ksh (modified) (2 diffs)

branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_vargas.ksh

@@ -63 +63 @@
 #=====================================================
 # Host and user names 
-# $hostname ou hostname
+
+  # $hostname ou hostname
 typeset  HOST=${HOST:=$( hostname )}
 # $username ou whoami
@@ -1172 +1173 @@
     IGCM_debug_PopStack "IGCM_sys_build_run_file"
 }
+
+
+############################################################
+# Build MPI/OMP scripts
+function IGCM_sys_build_execution_scripts 
+{
+    IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
+    if ( $DEBUG_sys ) ; then
+        echo "IGCM_sys_build_execution_scripts " $@
+    fi
+
+    typeset NbNodes_Job NbProc_Job comp_proc_mpi_loc comp_proc_omp_loc mpi_count
+    
+    # Verification with PBS parameter
+    if [ X${BATCH_NUM_PROC_TOT} != X ] ; then
+
+        if ( ${OK_PARA_MPI} ) ; then
+            EXECUTION=${HOST_MPIRUN_COMMAND}
+        else
+            if [ ${BATCH_NUM_PROC_TOT} -eq 1 ] ; then
+                EXECUTION=${MPIRUN_COMMAND:="time "}
+            fi
+        fi
+    else
+        if ( ! ${OK_PARA_MPI} ) ; then
+            EXECUTION=${MPIRUN_COMMAND:="time "}
+        fi
+    fi
+
+    if [ ${PROCESSUS_NUMBER} -gt 1 ] ; then
+   
+        EXECUTION="${EXECUTION} -pgmmodel mpmd -cmdfile ./runfile" 
+
+        IGCM_debug_Print 1 "sys vargas : La commande d execution est "
+        IGCM_debug_Print 1 $EXECUTION
+
+        if ( ${OK_PARA_OMP} ) ; then
+# NEW : 2 Noeuds
+# @ task_geometry={(0)(1,2,3)}
+# Nombre de processus demandes
+            cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g" 
+            cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g" | wc -c 
+            NbNodes_Job=$(( $( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "task_geometry" | sed -e "s/[^(]*([^(]*/(/g" | wc -c ) - 1 ))
+
+            if [ ${NbNodes_Job} -eq 0 ] ; then
+                IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no task_geometry defined with OpenMP run."
+                exit 1
+            fi
+        else
+# OLD : 
+# @ total_tasks = 3
+# @ environment = "BATCH_NUM_PROC_TOT=3"
+# Pas d OpenMP
+# @ resources = ConsumableCpus(1)
+            cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "total_tasks" | sed -e "s/.*total_tasks = //" 
+            NbProc_Job=$( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "total_tasks" | sed -e "s/.*total_tasks = //" )
+            NbProc_Job=${NbProc_Job:=0}
+            if [ ${NbProc_Job} -eq 0 ] ; then
+                IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no total_tasks defined with MPI only run."
+                exit 1
+            fi
+            if ( $( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "resources = ConsumableCpus(1)" ) ) ; then
+                IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no ressource defined with MPI only run."
+                exit 1
+            fi            
+        fi
+        if [ -f runfile ] ; then
+            IGCM_sys_Rm -f runfile
+            touch runfile
+        fi
+    fi
+
+# Then loop on the components 
+
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+            
+        # Only if we really have an executable for the component :
+        if [ "X${ExeNameOut}" != X\"\" ] ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+            
+            if ( ${OK_PARA_MPI} ) ; then 
+
+                (( mpi_count = 1 ))
+                until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
+                    if ( ${OK_PARA_OMP} ) ; then 
+                        echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} > out_${ExeNameOut}.out.${mpi_count} 2>out_${ExeNameOut}.err.${mpi_count}" >> runfile
+                    else
+                        echo "./${ExeNameOut} > out_${ExeNameOut}.out.${mpi_count} 2>out_${ExeNameOut}.err.${mpi_count}" >> runfile 
+                    fi
+                    (( mpi_count = mpi_count + 1 ))
+                done
+            else
+                if ( ${OK_PARA_OMP} ) ; then 
+                    echo "env OMP_NUM_THREADS=${comp_proc_omp_loc} ./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> runfile
+                else
+                    echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> runfile 
+                fi
+            fi
+        fi
+    done
+    IGCM_sys_Chmod u+x runfile 
+
+    IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
+}
+
+