Changeset 558 for branches/libIGCM_MPI_OpenMP

Timestamp:

02/24/12 18:16:41 (12 years ago)

Author:

mafoipsl

Message:

Add default number of processors in libIGCM_sys library. Adapted and tested on vargas and SX9.

Location:

branches/libIGCM_MPI_OpenMP

Files:

• libIGCM_config/libIGCM_config.ksh (modified) (3 diffs)
• libIGCM_sys/libIGCM_sys_mercurex9.ksh (modified) (2 diffs)
• libIGCM_sys/libIGCM_sys_vargas.ksh (modified) (1 diff)

branches/libIGCM_MPI_OpenMP/libIGCM_config/libIGCM_config.ksh

@@ -607 +607 @@
                         *[mM][pP][iI]*)
                             # Read MPI parameter for composante
-                            eval ${comp}_PROC_MPI=$( echo ${tempvar} | sed -e "s/MPI//" )
+                            eval ${comp}_PROC_MPI=$( echo ${tempvar} | tr '[a-z]' '[A-Z]' | sed -e "s/MPI//" )
                             OK_PARA_MPI=true;;
                         *[oO][mM][pP]*)
                             # Read OMP parameter for composante
-                            eval ${comp}_PROC_OMP=$( echo ${tempvar} | sed -e "s/OMP//" )
+                            eval ${comp}_PROC_OMP=$( echo ${tempvar} | tr '[a-z]' '[A-Z]' | sed -e "s/OMP//" )
                             OK_PARA_OMP=true;;
                         *[nN][oO][dD]*)
                             # Read NOD (NumBer of Nodes) parameter for composante
-                            eval ${comp}_PROC_NOD=$( echo ${tempvar} | sed -e "s/NOD//" )
+                            eval ${comp}_PROC_NOD=$( echo ${tempvar} | tr '[a-z]' '[A-Z]' | sed -e "s/NOD//" )
                             OK_PARA_NOD=true
                             OK_PARA_MPI=true;;
@@ -642 +642 @@
     if [ X${BATCH_NUM_PROC_TOT} != X ] ; then
         # BATCH_NUM_PROC_TOT is set
-        if ( ! ${OK_PARA_MPI} ) ; then
+        if ( ${OK_PARA_MPI} ) ; then
+            IGCM_debug_Print 1 "MPI/OMP/NOD fund into config.card and BATCH_NUM_PROC_TOT = ${BATCH_NUM_PROC_TOT} "
+        else
             # with previous method.
             if [ ${BATCH_NUM_PROC_TOT} -gt 1 ] ; then
@@ -648 +650 @@
                 if ( ${OK_PARA_MPMD} ) ; then
                      # with MPMD ie CPL/oasis method
-                     echo "We will use default number of MPI tasks : 5 for OCE, 1 for CPL and 26 for ATM for coupled configuration"
+                     echo "We will use default number of MPI tasks for this machine : "
+                     echo "${DEFAULT_NUM_PROC_OCE} for OCE; "
+                     echo "${DEFAULT_NUM_PROC_CPL} for CPL; "
+                     echo "${DEFAULT_NUM_PROC_ATM} for ATM; "
+                     echo "${DEFAULT_NUM_PROC_TOTAL} for the total number of processors; "
                      OK_PARA_MPI=true
-                     CPL_PROC_MPI=1
-                     OCE_PROC_MPI=5
-                     ATM_PROC_MPI=26
-                     PROCESSUS_NUMBER=32
+                     CPL_PROC_MPI=${DEFAULT_NUM_PROC_CPL}
+                     OCE_PROC_MPI=${DEFAULT_NUM_PROC_OCE}
+                     ATM_PROC_MPI=${DEFAULT_NUM_PROC_ATM}
+                     PROCESSUS_NUMBER=${DEFAULT_NUM_PROC_TOTAL}
                 else
                      # with have only one executable

branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_mercurex9.ksh

@@ -161 +161 @@
 #- Max number of arguments passed to nco operator or demigration command
 UNIX_MAX_LIMIT=360
+
+#====================================================
+#- Default number of MPI task for IPSL coupled model
+#- required for backward compatibility
+#-
+DEFAULT_NUM_PROC_OCE=1
+DEFAULT_NUM_PROC_CPL=1
+DEFAULT_NUM_PROC_ATM=3
+DEFAULT_NUM_PROC_TOTAL=4
 
 #D-#==================================================
@@ -1212 +1221 @@
 
 ############################################################
-# Build run file
+# Build MPI/OMP scripts
+function IGCM_sys_build_execution_scripts 
+{
+    IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
+    if ( $DEBUG_sys ) ; then
+        echo "IGCM_sys_build_execution_scripts " $@
+    fi
+
+    typeset NbNodes_Job NbProc_Job comp_proc_mpi_loc comp_proc_omp_loc mpi_count
+    
+    if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
+        IGCM_debug_Exit "IGCM_sys_mercurex9 build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
+    fi
+
+    if ( ${OK_PARA_MPMD} ) ; then
+
+        if [ -f run_file ] ; then
+            IGCM_sys_Rm -f run_file
+        fi
+        touch run_file
+
+        if ( ${OK_PARA_OMP} ) ; then
+                IGCM_debug_Print 2 "Error in config.card on SX9 : OpenMP not available"
+                exit 1
+        else
+# OLD :
+#PBS -v BATCH_NUM_PROC_TOT=4
+#PBS -l cpunum_job=${BATCH_NUM_PROC_TOT}
+            echo "Job_${config_UserChoices_JobName} includes BATCH_NUM_PROC_TOT = \c"
+            cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "BATCH_NUM_PROC_TOT" | sed -e "s/.*BATCH_NUM_PROC_TOT = //" 
+            NbProc_Job=$( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep "BATCH_NUM_PROC_TOT" | sed -e "s/.*BATCH_NUM_PROC_TOT = //" )
+            NbProc_Job=${NbProc_Job:=0}
+            if [ ${NbProc_Job} -eq 0 ] ; then
+                IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no BATCH_NUM_PROC_TOT defined with MPI only run."
+                exit 1
+            fi
+        fi
+
+# run_file construction
+
+# Then first loop on the components for the coupler ie oasis
+
+### the coupler ie oasis must be the first one
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+            
+        # for CPL component only
+        if [ "X${comp}" = "XCPL" ] ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+            
+            if ( ${OK_PARA_MPI} ) ; then 
+
+                (( mpi_count = 1 ))
+                until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
+                    echo "./${ExeNameOut}" >> run_file 
+                    (( mpi_count = mpi_count + 1 ))
+                done
+            fi
+        fi
+    done
+
+# Then second loop on the components
+
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+            
+        # Only if we really have an executable for the component and not the coupler ie oasis:
+        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+            
+            if ( ${OK_PARA_MPI} ) ; then 
+
+                (( mpi_count = 1 ))
+                until [ ${mpi_count} -gt ${comp_proc_mpi_loc} ] ; do
+                    echo "./${ExeNameOut}" >> run_file 
+                    (( mpi_count = mpi_count + 1 ))
+                done
+            fi
+        fi
+    done
+
+    EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
+
+    IGCM_sys_Chmod u+x run_file
+    if ( $DEBUG_sys ) ; then
+      echo "run_file contains : "
+      cat run_file
+    fi
+
+else # Only one executable. launch it.
+
+    for comp in ${config_ListOfComponents[*]} ; do
+
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        [ "X${ExeNameOut}" != X\"\" ] &&  EXECUTION="time mpirun -np ${BATCH_NUM_PROC_TOT} ./${ExeNameOut}"
+
+    done
+
+fi
+
+    IGCM_debug_Print 1 "sys mercurex9 : execution command is "
+    IGCM_debug_Print 1 "$EXECUTION"
+
+    IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
+}
 
 function IGCM_sys_build_run_file {
-    IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
-    if ( $DEBUG_sys ) ; then
-        echo "IGCM_sys_build_run_file " $@
-    fi
-    (( NUM_PROC_ATM = BATCH_NUM_PROC_TOT - 1 ))
-    (( NUM_PROC_OASIS = BATCH_NUM_PROC_TOT - NUM_PROC_ATM ))
-    (( NUM_PROC_OCE = BATCH_NUM_PROC_TOT - NUM_PROC_ATM ))
-    
-    if [ $1 = MPI2 ]; then
-        cat <<EOF > run_file
--p 1 -np 1 -e ./oasis
-EOF
-        (( NUM_PROCESS = BATCH_NUM_PROC_TOT + 1 ))
-        config_UserChoices_JobRunOptions='"-max_np ${NUM_PROCESS} -f"'
-
-    elif [ $1 = MPI1 ]; then
-        cat <<EOF > run_file
--p $NUM_PROC_OASIS -e ./oasis
--p $NUM_PROC_ATM -e ./lmdz.x
--p $NUM_PROC_OCE -e ./opa.xx
-EOF
- 
-    fi
-
-    IGCM_debug_PopStack "IGCM_sys_build_run_file"
-  
-}
+
+IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
+
+}
+

branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_vargas.ksh

@@ -138 +138 @@
 #- Max number of arguments passed to nco operator or demigration command
 UNIX_MAX_LIMIT=120
+
+#====================================================
+#- Default number of MPI task for IPSL coupled model
+#- required for backward compatibility
+#-
+DEFAULT_NUM_PROC_OCE=5
+DEFAULT_NUM_PROC_CPL=1
+DEFAULT_NUM_PROC_ATM=26
+DEFAULT_NUM_PROC_TOTAL=32
 
 #D-#==================================================