Changeset 687

Timestamp:

06/18/12 17:21:40 (12 years ago)

Author:

mafoipsl

Message:

On Titane : verification of MPI/OpenMP performances. Header checks added. module load openmp/4thds validated to distribute tasks and threads on different nodes. More information on : wiki:IntegrationOpenMP/EnteteCommandesTitane?

File:

• trunk/libIGCM/libIGCM_sys/libIGCM_sys_titane.ksh (modified) (5 diffs)

trunk/libIGCM/libIGCM_sys/libIGCM_sys_titane.ksh

@@ -1709 +1709 @@
   unset MPI_PLACING_MODE
 
+  # add module load nb of threads for core placement
+  (( max_omp = 0 ))
+
+  for comp in ${config_ListOfComponents[*]} ; do
+    eval ExeNameIn=\${config_Executable_${comp}[0]}
+    eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+    # Only if we really have an executable for the component :
+    if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
+      eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+      if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
+        (( max_omp = comp_proc_omp_loc ))
+      fi
+    fi
+  done
+
+  ( [ X${max_omp} = X4 ] || [ X${max_omp} = X8 ] ) &&  { module load openmp/${max_omp}thds ; IGCM_debug_Print 1 "module load openmp/${max_omp}thds" ; }
+
   IGCM_debug_PopStack "IGCM_sys_activ_variables"
 }
@@ -1743 +1761 @@
   typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
   typeset num_corempi nombre_restant_node nombre_restant_comp
+  typeset IGCM_OMP
 
   if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
@@ -1952 +1971 @@
     fi
 
-  else # Only one executable. launch it.
+  else # Only one executable. launch it after verifications. test done with :
+      # Job :
+      # #MSUB -n 8 # reservation des processeurs pour le job
+      # #MSUB -N 4 # reservation des nodes pour le job
+      # BATCH_NUM_PROC_TOT=32
+      #
+      # config.card :
+      # ATM= (gcm.e, lmdz.x, 8MPI, 4OMP)
 
       for comp in ${config_ListOfComponents[*]} ; do
@@ -1961 +1987 @@
           if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
 
+              IGCM_OMP=1
+
               if ( ${OK_PARA_OMP} ) ; then
                   eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
                   export OMP_NUM_THREADS=${comp_proc_omp_loc}
+                  IGCMG_OMP=${comp_proc_omp_loc}
               fi
               if  ( ${OK_PARA_MPI} ) ; then
                   eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+                  # Check if BATCH_NUM_PROC_TOT = (8*BRIDGE_MSUB_NNODE) 8 because we have 8 cores per node
+                  (( VERIF_BATCH_NUM_PROC_TOT = 8 * BRIDGE_MSUB_NNODE ))
+                  if ( [  ${VERIF_BATCH_NUM_PROC_TOT} -ne ${BATCH_NUM_PROC_TOT} ] ) ; then
+                      IGCM_debug_Print 1 "sys Titane : BATCH_NUM_PROC_TOT <> 8 * BRIDGE_MSUB_NNODE : ${BATCH_NUM_PROC_TOT} <> 8 * ${BRIDGE_MSUB_NNODE} "
+                      IGCM_debug_Print 1 "sys Titane : Check #MSUB -N xx into Job"
+                      IGCM_debug_Exit "IGCM_sys_build_execution_scripts"
+                  fi
+                  # Check if BATCH_NUM_PROC_TOT = (OMP_NUM_THREADS*BRIDGE_MSUB_NPROC)
+                  (( VERIF_BATCH_NUM_PROC_TOT = IGCMG_OMP * BRIDGE_MSUB_NPROC ))
+                  if ( [  ${VERIF_BATCH_NUM_PROC_TOT} -ne ${BATCH_NUM_PROC_TOT} ] ) ; then
+                      IGCM_debug_Print 1 "sys Titane : BATCH_NUM_PROC_TOT <> ${comp}_PROC_OMP * BRIDGE_MSUB_NPROC : ${BATCH_NUM_PROC_TOT} <> ${IGCMG_OMP} * ${BRIDGE_MSUB_NPROC} "
+                      IGCM_debug_Print 1 "sys Titane : Check #MSUB -n xx into Job"
+                      IGCM_debug_Exit "IGCM_sys_build_execution_scripts"
+                  fi
                   # Default : mpirun used if nb_proc gt 1
                   # pour sortie out/err par process ? more test required
@@ -1972 +2015 @@
                   EXECUTION="${HOST_MPIRUN_COMMAND} -np ${comp_proc_mpi_loc} ./${ExeNameOut}"
               else
+                  # Check if BATCH_NUM_PROC_TOT = IGCMG_OMP
+                  if ( [  ${IGCM_OMP} -ne ${BATCH_NUM_PROC_TOT} ] ) ; then
+                      IGCM_debug_Print 1 "sys Titane : BATCH_NUM_PROC_TOT <> ${comp}_PROC_OMP : ${BATCH_NUM_PROC_TOT} <> ${BRIDGE_MSUB_NPROC} "
+                      IGCM_debug_Print 1 "sys Titane : Check #MSUB -n xx into Job"
+                      IGCM_debug_Exit "IGCM_sys_build_execution_scripts"
+                  fi
                   # Default : mpirun is NOT used if nb_proc eq 1
                   # pour sortie out/err par process ? more test required
                   # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
-                  EXECUTION="time ./${ExeNameOut}"
+                  EXECUTION="/usr/bin/time ./${ExeNameOut}"
               fi
           fi