Changeset 621 for trunk/libIGCM/libIGCM_sys

Timestamp:

03/30/12 15:42:12 (12 years ago)

Author:

sdipsl

Message:

• I messed up the merge (episode 2 : it will be at least a trilogy) mercure sx9 and titane

Location:

trunk/libIGCM/libIGCM_sys

Files:

• libIGCM_sys_mercurex9.ksh (modified) (1 diff)
• libIGCM_sys_titane.ksh (modified) (2 diffs)

trunk/libIGCM/libIGCM_sys/libIGCM_sys_mercurex9.ksh

@@ -1478 +1478 @@
 
 ############################################################
-# Build run file
+# Build MPI/OMP scripts
+function IGCM_sys_build_execution_scripts 
+{
+    IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
+    if ( $DEBUG_sys ) ; then
+        echo "IGCM_sys_build_execution_scripts " $@
+    fi
+
+    typeset NbNodes_Job NbProc_Job comp_proc_mpi_loc comp_proc_omp_loc mpi_count
+    
+    if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
+        IGCM_debug_Exit "IGCM_sys_mercurex9 build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
+    fi
+
+    if ( ${OK_PARA_MPMD} ) ; then
+
+        if [ -f run_file ] ; then
+            IGCM_sys_Rm -f run_file
+        fi
+        touch run_file
+
+        if ( ${OK_PARA_OMP} ) ; then
+                IGCM_debug_Print 2 "Error in config.card on SX9 : OpenMP not available"
+                exit 1
+        else
+# OLD :
+#PBS -v BATCH_NUM_PROC_TOT=4
+#PBS -l cpunum_job=${BATCH_NUM_PROC_TOT}
+            echo "Job_${config_UserChoices_JobName} includes BATCH_NUM_PROC_TOT = \c"
+            cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep -- "-v *BATCH_NUM_PROC_TOT" | sed -e "s/.*BATCH_NUM_PROC_TOT *= *//" 
+            NbProc_Job=$( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep -- "-v *BATCH_NUM_PROC_TOT" | sed -e "s/.*BATCH_NUM_PROC_TOT *= *//" )
+            NbProc_Job=${NbProc_Job:=0}
+            if [ ${NbProc_Job} -eq 0 ] ; then
+                IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no BATCH_NUM_PROC_TOT defined with MPI only run."
+                exit 1
+            fi
+        fi
+
+# run_file construction
+
+# Then first loop on the components for the coupler ie oasis
+
+### the coupler ie oasis must be the first one
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+            
+        # for CPL component only
+        if [ "X${comp}" = "XCPL" ] ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+            
+            if ( ${OK_PARA_MPI} ) ; then 
+                echo "-p ${comp_proc_mpi_loc} -e ./${ExeNameOut}" >> run_file 
+            fi
+        fi
+    done
+
+# Then second loop on the components
+
+    for comp in ${config_ListOfComponents[*]} ; do
+        
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+            
+        # Only if we really have an executable for the component and not the coupler ie oasis:
+        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
+
+            eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+            
+            if ( ${OK_PARA_MPI} ) ; then 
+                    echo "-p ${comp_proc_mpi_loc} -e ./${ExeNameOut}" >> run_file 
+            fi
+        fi
+    done
+
+    EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
+
+    IGCM_sys_Chmod u+x run_file
+    if ( $DEBUG_sys ) ; then
+      echo "run_file contains : "
+      cat run_file
+    fi
+
+else # Only one executable. launch it.
+
+    for comp in ${config_ListOfComponents[*]} ; do
+
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        [ "X${ExeNameOut}" != X\"\" ] &&  EXECUTION="time mpirun -np ${BATCH_NUM_PROC_TOT} ./${ExeNameOut}"
+
+    done
+
+fi
+
+    IGCM_debug_Print 1 "sys mercurex9 : execution command is "
+    IGCM_debug_Print 1 "$EXECUTION"
+
+    IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
+
+}
 
 function IGCM_sys_build_run_file {
-  IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
-  if ( $DEBUG_sys ) ; then
-    echo "IGCM_sys_build_run_file " $@
-  fi
-  (( NUM_PROC_ATM = BATCH_NUM_PROC_TOT - 1 ))
-  (( NUM_PROC_OASIS = BATCH_NUM_PROC_TOT - NUM_PROC_ATM ))
-  (( NUM_PROC_OCE = BATCH_NUM_PROC_TOT - NUM_PROC_ATM ))
-  
-  if [ $1 = MPI2 ]; then
-    cat <<EOF > run_file
--p 1 -np 1 -e ./oasis
-EOF
-    (( NUM_PROCESS = BATCH_NUM_PROC_TOT + 1 ))
-    config_UserChoices_JobRunOptions='"-max_np ${NUM_PROCESS} -f"'
-
-  elif [ $1 = MPI1 ]; then
-    cat <<EOF > run_file
--p $NUM_PROC_OASIS -e ./oasis
--p $NUM_PROC_ATM -e ./lmdz.x
--p $NUM_PROC_OCE -e ./opa.xx
-EOF
-  fi
-  IGCM_debug_PopStack "IGCM_sys_build_run_file"
-}
+
+IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
+
+}

trunk/libIGCM/libIGCM_sys/libIGCM_sys_titane.ksh

@@ -1663 +1663 @@
     echo "IGCM_sys_activ_variables"
   fi
-  ulimit -s 2097152
+
+  typeset max_omp
+
+  ulimit -s unlimited
+
+  (( max_omp = 0 ))
+  for comp in ${config_ListOfComponents[*]} ; do
+        
+    eval ExeNameIn=\${config_Executable_${comp}[0]}
+    eval ExeNameOut=\${config_Executable_${comp}[1]}
+    
+    # Only if we really have an executable for the component :
+    if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
+      eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+      if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
+        (( max_omp = comp_proc_omp_loc ))
+      fi
+    fi
+  done
+  if [ ${max_omp} -gt 1 ] ; then
+    module load openmp/${max_omp}thds
+  fi
+
   IGCM_debug_PopStack "IGCM_sys_activ_variables"
 }
@@ -1679 +1701 @@
 
 ############################################################
-# Build run file
+# Build MPI/OMP scripts run file (dummy function)
 
 function IGCM_sys_build_run_file {
-  IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
-  if ( $DEBUG_sys ) ; then
-    echo "IGCM_sys_build_run_file " $@
-  fi
-
-  # set Number of processors for OCE here
-  NUM_PROC_OCE=5
-
-  (( NUM_PROC_ATM = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE - 1))
-  (( nb_tot_m1    = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE ))
-  if [ $1 = MPI1 ]; then
-    cat <<EOF > run_file
--np 1 ./oasis
--np ${NUM_PROC_ATM} ./lmdz.x
--np ${NUM_PROC_OCE} ./opa.xx
-EOF
-    config_UserChoices_JobRunOptions='"--app"'
-    IGCM_sys_Chmod u+x run_file
-  fi
-  IGCM_debug_PopStack "IGCM_sys_build_run_file"
-  
+
+IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
+
+}
+
+############################################################
+# Build MPI/OMP scripts
+function IGCM_sys_build_execution_scripts 
+{
+  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
+  if ( $DEBUG_sys ) ; then
+    echo "IGCM_sys_build_execution_scripts " $@
+  fi
+  typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut 
+  typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc 
+  typeset num_corempi nombre_restant_node nombre_restant_comp
+
+  if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
+    IGCM_debug_Exit "IGCM_sys_titane build_execution_scripts : Job_${config_UserChoices_JobName} doesn't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
+  fi
+
+  if ( ${OK_PARA_MPMD} ) ; then
+
+    if [ -f run_file ] ; then
+      IGCM_sys_Rm -f run_file
+    fi
+    touch run_file
+
+    if ( ${OK_PARA_OMP} ) ; then
+
+      #  Hosts treatment
+
+      ${HOST_MPIRUN_COMMAND} hostname | sort | uniq > hosts.tmp
+        
+      i=0
+      rm -f hosts
+      IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
+      for nodes in `cat hosts.tmp` ; do
+        host[$i]=$nodes
+        echo "${host[$i]} slots=8 max_slots=8" >> hosts
+        IGCM_debug_Print 1 ${host[$i]}
+        i=$((i+1))
+      done
+      rm -f hosts.tmp
+            
+      listnodes=${host[*]}
+            
+      EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts" 
+        
+      # Initialisation
+
+      init_node=y
+      node_num_current=0
+      start_num=0
+      init_exec=n
+
+      # Test : if oasis is there, we put it at the first position
+            
+      for comp in ${config_ListOfComponents[*]} ; do
+                
+        if [ "X${comp}" = "XCPL" ]  ; then
+                    
+          eval ExeNameIn=\${config_Executable_${comp}[0]}
+          eval ExeNameOut=\${config_Executable_${comp}[1]}
+                    
+          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+          echo ""  >> script_${ExeNameOut}.ksh
+          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
+          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+                    
+          init_node=n
+                    
+          (( nombre_restant_node = NUM_COREPERNODE - 1 ))
+          node_num_current=0
+          node_current=${host[${node_num_current}]}
+                    
+          EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh" 
+                    
+          init_exec=y
+          start_num=1
+                    
+        fi
+                
+      done
+            
+      # Then loop on the components (except for oasis)
+
+      for comp in ${config_ListOfComponents[*]} ; do
+                
+        eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+                
+        # Only if we really have an executable for the component :
+        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
+
+          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+                    
+          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+          # echo "set -vx" >> script_${ExeNameOut}.ksh
+          echo ""  >> script_${ExeNameOut}.ksh
+          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
+          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
+          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
+          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+                    
+          node_num=0
+                    
+          # We define the number of MPI process to be assigned for the component 
+
+          nombre_restant_comp=${comp_proc_mpi_loc}
+                    
+          # Loop on the allocated nodes
+
+          for node in ${listnodes} ; do
+
+            # We go to the current node
+            if [ ${node_num} = ${node_num_current} ] ; then
+
+              node_current=${host[${node_num_current}]}
+
+              # If first time on the node : initialisation 
+
+              if [ ${init_node} = y ] ; then
+                nombre_restant_node=${NUM_COREPERNODE}
+              fi
+                            
+              # Test on the number of OMP threads
+
+              if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
+                (( node_num = node_num + 1 ))
+                node_num_current=${node_num}
+                init_node=y
+                continue
+              fi
+
+              # Number of MPI process to assign
+
+              (( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
+                            
+              if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
+                num_corempi=${nombre_restant_comp}
+              fi
+
+              (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
+              (( nombre_restant_comp = nombre_restant_comp - num_corempi ))
+                            
+              if [ ${init_exec} = y ] ; then
+                EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
+              else      
+                EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
+                init_exec=y
+              fi
+
+              ((  start_num = num_corempi + start_num ))
+                            
+            else
+                            
+              (( node_num = node_num + 1 )) 
+              continue
+            fi
+                        
+            # Test on the number of core/process remaining on the node/component 
+                        
+            if [ ${nombre_restant_node} = 0 ] ; then
+              (( node_num = node_num + 1 ))
+              node_num_current=${node_num}
+              init_node=y
+
+              if [ ${nombre_restant_comp} = 0 ] ; then
+                break 1
+              fi
+            else
+
+              node_num_current=${node_num}
+              init_node=n
+                            
+              if [ ${nombre_restant_comp} = 0 ] ; then
+                break 1
+              fi
+            fi
+          done
+        fi
+      done
+            
+    else
+
+        # Then first loop on the components for the coupler ie oasis
+
+        ## the coupler ie oasis must be the first one
+        for comp in ${config_ListOfComponents[*]} ; do
+            
+            eval ExeNameOut=\${config_Executable_${comp}[1]}
+                
+        # for CPL component only
+            if [ "X${comp}" = "XCPL" ] ; then
+                eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+                echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file 
+            fi
+        done
+
+        # Then second loop on the components
+
+        for comp in ${config_ListOfComponents[*]} ; do
+                
+            eval ExeNameOut=\${config_Executable_${comp}[1]}
+                
+            # Only if we really have an executable for the component and not the coupler ie oasis:
+            if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
+                eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+                echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file 
+            fi
+        done
+        IGCM_sys_Chmod u+x run_file
+
+        EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
+          
+    fi
+
+  else # Only one executable. launch it.
+
+      for comp in ${config_ListOfComponents[*]} ; do
+
+          # Only if we really have an executable for the component :
+          eval ExeNameOut=\${config_Executable_${comp}[1]}
+          if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then   
+                
+              echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+              echo ""  >> script_${ExeNameOut}.ksh
+              if ( ${OK_PARA_OMP} ) ; then 
+                  eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+                  echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh       
+              fi
+              if  ( ${OK_PARA_MPI} ) ; then 
+                  # Default : mpirun used if nb_proc gt 1
+                  # pour sortie out/err par process
+                  # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}"  >> script_${ExeNameOut}.ksh
+                  echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
+                  IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+                  EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
+              else
+                  # Default : mpirun is NOT used if nb_proc eq 1
+                  # pour sortie out/err par process
+                  # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
+                  echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
+                  IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+                  EXECUTION="time ./script_${ExeNameOut}.ksh"
+              fi
+          fi
+      done
+      
+  fi
+
+  IGCM_debug_Print 1 "sys Titane : La commande d execution est "
+  IGCM_debug_Print 1 $EXECUTION
+
+  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
 }