Changeset 1381 for trunk/libIGCM

Timestamp:

03/03/17 17:52:43 (7 years ago)

Author:

mafoipsl

Message:

See ticket #323. Let use ccc_mprun instead of mpirun for MPMD/MPI/OpenMP execution on curie as suggested by hotline. More reliable and more simple to use.

File:

• trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh (modified) (6 diffs)

trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh

@@ -178 +178 @@
 #====================================================
 #- HOST_MPIRUN_COMMAND
-typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1"}
+typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun "}
 
 #====================================================
@@ -1187 +1187 @@
   if ( ${OK_PARA_MPMD} ) ; then
 
+    if [ -f run_file ] ; then
+      IGCM_sys_Rm -f run_file
+    fi
+    touch run_file
+
     # Only MPI (MPMD)
     if  ( ! ${OK_PARA_OMP} ) ; then
-
-      if [ -f run_file ] ; then
-        IGCM_sys_Rm -f run_file
-      fi
-      touch run_file
 
       # Build run_file
@@ -1228 +1228 @@
       done
 
-      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
-
-      IGCM_sys_Chmod u+x run_file
-      if ( $DEBUG_sys ) ; then
-        echo "run_file contains : "
-        cat run_file
-      fi
-
     # MPI-OpenMP (MPMD)
+    # example of run_file
+    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
+    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
+    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
+
     else
-
-      # Use of mpirun instead of ccc_mprun
-      EXECUTION="time mpirun"
-
-      #  Hosts treatment
-      ${EXECUTION} hostname | sort | uniq > hosts.tmp
-
-      i=0
-      rm -f hosts rankfile
-      IGCM_debug_Print 1 "sys Curie, Hosts available :"
-      for nodes in `cat hosts.tmp`
-      do
-        host[$i]=$nodes
-        echo "${host[$i]}" >> hosts
-        IGCM_debug_Print 1 ${host[$i]}
-        i=$((i+1))
-      done
-      rm -f hosts.tmp
-
-      listnodes=${host[*]}
-
-      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
-
-      # Initialisation
-      rank=0
-      current_core=0
-      core_per_node=16
-      init_exec=n
 
       # Loop on the components
@@ -1285 +1254 @@
           eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
           eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+          # Build script files
 
           echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
@@ -1302 +1273 @@
               ;;
             esac
+            ### only for LMDZ?
             echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
             echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
             echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
             echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
+
           fi
 
+          # to have out/err per process on different files
+          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
           #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
-          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
-          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
-          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
-          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
-
-          if [ ${init_exec} = y ] ; then
-            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
-          else
-            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
-            init_exec=y
-          fi
-
-          # Build rankfile : method used to assign cores and nodes for the MPI process
-          # Ex :
-          #rank 0=curie5296 slot=0,1,2,3
-          #rank 1=curie5296 slot=4,5,6,7
-          # Example of final command :
-          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
-          # with script_lmdz.x.ksh :
-          # #!/bin/ksh
-          #export KMP_STACKSIZE=3g
-          #export KMP_LIBRARY=turnaround
-          #export MKL_SERIAL=YES
-          #OMP_NUM_THREADS=4
-          #./lmdz.x
-
-          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
-            (( index_host = current_core / core_per_node ))
-            host_value=${host[${index_host}]}
-            (( slot =  current_core % core_per_node ))
-            virg=","
-            string_final=""
-            for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
-              string=$index$virg
-              string_final=$string_final$string
-            done
-            string_final=$( echo $string_final | sed "s/.$//" )
-            echo "rank $rank=$host_value slot=$string_final" >> rankfile
-            (( rank = rank + 1 ))
-            (( current_core = current_core + comp_proc_omp_loc ))
-          done
+
+          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+
+          # Complete run_file
+
+          echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
+
         fi
       done
+
+      # export system variables required with MPMD/MPI/OpenMP ccc_mprun method
+
+      export BRIDGE_HETEROGENEOUS_MPMD=1
+      export OMPI_MCA_ess=pmi                  # Necessaire pour bullxmpi
+
+    fi
+
+    # for both MPMD/MPI and MPMD/MPI/OpenMP mode
+
+    EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
+
+    IGCM_sys_Chmod u+x run_file
+    if ( $DEBUG_sys ) ; then
+      echo "run_file contains : "
+      cat run_file
     fi
 
@@ -1362 +1319 @@
       eval ExeNameOut=\${config_Executable_${comp}[1]}
       if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
+
+        # Build script files
 
         echo "#!/bin/ksh" > script_${ExeNameOut}.ksh