Index: /branches/libIGCM_concurrent/AA_job
===================================================================
--- /branches/libIGCM_concurrent/AA_job	(revision 1627)
+++ /branches/libIGCM_concurrent/AA_job	(revision 1628)
@@ -35,4 +35,5 @@
 #-Q- irene #MSUB -c ::openMPthreads::  # Number of openMP threads. To specify only for SMPD
 #-Q- irene #MSUB -x                    # exclusive node. To specify only for MPMD together with the one below
+#-Q- irene #MSUB -N ::NodeNumber::
 #-Q- irene #MSUB -E '--cpu_bind=none'
 #-Q- irene #MSUB -T ::WallTime::              # Wall clock limit (seconds)
@@ -547,5 +548,11 @@
 	  typeset RET
 	  RUN_DATE_BEGIN=$( date '+%Y-%m-%dT%H:%M:%S' )
+          if [ -f EXECUTION.exe ] ; then
+             cat EXECUTION.exe
+             ./EXECUTION.exe
+             IGCM_sys_Cd ${RUN_DIR}
+          else
 	  ${EXECUTION}  >> ${Exe_Output} 2>&1
+          fi
 	  RET=$?
 	  RUN_DATE_END=$( date '+%Y-%m-%dT%H:%M:%S' )
@@ -583,5 +590,12 @@
   echo "#######################################"
   echo
+  if [ -f EXECUTION.exe ] ; then
+        echo "Main DIR"
   ls -lrt
+        echo "2nd RUNDIR"
+        ls -lrt RUNDIR_2
+  else
+        ls -lrt
+  fi
 
   #D-
Index: /branches/libIGCM_concurrent/libIGCM_comp/libIGCM_comp.ksh
===================================================================
--- /branches/libIGCM_concurrent/libIGCM_comp/libIGCM_comp.ksh	(revision 1627)
+++ /branches/libIGCM_concurrent/libIGCM_comp/libIGCM_comp.ksh	(revision 1628)
@@ -316,4 +316,9 @@
     typeset file_in_ file_in file_out_ file_out do_init
     for comp in ${config_ListOfComponents[*]} ; do
+	number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+	if [ X${number_rundir} != X ] ; then 
+	    [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+	    cd RUNDIR_${number_rundir} ;
+	fi
       # Initialize
       do_init="y"
@@ -366,4 +371,7 @@
           done
         fi
+      fi
+      if [ X${number_rundir} != X ] ; then
+	  cd $RUN_DIR 
       fi
     done
@@ -440,4 +448,9 @@
 
   for comp in ${config_ListOfComponents[*]} ; do
+      	number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+	if [ X${number_rundir} != X ] ; then 
+	    [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+	    cd RUNDIR_${number_rundir} ;
+	   fi
     # Define component
     eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
@@ -490,4 +503,7 @@
       done
     fi
+    if [ X${number_rundir} != X ] ; then 
+	cd $RUN_DIR 
+    fi
   done
 
@@ -513,4 +529,9 @@
 
   for comp in ${config_ListOfComponents[*]} ; do
+ 	number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+	if [ X${number_rundir} != X ] ; then 
+	    [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+	    cd RUNDIR_${number_rundir} ;
+	fi
 
     # Define component
@@ -586,4 +607,7 @@
       fi
     fi
+    if [ X${number_rundir} != X ] ; then 
+	cd $RUN_DIR 
+    fi
   done
 
@@ -619,4 +643,9 @@
   typeset comp compname comptagname card ListFilesName FileName0 NbFiles i i_ file_in file_out
   for comp in ${config_ListOfComponents[*]} ; do
+	number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+	if [ X${number_rundir} != X ] ; then 
+	    [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+	    cd RUNDIR_${number_rundir} ;
+	fi
     # Define component
     eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
@@ -654,4 +683,7 @@
       done
     fi
+    if [ X${number_rundir} != X ] ; then 
+	cd $RUN_DIR 
+    fi
 
   done
@@ -680,4 +712,10 @@
 
   for comp in ${config_ListOfComponents[*]} ; do
+	number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+	if [ X${number_rundir} != X ] ; then 
+	    [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+	    cd RUNDIR_${number_rundir} ;
+	fi
+  
     # Define component
     eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
@@ -1047,4 +1085,7 @@
       fi
     fi
+    if [ X${number_rundir} != X ] ; then 
+	cd $RUN_DIR 
+    fi
   done
 
@@ -1074,4 +1115,10 @@
   byPass=false
   for comp in ${config_ListOfComponents[*]} ; do
+	number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+	if [ X${number_rundir} != X ] ; then 
+	    [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+	    cd RUNDIR_${number_rundir} ;
+	fi
+
     # Define component
 
@@ -1111,4 +1158,8 @@
       IGCM_debug_Print 1 "Compiler is ${compilerVersion}"
     fi
+    if [ X${number_rundir} != X ] ; then 
+	cd $RUN_DIR 
+    fi  
+
   done
 
@@ -1606,4 +1657,9 @@
   typeset comp compname comptagname
   for comp in ${config_ListOfComponents[*]} ; do
+ 	number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+	if [ X${number_rundir} != X ] ; then 
+	    [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+	    cd RUNDIR_${number_rundir} ;
+	fi
     # Define component
     eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
@@ -1657,4 +1713,7 @@
       fi
     fi
+    if [ X${number_rundir} != X ] ; then 
+	cd $RUN_DIR 
+    fi
   done
 
@@ -1709,6 +1768,11 @@
     fi
   fi
-
-  for comp in ${config_ListOfComponents[*]} ; do
+for comp in ${config_ListOfComponents[*]} ; do
+ 	number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+	if [ X${number_rundir} != X ] ; then 
+	    [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+	    cd RUNDIR_${number_rundir} ;
+	fi
+
     # Define component
     eval compname=\${config_ListOfComponents_${comp}[0]} > /dev/null 2>&1
@@ -2042,4 +2106,7 @@
     fi
     echo
+    if [ X${number_rundir} != X ] ; then 
+	cd $RUN_DIR 
+    fi
   done
   # Append the sync call and the copy sequence to the IGCM_FlushRebuild function if needed
Index: /branches/libIGCM_concurrent/libIGCM_config/libIGCM_config.ksh
===================================================================
--- /branches/libIGCM_concurrent/libIGCM_config/libIGCM_config.ksh	(revision 1627)
+++ /branches/libIGCM_concurrent/libIGCM_config/libIGCM_config.ksh	(revision 1628)
@@ -805,4 +805,5 @@
   openMPthreads=0
   NbExec=0
+  first_slurm_comp=0
 
   OK_PARA_MPI=false
@@ -812,4 +813,16 @@
 
   for comp in ${config_ListOfComponents[*]} ; do
+
+      # Add a node in case of many execution in concurrent mode
+      number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+      if [ X${number_rundir} != X ] ; then
+	  if [ ${first_slurm_comp} = "0" ] ; then
+	      if [ $(( $coreNumber % $NB_CORE_PER_NODE )) -ne 0 ] ; then
+		  (( coreNumber = coreNumber + NB_CORE_PER_NODE - coreNumber % NB_CORE_PER_NODE ))
+	      fi
+	      echo $coreNumber
+	      first_slurm_comp=1 ;
+	  fi
+      fi
 
     # Manage component executable
Index: /branches/libIGCM_concurrent/libIGCM_sys/libIGCM_sys_irene.ksh
===================================================================
--- /branches/libIGCM_concurrent/libIGCM_sys/libIGCM_sys_irene.ksh	(revision 1627)
+++ /branches/libIGCM_concurrent/libIGCM_sys/libIGCM_sys_irene.ksh	(revision 1628)
@@ -1181,62 +1181,12 @@
   typeset file
   file=$1
-
-  if [ ${executionType} -eq 1 ] ; then
-    # MPMD + MPI
-    sed -e "/::openMPthreads::/d"                  \
-        -e "s/::JobNumProcTot::/${coreNumber}/"    \
-        -e "/#MSUB -x/d"                           \
+    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
+	(( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
+	[ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
+      sed -e "/::openMPthreads::/d"                  \
+        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
+	  -e "s/::NodeNumber::/${nodeNumber}/"    \
         -e "/--cpu_bind=none/d"                    \
       ${file} > ${file}.tmp
-
-  elif [ ${executionType} -eq 2 ] ; then
-    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
-    if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
-      sed -e "/::openMPthreads::/d"                  \
-          -e "s/::JobNumProcTot::/${coreNumber}/"    \
-        ${file} > ${file}.tmp
-    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
-      sed -e "/::openMPthreads::/d"                  \
-          -e "s/::JobNumProcTot::/${coreNumber}/"    \
-          -e "/--cpu_bind=none/d"                    \
-        ${file} > ${file}.tmp
-    else
-     IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
-     IGCM_debug_Exit "IGCM_sys_updateHeaders"
-    fi
-  elif [ ${executionType} -eq 3 ] ; then
-    # SPMD + MPI/OMP
-    sed -e "s/::openMPthreads::/${openMPthreads}/" \
-        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
-        -e "/#MSUB -x/d"                           \
-        -e "/--cpu_bind=none/d"                    \
-      ${file} > ${file}.tmp
-
-  elif [ ${executionType} -eq 4 ] ; then
-    # SPMD + MPI only
-    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
-        -e "/::openMPthreads::/d"                  \
-        -e "/#MSUB -x/d"                           \
-        -e "/--cpu_bind=none/d"                    \
-      ${file} > ${file}.tmp
-
-  elif [ ${executionType} -eq 5 ] ; then
-    # SPMD + OMP only
-    sed -e "s/::openMPthreads::/${openMPthreads}/" \
-        -e "/::JobNumProcTot::/d"                  \
-        -e "/#MSUB -x/d"                           \
-        -e "/--cpu_bind=none/d"                    \
-      ${file} > ${file}.tmp
-
-  elif [ ${executionType} -eq 6 ] ; then
-    # SEQUENTIAL THEN
-    sed -e "s/::JobNumProcTot::/1/"                \
-        -e "/::openMPthreads::/d"                  \
-        -e "/#MSUB -x/d"                           \
-        -e "/--cpu_bind=none/d"                    \
-      ${file} > ${file}.tmp
-
-  fi
-
   IGCM_sys_Mv ${file}.tmp ${file}
 
@@ -1270,4 +1220,232 @@
     echo "IGCM_sys_build_execution_scripts " $@
   fi
+  if ( [ "X${config_UserChoices_ExecutionMode}" = "Xslurm" ] ) ; then
+
+      EXECUTION="/usr/bin/time srun "
+
+        if ( ${OK_PARA_MPMD} ) ; then
+
+    # MPMD mode
+    # 1 MPI only : executionType=1
+    # 2 MPI/OpenMP : executionType=2
+
+    if [ -f run_file ] ; then
+      IGCM_sys_Rm -f run_file
+    fi
+    if [ -f RUNDIR_2/run_file ] ; then
+	IGCM_sys_Rm -f RUNDIR_2/run_file
+    fi
+    touch run_file
+
+
+
+    # 2 MPI/OpenMP : executionType=2
+
+    # MPI-OpenMP (MPMD)
+    # export SLURM_HOSTFILE=./hostlist 
+    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
+    # example of  hostlist file :
+    # r3i3n33
+    # r3i3n33
+    # ...
+    # example of run_file :
+    # 0-70 ./prog_lmdz.x.sh %o %t
+    # 71-430 ./prog_opa.xx.sh %o %t
+    # 431-431 ./prog_xios.x.sh %o %t
+    # examples of prog_file :
+    # prog_lmdz.x.sh :
+    # (( init = 0 + $1 ))
+    # (( index = init * 10 ))
+    # (( slot = index % 40 ))
+    # taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
+    # that will become
+    # taskset -c 0-9 ./script_lmdz.x.ksh
+    # ...
+    # with script_lmdz.x.ksh
+    # export OMP_STACKSIZE=3g
+    # export OMP_PLACES=cores
+    # export OMP_NUM_THREADS=10
+    # ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
+
+    #  Hosts treatment
+    _bkIFS=$IFS;
+	IFS=$'\n'; set -f
+	listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
+	IFS=$_bkIFS; set +f
+	rm -f hostlist
+
+     # Loop on the components to build run_file and script_exec files
+	rank=0
+	current_core=0
+        current_core_mpi=0
+	current_core_tmp=0
+        current_core_mpi_tmp=0
+	first_slurm_comp=0
+
+      for comp in ${config_ListOfComponents[*]} ; do
+
+	number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
+	  if [ X${number_rundir} != X ] ; then
+	      [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
+	      cd RUNDIR_${number_rundir} 
+	      if [ ${first_slurm_comp} = "0" ] ; then
+		  (( NbMPItasks_run1 = current_core_mpi_tmp ))
+		  ### On change de noeud pour le prochain srun
+		  if [ $(( $current_core % $NB_CORE_PER_NODE )) -ne 0 ] ; then
+		      (( current_core = current_core + NB_CORE_PER_NODE - current_core % NB_CORE_PER_NODE ))
+		  fi
+		  current_core_tmp=0 ; current_core_mpi_tmp=0 ; first_slurm_comp=1 ;
+	      fi
+	  fi
+
+	eval ExeNameIn=\${config_Executable_${comp}[0]}
+        eval ExeNameOut=\${config_Executable_${comp}[1]}
+
+        # Not possible if oasis has an executable (i.e old version of oasis3)
+        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
+          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
+          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
+          IGCM_debug_Verif_Exit
+        fi
+
+        # Only if we really have an executable for the component :
+        if [ "X${ExeNameOut}" != X\"\" ] ; then
+
+          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
+     
+
+          # Build script files
+
+          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+          echo ""  >> script_${ExeNameOut}.ksh
+          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
+            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
+            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
+          fi
+
+          # to have out/err per process on different files
+          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
+
+          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+
+        # Build run_file
+        # Only if the component has an executable
+        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
+
+          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+          (( end_core = ${current_core_mpi_tmp} + ${comp_proc_mpi_loc} - 1 ))
+          echo "${current_core_mpi_tmp}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
+          (( current_core_mpi_tmp = ${end_core} + 1 ))
+        fi
+
+          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
+              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
+          else
+              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
+          fi
+
+          # Build configuration file
+ 
+	      echo "#!/bin/sh" > prog_${ExeNameOut}.sh
+          echo "(( init = $current_core_tmp + \$1 ))" >> prog_${ExeNameOut}.sh
+          echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
+          echo "(( slot = index % 40 ))" >> prog_${ExeNameOut}.sh
+	      echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
+          echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
+
+          IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
+
+        # Build hostlist file
+
+          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
+             (( index_host = current_core / NB_CORE_PER_NODE ))
+             host_value=${listnodes[${index_host}]}
+             echo "$host_value" >> hostlist
+	     if [ ${DRYRUN_DEBUG} = 4 ] ; then
+		 echo "node_${index_host}_X" >> hostlist_template
+	     fi
+             (( current_core = current_core + offset_comp_proc_loc ))
+             (( current_core_tmp = current_core_tmp + offset_comp_proc_loc ))
+          done
+        fi
+	if [ X${number_rundir} != X ] ; then 
+	    cd $RUN_DIR ;
+	fi
+      done
+
+      ## variable added to stop after 60s instead of 600s by default. 
+      ## This is used when no error comes from executables and when something stopped an executable without notice. 
+      export SLURM_WAIT=60
+
+      if [ X${number_rundir} != X ] ; then
+	  echo "cd $RUN_DIR ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${NbMPItasks_run1} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution  2>&1 &" > EXECUTION.exe
+	  echo "cd RUNDIR_2 ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${current_core_mpi_tmp} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution  2>&1 &" >> EXECUTION.exe
+	  echo "wait" >> EXECUTION.exe
+	  chmod u+x EXECUTION.exe 
+      else
+	  EXECUTION="/usr/bin/time srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
+      fi
+      IGCM_sys_Chmod u+x run_file
+      if ( $DEBUG_sys ) ; then
+        echo "run_file contains : "
+        cat run_file
+      fi
+
+#    fi # if ${OK_PARA_MPMD}
+
+  else
+  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6
+
+    for comp in ${config_ListOfComponents[*]} ; do
+
+      # Only if we really have an executable for the component :
+      eval ExeNameOut=\${config_Executable_${comp}[1]}
+      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
+
+        # Build script files
+
+        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+        echo ""  >> script_${ExeNameOut}.ksh
+        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+
+        if ( ${OK_PARA_OMP} ) ; then
+            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+	    # Check if the number of threads is correct
+	    case ${comp_proc_omp_loc} in
+		2|4|5|10|20)
+		    IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
+		    ;;
+		*)
+		    IGCM_debug_Exit "ERROR with OMP parameters !"
+		    IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
+		    IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
+		    IGCM_debug_Verif_Exit
+		    ;;
+	    esac 
+            echo ""  >> script_${ExeNameOut}.ksh
+            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
+            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
+        fi
+
+        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+        
+        # To have out/err per process on different files
+        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
+        EXECUTION="/usr/bin/time srun ./script_${ExeNameOut}.ksh"
+
+        IGCM_debug_Print 1 "sys Jean-Zay : script_${ExeNameOut}.ksh contains"
+        cat script_${ExeNameOut}.ksh
+
+      fi
+
+    done
+
+  fi # ${OK_PARA_MPMD}
+      
+  else
 
   EXECUTION=${HOST_MPIRUN_COMMAND}
@@ -1590,4 +1768,5 @@
   fi # ${OK_PARA_MPMD}
 
+  fi
   IGCM_debug_Print 1 "sys Irene : execution command is "
   IGCM_debug_Print 1 "$EXECUTION"
