Index: /branches/libIGCM_MPI_OpenMP/AA_job
===================================================================
--- /branches/libIGCM_MPI_OpenMP/AA_job	(revision 489)
+++ /branches/libIGCM_MPI_OpenMP/AA_job	(revision 490)
@@ -440,22 +440,13 @@
 
     echo "========================================================================"
-    if ( [ X${BATCH_NUM_PROC_TOT} != X ] && [ "${BATCH_NUM_PROC_TOT}" -gt 1 ] ) ; then
-	MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND}
-	eval MPIRUN_OPTIONS=${config_UserChoices_JobRunOptions}
-    else
-	MPIRUN_COMMAND=${MPIRUN_COMMAND:="time "}
-	MPIRUN_OPTIONS=${MPIRUN_OPTIONS:=""}
-    fi
-
     if [ ${DRYRUN} -le 1 ] ; then
 	REAL_DATE_INIT=$( date )
-	echo "EXECUTION of : ${MPIRUN_COMMAND} ${MPIRUN_OPTIONS} ./${config_Executable_Name}   > ${Exe_Output} 2>&1"
 	echo                                                                                   > ${Exe_Output}
 	echo "#######################################"                                        >> ${Exe_Output}
-        echo "EXECUTION of : ${MPIRUN_COMMAND} ${MPIRUN_OPTIONS} ./${config_Executable_Name}" >> ${Exe_Output}
+        echo "EXECUTION of : ${EXECUTION}"                                                    >> ${Exe_Output}
 	echo                                                                                  >> ${Exe_Output}
 	typeset RET
 	RUN_DATE_BEGIN=$( date '+%Y-%m-%dT%H:%M:%S' )
-	${MPIRUN_COMMAND} ${MPIRUN_OPTIONS} ./${config_Executable_Name} >> ${Exe_Output} 2>&1
+	${EXECUTION}  >> ${Exe_Output} 2>&1
 	RET=$?
 	RUN_DATE_END=$( date '+%Y-%m-%dT%H:%M:%S' )
@@ -473,5 +464,6 @@
 	echo                                            >> ${Exe_Output}
     else
-	echo "EXECUTION of : ${MPIRUN_COMMAND} ${MPIRUN_OPTIONS} ./${config_Executable_Name} simulated for DRYRUN = " $DRYRUN
+        echo "EXECUTION of : ${EXECUTION} simulated for DRYRUN = " $DRYRUN
+        echo "EXECUTION of : ${EXECUTION} simulated for DRYRUN = " $DRYRUN > ${Exe_Output}
 	if ( $DEBUG_debug ) ; then
 	    echo "FOR EXECUTION DRYRUN mode = " $DRYRUN >> stack
Index: /branches/libIGCM_MPI_OpenMP/libIGCM_config/libIGCM_config.ksh
===================================================================
--- /branches/libIGCM_MPI_OpenMP/libIGCM_config/libIGCM_config.ksh	(revision 489)
+++ /branches/libIGCM_MPI_OpenMP/libIGCM_config/libIGCM_config.ksh	(revision 490)
@@ -108,6 +108,8 @@
     eval Script_Output_Prefix=${config_UserChoices_Script_Output_Prefix:='Script_Output'}
     IGCM_debug_Print 3 "Script_Output_Prefix = ${Script_Output_Prefix}"
-    eval Exe_Output=out_${config_Executable_Name}
-    IGCM_debug_Print 3 "Exe_Output           = ${Exe_Output}"
+#MM : obsolete !
+#    eval Exe_Output=out_${config_Executable_Name}
+    eval Exe_Output=out_execution
+#    IGCM_debug_Print 3 "Exe_Output           = ${Exe_Output}"
 
     #==================================
@@ -536,4 +538,133 @@
     fi
 
+    typeset ExeNameIn
+    typeset comp i
+    typeset tempvar tempvarMPI tempvarNOD NbElts j
+
+    (( PROCESSUS_NUMBER = 0 ))
+    (( i=0 ))
+
+    OK_PARA_MPI=false
+    OK_PARA_OMP=false
+    OK_PARA_NOD=false
+    for comp in ${config_ListOfComponents[*]} ; do
+
+	IGCM_debug_Print 1 ${comp}
+
+	eval ExeNameIn=\${config_Executable_${comp}[0]}
+
+	# NO order in config.card for parallelized values ! 
+	# just use suffix : MPI , OMP and NOD (for number of NODes.)
+
+	# NOD is the number of NODes allocated
+	eval ${comp}_PROC_NOD=0
+
+	# MPI is the number of MPI processus per nodes
+	eval ${comp}_PROC_MPI=0
+
+	# OMP is the number of OpenMP threads per MPI processus
+	eval ${comp}_PROC_OMP=0
+	
+	# Only if we really have an executable for the component :
+	if [ X${ExeNameIn} != X\"\" ] ; then
+	    eval NbElts=\${#config_Executable_${comp}[@]}
+	    if [ ${NbElts} -gt 2 ] ; then
+		(( j = 2 ))
+		eval ${comp}_PROC_MPI=1
+		eval ${comp}_PROC_OMP=1
+		eval ${comp}_PROC_NOD=1
+
+		while [ $j -lt ${NbElts} ] ; do 
+		    eval tempvar=\${config_Executable_${comp}[${j}]}
+
+		    echo 
+#MM BUG impossible ?!
+		    if [ X${tempvar} == X ] ; then
+			echo "Error reading MPI/OMP parameters !!!" 
+			echo "Check your config.card." 
+		    # MM : Ã  enlever :
+			exit 1
+                        #break
+		    fi
+
+# pb		    eval ${comp}_PROC_MPI=1
+# pb 		    eval ${comp}_PROC_OMP=1
+# pb		    eval ${comp}_PROC_NOD=1
+
+		    eval IGCM_debug_Print 2 ${tempvar}
+
+		    case ${tempvar} in
+			*MPI)
+                            # Read MPI parameter for composante
+			    eval ${comp}_PROC_MPI=$( echo ${tempvar} | sed -e "s/MPI//" )
+			    OK_PARA_MPI=true;;
+			*OMP)
+                            # Read OMP parameter for composante
+			    eval ${comp}_PROC_OMP=$( echo ${tempvar} | sed -e "s/OMP//" )
+			    OK_PARA_OMP=true;;
+			*NOD)
+                            # Read NOD (NumBer of Nodes) parameter for composante
+			    eval ${comp}_PROC_NOD=$( echo ${tempvar} | sed -e "s/NOD//" )
+			    OK_PARA_NOD=true
+			    OK_PARA_MPI=true;;
+		    esac
+
+		    (( j = j + 1 ))
+		done
+
+		eval tempvarMPI=\${${comp}_PROC_MPI}
+		eval tempvarNOD=\${${comp}_PROC_NOD}
+		eval tempvarOMP=\${${comp}_PROC_OMP}
+		(( PROCESSUS_NUMBER = PROCESSUS_NUMBER + tempvarMPI * tempvarNOD * tempvarOMP ))
+ 	    fi
+	fi
+
+	(( i=i+1 ))
+    done
+
+    # sequential case !
+    if [ ${PROCESSUS_NUMBER} -eq 0 ] ; then
+	(( PROCESSUS_NUMBER = 1 ))
+	echo "PROCESSUS_NUMBER is all 0 (sequential use of old definition in config->Executable->list)."
+	echo "We set it to 1."
+    fi
+
+    IGCM_debug_Print 1 "MPI/OMP treatment PROCESSUS_NUMBER = ${PROCESSUS_NUMBER}"
+
+    # Verification with PBS parameter
+    if [ X${BATCH_NUM_PROC_TOT} != X ] ; then
+	if [ ${BATCH_NUM_PROC_TOT} -ne ${PROCESSUS_NUMBER} ] ; then
+	    echo "Warning with parallelization parameters !"
+	    echo "PBS variable BATCH_NUM_PROC_TOT = ${BATCH_NUM_PROC_TOT} "
+	    echo " is the total number of _processors_ reserved."
+	    echo "It is not equal to the sum of _processus_ in config.card = ${PROCESSUS_NUMBER}."
+	fi
+	if ( ! ${OK_PARA_MPI} ) ; then
+	    if [ ${BATCH_NUM_PROC_TOT} -gt 1 ] ; then
+		echo "You have given a reservation of ${BATCH_NUM_PROC_TOT} processors,"
+		echo "but you havn't filled the parallel parameter in config->Executable->list."
+		echo "This is the old method and it is no more supported."
+		echo "Please read documentation or config specific comments."
+		echo "We stop now."
+		exit 1
+	    else 
+		PROCESSUS_NUMBER=1
+	    fi
+	fi
+    else
+	if ( ${OK_PARA_MPI} ) ; then
+	    echo "You have not given a value of ${BATCH_NUM_PROC_TOT} processors,"
+	    echo "but you have filled the parallel parameter in config->Executable->list."
+	    echo "This is the old method and it is no more supported."
+	    echo "Please read documentation or config specific comments."
+	    echo "We stop now."
+	    exit 1
+	fi
+    fi
+
+#     if ( ${OK_PARA_MPI} || ${OK_PARA_OMP} || ${OK_PARA_NOD} ) ; then
+    IGCM_sys_build_execution_scripts
+#     fi
+
     ExecutionFail=false
 
Index: /branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_titane.ksh
===================================================================
--- /branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_titane.ksh	(revision 489)
+++ /branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_titane.ksh	(revision 490)
@@ -167,4 +167,7 @@
 #- Max number of arguments passed to nco operator or demigration command
 UNIX_MAX_LIMIT=360
+
+NUM_COREPERNODE=8
+	
 
 #D-#==================================================
@@ -1266,5 +1269,31 @@
 	echo "IGCM_sys_activ_variables"
     fi
-    ulimit -s 2097152
+#    ulimit -s 2097152
+
+    typeset max_omp
+
+    ulimit -s unlimited
+
+    (( max_omp = 0 ))
+    for comp in ${config_ListOfComponents[*]} ; do
+	
+	eval ExeNameIn=\${config_Executable_${comp}[0]}
+	eval ExeNameOut=\${config_Executable_${comp}[1]}
+	
+    
+	# Only if we really have an executable for the component :
+	if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
+	    eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+
+	    if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then
+
+		(( max_omp = comp_proc_omp_loc ))
+	    fi
+	fi
+    done
+    if [ ${max_omp} -gt 1 ] ; then
+	module load openmp/${max_omp}thds
+    fi
+
     IGCM_debug_PopStack "IGCM_sys_activ_variables"
 }
@@ -1281,30 +1310,237 @@
 }
 
+
+# function IGCM_sys_build_run_file {
 ############################################################
 # Build run file
 
-function IGCM_sys_build_run_file {
-    IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
-    if ( $DEBUG_sys ) ; then
-	echo "IGCM_sys_build_run_file " $@
-    fi
-
-    # set Number of processors for OCE here
-    NUM_PROC_OCE=5
-
-    (( NUM_PROC_ATM = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE - 1))
-    (( nb_tot_m1    = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE ))
-    if [ $1 = MPI1 ]; then
-	cat <<EOF > run_file
--np 1 ./oasis
--np ${NUM_PROC_ATM} ./lmdz.x
--np ${NUM_PROC_OCE} ./opa.xx
-EOF
-	config_UserChoices_JobRunOptions='"--app"'
-	IGCM_sys_Chmod u+x run_file
-    fi
-    IGCM_debug_PopStack "IGCM_sys_build_run_file"
+#     IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
+#     if ( $DEBUG_sys ) ; then
+# 	echo "IGCM_sys_build_run_file " $@
+#     fi
+
+#     # set Number of processors for OCE here
+#     NUM_PROC_OCE=5
+
+#     (( NUM_PROC_ATM = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE - 1))
+#     (( nb_tot_m1    = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE ))
+#     if [ $1 = MPI1 ]; then
+# 	cat <<EOF > run_file
+# -np 1 ./oasis
+# -np ${NUM_PROC_ATM} ./lmdz.x
+# -np ${NUM_PROC_OCE} ./opa.xx
+# EOF
+# 	config_UserChoices_JobRunOptions='"--app"'
+# 	IGCM_sys_Chmod u+x run_file
+#     fi
+#     IGCM_debug_PopStack "IGCM_sys_build_run_file"
   
-}
+# }
+
+############################################################
+# Build MPI/OMP scripts
+function IGCM_sys_build_execution_scripts 
+{
+    IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
+    if ( $DEBUG_sys ) ; then
+        echo "IGCM_sys_build_execution_scripts " $@
+    fi
+
+    # Verification with PBS parameter
+    if [ X${BATCH_NUM_PROC_TOT} != X ] ; then
+
+	if ( ${OK_PARA_MPI} ) ; then
+	    EXECUTION=${HOST_MPIRUN_COMMAND}
+	else
+	    if [ ${BATCH_NUM_PROC_TOT} -eq 1 ] ; then
+		EXECUTION=${MPIRUN_COMMAND:="time "}
+	    fi
+	fi
+    else
+	if ( ! ${OK_PARA_MPI} ) ; then
+            EXECUTION=${MPIRUN_COMMAND:="time "}
+	fi
+    fi
+
+    if [ ${PROCESSUS_NUMBER} -gt 1 ] ; then
+
+    #  Hosts treatment
+
+	${EXECUTION} hostname | sort | uniq > hosts.tmp
+	
+	i=0
+	rm -f hosts
+	IGCM_debug_Print 1 "sys Titane, Hosts avaible :"
+	for nodes in `cat hosts.tmp`
+	do
+	    host[$i]=$nodes
+	    echo "${host[$i]} slots=8 max_slots=8" >> hosts
+	    IGCM_debug_Print 1 ${host[$i]}
+	    i=$((i+1))
+	done
+	rm -f hosts.tmp
+
+	listnodes=${host[*]}
+    
+	EXECUTION="${EXECUTION} -hostfile hosts" 
+    fi
+
+# Initialisation
+
+    init_node=y
+    node_num_current=0
+    start_num=0
+    init_exec=n
+
+
+# Test : if oasis is there, we put it at the first position
+		
+    for comp in ${config_ListOfComponents[*]} ; do
+	
+	if [ "X${comp}" = "XCPL" ]  ; then
+	    
+	    eval ExeNameIn=\${config_Executable_${comp}[0]}
+	    eval ExeNameOut=\${config_Executable_${comp}[1]}
+	    
+	    echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+	    echo ""  >> script_${ExeNameOut}.ksh
+	    echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+	    echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+	    echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+	    echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
+	    IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+	    
+	    init_node=n
+	    
+	    (( nombre_restant_node = NUM_COREPERNODE - 1 ))
+	    node_num_current=0
+	    node_current=${host[${node_num_current}]}
+	    
+	    EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh" 
+	    
+            init_exec=y
+	    start_num=1
+	    
+	fi
+	
+    done
+    
+# Then loop on the components (except for oasis)
+
+    for comp in ${config_ListOfComponents[*]} ; do
+	
+	eval ExeNameIn=\${config_Executable_${comp}[0]}
+	eval ExeNameOut=\${config_Executable_${comp}[1]}
+	
+    
+	# Only if we really have an executable for the component :
+	if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
+
+	    eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
+	    eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
+	    
+	    if ( ${OK_PARA_MPI} ) ; then 
+		
+		echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
+#		echo "set -vx" >> script_${ExeNameOut}.ksh
+		echo ""  >> script_${ExeNameOut}.ksh
+		echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
+		echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
+		echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
+		echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
+ 		echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
+		echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
+		IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
+		
+		node_num=0
+		
+# We define the number of MPI process to be assigned for the component 
+
+		nombre_restant_comp=${comp_proc_mpi_loc}
+		
+# Loop on the allocated nodes
+
+		for node in ${listnodes} ; do
+
+# We go to the current node
+
+		    if [ ${node_num} = ${node_num_current} ] ; then
+
+			node_current=${host[${node_num_current}]}
+
+# If first time on the node : initialisation 
+
+			if [ ${init_node} = y ] ; then
+			    nombre_restant_node=${NUM_COREPERNODE}
+			fi
+	
+# Test on the number of OMP threads
+
+			if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
+			    (( node_num = node_num + 1 ))
+			    node_num_current=${node_num}
+			    init_node=y
+			    continue
+			fi
+
+# Number of MPI process to assign
+
+			(( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
+ 
+			if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
+			    num_corempi=${nombre_restant_comp}
+			fi
+
+			(( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
+			(( nombre_restant_comp = nombre_restant_comp - num_corempi ))
+		    
+			if [ ${init_exec} = y ] ; then
+			    EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
+			else	
+			    EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
+			    init_exec=y
+			fi
+
+			((  start_num = num_corempi + start_num ))
+	
+		    else
+			
+			(( node_num = node_num + 1 )) 
+			continue
+		    fi
+		
+# Test on the number of core/process remaining on the node/component 
+		
+		    if [ ${nombre_restant_node} = 0 ] ; then
+			(( node_num = node_num + 1 ))
+			node_num_current=${node_num}
+			init_node=y
+
+			if [ ${nombre_restant_comp} = 0 ] ; then
+			    break 1
+			fi
+		    else
+
+			node_num_current=${node_num}
+			init_node=n
+			
+			if [ ${nombre_restant_comp} = 0 ] ; then
+			    break 1
+			fi
+		    fi
+		done
+	    else
+		EXECUTION="${EXECUTION} ./${ExeNameOut}"
+	    fi
+	fi
+    done
+
+    IGCM_debug_Print 1 "sys Titane : La commande d execution est "
+    IGCM_debug_Print 1 $EXECUTION
+
+    IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
+}
+
+
 
 ##############################################################
