- Timestamp:
- 02/28/12 15:20:54 (12 years ago)
- Location:
- branches/libIGCM_MPI_OpenMP
- Files:
-
- 12 edited
- 1 copied
Legend:
- Unmodified
- Added
- Removed
-
branches/libIGCM_MPI_OpenMP/AA_additionnal
r519 r563 10 10 #-Q- cesium #MSUB -E "-j o" 11 11 #-Q- cesium #MSUB -E "-S /bin/ksh" 12 #-Q- curie ###################### 13 #-Q- curie ## CURIE TGCC/CEA ## 14 #-Q- curie ###################### 15 #-Q- curie #MSUB -r TS # Nom du job 16 #-Q- curie #MSUB -eo 17 #-Q- curie #MSUB -n 1 # Reservation du processus 18 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 12 19 #-Q- platine #!/usr/bin/ksh 13 20 #-Q- platine ################## -
branches/libIGCM_MPI_OpenMP/AA_atlas_LMDZ
r519 r563 9 9 #-Q- cesium #MSUB -E "-j o" 10 10 #-Q- cesium #MSUB -E "-S /bin/ksh" 11 #-Q- curie ###################### 12 #-Q- curie ## CURIE TGCC/CEA ## 13 #-Q- curie ###################### 14 #-Q- curie #MSUB -r TS # Nom du job 15 #-Q- curie #MSUB -eo 16 #-Q- curie #MSUB -n 1 # Reservation du processus 17 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 11 18 #-Q- platine #!/usr/bin/ksh 12 19 #-Q- platine ################## -
branches/libIGCM_MPI_OpenMP/AA_atlas_ORCA_LIM
r519 r563 9 9 #-Q- cesium #MSUB -E "-j o" 10 10 #-Q- cesium #MSUB -E "-S /bin/ksh" 11 #-Q- curie ###################### 12 #-Q- curie ## CURIE TGCC/CEA ## 13 #-Q- curie ###################### 14 #-Q- curie #MSUB -r TS # Nom du job 15 #-Q- curie #MSUB -eo 16 #-Q- curie #MSUB -n 1 # Reservation du processus 17 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 11 18 #-Q- platine #!/usr/bin/ksh 12 19 #-Q- platine ################## -
branches/libIGCM_MPI_OpenMP/AA_atlas_ORCHIDEE
r519 r563 9 9 #-Q- cesium #MSUB -E "-j o" 10 10 #-Q- cesium #MSUB -E "-S /bin/ksh" 11 #-Q- curie ###################### 12 #-Q- curie ## CURIE TGCC/CEA ## 13 #-Q- curie ###################### 14 #-Q- curie #MSUB -r TS # Nom du job 15 #-Q- curie #MSUB -eo 16 #-Q- curie #MSUB -n 1 # Reservation du processus 17 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 11 18 #-Q- platine #!/usr/bin/ksh 12 19 #-Q- platine ################## -
branches/libIGCM_MPI_OpenMP/AA_atlas_PISCES
r519 r563 9 9 #-Q- cesium #MSUB -E "-j o" 10 10 #-Q- cesium #MSUB -E "-S /bin/ksh" 11 #-Q- curie ###################### 12 #-Q- curie ## CURIE TGCC/CEA ## 13 #-Q- curie ###################### 14 #-Q- curie #MSUB -r TS # Nom du job 15 #-Q- curie #MSUB -eo 16 #-Q- curie #MSUB -n 1 # Reservation du processus 17 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 11 18 #-Q- platine #!/usr/bin/ksh 12 19 #-Q- platine ################## -
branches/libIGCM_MPI_OpenMP/AA_create_multi_se
r519 r563 9 9 #-Q- cesium #MSUB -E "-j o" 10 10 #-Q- cesium #MSUB -E "-S /bin/ksh" 11 #-Q- curie ###################### 12 #-Q- curie ## CURIE TGCC/CEA ## 13 #-Q- curie ###################### 14 #-Q- curie #MSUB -r TS # Nom du job 15 #-Q- curie #MSUB -eo 16 #-Q- curie #MSUB -n 1 # Reservation du processus 17 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 11 18 #-Q- platine #!/usr/bin/ksh 12 19 #-Q- platine ################### -
branches/libIGCM_MPI_OpenMP/AA_create_se
r519 r563 9 9 #-Q- cesium #MSUB -E "-j o" 10 10 #-Q- cesium #MSUB -E "-S /bin/ksh" 11 #-Q- curie ###################### 12 #-Q- curie ## CURIE TGCC/CEA ## 13 #-Q- curie ###################### 14 #-Q- curie #MSUB -r TS # Nom du job 15 #-Q- curie #MSUB -eo 16 #-Q- curie #MSUB -n 1 # Reservation du processus 17 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 11 18 #-Q- platine #!/usr/bin/ksh 12 19 #-Q- platine ################### -
branches/libIGCM_MPI_OpenMP/AA_create_ts
r515 r563 9 9 #-Q- cesium #MSUB -E "-j o" 10 10 #-Q- cesium #MSUB -E "-S /bin/ksh" 11 #-Q- curie ###################### 12 #-Q- curie ## CURIE TGCC/CEA ## 13 #-Q- curie ###################### 14 #-Q- curie #MSUB -r TS # Nom du job 15 #-Q- curie #MSUB -eo 16 #-Q- curie #MSUB -n 1 # Reservation du processus 17 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 11 18 #-Q- platine #!/usr/bin/ksh 12 19 #-Q- platine ################### -
branches/libIGCM_MPI_OpenMP/AA_job
r512 r563 10 10 #-Q- cesium #MSUB -E "-j o" 11 11 #-Q- cesium #MSUB -E "-S /bin/ksh" 12 #-Q- curie #!/usr/bin/ksh 13 #-Q- curie ###################### 14 #-Q- curie ## CURIE TGCC/CEA ## 15 #-Q- curie ###################### 16 #-Q- curie #MSUB -r ::Jobname:: # nom de la requete 17 #-Q- curie #MSUB -o Script_Output_::Jobname::.000001 # nom du fichier de sortie 18 #-Q- curie #MSUB -e Script_Output_::Jobname::.000001 # nom du fichier de sortie 19 #-Q- curie #MSUB -eo 20 #-Q- curie #MSUB -n ::JobNumProcTot:: # reservation des processeurs pour le job 21 #-Q- curie #MSUB -T 86400 # Limite temps (en secondes) 22 #-Q- curie ##MSUB -p XXXXXXX 23 #-Q- curie BATCH_NUM_PROC_TOT=$BRIDGE_MSUB_NPROC 12 24 #-Q- platine #!/usr/bin/ksh 13 25 #-Q- platine ################## -
branches/libIGCM_MPI_OpenMP/AA_monitoring
r519 r563 9 9 #-Q- cesium #MSUB -E "-j o" 10 10 #-Q- cesium #MSUB -E "-S /bin/ksh" 11 #-Q- curie ###################### 12 #-Q- curie ## CURIE TGCC/CEA ## 13 #-Q- curie ###################### 14 #-Q- curie #MSUB -r TS # Nom du job 15 #-Q- curie #MSUB -eo 16 #-Q- curie #MSUB -n 1 # Reservation du processus 17 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 11 18 #-Q- platine #!/usr/bin/ksh 12 19 #-Q- platine ################## -
branches/libIGCM_MPI_OpenMP/AA_rebuild_fromArchive
r519 r563 9 9 #-Q- cesium #MSUB -E "-j o" 10 10 #-Q- cesium #MSUB -E "-S /bin/ksh" 11 #-Q- curie ###################### 12 #-Q- curie ## CURIE TGCC/CEA ## 13 #-Q- curie ###################### 14 #-Q- curie #MSUB -r TS # Nom du job 15 #-Q- curie #MSUB -eo 16 #-Q- curie #MSUB -n 1 # Reservation du processus 17 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 11 18 #-Q- platine #!/usr/bin/ksh 12 19 #-Q- platine ################### -
branches/libIGCM_MPI_OpenMP/AA_rebuild_fromWorkdir
r519 r563 9 9 #-Q- cesium #MSUB -E "-j o" 10 10 #-Q- cesium #MSUB -E "-S /bin/ksh" 11 #-Q- curie ###################### 12 #-Q- curie ## CURIE TGCC/CEA ## 13 #-Q- curie ###################### 14 #-Q- curie #MSUB -r TS # Nom du job 15 #-Q- curie #MSUB -eo 16 #-Q- curie #MSUB -n 1 # Reservation du processus 17 #-Q- curie #MSUB -T 86400 # Limite de temps elapsed du job 11 18 #-Q- platine #!/usr/bin/ksh 12 19 #-Q- platine ################### -
branches/libIGCM_MPI_OpenMP/libIGCM_sys/libIGCM_sys_curie.ksh
r562 r563 68 68 typeset LOGIN=${LOGIN:=$( whoami )} 69 69 # $hostname of the MASTER job 70 typeset MASTER= titane70 typeset MASTER=curie 71 71 72 72 #D- … … 80 80 typeset -r RSYNC_opt="-va" 81 81 # ie storage filesystem 82 typeset -r RHOST= titane82 typeset -r RHOST=curie 83 83 84 84 #==================================================== … … 90 90 # Set environment tools (ferret, nco, cdo) 91 91 #==================================================== 92 . / home/cont003/p86ipsl/.atlas_env_titane_ksh92 . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_curie_ksh 93 93 94 94 #==================================================== … … 107 107 #==================================================== 108 108 #- SUBMIT_DIR : submission dir 109 typeset SUBMIT_DIR=${SUBMIT_DIR:=${ LS_SUBCWD}}109 typeset SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}} 110 110 111 111 #==================================================== 112 112 #- ARCHIVE 113 typeset -r ARCHIVE=${DMFDIR} 114 115 #==================================================== 116 #- Mirror libIGCM from titane to cesium if needed 117 ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $2}' ) 118 if [ ! ${ROOTSYS} = "home" ] ; then 119 typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true} 120 else 121 typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false} 122 fi 113 typeset -r ARCHIVE=${CCCSTOREDIR} 114 115 #==================================================== 116 typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false} 123 117 124 118 #==================================================== 125 119 #- libIGCM_POST 126 if ( ${MirrorlibIGCM} ) ; then 127 PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" ) 128 typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM 129 else 130 typeset -r libIGCM_POST=${libIGCM} 131 fi 120 typeset -r libIGCM_POST=${libIGCM} 132 121 133 122 #==================================================== 134 123 #- IN 135 typeset -r R_IN=${R_IN:=/ dmnfs/cont003/p86ipsl/IGCM}136 typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ dmnfs/cont003/p24data}124 typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM} 125 typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data} 137 126 138 127 #==================================================== … … 146 135 #==================================================== 147 136 #- OUT_POST 148 typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT 137 typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT_POST 149 138 150 139 #==================================================== 151 140 #- RUN_DIR_PATH : Temporary working directory (=> TMP) 152 if [ ! X${ LSB_QUEUE} = Xmono] ; then153 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${ LSB_JOBID}}141 if [ ! X${BRIDGE_MSUB_NPROC} = X1 ] ; then 142 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}} 154 143 else 155 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${ LSB_JOBID}}144 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/TMPDIR_IGCM/${BRIDGE_MSUB_JOBID}} 156 145 fi 157 146 … … 162 151 #==================================================== 163 152 #- HOST_MPIRUN_COMMAND 164 typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}153 typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time ccc_mprun"} 165 154 166 155 #==================================================== … … 224 213 fi 225 214 226 typeset NB_ESSAI DELAI status i 227 # keep standard input for the loop onto temporary file 228 cat >tmp_IGCM_sys_RshPost_$$ 229 230 #echo cat tmp_IGCM_sys_RshPost_$$ INITIAL 231 #cat tmp_IGCM_sys_RshPost_$$ 232 233 if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then 234 # little hack so that rebuild submission is done on titane not an cesium 235 # 236 libIGCM_POST_sed=$( echo $libIGCM_POST | sed 's/\//\\\//g' ) 237 POST_DIR_sed=$( echo ${POST_DIR} | sed 's/\//\\\//g' ) 238 sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt 239 sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$ 240 sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt 241 sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$ 242 #\mv tmp.txt tmp_IGCM_sys_RshPost_$$ 243 # 244 #echo cat tmp_IGCM_sys_RshPost_$$ AFTER 245 #cat tmp_IGCM_sys_RshPost_$$ 246 /bin/ksh <tmp_IGCM_sys_RshPost_$$ 215 /bin/ksh ${@} 247 216 if [ $? -gt 0 ] ; then 248 217 echo "IGCM_sys_RshPost : erreur." 249 218 IGCM_debug_Exit "IGCM_sys_RshPost" 250 219 fi 251 # delete temporary file252 \rm tmp_IGCM_sys_RshPost_$$253 else254 # number of tentative255 NB_ESSAI=10256 # time delay between tentative257 DELAI=10258 i=0259 while [ $i -ne $NB_ESSAI ] ; do260 ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$261 status=$?262 if [ ${status} -ne 0 ];263 then264 sleep $DELAI265 else266 break267 fi268 let i=$i+1269 done270 # delete temporary file271 \rm tmp_IGCM_sys_RshPost_$$272 273 if [ ${status} -gt 0 ] ; then274 echo "IGCM_sys_RshPost : erreur."275 IGCM_debug_Exit "IGCM_sys_RshPost"276 fi277 fi278 220 IGCM_debug_PopStack "IGCM_sys_RshPost" 279 221 } … … 295 237 status=completed 296 238 fi 297 298 ssh -t titane996 ssh platine /bin/ksh <<-EOF299 export LOGIN=${LOGIN}300 export config_UserChoices_JobName=${config_UserChoices_JobName}301 export config_UserChoices_MailName=${config_UserChoices_MailName}302 export DateBegin=${DateBegin}303 export DateEnd=${DateEnd}304 export R_SAVE=${R_SAVE}305 export SUBMIT_DIR=${SUBMIT_DIR}306 export status=${status}307 239 308 240 cat << END_MAIL > job_end.mail … … 317 249 318 250 if [ ! -z ${config_UserChoices_MailName} ] ; then 319 mail x-s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail251 mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < job_end.mail 320 252 elif [ -f ~/.forward ] ; then 321 mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail 322 fi 323 324 sleep 10 325 rm -f job_end.mail 326 EOF 253 mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail 254 else 255 mail -s "${config_UserChoices_JobName} ${status}" ${USER} < job_end.mail 256 fi 327 257 328 258 if [ $? -gt 0 ] ; then … … 350 280 fi 351 281 fi 352 # v érification :282 # verification : 353 283 if [ ! -d ${1} ] ; then 354 284 echo "IGCM_sys_Mkdir : erreur." … … 584 514 585 515 if [ ${config_UserChoices_JobName}.${CumulPeriod} = "." ] ; then 586 /usr/ local/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} < $1516 /usr/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} < $1 587 517 else 588 /usr/ local/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod} < $1518 /usr/bin/ccc_msub -o ${Script_Output} -e ${Script_Output} -r ${config_UserChoices_JobName}.${CumulPeriod} < $1 589 519 fi 590 520 … … 606 536 echo "IGCM_sys_QsubPost :" $@ 607 537 fi 608 /usr/ local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}"${libIGCM_POST}/$1.job538 /usr/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out ${libIGCM_POST}/$1.job 609 539 if [ $? -gt 0 ] ; then 610 540 echo "IGCM_sys_QsubPost : erreur " $@ … … 940 870 # 941 871 #USUAL WAY 942 # add dmget (to demigrate all offline files) to reduce time of this command :943 dmget $1/*872 # add get (to demigrate all offline files) to reduce time of this command : 873 ccc_hsm get $1/* 944 874 \cp -r $1 $2 > out_rsync 2>&1 945 875 RET=$? … … 1167 1097 fi 1168 1098 1169 dmget ${dm_liste[*]} > out_rsync 2>&11099 ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1 1170 1100 RET=$? 1171 1101 … … 1273 1203 echo "IGCM_sys_rebuild :" $@ 1274 1204 fi 1275 /home/cont003/p86ipsl/X64/bin/rebuild -f -o $@1205 ~p86ipsl/CESIUM/bin/rebuild -f -o $@ 1276 1206 if [ $? -gt 0 ] ; then 1277 1207 echo "IGCM_sys_rebuild : erreur ${@}." … … 1290 1220 echo "IGCM_sys_activ_variables" 1291 1221 fi 1292 # ulimit -s 20971521293 1294 typeset max_omp1295 1222 1296 1223 ulimit -s unlimited 1297 1298 (( max_omp = 0 ))1299 for comp in ${config_ListOfComponents[*]} ; do1300 1301 eval ExeNameIn=\${config_Executable_${comp}[0]}1302 eval ExeNameOut=\${config_Executable_${comp}[1]}1303 1304 1305 # Only if we really have an executable for the component :1306 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then1307 eval comp_proc_omp_loc=\${${comp}_PROC_OMP}1308 1309 if [ ${comp_proc_omp_loc} -gt ${max_omp} ] ; then1310 1311 (( max_omp = comp_proc_omp_loc ))1312 fi1313 fi1314 done1315 if [ ${max_omp} -gt 1 ] ; then1316 module load openmp/${max_omp}thds1317 fi1318 1224 1319 1225 IGCM_debug_PopStack "IGCM_sys_activ_variables" … … 1332 1238 1333 1239 1334 #function IGCM_sys_build_run_file {1240 function IGCM_sys_build_run_file { 1335 1241 ############################################################ 1336 1242 # Build run file 1337 1243 1338 # IGCM_debug_PushStack "IGCM_sys_build_run_file" $@ 1339 # if ( $DEBUG_sys ) ; then 1340 # echo "IGCM_sys_build_run_file " $@ 1341 # fi 1342 1343 # # set Number of processors for OCE here 1344 # NUM_PROC_OCE=5 1345 1346 # (( NUM_PROC_ATM = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE - 1)) 1347 # (( nb_tot_m1 = $BATCH_NUM_PROC_TOT - NUM_PROC_OCE )) 1348 # if [ $1 = MPI1 ]; then 1349 # cat <<EOF > run_file 1350 # -np 1 ./oasis 1351 # -np ${NUM_PROC_ATM} ./lmdz.x 1352 # -np ${NUM_PROC_OCE} ./opa.xx 1353 # EOF 1354 # config_UserChoices_JobRunOptions='"--app"' 1355 # IGCM_sys_Chmod u+x run_file 1356 # fi 1357 # IGCM_debug_PopStack "IGCM_sys_build_run_file" 1358 1359 # } 1244 IGCM_debug_Print 1 "Dummy IGCM_sys_build_run_file" 1245 1246 } 1360 1247 1361 1248 ############################################################ … … 1368 1255 fi 1369 1256 1370 typeset nodes listnodes init_node node_num_current start_num init_exec comp ExeNameIn ExeNameOut 1371 typeset nombre_restant_node node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc 1372 typeset num_corempi nombre_restant_node nombre_restant_comp 1373 1374 # Verification with PBS parameter 1375 if [ X${BATCH_NUM_PROC_TOT} != X ] ; then 1376 1377 if ( ${OK_PARA_MPI} ) ; then 1378 EXECUTION=${HOST_MPIRUN_COMMAND} 1379 else 1380 if [ ${BATCH_NUM_PROC_TOT} -eq 1 ] ; then 1381 EXECUTION=${MPIRUN_COMMAND:="time "} 1382 fi 1383 fi 1384 else 1385 if ( ! ${OK_PARA_MPI} ) ; then 1386 EXECUTION=${MPIRUN_COMMAND:="time "} 1387 fi 1388 fi 1389 1390 if [ ${PROCESSUS_NUMBER} -gt 1 ] ; then 1391 1392 # Hosts treatment 1393 1394 ${EXECUTION} hostname | sort | uniq > hosts.tmp 1395 1396 i=0 1397 rm -f hosts 1398 IGCM_debug_Print 1 "sys Titane, Hosts avaible :" 1399 for nodes in `cat hosts.tmp` 1400 do 1401 host[$i]=$nodes 1402 echo "${host[$i]} slots=8 max_slots=8" >> hosts 1403 IGCM_debug_Print 1 ${host[$i]} 1404 i=$((i+1)) 1405 done 1406 rm -f hosts.tmp 1407 1408 listnodes=${host[*]} 1409 1410 EXECUTION="${EXECUTION} -hostfile hosts" 1411 fi 1412 1413 # Initialisation 1414 1415 init_node=y 1416 node_num_current=0 1417 start_num=0 1418 init_exec=n 1419 1420 1421 # Test : if oasis is there, we put it at the first position 1422 1257 # Verification with MSUB parameter 1258 EXECUTION=${HOST_MPIRUN_COMMAND} 1259 1260 if ( ${OK_PARA_MPMD} ) ; then 1261 1262 if [ -f run_file ] ; then 1263 IGCM_sys_Rm -f run_file 1264 fi 1265 touch run_file 1266 1267 # run_file construction 1268 1269 # Then first loop on the components for the coupler ie oasis 1270 1271 ### the coupler ie oasis must be the first one 1423 1272 for comp in ${config_ListOfComponents[*]} ; do 1424 1425 if [ "X${comp}" = "XCPL" ] ; then 1426 1427 eval ExeNameIn=\${config_Executable_${comp}[0]} 1428 eval ExeNameOut=\${config_Executable_${comp}[1]} 1429 1430 echo "#!/bin/ksh" > script_${ExeNameOut}.ksh 1431 echo "" >> script_${ExeNameOut}.ksh 1432 echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 1433 echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh 1434 echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh 1435 echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh 1436 IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh 1437 1438 init_node=n 1439 1440 (( nombre_restant_node = NUM_COREPERNODE - 1 )) 1441 node_num_current=0 1442 node_current=${host[${node_num_current}]} 1443 1444 EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh" 1445 1446 init_exec=y 1447 start_num=1 1448 1449 fi 1450 1273 1274 eval ExeNameIn=\${config_Executable_${comp}[0]} 1275 eval ExeNameOut=\${config_Executable_${comp}[1]} 1276 1277 # for CPL component only 1278 if [ "X${comp}" = "XCPL" ] ; then 1279 1280 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 1281 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 1282 echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file 1283 fi 1451 1284 done 1452 1453 # Then loop on the components (except for oasis)1285 1286 # Then second loop on the components 1454 1287 1455 1288 for comp in ${config_ListOfComponents[*]} ; do 1456 1457 eval ExeNameIn=\${config_Executable_${comp}[0]} 1458 eval ExeNameOut=\${config_Executable_${comp}[1]} 1459 1460 1461 # Only if we really have an executable for the component : 1462 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then 1463 1464 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 1465 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 1466 1467 if ( ${OK_PARA_MPI} ) ; then 1468 1469 echo "#!/bin/ksh" > script_${ExeNameOut}.ksh 1470 # echo "set -vx" >> script_${ExeNameOut}.ksh 1471 echo "" >> script_${ExeNameOut}.ksh 1472 echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 1473 echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh 1474 echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh 1475 echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh 1476 echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh 1477 echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh 1478 IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh 1479 1480 node_num=0 1481 1482 # We define the number of MPI process to be assigned for the component 1483 1484 nombre_restant_comp=${comp_proc_mpi_loc} 1485 1486 # Loop on the allocated nodes 1487 1488 for node in ${listnodes} ; do 1489 1490 # We go to the current node 1491 1492 if [ ${node_num} = ${node_num_current} ] ; then 1493 1494 node_current=${host[${node_num_current}]} 1495 1496 # If first time on the node : initialisation 1497 1498 if [ ${init_node} = y ] ; then 1499 nombre_restant_node=${NUM_COREPERNODE} 1500 fi 1501 1502 # Test on the number of OMP threads 1503 1504 if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then 1505 (( node_num = node_num + 1 )) 1506 node_num_current=${node_num} 1507 init_node=y 1508 continue 1509 fi 1510 1511 # Number of MPI process to assign 1512 1513 (( num_corempi = nombre_restant_node / comp_proc_omp_loc )) 1514 1515 if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then 1516 num_corempi=${nombre_restant_comp} 1517 fi 1518 1519 (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc )) 1520 (( nombre_restant_comp = nombre_restant_comp - num_corempi )) 1521 1522 if [ ${init_exec} = y ] ; then 1523 EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh" 1524 else 1525 EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh" 1526 init_exec=y 1527 fi 1528 1529 (( start_num = num_corempi + start_num )) 1530 1531 else 1532 1533 (( node_num = node_num + 1 )) 1534 continue 1535 fi 1536 1537 # Test on the number of core/process remaining on the node/component 1538 1539 if [ ${nombre_restant_node} = 0 ] ; then 1540 (( node_num = node_num + 1 )) 1541 node_num_current=${node_num} 1542 init_node=y 1543 1544 if [ ${nombre_restant_comp} = 0 ] ; then 1545 break 1 1546 fi 1547 else 1548 1549 node_num_current=${node_num} 1550 init_node=n 1551 1552 if [ ${nombre_restant_comp} = 0 ] ; then 1553 break 1 1554 fi 1555 fi 1556 done 1557 else 1558 EXECUTION="${EXECUTION} ./${ExeNameOut}" 1559 fi 1560 fi 1289 1290 eval ExeNameIn=\${config_Executable_${comp}[0]} 1291 eval ExeNameOut=\${config_Executable_${comp}[1]} 1292 1293 # Only if we really have an executable for the component and not the coupler ie oasis: 1294 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then 1295 1296 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 1297 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 1298 echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file 1299 fi 1561 1300 done 1562 1301 1563 IGCM_debug_Print 1 "sys Titane : La commande d execution est " 1564 IGCM_debug_Print 1 $EXECUTION 1302 EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file" 1303 1304 IGCM_sys_Chmod u+x run_file 1305 if ( $DEBUG_sys ) ; then 1306 echo "run_file contains : " 1307 cat run_file 1308 fi 1309 1310 else # Only one executable. launch it. 1311 1312 # 1313 for comp in ${config_ListOfComponents[*]} ; do 1314 1315 eval ExeNameIn=\${config_Executable_${comp}[0]} 1316 eval ExeNameOut=\${config_Executable_${comp}[1]} 1317 1318 if [ "X${ExeNameOut}" != X\"\" ] ; then 1319 if ( ${OK_PARA_MPI} ) ; then 1320 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 1321 EXECUTION="${EXECUTION} -n ${comp_proc_mpi_loc} ./${ExeNameOut}" 1322 else 1323 EXECUTION="${EXECUTION} ./${ExeNameOut}" 1324 fi 1325 if ( ${OK_PARA_OMP} ) ; then 1326 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 1327 EXECUTION="export OMP_NUM_THREADS=${comp_proc_omp_loc} ; ${EXECUTION} " 1328 fi 1329 fi 1330 1331 done 1332 1333 fi 1334 1335 IGCM_debug_Print 1 "sys Curie : execution command is " 1336 IGCM_debug_Print 1 "$EXECUTION" 1565 1337 1566 1338 IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
Note: See TracChangeset
for help on using the changeset viewer.