- Timestamp:
- 10/23/19 18:15:14 (4 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh
r1492 r1493 1136 1136 sed -e "/::openMPthreads::/d" \ 1137 1137 -e "s/::JobNumProcTot::/${coreNumber}/" \ 1138 -e "/#MSUB -x/d" \1139 -e "/--cpu_bind=none/d" \1140 1138 ${file} > ${file}.tmp 1141 1139 1142 1140 elif [ ${executionType} -eq 2 ] ; then 1143 # MPMD + MPI + OMP : mpirun/ccc_mprun/error 1144 if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then 1145 sed -e "/::openMPthreads::/d" \ 1146 -e "s/::JobNumProcTot::/${coreNumber}/" \ 1147 ${file} > ${file}.tmp 1148 elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then 1149 sed -e "/::openMPthreads::/d" \ 1150 -e "s/::JobNumProcTot::/${coreNumber}/" \ 1151 -e "/--cpu_bind=none/d" \ 1152 ${file} > ${file}.tmp 1153 else 1154 IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card" 1155 IGCM_debug_Exit "IGCM_sys_updateHeaders" 1156 fi 1141 # MPMD + MPI + OMP 1142 sed -e "s/::openMPthreads::/${openMPthreads}/" \ 1143 -e "s/::JobNumProcTot::/${mpiTasks}/" \ 1144 ${file} > ${file}.tmp 1145 1157 1146 elif [ ${executionType} -eq 3 ] ; then 1158 1147 # SPMD + MPI/OMP 1159 1148 sed -e "s/::openMPthreads::/${openMPthreads}/" \ 1160 1149 -e "s/::JobNumProcTot::/${mpiTasks}/" \ 1161 -e "/#MSUB -x/d" \1162 -e "/--cpu_bind=none/d" \1163 1150 ${file} > ${file}.tmp 1164 1151 … … 1167 1154 sed -e "s/::JobNumProcTot::/${mpiTasks}/" \ 1168 1155 -e "/::openMPthreads::/d" \ 1169 -e "/#MSUB -x/d" \1170 -e "/--cpu_bind=none/d" \1171 1156 ${file} > ${file}.tmp 1172 1157 … … 1175 1160 sed -e "s/::openMPthreads::/${openMPthreads}/" \ 1176 1161 -e "/::JobNumProcTot::/d" \ 1177 -e "/#MSUB -x/d" \1178 -e "/--cpu_bind=none/d" \1179 1162 ${file} > ${file}.tmp 1180 1163 … … 1183 1166 sed -e "s/::JobNumProcTot::/1/" \ 1184 1167 -e "/::openMPthreads::/d" \ 1185 -e "/#MSUB -x/d" \1186 -e "/--cpu_bind=none/d" \1187 1168 ${file} > ${file}.tmp 1188 1169 … … 1220 1201 echo "IGCM_sys_build_execution_scripts " $@ 1221 1202 fi 1222 1203 IGCM_debug_Print 1 "executionType= ${executionType}" 1223 1204 EXECUTION=${HOST_MPIRUN_COMMAND} 1224 1205 … … 1226 1207 1227 1208 # MPMD mode 1228 # 1 MPI only 1229 # 2 MPI/OpenMP mpirun method 1230 # 3 MPI/OpenMP ccc_mprun method 1209 # 1 MPI only : executionType=1 1210 # 2 MPI/OpenMP : executionType=2 1231 1211 1232 1212 if [ -f run_file ] ; then … … 1264 1244 fi 1265 1245 1266 elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then 1267 1268 # 2 MPI/OpenMP mpirun method 1269 1270 # Use of mpirun instead of ccc_mprun 1271 EXECUTION="time mpirun" 1272 1273 # Hosts treatment 1274 ${EXECUTION} hostname | sort | uniq > hosts.tmp 1275 1276 i=0 1277 rm -f hosts rankfile 1278 IGCM_debug_Print 1 "sys Irene, Hosts available :" 1279 for nodes in `cat hosts.tmp` 1280 do 1281 host[$i]=$nodes 1282 echo "${host[$i]}" >> hosts 1283 IGCM_debug_Print 1 ${host[$i]} 1284 i=$((i+1)) 1285 done 1286 rm -f hosts.tmp 1287 1288 listnodes=${host[*]} 1289 1290 EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile" 1291 1292 # Initialisation 1293 rank=0 1246 else 1247 1248 # 2 MPI/OpenMP : executionType=2 1249 1250 # MPI-OpenMP (MPMD) 1251 # example of run_file 1252 # 0-70 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh 1253 # 71-71 env OMP_NUM_THREADS=1 ./script_xios.x.ksh 1254 1255 # Loop on the components to build run_file and script_exec files 1294 1256 current_core=0 1295 init_exec=n1296 1297 # Loop on the components1298 1257 for comp in ${config_ListOfComponents[*]} ; do 1299 1258 … … 1313 1272 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 1314 1273 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 1315 eval comp_proc_nod_loc=\${${comp}_PROC_NOD} 1274 1275 # Build script files 1316 1276 1317 1277 echo "#!/bin/ksh" > script_${ExeNameOut}.ksh 1318 1278 echo "" >> script_${ExeNameOut}.ksh 1319 1279 if [ ${comp_proc_omp_loc} -gt 1 ] ; then 1320 1321 # Check if the number of threads is correct 1322 case ${comp_proc_omp_loc} in 1323 2|4|8|16) 1324 IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads" 1325 ;; 1326 *) 1327 IGCM_debug_Exit "ERROR with OMP parameters !" 1328 IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads" 1329 IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible " 1330 IGCM_debug_Verif_Exit 1331 ;; 1332 esac 1333 echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 1334 echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh 1335 echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh 1336 echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh 1337 fi 1338 1339 #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh 1340 echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh 1341 echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh 1342 echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh 1343 IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh 1344 1345 if [ ${init_exec} = y ] ; then 1346 EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh" 1347 else 1348 EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh" 1349 init_exec=y 1350 fi 1351 1352 # Build rankfile : method used to assign cores and nodes for the MPI process 1353 # Ex : 1354 #rank 0=curie5296 slot=0,1,2,3 1355 #rank 1=curie5296 slot=4,5,6,7 1356 # Example of final command : 1357 # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh 1358 # with script_lmdz.x.ksh : 1359 # #!/bin/ksh 1360 #export KMP_STACKSIZE=3g 1361 #export KMP_LIBRARY=turnaround 1362 #export MKL_SERIAL=YES 1363 #OMP_NUM_THREADS=4 1364 #./lmdz.x 1365 if [ ${comp_proc_nod_loc} -gt 1 ] ; then 1366 (( offset_comp_proc_loc = NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) )) 1367 else 1368 (( offset_comp_proc_loc = comp_proc_omp_loc )) 1369 fi 1370 for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do 1371 (( index_host = current_core / NB_CORE_PER_NODE )) 1372 host_value=${host[${index_host}]} 1373 (( slot = current_core % NB_CORE_PER_NODE )) 1374 virg="," 1375 string_final="" 1376 for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do 1377 string=$index$virg 1378 string_final=$string_final$string 1379 done 1380 string_final=$( echo $string_final | sed "s/.$//" ) 1381 echo "rank $rank=$host_value slot=$string_final" >> rankfile 1382 (( rank = rank + 1 )) 1383 (( current_core = current_core + offset_comp_proc_loc )) 1384 done 1385 fi 1386 done 1387 1388 elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then 1389 1390 # 3 MPI/OpenMP ccc_mprun method 1391 1392 # MPI-OpenMP (MPMD) 1393 # example of run_file 1394 # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh 1395 # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh 1396 # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh 1397 1398 # Loop on the components 1399 for comp in ${config_ListOfComponents[*]} ; do 1400 1401 eval ExeNameIn=\${config_Executable_${comp}[0]} 1402 eval ExeNameOut=\${config_Executable_${comp}[1]} 1403 1404 # Not possible if oasis has an executable (i.e old version of oasis3) 1405 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then 1406 IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version" 1407 IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler" 1408 IGCM_debug_Verif_Exit 1409 fi 1410 1411 # Only if we really have an executable for the component : 1412 if [ "X${ExeNameOut}" != X\"\" ] ; then 1413 1414 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 1415 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 1416 1417 # Build script files 1418 1419 echo "#!/bin/ksh" > script_${ExeNameOut}.ksh 1420 echo "" >> script_${ExeNameOut}.ksh 1421 if [ ${comp_proc_omp_loc} -gt 1 ] ; then 1422 1423 # Check if the number of threads is correct 1424 case ${comp_proc_omp_loc} in 1425 2|4|8|12|16|24) 1426 IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads" 1427 ;; 1428 *) 1429 IGCM_debug_Exit "ERROR with OMP parameters !" 1430 IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads" 1431 IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible " 1432 IGCM_debug_Verif_Exit 1433 ;; 1434 esac 1435 ### only for LMDZ? 1436 echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 1437 echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh 1438 echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh 1439 echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh 1440 1280 echo "export OMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 1281 echo "export OMP_PLACES=cores" >> script_${ExeNameOut}.ksh 1441 1282 fi 1442 1283 1443 1284 # to have out/err per process on different files 1444 1285 echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh 1445 #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh1446 1286 1447 1287 IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh 1448 1288 1449 1289 # Complete run_file 1450 1451 echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file 1290 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 1291 (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 )) 1292 echo "${current_core}-${end_core} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >> run_file 1293 (( current_core = ${end_core} + 1 )) 1452 1294 1453 1295 fi … … 1458 1300 export SLURM_WAIT=60 1459 1301 1460 ## module advised by TGCC (instead of 2 variables) 1461 module load feature/bridge/heterogenous_mpmd 1462 1463 # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method) 1464 1465 EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file" 1302 EXECUTION="${HOST_MPIRUN_COMMAND} --multi-prog ./run_file" 1466 1303 1467 1304 IGCM_sys_Chmod u+x run_file … … 1471 1308 fi 1472 1309 1473 else 1474 1475 IGCM_debug_Print 1 "sys Curie : choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie" 1476 1477 fi # 1 2 or 3 if ${OK_PARA_MPMD} 1310 fi # if ${OK_PARA_MPMD} 1478 1311 1479 1312 else 1480 # Only one executable (SPMD mode) .1313 # Only one executable (SPMD mode): executionType=3, 4, 5 and 6 1481 1314 1482 1315 for comp in ${config_ListOfComponents[*]} ; do … … 1495 1328 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 1496 1329 echo "" >> script_${ExeNameOut}.ksh 1497 echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 1498 echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh 1499 echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh 1330 echo "export OMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 1331 echo "export OMP_PLACES=cores" >> script_${ExeNameOut}.ksh 1500 1332 echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh 1501 1333 fi 1502 1334 1503 if ( ${OK_PARA_MPI} ) ; then 1504 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 1505 # Default : ccc_mprun used if nb_proc gt 1 1506 # to have out/err per process on different files 1507 echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh 1508 #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh 1509 EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh" 1510 else 1511 # Default : ccc_mprun is NOT used if nb_proc eq 1 1512 # to have out/err per process on different files 1513 echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh 1514 #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh 1515 EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh" 1516 fi 1335 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 1336 1337 # To have out/err per process on different files 1338 echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh 1339 EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh" 1517 1340 1518 1341 IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains"
Note: See TracChangeset
for help on using the changeset viewer.