Changeset 1493 for trunk


Ignore:
Timestamp:
10/23/19 18:15:14 (4 years ago)
Author:
jgipsl
Message:

Adapted executaionType=1 and 2 for jean-zay but only tested for 2 executables for now (lmdz+xios).

File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh

    r1492 r1493  
    11361136    sed -e "/::openMPthreads::/d"                  \ 
    11371137        -e "s/::JobNumProcTot::/${coreNumber}/"    \ 
    1138         -e "/#MSUB -x/d"                           \ 
    1139         -e "/--cpu_bind=none/d"                    \ 
    11401138      ${file} > ${file}.tmp 
    11411139 
    11421140  elif [ ${executionType} -eq 2 ] ; then 
    1143     # MPMD + MPI + OMP : mpirun/ccc_mprun/error 
    1144     if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then 
    1145       sed -e "/::openMPthreads::/d"                  \ 
    1146           -e "s/::JobNumProcTot::/${coreNumber}/"    \ 
    1147         ${file} > ${file}.tmp 
    1148     elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then 
    1149       sed -e "/::openMPthreads::/d"                  \ 
    1150           -e "s/::JobNumProcTot::/${coreNumber}/"    \ 
    1151           -e "/--cpu_bind=none/d"                    \ 
    1152         ${file} > ${file}.tmp 
    1153     else 
    1154      IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card" 
    1155      IGCM_debug_Exit "IGCM_sys_updateHeaders" 
    1156     fi 
     1141    # MPMD + MPI + OMP 
     1142      sed -e "s/::openMPthreads::/${openMPthreads}/" \ 
     1143          -e "s/::JobNumProcTot::/${mpiTasks}/"    \ 
     1144          ${file} > ${file}.tmp 
     1145 
    11571146  elif [ ${executionType} -eq 3 ] ; then 
    11581147    # SPMD + MPI/OMP 
    11591148    sed -e "s/::openMPthreads::/${openMPthreads}/" \ 
    11601149        -e "s/::JobNumProcTot::/${mpiTasks}/"      \ 
    1161         -e "/#MSUB -x/d"                           \ 
    1162         -e "/--cpu_bind=none/d"                    \ 
    11631150      ${file} > ${file}.tmp 
    11641151 
     
    11671154    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \ 
    11681155        -e "/::openMPthreads::/d"                  \ 
    1169         -e "/#MSUB -x/d"                           \ 
    1170         -e "/--cpu_bind=none/d"                    \ 
    11711156      ${file} > ${file}.tmp 
    11721157 
     
    11751160    sed -e "s/::openMPthreads::/${openMPthreads}/" \ 
    11761161        -e "/::JobNumProcTot::/d"                  \ 
    1177         -e "/#MSUB -x/d"                           \ 
    1178         -e "/--cpu_bind=none/d"                    \ 
    11791162      ${file} > ${file}.tmp 
    11801163 
     
    11831166    sed -e "s/::JobNumProcTot::/1/"                \ 
    11841167        -e "/::openMPthreads::/d"                  \ 
    1185         -e "/#MSUB -x/d"                           \ 
    1186         -e "/--cpu_bind=none/d"                    \ 
    11871168      ${file} > ${file}.tmp 
    11881169 
     
    12201201    echo "IGCM_sys_build_execution_scripts " $@ 
    12211202  fi 
    1222  
     1203  IGCM_debug_Print 1 "executionType= ${executionType}" 
    12231204  EXECUTION=${HOST_MPIRUN_COMMAND} 
    12241205 
     
    12261207 
    12271208    # MPMD mode 
    1228     # 1 MPI only 
    1229     # 2 MPI/OpenMP mpirun method 
    1230     # 3 MPI/OpenMP ccc_mprun method 
     1209    # 1 MPI only : executionType=1 
     1210    # 2 MPI/OpenMP : executionType=2 
    12311211 
    12321212    if [ -f run_file ] ; then 
     
    12641244      fi 
    12651245 
    1266     elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then 
    1267  
    1268     # 2 MPI/OpenMP mpirun method 
    1269  
    1270       # Use of mpirun instead of ccc_mprun 
    1271       EXECUTION="time mpirun" 
    1272  
    1273       #  Hosts treatment 
    1274       ${EXECUTION} hostname | sort | uniq > hosts.tmp 
    1275  
    1276       i=0 
    1277       rm -f hosts rankfile 
    1278       IGCM_debug_Print 1 "sys Irene, Hosts available :" 
    1279       for nodes in `cat hosts.tmp` 
    1280       do 
    1281         host[$i]=$nodes 
    1282         echo "${host[$i]}" >> hosts 
    1283         IGCM_debug_Print 1 ${host[$i]} 
    1284         i=$((i+1)) 
    1285       done 
    1286       rm -f hosts.tmp 
    1287  
    1288       listnodes=${host[*]} 
    1289  
    1290       EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile" 
    1291  
    1292       # Initialisation 
    1293       rank=0 
     1246    else  
     1247 
     1248    # 2 MPI/OpenMP : executionType=2 
     1249 
     1250    # MPI-OpenMP (MPMD) 
     1251    # example of run_file 
     1252    # 0-70   env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh 
     1253    # 71-71  env OMP_NUM_THREADS=1 ./script_xios.x.ksh 
     1254 
     1255      # Loop on the components to build run_file and script_exec files 
    12941256      current_core=0 
    1295       init_exec=n 
    1296  
    1297       # Loop on the components 
    12981257      for comp in ${config_ListOfComponents[*]} ; do 
    12991258 
     
    13131272          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 
    13141273          eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 
    1315           eval comp_proc_nod_loc=\${${comp}_PROC_NOD} 
     1274 
     1275          # Build script files 
    13161276 
    13171277          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh 
    13181278          echo ""  >> script_${ExeNameOut}.ksh 
    13191279          if [ ${comp_proc_omp_loc} -gt 1 ] ; then 
    1320  
    1321             # Check if the number of threads is correct 
    1322             case ${comp_proc_omp_loc} in 
    1323             2|4|8|16) 
    1324               IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads" 
    1325               ;; 
    1326             *) 
    1327               IGCM_debug_Exit "ERROR with OMP parameters !" 
    1328               IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads" 
    1329               IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible " 
    1330               IGCM_debug_Verif_Exit 
    1331               ;; 
    1332             esac 
    1333             echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh 
    1334             echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh 
    1335             echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh 
    1336             echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh 
    1337           fi 
    1338  
    1339           #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh 
    1340           echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh 
    1341           echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh 
    1342           echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh 
    1343           IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh 
    1344  
    1345           if [ ${init_exec} = y ] ; then 
    1346             EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh" 
    1347           else 
    1348             EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh" 
    1349             init_exec=y 
    1350           fi 
    1351  
    1352           # Build rankfile : method used to assign cores and nodes for the MPI process 
    1353           # Ex : 
    1354           #rank 0=curie5296 slot=0,1,2,3 
    1355           #rank 1=curie5296 slot=4,5,6,7 
    1356           # Example of final command : 
    1357           # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh 
    1358           # with script_lmdz.x.ksh : 
    1359           # #!/bin/ksh 
    1360           #export KMP_STACKSIZE=3g 
    1361           #export KMP_LIBRARY=turnaround 
    1362           #export MKL_SERIAL=YES 
    1363           #OMP_NUM_THREADS=4 
    1364           #./lmdz.x 
    1365           if [ ${comp_proc_nod_loc} -gt 1 ] ; then 
    1366               (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) )) 
    1367           else 
    1368               (( offset_comp_proc_loc =  comp_proc_omp_loc )) 
    1369           fi 
    1370           for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do 
    1371              (( index_host = current_core / NB_CORE_PER_NODE )) 
    1372              host_value=${host[${index_host}]} 
    1373              (( slot =  current_core % NB_CORE_PER_NODE )) 
    1374              virg="," 
    1375              string_final="" 
    1376              for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do 
    1377                  string=$index$virg 
    1378                  string_final=$string_final$string 
    1379              done 
    1380              string_final=$( echo $string_final | sed "s/.$//" ) 
    1381              echo "rank $rank=$host_value slot=$string_final" >> rankfile 
    1382              (( rank = rank + 1 )) 
    1383              (( current_core = current_core + offset_comp_proc_loc )) 
    1384           done 
    1385         fi 
    1386       done 
    1387  
    1388     elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then 
    1389  
    1390     # 3 MPI/OpenMP ccc_mprun method 
    1391  
    1392     # MPI-OpenMP (MPMD) 
    1393     # example of run_file 
    1394     # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh 
    1395     # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh 
    1396     # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh 
    1397  
    1398       # Loop on the components 
    1399       for comp in ${config_ListOfComponents[*]} ; do 
    1400  
    1401         eval ExeNameIn=\${config_Executable_${comp}[0]} 
    1402         eval ExeNameOut=\${config_Executable_${comp}[1]} 
    1403  
    1404         # Not possible if oasis has an executable (i.e old version of oasis3) 
    1405         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then 
    1406           IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version" 
    1407           IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler" 
    1408           IGCM_debug_Verif_Exit 
    1409         fi 
    1410  
    1411         # Only if we really have an executable for the component : 
    1412         if [ "X${ExeNameOut}" != X\"\" ] ; then 
    1413  
    1414           eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 
    1415           eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 
    1416  
    1417           # Build script files 
    1418  
    1419           echo "#!/bin/ksh" > script_${ExeNameOut}.ksh 
    1420           echo ""  >> script_${ExeNameOut}.ksh 
    1421           if [ ${comp_proc_omp_loc} -gt 1 ] ; then 
    1422  
    1423             # Check if the number of threads is correct 
    1424             case ${comp_proc_omp_loc} in 
    1425             2|4|8|12|16|24) 
    1426               IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads" 
    1427               ;; 
    1428             *) 
    1429               IGCM_debug_Exit "ERROR with OMP parameters !" 
    1430               IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads" 
    1431               IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible " 
    1432               IGCM_debug_Verif_Exit 
    1433               ;; 
    1434             esac 
    1435             ### only for LMDZ? 
    1436             echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh 
    1437             echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh 
    1438             echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh 
    1439             echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh 
    1440  
     1280            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh 
     1281            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh 
    14411282          fi 
    14421283 
    14431284          # to have out/err per process on different files 
    14441285          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh 
    1445           #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh 
    14461286 
    14471287          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh 
    14481288 
    14491289          # Complete run_file 
    1450  
    1451           echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file 
     1290          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 
     1291          (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 )) 
     1292          echo "${current_core}-${end_core}  env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >> run_file 
     1293          (( current_core = ${end_core} + 1 )) 
    14521294 
    14531295        fi 
     
    14581300      export SLURM_WAIT=60 
    14591301 
    1460       ## module advised by TGCC (instead of 2 variables) 
    1461       module load feature/bridge/heterogenous_mpmd 
    1462  
    1463       # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)  
    1464  
    1465       EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file" 
     1302      EXECUTION="${HOST_MPIRUN_COMMAND} --multi-prog ./run_file" 
    14661303 
    14671304      IGCM_sys_Chmod u+x run_file 
     
    14711308      fi 
    14721309 
    1473     else 
    1474  
    1475       IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie" 
    1476  
    1477     fi # 1 2 or 3 if ${OK_PARA_MPMD} 
     1310    fi # if ${OK_PARA_MPMD} 
    14781311 
    14791312  else 
    1480   # Only one executable (SPMD mode). 
     1313  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6 
    14811314 
    14821315    for comp in ${config_ListOfComponents[*]} ; do 
     
    14951328          eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 
    14961329          echo ""  >> script_${ExeNameOut}.ksh 
    1497           echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh 
    1498           echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh 
    1499           echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh 
     1330          echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh 
     1331          echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh 
    15001332          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh 
    15011333        fi 
    15021334 
    1503         if  ( ${OK_PARA_MPI} ) ; then 
    1504           eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 
    1505           # Default : ccc_mprun used if nb_proc gt 1 
    1506           # to have out/err per process on different files 
    1507           echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh 
    1508           #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh 
    1509           EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh" 
    1510         else 
    1511           # Default : ccc_mprun is NOT used if nb_proc eq 1 
    1512           # to have out/err per process on different files 
    1513           echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh 
    1514           #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh 
    1515           EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh" 
    1516         fi 
     1335        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 
     1336         
     1337        # To have out/err per process on different files 
     1338        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh 
     1339        EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh" 
    15171340 
    15181341        IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains" 
Note: See TracChangeset for help on using the changeset viewer.