Changeset 1296 for trunk/libIGCM/libIGCM_sys
- Timestamp:
- 03/11/16 12:15:32 (8 years ago)
- Location:
- trunk/libIGCM/libIGCM_sys
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/libIGCM/libIGCM_sys/libIGCM_sys_ada.ksh
r1286 r1296 145 145 #- set PackDefault to true on ada 146 146 PackDefault=true 147 148 #====================================================149 #- Number of core per node (max number of OpenMP task)150 NUM_COREPERNODE=8151 147 152 148 #==================================================== … … 1255 1251 #D- 1256 1252 function IGCM_sys_ListJobInQueue { 1257 IGCM_debug_PushStack "IGCM_sys_ListJobInQueue" 1253 IGCM_debug cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 1254 \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 1255 IGCM_debug_PopStack "IGCM_sys_atlas" 1256 return 1 1257 else 1258 IGCM_debug_PopStack "IGCM_sys_atlas" 1259 return 0 1260 fi 1261 1262 IGCM_debug_PopStack "IGCM_sys_atlas" 1263 } 1264 1265 _PushStack "IGCM_sys_ListJobInQueue" 1258 1266 if ( $DEBUG_sys ) ; then 1259 1267 echo "IGCM_sys_ListJobInQueue" … … 1290 1298 if [ ${status} -gt 0 ] ; then 1291 1299 echo "IGCM_sys_atlas : error code ${status}" 1292 cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 1293 \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 1294 IGCM_debug_PopStack "IGCM_sys_atlas" 1295 return 1 1296 else 1297 IGCM_debug_PopStack "IGCM_sys_atlas" 1298 return 0 1299 fi 1300 1301 IGCM_debug_PopStack "IGCM_sys_atlas" 1302 } 1303 1300 -
trunk/libIGCM/libIGCM_sys/libIGCM_sys_ciclad.ksh
r1293 r1296 62 62 63 63 #===================================================== 64 # Host and user names64 # Host user names project maxCpuTime 65 65 # $hostname ou hostname 66 66 typeset HOST=${HOST:=$( hostname )} … … 156 156 157 157 #==================================================== 158 #- Number of core per node (max number of OpenMP task)159 NUM_COREPERNODE=4160 161 #====================================================162 158 #- Default number of MPI task for IPSL coupled model 163 159 #- required for backward compatibility 164 160 #- 165 DEFAULT_NUM_PROC_OCE= 1161 DEFAULT_NUM_PROC_OCE=5 166 162 DEFAULT_NUM_PROC_CPL=1 167 163 (( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL )) … … 350 346 fi 351 347 352 \ tree -f $@348 \ls -lR ${@} 353 349 354 350 IGCM_debug_PopStack "IGCM_sys_Tree" … … 728 724 #D-#================================================== 729 725 #D-function IGCM_sys_sync 730 #D-* Purpose: flush buffer on disk (dummy function on Ada)726 #D-* Purpose: flush buffer on disk (dummy function on Ciclad) 731 727 #D-* Examples: 732 728 #D- … … 859 855 #D- 860 856 function IGCM_sys_build_run_file { 861 862 857 IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file " 863 864 858 } 865 859 … … 879 873 fi 880 874 881 typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut 882 typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc 883 typeset num_corempi nombre_restant_node nombre_restant_comp 884 875 EXECUTION=${HOST_MPIRUN_COMMAND} 876 877 # MPMD mode 885 878 if ( ${OK_PARA_MPMD} ) ; then 886 879 887 if [ -f run_file ] ; then 888 IGCM_sys_Rm -f run_file 889 fi 890 touch run_file 891 892 if ( ${OK_PARA_OMP} ) ; then 880 # Only MPI (MPMD) 881 if ( ! ${OK_PARA_OMP} ) ; then 882 883 if [ -f run_file ] ; then 884 IGCM_sys_Rm -f run_file 885 fi 886 touch run_file 887 888 # Build run_file 889 890 # First loop on the components for the coupler ie oasis (only if oasis3) 891 # the coupler ie oasis3 must be the first one 892 for comp in ${config_ListOfComponents[*]} ; do 893 894 eval ExeNameIn=\${config_Executable_${comp}[0]} 895 eval ExeNameOut=\${config_Executable_${comp}[1]} 896 897 # for CPL component only 898 if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then 899 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 900 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 901 echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file 902 fi 903 done 904 905 # Then second loop on the components 906 for comp in ${config_ListOfComponents[*]} ; do 907 908 eval ExeNameIn=\${config_Executable_${comp}[0]} 909 eval ExeNameOut=\${config_Executable_${comp}[1]} 910 911 # Only if we really have an executable for the component and not the coupler ie oasis: 912 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then 913 914 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 915 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 916 echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file 917 fi 918 done 919 920 EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file" 921 922 IGCM_sys_Chmod u+x run_file 923 if ( $DEBUG_sys ) ; then 924 echo "run_file contains : " 925 cat run_file 926 fi 927 928 # MPI-OpenMP (MPMD) 929 else 893 930 894 931 # Hosts treatment 895 896 932 ${HOST_MPIRUN_COMMAND} hostname | sort | uniq > hosts.tmp 897 933 898 934 i=0 899 935 rm -f hosts 900 IGCM_debug_Print 1 "sys Ciclad, Hosts avai ble :"936 IGCM_debug_Print 1 "sys Ciclad, Hosts available :" 901 937 for nodes in `cat hosts.tmp` ; do 902 938 host[$i]=$nodes … … 912 948 913 949 # Initialisation 914 915 init_node=y 916 node_num_current=0 917 start_num=0 950 rank=0 951 current_core=0 952 core_per_node=8 918 953 init_exec=n 919 954 920 # Build run_file 921 922 # First loop on the components for the coupler ie oasis (only if oasis3) 923 # the coupler ie oasis3 must be the first one 955 # Loop on the components 924 956 for comp in ${config_ListOfComponents[*]} ; do 925 957 926 if [ "X${comp}" = "XCPL" ] ; then 927 928 eval ExeNameIn=\${config_Executable_${comp}[0]} 929 eval ExeNameOut=\${config_Executable_${comp}[1]} 958 eval ExeNameIn=\${config_Executable_${comp}[0]} 959 eval ExeNameOut=\${config_Executable_${comp}[1]} 960 961 # Not possible if oasis has an executable (i.e old version of oasis3) 962 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" = "XCPL" ] ) ; then 963 IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version" 964 IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler" 965 IGCM_debug_Verif_Exit 966 fi 967 968 # Only if we really have an executable for the component : 969 if [ "X${ExeNameOut}" != X\"\" ] ; then 970 971 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 972 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 930 973 931 974 echo "#!/bin/ksh" > script_${ExeNameOut}.ksh 932 975 echo "" >> script_${ExeNameOut}.ksh 933 #echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 934 #echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh 935 #echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh 936 echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh 976 if [ ${comp_proc_omp_loc} -gt 1 ] ; then 977 978 # Check if the number of threads is correct 979 case ${comp_proc_omp_loc} in 980 2|4) 981 IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads" 982 ;; 983 *) 984 IGCM_debug_Exit "ERROR with OMP parameters !" 985 IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads" 986 IGCM_debug_Print 2 "Only 2,4 as number of OMP threads are possible " 987 IGCM_debug_Verif_Exit 988 ;; 989 esac 990 #echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 991 #echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh 992 #echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh 993 echo "export OMP_STACKSIZE=200M" >> script_${ExeNameOut}.ksh 994 echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh 995 fi 996 #echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh 997 #echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh 998 echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh 937 999 IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh 938 1000 939 init_node=n 940 941 (( nombre_restant_node = NUM_COREPERNODE - 1 )) 942 node_num_current=0 943 node_current=${host[${node_num_current}]} 944 945 EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh" 946 947 init_exec=y 948 start_num=1 949 950 fi 951 952 done 953 954 # Then loop on the components (except for oasis) 955 956 for comp in ${config_ListOfComponents[*]} ; do 957 958 eval ExeNameIn=\${config_Executable_${comp}[0]} 959 eval ExeNameOut=\${config_Executable_${comp}[1]} 960 961 # Only if we really have an executable for the component : 962 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then 963 964 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI} 965 eval comp_proc_omp_loc=\${${comp}_PROC_OMP} 966 967 echo "#!/bin/ksh" > script_${ExeNameOut}.ksh 968 # echo "set -vx" >> script_${ExeNameOut}.ksh 969 echo "" >> script_${ExeNameOut}.ksh 970 #echo "export KMP_STACKSIZE=3g" >> script_${ExeNameOut}.ksh 971 #echo "export KMP_LIBRARY=turnaround" >> script_${ExeNameOut}.ksh 972 #echo "export MKL_SERIAL=YES" >> script_${ExeNameOut}.ksh 973 echo "export OMP_STACKSIZE=200M" >> script_${ExeNameOut}.ksh 974 echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh 975 echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >> script_${ExeNameOut}.ksh 976 echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh 977 IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh 978 979 node_num=0 980 981 # We define the number of MPI process to be assigned for the component 982 983 nombre_restant_comp=${comp_proc_mpi_loc} 984 985 # Loop on the allocated nodes 986 987 for node in ${listnodes} ; do 988 989 # We go to the current node 990 if [ ${node_num} = ${node_num_current} ] ; then 991 992 node_current=${host[${node_num_current}]} 993 994 # If first time on the node : initialisation 995 996 if [ ${init_node} = y ] ; then 997 nombre_restant_node=${NUM_COREPERNODE} 998 fi 999 1000 # Test on the number of OMP threads 1001 1002 if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then 1003 (( node_num = node_num + 1 )) 1004 node_num_current=${node_num} 1005 init_node=y 1006 continue 1007 fi 1008 1009 # Number of MPI process to assign 1010 1011 (( num_corempi = nombre_restant_node / comp_proc_omp_loc )) 1012 1013 if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then 1014 num_corempi=${nombre_restant_comp} 1015 fi 1016 1017 (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc )) 1018 (( nombre_restant_comp = nombre_restant_comp - num_corempi )) 1019 1020 if [ ${init_exec} = y ] ; then 1021 EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh" 1022 else 1023 EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh" 1024 init_exec=y 1025 fi 1026 1027 (( start_num = num_corempi + start_num )) 1028 1029 else 1030 1031 (( node_num = node_num + 1 )) 1032 continue 1033 fi 1034 1035 # Test on the number of core/process remaining on the node/component 1036 1037 if [ ${nombre_restant_node} = 0 ] ; then 1038 (( node_num = node_num + 1 )) 1039 node_num_current=${node_num} 1040 init_node=y 1041 1042 if [ ${nombre_restant_comp} = 0 ] ; then 1043 break 1 1044 fi 1045 else 1046 1047 node_num_current=${node_num} 1048 init_node=n 1049 1050 if [ ${nombre_restant_comp} = 0 ] ; then 1051 break 1 1052 fi 1053 fi 1001 if [ ${init_exec} = y ] ; then 1002 EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh" 1003 else 1004 EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh" 1005 init_exec=y 1006 fi 1007 1008 # Build rankfile : method used to assign cores and nodes for the MPI process 1009 # Ex : 1010 #rank 0=curie5296 slot=0,1,2,3 1011 #rank 1=curie5296 slot=4,5,6,7 1012 # Example of final command : 1013 # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh 1014 # with script_lmdz.x.ksh : 1015 # #!/bin/ksh 1016 #export OMP_STACKSIZE=200M 1017 #export OMP_NUM_THREADS=4 1018 #./lmdz.x 1019 1020 for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do 1021 (( index_host = current_core / core_per_node )) 1022 host_value=${host[${index_host}]} 1023 (( slot = current_core % core_per_node )) 1024 virg="," 1025 string_final="" 1026 for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do 1027 string=$index$virg 1028 string_final=$string_final$string 1029 done 1030 string_final=$( echo $string_final | sed "s/.$//" ) 1031 echo "rank $rank=$host_value slot=$string_final" >> rankfile 1032 (( rank = rank + 1 )) 1033 (( current_core = current_core + comp_proc_omp_loc )) 1054 1034 done 1055 1035 fi 1056 1036 done 1057 1058 else1059 1060 # Then first loop on the components for the coupler ie oasis1061 1062 ## the coupler ie oasis must be the first one1063 for comp in ${config_ListOfComponents[*]} ; do1064 1065 eval ExeNameOut=\${config_Executable_${comp}[1]}1066 1067 # for CPL component only1068 if [ "X${comp}" = "XCPL" ] ; then1069 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}1070 echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file1071 fi1072 done1073 1074 # Then second loop on the components1075 1076 for comp in ${config_ListOfComponents[*]} ; do1077 1078 eval ExeNameOut=\${config_Executable_${comp}[1]}1079 1080 # Only if we really have an executable for the component and not the coupler ie oasis:1081 if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then1082 eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}1083 echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file1084 fi1085 done1086 IGCM_sys_Chmod u+x run_file1087 1088 EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"1089 1090 1037 fi 1091 1038 1092 else # Only one executable. launch it. 1039 # Only one executable (SPMD mode). 1040 else 1093 1041 1094 1042 for comp in ${config_ListOfComponents[*]} ; do -
trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh
r1288 r1296 181 181 #- set PackDefault to true on curie 182 182 PackDefault=true 183 184 #====================================================185 #- Number of core per node (max number of OpenMP task)186 NUM_COREPERNODE=8187 183 188 184 #==================================================== … … 1317 1313 fi 1318 1314 1319 # Build rankfile : method used to assign cores and nodes for the MPI process 1320 # Ex : 1321 #rank 0=curie5296 slot=0,1,2,3 1322 #rank 1=curie5296 slot=4,5,6,7 1323 # Example of final command : 1324 # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh 1325 # with script_lmdz.x.ksh : 1326 # #!/bin/ksh 1327 #export KMP_STACKSIZE=3g 1328 #export KMP_LIBRARY=turnaround 1329 #export MKL_SERIAL=YES 1330 #OMP_NUM_THREADS=4 1331 #./lmdz.x 1332 # 1315 # Build rankfile : method used to assign cores and nodes for the MPI process 1316 # Ex : 1317 #rank 0=curie5296 slot=0,1,2,3 1318 #rank 1=curie5296 slot=4,5,6,7 1319 # Example of final command : 1320 # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh 1321 # with script_lmdz.x.ksh : 1322 # #!/bin/ksh 1323 #export KMP_STACKSIZE=3g 1324 #export KMP_LIBRARY=turnaround 1325 #export MKL_SERIAL=YES 1326 #OMP_NUM_THREADS=4 1327 #./lmdz.x 1333 1328 1334 1329 for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do … … 1348 1343 done 1349 1344 fi 1350 1351 1345 done 1352 1346 fi
Note: See TracChangeset
for help on using the changeset viewer.