[1888] | 1 | import glob |
---|
| 2 | import sys |
---|
| 3 | import subprocess |
---|
| 4 | import os |
---|
| 5 | import json |
---|
| 6 | import itertools |
---|
| 7 | import copy |
---|
| 8 | |
---|
| 9 | param_list = ["ATMdomain", "UsingServer2", "NumberClients", "NumberServers", "RatioServer2", "NumberPoolsServer2", "Duration"] |
---|
| 10 | param_short_list = ["ATMdom", "Srv2", "NbClnt", "NbSrv", "RatioSrv2", "NbPlSrv2", "Duration"] |
---|
| 11 | |
---|
| 12 | mode=os.getenv("mode") |
---|
| 13 | arch=os.getenv("arch") |
---|
[1893] | 14 | machine=os.getenv("xios_machine_name") |
---|
[1888] | 15 | |
---|
[1893] | 16 | nb_proc_irene=20 # this must be >= NumberClients+NumberServers for all configs for all test folders |
---|
| 17 | nb_proc_jz=36 # this must be >= sum(NumberClients+NumberServers) for each test folder |
---|
| 18 | |
---|
[1888] | 19 | def OSinfo(runthis): |
---|
| 20 | red = lambda text: '\033[0;31m' + text + '\033[0m' |
---|
| 21 | osstdout = subprocess.Popen(runthis, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) |
---|
| 22 | theInfo = osstdout.communicate()[0].strip() |
---|
| 23 | if osstdout.returncode!=0: |
---|
| 24 | print(red(runthis+" FAILED")) |
---|
[1891] | 25 | print(theInfo) |
---|
[1888] | 26 | sys.exit() |
---|
| 27 | |
---|
| 28 | def product_dict(**kwargs): |
---|
| 29 | keys = kwargs.keys() |
---|
| 30 | vals = kwargs.values() |
---|
| 31 | for instance in itertools.product(*vals): |
---|
| 32 | yield dict(zip(keys, instance)) |
---|
| 33 | |
---|
| 34 | def get_default_param(): |
---|
| 35 | f=open("default_param.json", 'r') |
---|
| 36 | default_param = json.load(f) |
---|
| 37 | f.close() |
---|
| 38 | return default_param[0] |
---|
| 39 | |
---|
[1893] | 40 | def generate_job(fn, n): |
---|
| 41 | if machine=="irene": |
---|
| 42 | with open(fn, "w") as fh: |
---|
| 43 | fh.write("#!/bin/bash\n") |
---|
| 44 | fh.write("#MSUB -r XIOS\n") |
---|
| 45 | fh.write("#MSUB -eo\n") |
---|
| 46 | fh.write("#MSUB -o client_output.out\n") |
---|
| 47 | fh.write("#MSUB -e client_error.out\n") |
---|
| 48 | fh.write("#MSUB -c 1\n") |
---|
| 49 | fh.write("#MSUB -n "+str(n)+"\n") |
---|
| 50 | fh.write("#MSUB -X\n") |
---|
| 51 | fh.write("#MSUB -x\n") |
---|
| 52 | fh.write("#MSUB -T 1800\n") |
---|
| 53 | fh.write("#MSUB -q skylake\n") |
---|
| 54 | fh.write("#MSUB -A gen0826\n") |
---|
| 55 | fh.write("#MSUB -Q test\n") |
---|
| 56 | fh.write("#MSUB -m work,scratch\n") |
---|
| 57 | fh.write("source ../../../BUILD/build_"+arch+"_"+mode+"/arch.env\n") |
---|
| 58 | fh.write("ccc_mprun -n "+str(n)+" generic_testcase.exe\n") |
---|
[1888] | 59 | |
---|
[1893] | 60 | if machine=="jeanzay": |
---|
| 61 | with open(fn, "w") as fh: |
---|
| 62 | fh.write("#!/bin/bash\n") |
---|
| 63 | fh.write("#SBATCH --ntasks="+str(n)+"\n") |
---|
| 64 | fh.write("#SBATCH --hint=nomultithread\n") |
---|
| 65 | fh.write("#SBATCH -o output.out\n") |
---|
| 66 | fh.write("#SBATCH -e error.out\n") |
---|
| 67 | fh.write("#SBATCH -t 00:10:00\n") |
---|
| 68 | fh.write("#SBATCH --account=psl@cpu\n") |
---|
| 69 | fh.write("ulimit -c 0\n") |
---|
| 70 | fh.write("cd ${SLURM_SUBMIT_DIR}\n") |
---|
| 71 | fh.write("source ../../../BUILD/build_"+arch+"_"+mode+"/arch.env\n") |
---|
| 72 | fh.write("source $I_MPI_ROOT/intel64/bin/mpivars.sh release_mt") |
---|
| 73 | fh.write("srun generic_testcase.exe") |
---|
[1888] | 74 | |
---|
[1893] | 75 | |
---|
[1888] | 76 | def update_full_job(location, n): |
---|
[1893] | 77 | global my_counter |
---|
| 78 | if machine=="irene": |
---|
| 79 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
[1888] | 80 | fh.write("\ncd ${location}/"+location+"; ccc_mprun -E \'--exclusive\' -n "+str(n)+" generic_testcase.exe > output.out 2> error.out &\n") |
---|
| 81 | fh.write("PIDS+=($!)\n") |
---|
| 82 | fh.write("CONFIGS+=("+location+")\n") |
---|
[1893] | 83 | if machine=="jeanzay": |
---|
| 84 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
| 85 | fh.write("\ncd ${location}/"+location+"; srun --exclusive -n "+str(n)+" generic_testcase.exe > output.out 2> error.out &\n") |
---|
[1888] | 86 | |
---|
[1893] | 87 | def main(): |
---|
[1888] | 88 | |
---|
[1893] | 89 | if machine=="irene": |
---|
| 90 | with open("full_job_"+arch+"_"+mode+".sh", "w") as fh: |
---|
| 91 | fh.write("#!/bin/bash\n") |
---|
| 92 | fh.write("#MSUB -r XIOS\n") |
---|
| 93 | fh.write("#MSUB -eo\n") |
---|
| 94 | fh.write("#MSUB -o client_output.out\n") |
---|
| 95 | fh.write("#MSUB -e client_error.err\n") |
---|
| 96 | fh.write("#MSUB -c 1\n") |
---|
| 97 | fh.write("#MSUB -n "+str(nb_proc_irene)+"\n") |
---|
| 98 | fh.write("#MSUB -X\n") |
---|
| 99 | fh.write("#MSUB -x\n") |
---|
| 100 | fh.write("#MSUB -T 1800\n") |
---|
| 101 | fh.write("#MSUB -q skylake\n") |
---|
| 102 | fh.write("#MSUB -A gen0826\n") |
---|
| 103 | fh.write("#MSUB -Q test\n") |
---|
| 104 | fh.write("#MSUB -m work,scratch\n") |
---|
| 105 | fh.write("export location="+os.getcwd()+"\n") |
---|
| 106 | fh.write("export log_location="+os.getcwd()+"\n") |
---|
| 107 | fh.write("source ../BUILD/build_"+arch+"_"+mode+"/arch.env\n") |
---|
| 108 | fh.write("echo \"parallel launch arch="+arch+" mode="+mode+"\" >> ${log_location}/Log.txt\n") |
---|
| 109 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
| 110 | |
---|
| 111 | if machine=="jeanzay": |
---|
| 112 | with open("full_job_"+arch+"_"+mode+".sh", "w") as fh: |
---|
| 113 | fh.write("#!/bin/bash\n") |
---|
| 114 | fh.write("#SBATCH --ntasks="+str(nb_proc_jz)+"\n") |
---|
| 115 | fh.write("#SBATCH --hint=nomultithread\n") |
---|
| 116 | fh.write("#SBATCH -t 00:10:00\n") |
---|
| 117 | fh.write("#SBATCH --account=psl@cpu\n") |
---|
| 118 | fh.write("ulimit -c 0\n") |
---|
| 119 | fh.write("cd ${SLURM_SUBMIT_DIR}\n") |
---|
| 120 | fh.write("source ../BUILD/build_"+arch+"_"+mode+"/arch.env\n") |
---|
| 121 | fh.write("source $I_MPI_ROOT/intel64/bin/mpivars.sh release_mt\n") |
---|
| 122 | fh.write("export location="+os.getcwd()+"\n") |
---|
| 123 | fh.write("export log_location="+os.getcwd()+"\n") |
---|
| 124 | fh.write("echo \"parallel launch arch="+arch+" mode="+mode+"\" >> ${log_location}/Log.txt\n") |
---|
| 125 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
| 126 | |
---|
| 127 | |
---|
| 128 | |
---|
[1888] | 129 | test_folder_list = glob.glob('test_*') |
---|
| 130 | all_config=dict() |
---|
| 131 | |
---|
| 132 | default_param = get_default_param() |
---|
| 133 | # print(default_param) |
---|
| 134 | |
---|
| 135 | for test_folder in test_folder_list: |
---|
| 136 | config_list=[] |
---|
| 137 | config_name=[] |
---|
| 138 | with open(test_folder+"/user_param.json", "r") as f: |
---|
| 139 | config_dict = json.load(f) |
---|
| 140 | |
---|
| 141 | for i in range(len(config_dict)): |
---|
| 142 | config_list.extend(list(product_dict(**config_dict[i]))) |
---|
| 143 | #print(config_list) |
---|
| 144 | for i in range(len(config_list)): |
---|
| 145 | # print(config_list[i]) |
---|
| 146 | keylist = list(config_list[i].keys()) |
---|
| 147 | # print(keylist) |
---|
| 148 | full_config = copy.deepcopy(default_param) |
---|
| 149 | for j in range(len(keylist)): |
---|
| 150 | full_config[ keylist[j] ] = config_list[i][keylist[j]] |
---|
| 151 | |
---|
| 152 | #print(full_config) |
---|
| 153 | mystr = str(full_config) |
---|
| 154 | mystr = mystr.replace("{", "") |
---|
| 155 | mystr = mystr.replace("}", "") |
---|
| 156 | mystr = mystr.replace("[", "") |
---|
| 157 | mystr = mystr.replace("]", "") |
---|
| 158 | mystr = mystr.replace(",", "") |
---|
| 159 | mystr = mystr.replace(":", "") |
---|
| 160 | mystr = mystr.replace("'", "") |
---|
| 161 | mystr = mystr.replace("b'", "") |
---|
| 162 | mystr = mystr.replace(" ", "_") |
---|
| 163 | for j in range(len(param_list)): |
---|
| 164 | mystr = mystr.replace(param_list[j], param_short_list[j]) |
---|
| 165 | if not mystr in config_name: |
---|
| 166 | config_name.append(mystr) |
---|
| 167 | #print(mystr) |
---|
| 168 | OSinfo("mkdir -p "+test_folder+"/CONFIG_"+mystr) |
---|
| 169 | OSinfo("cp build_"+arch+"_"+mode+"/bin/generic_testcase.exe "+test_folder+"/CONFIG_"+mystr) |
---|
| 170 | OSinfo("cp iodef.xml "+test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp") |
---|
| 171 | |
---|
| 172 | with open(test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp", "r") as f: |
---|
| 173 | lines = f.readlines() |
---|
| 174 | for i in range(len(lines)): |
---|
| 175 | if "XIOS::" in lines[i]: |
---|
| 176 | config_keys = list(full_config.keys()) |
---|
| 177 | for idx in range(len(config_keys)): |
---|
| 178 | lines[i] = lines[i].replace("XIOS::"+config_keys[idx], str(full_config[config_keys[idx]])) |
---|
| 179 | with open(test_folder+"/CONFIG_"+mystr+"/iodef.xml", "w") as g: |
---|
| 180 | for line in lines: |
---|
| 181 | g.write(line) |
---|
| 182 | |
---|
| 183 | OSinfo("rm -f "+test_folder+"/CONFIG_"+mystr+"/iodef.xml.tmp") |
---|
| 184 | OSinfo("cp "+test_folder+"/context_atm.xml "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 185 | OSinfo("cp context_grid_dynamico.xml "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 186 | OSinfo("cp dynamico_grid.nc "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 187 | OSinfo("cp "+test_folder+"/checkfile.def "+test_folder+"/CONFIG_"+mystr+"/") |
---|
| 188 | with open(test_folder+"/CONFIG_"+mystr+"/param.def", "w") as fh: |
---|
| 189 | fh.write("¶ms_run\n") |
---|
| 190 | fh.write("duration=\'"+full_config["Duration"]+"\'\n") |
---|
| 191 | fh.write("nb_proc_atm="+str(full_config["NumberClients"])+"\n") |
---|
| 192 | fh.write("/\n") |
---|
| 193 | with open(test_folder+"/CONFIG_"+mystr+"/all_param.def", "w") as fh: |
---|
| 194 | fh.write("¶ms_run\n") |
---|
| 195 | for param in param_list: |
---|
| 196 | fh.write(param+"="+str(full_config[param])+"\n") |
---|
| 197 | fh.write("/\n") |
---|
| 198 | |
---|
[1893] | 199 | generate_job(test_folder+"/CONFIG_"+mystr+"/job.sh", full_config['NumberClients']+full_config['NumberServers']) |
---|
[1888] | 200 | update_full_job(test_folder+"/CONFIG_"+mystr, full_config['NumberClients']+full_config['NumberServers']) |
---|
| 201 | |
---|
| 202 | #print(config_name) |
---|
[1895] | 203 | if machine=="jeanzay": |
---|
| 204 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
| 205 | fh.write("wait\nwait\n") |
---|
| 206 | fh.write("echo \"tests in "+test_folder+" finished\"\n") |
---|
[1888] | 207 | all_config[test_folder] = config_name |
---|
[1893] | 208 | |
---|
[1888] | 209 | |
---|
[1893] | 210 | if machine=="irene": |
---|
| 211 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
| 212 | fh.write("\nfor pid in ${PIDS[@]}; do\n") |
---|
| 213 | fh.write("wait ${pid}\n") |
---|
| 214 | fh.write("STATUS+=($?)\ndone\n") |
---|
| 215 | fh.write("\ni=0\n") |
---|
| 216 | fh.write("#for st in ${STATUS[@]}; do\n") |
---|
| 217 | fh.write("#if [[ ${st} -ne 0 ]]; then\n") |
---|
| 218 | fh.write("#echo \"${CONFIGS[${i}]} -1\" >> ${location}/plain_report.txt\n") |
---|
| 219 | fh.write("#else\n") |
---|
| 220 | fh.write("#echo \"${CONFIGS[${i}]} 1\" >> ${location}/plain_report.txt\n") |
---|
| 221 | fh.write("#fi\n") |
---|
| 222 | fh.write("#((i+=1))\n") |
---|
| 223 | fh.write("#done\n\n") |
---|
| 224 | fh.write("#wait\n") |
---|
| 225 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
| 226 | if machine=="jeanzay": |
---|
| 227 | with open("full_job_"+arch+"_"+mode+".sh", "a") as fh: |
---|
| 228 | fh.write("date >> ${log_location}/Log.txt\n") |
---|
[1888] | 229 | |
---|
[1893] | 230 | |
---|
[1888] | 231 | #print(all_config) |
---|
| 232 | |
---|
| 233 | if __name__== "__main__": |
---|
| 234 | main() |
---|