Changeset 1838
- Timestamp:
- 01/02/20 16:07:39 (4 years ago)
- Location:
- XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE
- Files:
-
- 7 added
- 10 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/check.py
r1781 r1838 7 7 8 8 9 from default_param import * 10 from user_param import * 9 # from default_param import * 10 # from user_param import * 11 from param_list import * 11 12 12 13 # load default_param first, variables defines in user_param will overwrite the default value … … 26 27 27 28 28 def write_jz_job(configName, nbproc=8):29 red = lambda text: '\033[0;31m' + text + '\033[0m'30 f=open("jz_job_"+configName+".sh", "w")31 f.write("#!/bin/bash\n")32 f.write("#SBATCH --ntasks="+repr(nbproc)+" # Nombre total de processus MPI\n")33 f.write("#SBATCH --ntasks-per-node=16\n")34 f.write("#SBATCH --hint=nomultithread\n")35 f.write("#SBATCH -t 00:10:00 # Temps elapsed maximum d'execution\n")36 f.write("#SBATCH -o single_config.out # Nom du fichier de sortie\n")37 f.write("#SBATCH -e single_config.err # Nom du fichier d'erreur\n")38 f.write("#SBATCH --account=psl@cpu\n")39 f.write("cd ${SLURM_SUBMIT_DIR}\n")40 f.write("module purge\n")41 f.write("module load intel-all/19.0.4\n")42 f.write("module load netcdf/4.7.0/intel-19.0.4-mpi\n")43 f.write("module load netcdf-cxx/4.2/intel-19.0.4-mpi\n")44 f.write("module load netcdf-fortran/4.4.5/intel-19.0.4-mpi\n")45 f.write("module load gcc/6.5.0/gcc-4.8.5\n")46 f.write("module load cmake\n")47 f.write("module load cdo\n")48 f.write("ulimit -c 0\n")49 xios_root=os.getenv('xios_dir')50 build_folder=os.getenv('build_dir')51 f.write("export xios_dir="+xios_root+"\n")52 f.write("export build_dir="+build_folder+"\n")53 f.write("export machine_name=jeanzay\n")54 f.write("cp ../setup.py ./\n")55 f.write("cp ../run_sub_test.sh ./\n")56 f.write("mv user_params.def user_params.def_bkp\n")57 f.write("cp "+configName+"/setup/user_param.py user_params.def\n")58 f.write("python setup.py\n")59 f.write("cmake .\n")60 f.write("ctest -V #enable all output\n")61 f.write("rm -f setup.py\n")62 f.write("rm -f run_sub_test.sh\n")63 f.write("rm -f run_test_*.py\n")64 f.write("rm -f CMakeLists.txt\n")65 f.write("rm -f context_grid_dynamico.xml\n")66 f.write("rm -f dynamico_grid.nc\n")67 f.write("rm -f default_param.*\n")68 f.write("rm -f user_param.*\n")69 f.write("mv user_params.def_bkp user_params.def\n")70 f.write("\n")71 f.close72 29 73 def write_irene_job(configName, nbproc=8): 74 red = lambda text: '\033[0;31m' + text + '\033[0m' 75 f=open("irene_job_"+configName+".sh", "w") 76 f.write("#!/bin/bash\n") 77 f.write("#MSUB -r XIOS\n") 78 f.write("#MSUB -o client_output.out # standard output\n") 79 f.write("#MSUB -e client_error.err #error output\n") 80 f.write("#MSUB -eo\n") 81 f.write("#MSUB -c 1\n") 82 f.write("#MSUB -n "+ repr(nbproc) + " # Number of MPI tasks (SPMD case) or cores (MPMD case)\n") 83 f.write("#MSUB -X \n") 84 f.write("#MSUB -x \n") 85 f.write("#MSUB -T 600 # Wall clock limit (seconds)\n") 86 f.write("#MSUB -q skylake # thin nodes\n") 87 f.write("#MSUB -A devcmip6\n") 88 f.write("#MSUB -Q test\n") 89 f.write("#MSUB -m work\n") 90 f.write("cd $BRIDGE_MSUB_PWD\n") 91 f.write("module unload netcdf-c netcdf-fortran hdf5 flavor perl hdf5 boost blitz mpi gnu\n") 92 f.write("module load gnu\n") 93 f.write("module load mpi/intelmpi/2017.0.6.256\n") 94 f.write("module load flavor/buildcompiler/intel/17\n") 95 f.write("module load flavor/hdf5/parallel\n") 96 f.write("module load netcdf-fortran/4.4.4\n") 97 f.write("module load hdf5/1.8.20\n") 98 f.write("module load boost\n") 99 f.write("module load blitz\n") 100 f.write("module load feature/bridge/heterogenous_mpmd\n") 101 f.write("module load nco\n") 102 f.write("module load cdo\n") 103 f.write("export KMP_STACKSIZE=3g\n") 104 f.write("export KMP_LIBRARY=turnaround\n") 105 f.write("export MKL_SERIAL=YES\n") 106 f.write("export OMP_NUM_THREADS=${BRIDGE_MSUB_NCORE}\n") 107 f.write("set -x \n") 108 xios_root=os.getenv('xios_dir') 109 build_folder=os.getenv('build_dir') 110 f.write("export xios_dir="+xios_root+"\n") 111 f.write("export build_dir="+build_folder+"\n") 112 f.write("export machine_name=irene\n") 113 f.write("cp ../setup.py ./\n") 114 f.write("cp ../run_sub_test.sh ./\n") 115 f.write("mv user_params.def user_params.def_bkp\n") 116 f.write("cp "+configName+"/setup/user_param.py user_params.def\n") 117 f.write("python setup.py\n") 118 f.write("cmake .\n") 119 f.write("ctest -V #enable all output\n") 120 f.write("rm -f setup.py\n") 121 f.write("rm -f run_sub_test.sh\n") 122 f.write("rm -f run_test_*.py\n") 123 f.write("rm -f CMakeLists.txt\n") 124 f.write("rm -f context_grid_dynamico.xml\n") 125 f.write("rm -f dynamico_grid.nc\n") 126 f.write("rm -f default_param.*\n") 127 f.write("rm -f user_param.*\n") 128 f.write("mv user_params.def_bkp user_params.def\n") 129 f.write("\n") 130 f.close 131 132 30 blue = lambda text: '\033[0;34m' + text + '\033[0m' 31 yellow = lambda text: '\033[0;33m' + text + '\033[0m' 32 green = lambda text: '\033[0;32m' + text + '\033[0m' 33 # blue = lambda text: text 34 # yellow = lambda text: text 35 # green = lambda text: text 133 36 134 37 def main(): 135 38 136 black = lambda text: '\033[0;30m' + text + '\033[0m' 137 red = lambda text: '\033[0;31m' + text + '\033[0m' 138 green = lambda text: '\033[0;32m' + text + '\033[0m' 139 yellow = lambda text: '\033[0;33m' + text + '\033[0m' 140 blue = lambda text: '\033[0;34m' + text + '\033[0m' 141 magenta = lambda text: '\033[0;35m' + text + '\033[0m' 142 cyan = lambda text: '\033[0;36m' + text + '\033[0m' 143 lgray = lambda text: '\033[0;37m' + text + '\033[0m' 144 dgray = lambda text: '\033[1;30m' + text + '\033[0m' 145 lred = lambda text: '\033[1;32m' + text + '\033[0m' 146 lgreen = lambda text: '\033[1;32m' + text + '\033[0m' 147 lyellow = lambda text: '\033[1;33m' + text + '\033[0m' 148 lblue = lambda text: '\033[1;34m' + text + '\033[0m' 149 lpurple = lambda text: '\033[1;35m' + text + '\033[0m' 150 lcyan = lambda text: '\033[1;36m' + text + '\033[0m' 151 white = lambda text: '\033[1;37m' + text + '\033[0m' 39 defaultConfig = True 40 configNamePattern=[] 41 for i in range(len(param_list)): 42 f=open("user_param.py", "r") 43 for line in f: 44 line = line.strip().replace(" ","") 45 if line.startswith(param_list[i]): 46 configNamePattern.append('_'+param_short_list[i]+"="+line.split("=")[1].replace("'","")) 47 defaultConfig = False 152 48 153 configNamePattern=[0]*7 49 f.close() 50 51 configName = "config" 52 53 for i in range(len(configNamePattern)): 54 configName = configName+configNamePattern[i] 154 55 155 f=open("user_param.py", "r") 56 checkfile = "checkfile.def" 57 58 f=open(checkfile, "r") 156 59 for line in f: 157 60 line = line.strip() 158 if line.startswith("UsingServer2"): 159 configNamePattern[0] = '_UsingSrv2=' + UsingServer2 160 elif line.startswith("RatioServer2"): 161 configNamePattern[1] = '_RatioSrv2=' + repr(RatioServer2) 162 elif line.startswith("NumberPoolsServer2"): 163 configNamePattern[2] = '_NbPoolsSrv2=' + repr(NumberPoolsServer2) 164 elif line.startswith("NumberClients"): 165 configNamePattern[3] = '_NbClients=' + repr(NumberClients) 166 elif line.startswith("NumberServers"): 167 configNamePattern[4] = '_NbServers=' + repr(NumberServers) 168 elif line.startswith("Duration"): 169 configNamePattern[5] = '_Duration=' + Duration 170 elif line.startswith("ATMdomain"): 171 configNamePattern[5] = '_ATMdomain=' + ATMdomain 61 line = line.rstrip() 62 if line.startswith("all"): 63 allfiles = os.listdir(os.getcwd()) 64 g=open("checkfile_new.def", "w") 65 for thisfile in allfiles: 66 if thisfile.endswith(".nc") and thisfile != "dynamico_grid.nc": 67 g.write(thisfile+"\n") 68 checkfile = "checkfile_new.def" 69 g.close() 70 break 172 71 f.close() 72 73 if not os.path.exists(configName): 74 OSinfo("mkdir -p "+configName) 75 76 print(blue("\n ****************************************************************")) 77 print(blue(" * Directory " + repr(configName) + " Created ")) 78 print(blue(" * Copy results to \'not_validated_reference\' folder")) 79 print(blue(" ****************************************************************\n")) 80 81 OSinfo("mkdir -p "+configName+"/setup") 82 OSinfo("mkdir -p "+configName+"/tmp_reference") 83 84 OSinfo("cp user_param.py user_param.def") 85 OSinfo("cp user_param.py "+configName+"/setup") 86 OSinfo("mv user_param.def "+configName+"/setup") 87 88 OSinfo("cp default_param.json "+configName+"/setup") 173 89 174 175 176 configName = "config" 177 defaultConfig = True 178 179 for i in range(len(configNamePattern)): 180 if configNamePattern[i] != 0: 181 configName = configName + configNamePattern[i] 182 defaultConfig = False 183 184 if defaultConfig: 185 configName = "config_default" 186 187 90 OSinfo("mv all_param.def "+configName+"/setup") 91 OSinfo("mv iodef.xml "+configName+"/setup/") 92 OSinfo("cp context_atm.xml "+configName+"/setup/") 93 OSinfo("cp context_grid_dynamico.xml "+configName+"/setup/") 188 94 189 95 190 if not os.path.exists(configName): 96 f=open(checkfile, "r") 97 reportfile = open("../plain_report.txt", "a+") 98 for line in f: 99 line = line.strip() 100 line = line.rstrip() 101 if not line.startswith("#") and len(line) != 0: 102 OSinfo("cp "+line+" "+configName+"/tmp_reference") 103 reportfile.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(0)+" \n") 104 print(yellow("\n ****************************************")) 105 print(yellow(" ** "+line+" is stored as temporal reference !!! **")) 106 print(yellow("\n ****************************************")) 107 f.close() 108 reportfile.close() 191 109 192 193 194 195 os.mkdir(configName) 196 print blue("\n ****************************************************************") 197 print blue(" * Directory " + repr(configName) + " Created ") 198 print blue(" * Copy results to \'not_validated_reference\' folder") 199 print blue(" ****************************************************************\n") 200 201 OSinfo("mkdir "+configName+'/setup') 202 OSinfo('mkdir '+configName+'/tmp_reference') 203 OSinfo('mkdir '+configName+'/xios_output') 204 205 OSinfo('mv param.def '+configName+'/setup/') 206 OSinfo('mv all_param.def '+configName+'/setup/') 207 OSinfo('mv user_param.py '+configName+'/setup/') 208 OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ') 209 OSinfo('mv iodef.xml '+configName+'/setup/') 210 OSinfo('cp context_atm.xml '+configName+'/setup/') 211 OSinfo('cp context_grid_dynamico.xml '+configName+'/setup/') 212 OSinfo('cp dynamico_grid.nc '+configName+'/setup/') 213 OSinfo('mv xios_*.out '+configName+'/xios_output/') 214 OSinfo('mv xios_*.err '+configName+'/xios_output/') 215 216 217 f=open("checkfile.def", "r") 218 h=open("report.txt", "a+") 219 l=open("report.html", "a+") 220 preport=open("../plein_report.txt", "a+") 221 h.write("Config : "+configName+"\n") 222 l.write("<p>Config : "+configName+"</p>") 223 for line in f: 224 line=line.strip() 225 line=line.rstrip() 226 227 if line.startswith("all"): 228 allfiles = os.listdir(os.getcwd()) 229 for thisfile in allfiles: 230 if thisfile.endswith(".nc") and thisfile!="dynamico_grid.nc": 231 cmd = 'mv '+thisfile + ' '+configName+'/tmp_reference/' 232 OSinfo(cmd) 233 234 h.write("test for "+lpurple("all NC files")+" \t"+yellow('INITIALIZED')+"\n") 235 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"_all_nc_files"+" "+repr(0)+" \n") 236 l.write("<p>test for <font color=\"purple\">"+"all NC files"+"</font> <font color=\"darkorange\">INITIALIZED</font></p>") 237 print yellow("\n ****************************************") 238 print yellow(" ** "+"all NC files"+" is stored as temporal reference !!! **") 239 print yellow("\n ****************************************") 240 241 break 242 243 if not line.startswith("#") and len(line) != 0: 244 cmd = 'mv '+line + ' '+configName+'/tmp_reference/' 245 OSinfo(cmd) 246 247 h.write("test for "+lpurple(line)+" \t"+yellow('INITIALIZED')+"\n") 248 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(0)+" \n") 249 l.write("<p>test for <font color=\"purple\">"+line+"</font> <font color=\"darkorange\">INITIALIZED</font></p>") 250 print yellow("\n ****************************************") 251 print yellow(" ** "+line+" is stored as temporal reference !!! **") 252 print yellow("\n ****************************************") 253 254 f.close() 255 h.write("\n") 256 l.write("<br>") 257 h.close() 258 l.close() 259 preport.close() 260 #generate job scripts 261 hostname=os.getenv('machine_name') 262 if hostname.startswith("jeanzay"): 263 write_jz_job(configName, NumberClients+NumberServers) 264 265 elif hostname.startswith("irene"): 266 write_irene_job(configName, NumberClients+NumberServers) 267 268 else : 269 print("machine unknown") 270 print >> sys.stderr, 1 271 sys.exit() 272 273 274 else: 275 276 277 278 print blue("\n ********************************************************") 279 print blue(" * Directory " + repr(configName) + " already exists") 280 print blue(" * Start comparing results with references") 281 print blue(" ********************************************************\n") 110 else: 111 print(blue("\n ****************************************************************")) 112 print(blue(" * Directory " + repr(configName) + " already exists ")) 282 113 283 114 tmp_ref = os.path.isdir(configName+'/tmp_reference') 284 115 ref = os.path.isdir(configName+'/reference') and not tmp_ref 116 117 if tmp_ref : 118 print(blue(" * Start comparing results with NON-validated references")) 119 path = configName+"/tmp_reference" 120 if ref : 121 print(blue(" * Start comparing results with validated references")) 122 path = configName+"reference" 123 124 OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ') 125 print(blue(" ****************************************************************\n")) 285 126 286 287 if ref: 288 print blue("\n =====================================") 289 print blue(" | Comparing results with references |") 290 print blue(" =====================================\n") 291 path = configName+'/reference/' 292 OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ') 293 elif tmp_ref: 294 print blue("\n ===================================================") 295 print blue(" | Comparing results with NON-Validated references |") 296 print blue(" ===================================================\n") 297 path = configName+'/tmp_reference/' 298 OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ') 299 300 f=open("checkfile.def", "r") 301 h=open("report.txt", "a+") 302 l=open("report.html", "a+") 303 preport=open("../plein_report.txt", "a+") 304 305 306 h.write("Config : "+configName+"\n") 307 l.write("<p>Config : "+configName+"</p>") 127 f=open(checkfile, "r") 128 reportfile = open("../plain_report.txt", "a+") 308 129 for line in f: 309 310 line=line.strip() 311 line=line.rstrip() 312 313 if line.startswith("all") : 314 for thisfile in os.listdir(path): 315 if thisfile.endswith(".nc"): 316 cmd = 'cdo -W diffn '+ thisfile +' '+ path + thisfile + ' | tail -1 > diff.txt' 317 318 OSinfo(cmd) 319 320 if os.stat("diff.txt").st_size==0: 321 h.write("test for "+lpurple(thisfile)+" \t"+green('PASSED')+"\n") 322 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(1)+" \n") 323 l.write("<p>test for <font color=\"purple\">"+thisfile+"</font> <font color=\"green\">PASSED</font></p>") 324 print green("\n ****************************************") 325 print green(" ** "+thisfile+" is valid !!! **") 326 print green("\n ****************************************") 327 328 else: 329 g=open("diff.txt", "r") 330 for gline in g: 331 if gline.strip().startswith("0") or ":" in gline : 332 h.write("test for "+lpurple(thisfile)+" \t"+green('PASSED')+"\n") 333 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(1)+" \n") 334 l.write("<p>test for <font color=\"purple\">"+thisfile+"</font> <font color=\"green\">PASSED</font></p>") 335 print green("\n ****************************************") 336 print green(" ** "+thisfile+" is valid !!! **") 337 print green("\n ****************************************") 338 else: 339 h.write("test for "+lpurple(thisfile)+" \t"+red('FAILED')+"\n") 340 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(-1)+" \n") 341 l.write("<p>test for <font color=\"purple\">"+thisfile+"</font> <font color=\"red\">FAILED</font></p>") 342 print red("\n **************************************************") 343 print red(" ** "+thisfile+" is NOT valid. Please debugging.. **") 344 print red("\n **************************************************") 345 g.close() 346 347 # generate job submission script 348 hostname=os.getenv('machine_name') 349 350 if hostname.startswith("jeanzay"): 351 write_jz_job(configName, NumberClients+NumberServers) 352 353 elif hostname.startswith("irene"): 354 write_irene_job(configName, NumberClients+NumberServers) 355 356 else : 357 print("machine unknown") 358 print >> sys.stderr, 1 359 sys.exit() 360 break 361 362 363 elif not line.startswith("#") and len(line) != 0: 364 cmd = 'cdo -W diffn '+ line +' '+ path + line + ' | tail -1 > diff.txt' 365 366 OSinfo(cmd) 130 line = line.strip() 131 line = line.rstrip() 132 if not line.startswith("#") and len(line) != 0: 133 OSinfo("cdo -W diffn "+line+" "+path+line+" | tail -1 > diff.txt") 367 134 368 135 if os.stat("diff.txt").st_size==0: 369 h.write("test for "+lpurple(line)+" \t"+green('PASSED')+"\n") 370 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n") 371 l.write("<p>test for <font color=\"purple\">"+line+"</font> <font color=\"green\">PASSED</font></p>") 372 print green("\n ****************************************") 373 print green(" ** "+line+" is valid !!! **") 374 print green("\n ****************************************") 375 376 else: 136 reportfile.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n") 137 print(green("\n ****************************************")) 138 print(green(" ** "+line+" is valid !!! **")) 139 print(green("\n ****************************************")) 140 else: 377 141 g=open("diff.txt", "r") 378 142 for gline in g: 379 if gline.strip().startswith("0") or ":" in gline : 380 h.write("test for "+lpurple(line)+" \t"+green('PASSED')+"\n") 381 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n") 382 l.write("<p>test for <font color=\"purple\">"+line+"</font> <font color=\"green\">PASSED</font></p>") 383 print green("\n ****************************************") 384 print green(" ** "+line+" is valid !!! **") 385 print green("\n ****************************************") 143 if gline.strip().startswith("0") or ":" in gline: 144 reportfile.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n") 145 print(green("\n ****************************************")) 146 print(green(" ** "+line+" is valid !!! **")) 147 print(green("\n ****************************************")) 386 148 else: 387 h.write("test for "+lpurple(line)+" \t"+red('FAILED')+"\n") 388 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(-1)+" \n") 389 l.write("<p>test for <font color=\"purple\">"+line+"</font> <font color=\"red\">FAILED</font></p>") 390 print red("\n **************************************************") 391 print red(" ** "+line+" is NOT valid. Please debugging.. **") 392 print red("\n **************************************************") 149 reportfile.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(-1)+" \n") 150 print(green("\n ****************************************")) 151 print(green(" ** "+line+" is NOT valid. Please debugging.. **")) 152 print(green("\n ****************************************")) 393 153 g.close() 154 f.close() 155 reportfile.close() 394 156 395 # generate job submission script 396 hostname=os.getenv('machine_name') 397 398 if hostname.startswith("jeanzay"): 399 write_jz_job(configName, NumberClients+NumberServers) 400 401 elif hostname.startswith("irene"): 402 write_irene_job(configName, NumberClients+NumberServers) 403 404 else : 405 print("machine unknown") 406 print >> sys.stderr, 1 407 sys.exit() 408 409 410 f.close() 411 h.write("\n") 412 l.write("<br>") 413 h.close() 414 l.close() 415 preport.close() 416 417 if not os.path.exists('current_run'): 418 OSinfo('mkdir current_run') 419 420 if not os.path.exists('current_run/'+configName): 421 OSinfo('mkdir current_run/'+configName) 422 423 OSinfo('mv -f param.def current_run/'+configName) 424 OSinfo('mv -f all_param.def current_run/'+configName) 425 OSinfo('mv -f user_param.py current_run/'+configName) 426 OSinfo('mv -f iodef.xml current_run/'+configName) 427 OSinfo('rm -f current_run/'+configName+'/context_atm.xml') 428 OSinfo('cp -n context_atm.xml current_run/'+configName) 429 OSinfo('cp -f context_grid_dynamico.xml current_run/'+configName) 430 OSinfo('cp -f dynamico_grid.nc current_run/'+configName) 431 OSinfo('mv -f xios_*.out current_run/'+configName) 432 OSinfo('mv -f xios_*.err current_run/'+configName) 433 157 if os.path.exists("checkfile_new.def"): 158 OSinfo("rm -f checkfile_new.def") 434 159 435 160 -
XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/job_jeanzay.sh
r1795 r1838 34 34 35 35 source ../arch/arch-${arch}.env 36 source $I_MPI_ROOT/intel64/bin/mpivars.sh release_mt 36 37 module load cmake 37 38 module load cdo … … 58 59 59 60 60 rm -f pl ein_report.txt61 62 echo "#revision" ${xios_revision} >> pl ein_report.txt63 echo "#relurl" ${relurl} >> pl ein_report.txt64 echo "#machine" ${machine_name} >> pl ein_report.txt65 echo "#build_dir" ${xios_dir}/${build_dir} >> pl ein_report.txt66 echo "#arch" ${arch} >> pl ein_report.txt67 echo "#mode" ${mode} >> pl ein_report.txt61 rm -f plain_report.txt 62 63 echo "#revision" ${xios_revision} >> plain_report.txt 64 echo "#relurl" ${relurl} >> plain_report.txt 65 echo "#machine" ${machine_name} >> plain_report.txt 66 echo "#build_dir" ${xios_dir}/${build_dir} >> plain_report.txt 67 echo "#arch" ${arch} >> plain_report.txt 68 echo "#mode" ${mode} >> plain_report.txt 68 69 69 70 … … 74 75 do 75 76 cp setup.py ${i%%/} 77 cp default_param.json ${i%%/} 78 cp param_list.py ${i%%/} 76 79 cp run_sub_test.sh ${i%%/} 77 80 echo "bash -c \"cd " ${i%%/} " && python setup.py\" ">> setup.sh … … 107 110 mkdir -p ../html 108 111 mkdir -p ../html/test_${machine_name} 109 cp pl ein_report.txt ../html/test_${machine_name}/test_${xios_revision}_${machine_name}_${arch}_${mode}.txt112 cp plain_report.txt ../html/test_${machine_name}/test_${xios_revision}_${machine_name}_${arch}_${mode}.txt 110 113 111 114 … … 116 119 do 117 120 mkdir -p ../html/def_files/${xios_revision}/${i%%} 118 cp ${i%% }/user_params.def../html/def_files/${xios_revision}/${i%%}121 cp ${i%%/}/param.json ../html/def_files/${xios_revision}/${i%%} 119 122 for j in $(ls -d ${i%%/}/config_*) 120 123 do … … 126 129 127 130 bash -c "cd ../html && python generate_test.py" 128 129 130 131 131 132 #============================= X64_JEANZAY_debug ============================= … … 146 147 147 148 source ../arch/arch-${arch}.env 149 source $I_MPI_ROOT/intel64/bin/mpivars.sh release_mt 148 150 module load cmake 149 151 module load cdo … … 170 172 171 173 172 rm -f pl ein_report.txt173 174 echo "#revision" ${xios_revision} >> pl ein_report.txt175 echo "#relurl" ${relurl} >> pl ein_report.txt176 echo "#machine" ${machine_name} >> pl ein_report.txt177 echo "#build_dir" ${xios_dir}/${build_dir} >> pl ein_report.txt178 echo "#arch" ${arch} >> pl ein_report.txt179 echo "#mode" ${mode} >> pl ein_report.txt174 rm -f plain_report.txt 175 176 echo "#revision" ${xios_revision} >> plain_report.txt 177 echo "#relurl" ${relurl} >> plain_report.txt 178 echo "#machine" ${machine_name} >> plain_report.txt 179 echo "#build_dir" ${xios_dir}/${build_dir} >> plain_report.txt 180 echo "#arch" ${arch} >> plain_report.txt 181 echo "#mode" ${mode} >> plain_report.txt 180 182 181 183 … … 186 188 do 187 189 cp setup.py ${i%%/} 190 cp default_param.json ${i%%/} 191 cp param_list.py ${i%%/} 188 192 cp run_sub_test.sh ${i%%/} 189 193 echo "bash -c \"cd " ${i%%/} " && python setup.py\" ">> setup.sh … … 219 223 mkdir -p ../html 220 224 mkdir -p ../html/test_${machine_name} 221 222 cp plein_report.txt ../html/test_${machine_name}/test_${xios_revision}_${machine_name}_${arch}_${mode}.txt 225 cp plain_report.txt ../html/test_${machine_name}/test_${xios_revision}_${machine_name}_${arch}_${mode}.txt 226 223 227 224 228 mkdir -p ../html/def_files … … 228 232 do 229 233 mkdir -p ../html/def_files/${xios_revision}/${i%%} 230 cp ${i%% }/user_params.def../html/def_files/${xios_revision}/${i%%}234 cp ${i%%/}/param.json ../html/def_files/${xios_revision}/${i%%} 231 235 for j in $(ls -d ${i%%/}/config_*) 232 236 do … … 239 243 bash -c "cd ../html && python generate_test.py" 240 244 241 -
XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/run_sub_test.sh
r1792 r1838 5 5 cp ../config.py sub_config.py 6 6 cp ../check.py sub_check.py 7 cp ../default_param.py default_param.py7 # cp ../default_param.py default_param.py 8 8 9 9 -
XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/setup.py
r1792 r1838 1 1 import os 2 2 import sys 3 import json 4 import itertools 3 5 6 from param_list import * 4 7 5 def main(): 8 def getlabel(sub_list_element, level): 9 label="" 10 for l in range(level): 11 label = label+sub_list_element[l]+"_" 12 return label 6 13 7 black = lambda text: '\033[0;30m' + text + '\033[0m' 8 red = lambda text: '\033[0;31m' + text + '\033[0m' 9 green = lambda text: '\033[0;32m' + text + '\033[0m' 10 yellow = lambda text: '\033[0;33m' + text + '\033[0m' 11 blue = lambda text: '\033[0;34m' + text + '\033[0m' 12 magenta = lambda text: '\033[0;35m' + text + '\033[0m' 13 cyan = lambda text: '\033[0;36m' + text + '\033[0m' 14 lgray = lambda text: '\033[0;37m' + text + '\033[0m' 15 dgray = lambda text: '\033[1;30m' + text + '\033[0m' 16 lred = lambda text: '\033[1;32m' + text + '\033[0m' 17 lgreen = lambda text: '\033[1;32m' + text + '\033[0m' 18 lyellow = lambda text: '\033[1;33m' + text + '\033[0m' 19 lblue = lambda text: '\033[1;34m' + text + '\033[0m' 20 lpurple = lambda text: '\033[1;35m' + text + '\033[0m' 21 lcyan = lambda text: '\033[1;36m' + text + '\033[0m' 22 white = lambda text: '\033[1;37m' + text + '\033[0m' 14 def MySort(sub_list, level): 15 l=len(sub_list) 16 for i in range(0,l): 17 for j in range(0, l-i-1): 18 if(getlabel(sub_list[j], level) > getlabel(sub_list[j+1], level)): 19 tempo = sub_list[j] 20 sub_list[j]= sub_list[j+1] 21 sub_list[j+1]= tempo 22 return sub_list 23 23 24 default_nb_config = 7 24 def main() : 25 26 param_length = len(param_list) 27 param = [""]*param_length 28 29 30 f=open("user_param.json", 'r') 31 config_dict = json.load(f) 32 f.close() 33 34 f=open("default_param.json", 'r') 35 def_config = json.load(f) 36 f.close() 37 38 default_config=[""]*param_length 39 f=open("default_param.py", "w") 40 for i in range(param_length): 41 default_config[i] = str(def_config[0][param_list[i]]) 42 f.write(param_list[i]+" = "+str(def_config[0][param_list[i]])+"\n") 43 f.close() 44 45 config_list=[] 46 47 for config_set in config_dict: 48 sub_list=[] 49 for l in range(param_length): 50 if param_list[l] in config_set: 51 param[l]=config_set[param_list[l]].split(" ") 52 for i in range(len(param[l])) : 53 param[l][i] = [str(param[l][i])] 54 else : 55 param[l]=['_'] 56 57 for l in range(len(param_list)): 58 if not sub_list: 59 sub_list = list(param[l]) 60 else: 61 tmp_sub_list = list(itertools.product(sub_list,list(param[l]))) 62 sub_list=[] 63 for i in range(len(tmp_sub_list)): 64 sub_list.append(list(tmp_sub_list[i])) 65 for i in range(len(sub_list)): 66 sub_list[i] = list(itertools.chain(*sub_list[i])) 67 68 for i in range(len(sub_list)): 69 config_list.append(sub_list[i]) 70 71 72 param=[""]*param_length 73 74 config_list=MySort(config_list, param_length) 75 76 77 for i in range(len(config_list)): 78 print(config_list[i]) 79 80 81 full_config_list=[[]]*len(config_list) 82 83 for i in range(len(config_list)): 84 full_config_list[i] = list(config_list[i]) 85 for j in range(param_length): 86 if full_config_list[i][j]=='_': 87 full_config_list[i][j] = default_config[j] 88 # print(full_config_list[i]) 89 90 for i in range(len(config_list)): 91 f=open("user_param.py."+repr(i), "w") 92 for j in range(param_length): 93 if not config_list[i][j] == "_": 94 f.write(param_list[j]+" = "+config_list[i][j]+"\n") 95 f.close() 96 97 f=open("CMakeLists.txt", "w") 98 f.write("##############################\n") 99 f.write("# file generated by setup.py #\n") 100 f.write("# DO NOT modify #\n") 101 f.write("##############################\n\n") 102 f.write("cmake_minimum_required(VERSION 2.8.12.2)\n\n") 103 f.write("project(generic_testcase)\n\n") 104 f.write("find_package(PythonInterp REQUIRED)\n\n") 105 f.write("enable_testing()\n\n") 106 dirpath = os.getcwd() 107 foldername = os.path.basename(os.getcwd()) 108 print(foldername) 109 110 for x in range(len(config_list)): 111 f.write("add_test( NAME "+foldername+"_config"+repr(x)+'\n') 112 f.write(" COMMAND ${PYTHON_EXECUTABLE}\n") 113 f.write(" ${CMAKE_CURRENT_BINARY_DIR}/run_test_"+repr(x)+".py\n") 114 f.write(" "+foldername+"_config"+repr(x)+")\n\n") 115 f.write("set_tests_properties("+foldername+"_config"+repr(x)+" PROPERTIES\n") 116 f.write(" FAIL_REGULAR_EXPRESSION \"check.py failed;execution failed;failed;setup.sh failed; machine unknown\")\n\n") 25 117 26 nb_config = 1 27 configs = [0]*default_nb_config 28 29 configs_arg = [[], [], [], [], [], [], []] 118 g=open("run_test_"+repr(x)+".py", "w") 119 g.write("##############################\n") 120 g.write("# file generated by setup.py #\n") 121 g.write("# DO NOT modify #\n") 122 g.write("##############################\n\n") 123 g.write("import os\n") 124 g.write("import sys\n") 125 g.write("import subprocess\n") 126 30 127 31 f=open("user_params.def", "r") 32 for line in f: 33 line = line.strip() 34 if len(line) != 0: 35 if not line.startswith("#"): 36 nb_config = nb_config * (line.count(",")+1) 37 line = line.replace(",", "") 38 line = line.replace("=", "") 39 if line.startswith("UsingServer2"): 40 line = line.replace("UsingServer2", "") 41 for word in line.split(): 42 configs[0] += 1 43 configs_arg[0].append('UsingServer2 = '+word) 44 elif line.startswith("RatioServer2"): 45 line = line.replace("RatioServer2", "") 46 for word in line.split(): 47 configs[1] += 1 48 configs_arg[1].append('RatioServer2 = '+word) 49 elif line.startswith("NumberPoolsServer2"): 50 line = line.replace("NumberPoolsServer2", "") 51 for word in line.split(): 52 configs[2] += 1 53 configs_arg[2].append('NumberPoolsServer2 = '+word) 54 elif line.startswith("NumberClients"): 55 line = line.replace("NumberClients", "") 56 for word in line.split(): 57 configs[3] += 1 58 configs_arg[3].append('NumberClients = '+word) 59 elif line.startswith("NumberServers"): 60 line = line.replace("NumberServers", "") 61 for word in line.split(): 62 configs[4] += 1 63 configs_arg[4].append('NumberServers = '+word) 64 elif line.startswith("Duration"): 65 line = line.replace("Duration", "") 66 for word in line.split(): 67 configs[5] += 1 68 configs_arg[5].append('Duration = '+word) 69 elif line.startswith("ATMdomain"): 70 line = line.replace("ATMdomain", "") 71 for word in line.split(): 72 configs[6] += 1 73 configs_arg[6].append('ATMdomain = '+word) 128 g.write("\ndef main():\n") 129 g.write(" os.system(\'rm -f user_param.py\')\n") 130 g.write(" os.system(\'cp user_param.py."+repr(x)+" user_param.py\')\n") 131 g.write(" os.system(\'./run_sub_test.sh\')\n") 132 g.write("if __name__== \"__main__\":\n") 133 g.write(" main()\n") 134 g.close() 135 136 f.close() 74 137 75 76 for x in range(default_nb_config): 77 if configs[x]==0: 78 configs[x] =1 79 80 config_id=0 81 for i0 in range(configs[0]): 82 hasContent0 = len(configs_arg[0])>0 83 for i1 in range(configs[1]): 84 hasContent1 = len(configs_arg[1])>0 85 for i2 in range(configs[2]): 86 hasContent2 = len(configs_arg[2])>0 87 for i3 in range(configs[3]): 88 hasContent3 = len(configs_arg[3])>0 89 for i4 in range(configs[4]): 90 hasContent4 = len(configs_arg[4])>0 91 for i5 in range(configs[5]): 92 hasContent5 = len(configs_arg[5])>0 93 for i6 in range(configs[6]): 94 hasContent6 = len(configs_arg[6])>0 95 96 filename = "user_param.py."+repr(config_id) 97 g = open(filename, "w") 98 99 if hasContent0: g.write(configs_arg[0][i0]+'\n') 100 if hasContent1: g.write(configs_arg[1][i1]+'\n') 101 if hasContent2: g.write(configs_arg[2][i2]+'\n') 102 if hasContent3: g.write(configs_arg[3][i3]+'\n') 103 if hasContent4: g.write(configs_arg[4][i4]+'\n') 104 if hasContent5: g.write(configs_arg[5][i5]+'\n') 105 if hasContent6: g.write(configs_arg[6][i6]+'\n') 106 107 config_id +=1 108 g.close 138 if __name__== "__main__": 139 main() 109 140 110 141 111 142 112 f=open("CMakeLists.txt", "w")113 f.write("##############################\n")114 f.write("# file generated by setup.py #\n")115 f.write("# DO NOT modify #\n")116 f.write("##############################\n\n")117 f.write("cmake_minimum_required(VERSION 2.8.12.2)\n\n")118 f.write("project(generic_testcase)\n\n")119 f.write("find_package(PythonInterp REQUIRED)\n\n")120 f.write("enable_testing()\n\n")121 dirpath = os.getcwd()122 foldername = os.path.basename(os.getcwd())123 124 125 for x in range(nb_config):126 f.write("add_test( NAME "+foldername+"_config"+repr(x)+'\n')127 f.write(" COMMAND ${PYTHON_EXECUTABLE}\n")128 f.write(" ${CMAKE_CURRENT_BINARY_DIR}/run_test_"+repr(x)+".py\n")129 f.write(" "+foldername+"_config"+repr(x)+")\n\n")130 131 f.write("set_tests_properties("+foldername+"_config"+repr(x)+" PROPERTIES\n")132 f.write(" FAIL_REGULAR_EXPRESSION \"check.py failed;execution failed;failed;setup.sh failed; machine unknown\")\n\n")133 134 g=open("run_test_"+repr(x)+".py", "w")135 g.write("##############################\n")136 g.write("# file generated by setup.py #\n")137 g.write("# DO NOT modify #\n")138 g.write("##############################\n\n")139 g.write("import os\n")140 g.write("import sys\n")141 g.write("import subprocess\n")142 # g.write("\ndef OSinfo(runthis):\n")143 # g.write(" osstdout = subprocess.Popen(runthis, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)\n")144 # g.write(" theInfo = osstdout.communicate()[0].strip()\n")145 # g.write(" if osstdout.returncode!=0:\n")146 # g.write(" print(runthis+\" failed\")\n")147 # g.write(" print >> sys.stderr, osstdout.returncode\n")148 # g.write(" sys.exit()\n")149 # g.write(" else:\n")150 # g.write(" print(runthis+\" OK\")\n")151 152 g.write("\ndef main():\n")153 g.write(" os.system(\'rm -f user_param.py\')\n")154 g.write(" os.system(\'cp user_param.py."+repr(x)+" user_param.py\')\n")155 g.write(" os.system(\'./run_sub_test.sh\')\n")156 g.write("if __name__== \"__main__\":\n")157 g.write(" main()\n")158 g.close159 160 f.close161 162 f=open("report.txt", "w")163 g=open("report.html", "w")164 f.write("\n\n\nin "+cyan(foldername)+" ...\n\n")165 g.write("<br><br><br><p>in <font color=\"blue\">"+foldername+"</font> ...</p><br><br>")166 f.close167 g.close168 169 170 if __name__== "__main__":171 main() -
XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/test_axis_algo/checkfile.def
r1734 r1838 1 all1 #all 2 2 atm_output_zoom.nc 3 3 #atm_output_inverse.nc -
XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/test_domain_algo/checkfile.def
r1734 r1838 1 1 #all 2 3 2 atm_output.nc 4 5 3 atm_output_expand.nc 6 7 8 4 atm_output_extract.nc 9 10 5 atm_output_interpolate.nc 11 12 6 atm_output_zoom.nc -
XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/test_function/checkfile.def
r1733 r1838 1 #all 2 3 atm_output.nc 1 4 2 atm_output.nc3 5 #atm_output_ensemble.nc -
XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/test_grid_algo/checkfile.def
r1714 r1838 1 1 #atm_output_1.nc 2 3 2 atm_output.nc 4 5 3 atm_output_zoom.nc -
XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/test_scalar_algo/checkfile.def
r1714 r1838 1 1 #atm_output1.nc 2 3 2 atm_output.nc 4 3 atm_output_zoom.nc -
XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/user_config.py
r1771 r1838 41 41 f.write(" COMMAND rm -f report.txt\n") 42 42 f.write(" COMMAND rm -f report.html\n") 43 44 45 for folder in test_list:46 f.write(" COMMAND cat "+folder+"/report.txt >> report.txt\n")47 f.write(" COMMAND cat "+folder+"/report.html >> report.html\n")48 49 43 f.write(" )\n\n") 50 44 f.close
Note: See TracChangeset
for help on using the changeset viewer.