Changeset 1841 for XIOS/trunk/xios_test_suite/TEST_SUITE/check.py
- Timestamp:
- 01/02/20 19:10:21 (4 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/trunk/xios_test_suite/TEST_SUITE/check.py
r1824 r1841 7 7 8 8 9 from default_param import * 10 from user_param import * 9 # from default_param import * 10 # from user_param import * 11 from param_list import * 11 12 12 13 # load default_param first, variables defines in user_param will overwrite the default value … … 26 27 27 28 28 def write_jz_job(configName, nbproc=8):29 red = lambda text: '\033[0;31m' + text + '\033[0m'30 f=open("jz_job_"+configName+".sh", "w")31 f.write("#!/bin/bash\n")32 f.write("#SBATCH --ntasks="+repr(nbproc)+" # Nombre total de processus MPI\n")33 f.write("#SBATCH --ntasks-per-node=16\n")34 f.write("#SBATCH --hint=nomultithread\n")35 f.write("#SBATCH -t 00:10:00 # Temps elapsed maximum d'execution\n")36 f.write("#SBATCH -o single_config.out # Nom du fichier de sortie\n")37 f.write("#SBATCH -e single_config.err # Nom du fichier d'erreur\n")38 f.write("#SBATCH --account=psl@cpu\n")39 f.write("cd ${SLURM_SUBMIT_DIR}\n")40 f.write("module purge\n")41 f.write("module load intel-all/19.0.4\n")42 f.write("module load netcdf/4.7.0/intel-19.0.4-mpi\n")43 f.write("module load netcdf-cxx/4.2/intel-19.0.4-mpi\n")44 f.write("module load netcdf-fortran/4.4.5/intel-19.0.4-mpi\n")45 f.write("module load gcc/6.5.0/gcc-4.8.5\n")46 f.write("module load cmake\n")47 f.write("module load cdo\n")48 f.write("ulimit -c 0\n")49 xios_root=os.getenv('xios_dir')50 build_folder=os.getenv('build_dir')51 f.write("export xios_dir="+xios_root+"\n")52 f.write("export build_dir="+build_folder+"\n")53 f.write("export machine_name=jeanzay\n")54 f.write("cp ../setup.py ./\n")55 f.write("cp ../run_sub_test.sh ./\n")56 f.write("mv user_params.def user_params.def_bkp\n")57 f.write("cp "+configName+"/setup/user_param.py user_params.def\n")58 f.write("python setup.py\n")59 f.write("cmake .\n")60 f.write("ctest -V #enable all output\n")61 f.write("rm -f setup.py\n")62 f.write("rm -f run_sub_test.sh\n")63 f.write("rm -f run_test_*.py\n")64 f.write("rm -f CMakeLists.txt\n")65 f.write("rm -f context_grid_dynamico.xml\n")66 f.write("rm -f dynamico_grid.nc\n")67 f.write("rm -f default_param.*\n")68 f.write("rm -f user_param.*\n")69 f.write("mv user_params.def_bkp user_params.def\n")70 f.write("\n")71 f.close72 29 73 def write_irene_job(configName, nbproc=8): 74 red = lambda text: '\033[0;31m' + text + '\033[0m' 75 f=open("irene_job_"+configName+".sh", "w") 76 f.write("#!/bin/bash\n") 77 f.write("#MSUB -r XIOS\n") 78 f.write("#MSUB -o client_output.out # standard output\n") 79 f.write("#MSUB -e client_error.err #error output\n") 80 f.write("#MSUB -eo\n") 81 f.write("#MSUB -c 1\n") 82 f.write("#MSUB -n "+ repr(nbproc) + " # Number of MPI tasks (SPMD case) or cores (MPMD case)\n") 83 f.write("#MSUB -X \n") 84 f.write("#MSUB -x \n") 85 f.write("#MSUB -T 600 # Wall clock limit (seconds)\n") 86 f.write("#MSUB -q skylake # thin nodes\n") 87 f.write("#MSUB -A devcmip6\n") 88 f.write("#MSUB -Q test\n") 89 f.write("#MSUB -m work\n") 90 f.write("cd $BRIDGE_MSUB_PWD\n") 91 f.write("module unload netcdf-c netcdf-fortran hdf5 flavor perl hdf5 boost blitz mpi gnu\n") 92 f.write("module load gnu\n") 93 f.write("module load mpi/intelmpi/2017.0.6.256\n") 94 f.write("module load flavor/buildcompiler/intel/17\n") 95 f.write("module load flavor/hdf5/parallel\n") 96 f.write("module load netcdf-fortran/4.4.4\n") 97 f.write("module load hdf5/1.8.20\n") 98 f.write("module load boost\n") 99 f.write("module load blitz\n") 100 f.write("module load feature/bridge/heterogenous_mpmd\n") 101 f.write("module load nco\n") 102 f.write("module load cdo\n") 103 f.write("export KMP_STACKSIZE=3g\n") 104 f.write("export KMP_LIBRARY=turnaround\n") 105 f.write("export MKL_SERIAL=YES\n") 106 f.write("export OMP_NUM_THREADS=${BRIDGE_MSUB_NCORE}\n") 107 f.write("set -x \n") 108 xios_root=os.getenv('xios_dir') 109 build_folder=os.getenv('build_dir') 110 f.write("export xios_dir="+xios_root+"\n") 111 f.write("export build_dir="+build_folder+"\n") 112 f.write("export machine_name=irene\n") 113 f.write("cp ../setup.py ./\n") 114 f.write("cp ../run_sub_test.sh ./\n") 115 f.write("mv user_params.def user_params.def_bkp\n") 116 f.write("cp "+configName+"/setup/user_param.py user_params.def\n") 117 f.write("python setup.py\n") 118 f.write("cmake .\n") 119 f.write("ctest -V #enable all output\n") 120 f.write("rm -f setup.py\n") 121 f.write("rm -f run_sub_test.sh\n") 122 f.write("rm -f run_test_*.py\n") 123 f.write("rm -f CMakeLists.txt\n") 124 f.write("rm -f context_grid_dynamico.xml\n") 125 f.write("rm -f dynamico_grid.nc\n") 126 f.write("rm -f default_param.*\n") 127 f.write("rm -f user_param.*\n") 128 f.write("mv user_params.def_bkp user_params.def\n") 129 f.write("\n") 130 f.close 131 132 30 blue = lambda text: '\033[0;34m' + text + '\033[0m' 31 yellow = lambda text: '\033[0;33m' + text + '\033[0m' 32 green = lambda text: '\033[0;32m' + text + '\033[0m' 33 # blue = lambda text: text 34 # yellow = lambda text: text 35 # green = lambda text: text 133 36 134 37 def main(): 135 38 136 black = lambda text: '\033[0;30m' + text + '\033[0m' 137 red = lambda text: '\033[0;31m' + text + '\033[0m' 138 green = lambda text: '\033[0;32m' + text + '\033[0m' 139 yellow = lambda text: '\033[0;33m' + text + '\033[0m' 140 blue = lambda text: '\033[0;34m' + text + '\033[0m' 141 magenta = lambda text: '\033[0;35m' + text + '\033[0m' 142 cyan = lambda text: '\033[0;36m' + text + '\033[0m' 143 lgray = lambda text: '\033[0;37m' + text + '\033[0m' 144 dgray = lambda text: '\033[1;30m' + text + '\033[0m' 145 lred = lambda text: '\033[1;32m' + text + '\033[0m' 146 lgreen = lambda text: '\033[1;32m' + text + '\033[0m' 147 lyellow = lambda text: '\033[1;33m' + text + '\033[0m' 148 lblue = lambda text: '\033[1;34m' + text + '\033[0m' 149 lpurple = lambda text: '\033[1;35m' + text + '\033[0m' 150 lcyan = lambda text: '\033[1;36m' + text + '\033[0m' 151 white = lambda text: '\033[1;37m' + text + '\033[0m' 39 defaultConfig = True 40 configNamePattern=[] 41 for i in range(len(param_list)): 42 f=open("user_param.py", "r") 43 for line in f: 44 line = line.strip().replace(" ","") 45 if line.startswith(param_list[i]): 46 configNamePattern.append('_'+param_short_list[i]+"="+line.split("=")[1].replace("'","")) 47 defaultConfig = False 152 48 153 configNamePattern=[0]*7 49 f.close() 50 51 configName = "config" 52 53 for i in range(len(configNamePattern)): 54 configName = configName+configNamePattern[i] 154 55 155 f=open("user_param.py", "r") 56 checkfile = "checkfile.def" 57 58 f=open(checkfile, "r") 156 59 for line in f: 157 60 line = line.strip() 158 if line.startswith("UsingServer2"): 159 configNamePattern[0] = '_UsingSrv2=' + UsingServer2 160 elif line.startswith("RatioServer2"): 161 configNamePattern[1] = '_RatioSrv2=' + repr(RatioServer2) 162 elif line.startswith("NumberPoolsServer2"): 163 configNamePattern[2] = '_NbPoolsSrv2=' + repr(NumberPoolsServer2) 164 elif line.startswith("NumberClients"): 165 configNamePattern[3] = '_NbClients=' + repr(NumberClients) 166 elif line.startswith("NumberServers"): 167 configNamePattern[4] = '_NbServers=' + repr(NumberServers) 168 elif line.startswith("Duration"): 169 configNamePattern[5] = '_Duration=' + Duration 170 elif line.startswith("ATMdomain"): 171 configNamePattern[5] = '_ATMdomain=' + ATMdomain 61 line = line.rstrip() 62 if line.startswith("all"): 63 allfiles = os.listdir(os.getcwd()) 64 g=open("checkfile_new.def", "w") 65 for thisfile in allfiles: 66 if thisfile.endswith(".nc") and thisfile != "dynamico_grid.nc": 67 g.write(thisfile+"\n") 68 checkfile = "checkfile_new.def" 69 g.close() 70 break 172 71 f.close() 72 73 if not os.path.exists(configName): 74 OSinfo("mkdir -p "+configName) 75 76 print(blue("\n ****************************************************************")) 77 print(blue(" * Directory " + repr(configName) + " Created ")) 78 print(blue(" * Copy results to \'not_validated_reference\' folder")) 79 print(blue(" ****************************************************************\n")) 80 81 OSinfo("mkdir -p "+configName+"/setup") 82 OSinfo("mkdir -p "+configName+"/tmp_reference") 83 84 OSinfo("cp user_param.py user_param.def") 85 OSinfo("cp user_param.py "+configName+"/setup") 86 OSinfo("mv user_param.def "+configName+"/setup") 87 88 OSinfo("cp default_param.json "+configName+"/setup") 173 89 174 175 176 configName = "config" 177 defaultConfig = True 178 179 for i in range(len(configNamePattern)): 180 if configNamePattern[i] != 0: 181 configName = configName + configNamePattern[i] 182 defaultConfig = False 183 184 if defaultConfig: 185 configName = "config_default" 186 187 90 OSinfo("mv all_param.def "+configName+"/setup") 91 OSinfo("mv iodef.xml "+configName+"/setup/") 92 OSinfo("cp context_atm.xml "+configName+"/setup/") 93 OSinfo("cp context_grid_dynamico.xml "+configName+"/setup/") 188 94 189 95 190 if not os.path.exists(configName): 96 f=open(checkfile, "r") 97 reportfile = open("../plain_report.txt", "a+") 98 for line in f: 99 line = line.strip() 100 line = line.rstrip() 101 if not line.startswith("#") and len(line) != 0: 102 OSinfo("cp "+line+" "+configName+"/tmp_reference") 103 reportfile.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(0)+" \n") 104 print(yellow("\n ****************************************")) 105 print(yellow(" ** "+line+" is stored as temporal reference !!! **")) 106 print(yellow("\n ****************************************")) 107 f.close() 108 reportfile.close() 191 109 192 193 194 195 os.mkdir(configName) 196 print blue("\n ****************************************************************") 197 print blue(" * Directory " + repr(configName) + " Created ") 198 print blue(" * Copy results to \'not_validated_reference\' folder") 199 print blue(" ****************************************************************\n") 200 201 OSinfo("mkdir "+configName+'/setup') 202 OSinfo('mkdir '+configName+'/tmp_reference') 203 OSinfo('mkdir '+configName+'/xios_output') 204 205 OSinfo('mv param.def '+configName+'/setup/') 206 OSinfo('mv all_param.def '+configName+'/setup/') 207 OSinfo('mv user_param.py '+configName+'/setup/') 208 OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ') 209 OSinfo('mv iodef.xml '+configName+'/setup/') 210 OSinfo('cp context_atm.xml '+configName+'/setup/') 211 OSinfo('cp context_grid_dynamico.xml '+configName+'/setup/') 212 OSinfo('cp dynamico_grid.nc '+configName+'/setup/') 213 OSinfo('mv xios_*.out '+configName+'/xios_output/') 214 OSinfo('mv xios_*.err '+configName+'/xios_output/') 215 216 217 f=open("checkfile.def", "r") 218 preport=open("../plain_report.txt", "a+") 219 for line in f: 220 line=line.strip() 221 line=line.rstrip() 222 223 if line.startswith("all"): 224 allfiles = os.listdir(os.getcwd()) 225 for thisfile in allfiles: 226 if thisfile.endswith(".nc") and thisfile!="dynamico_grid.nc": 227 cmd = 'mv '+thisfile + ' '+configName+'/tmp_reference/' 228 OSinfo(cmd) 229 230 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"_all_nc_files"+" "+repr(0)+" \n") 231 print yellow("\n ****************************************") 232 print yellow(" ** "+"all NC files"+" is stored as temporal reference !!! **") 233 print yellow("\n ****************************************") 234 235 break 236 237 if not line.startswith("#") and len(line) != 0: 238 cmd = 'mv '+line + ' '+configName+'/tmp_reference/' 239 OSinfo(cmd) 240 241 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(0)+" \n") 242 print yellow("\n ****************************************") 243 print yellow(" ** "+line+" is stored as temporal reference !!! **") 244 print yellow("\n ****************************************") 245 246 f.close() 247 preport.close() 248 #generate job scripts 249 hostname=os.getenv('machine_name') 250 if hostname.startswith("jeanzay"): 251 write_jz_job(configName, NumberClients+NumberServers) 252 253 elif hostname.startswith("irene"): 254 write_irene_job(configName, NumberClients+NumberServers) 255 256 else : 257 print("machine unknown") 258 print >> sys.stderr, 1 259 sys.exit() 260 261 262 else: 263 264 265 266 print blue("\n ********************************************************") 267 print blue(" * Directory " + repr(configName) + " already exists") 268 print blue(" * Start comparing results with references") 269 print blue(" ********************************************************\n") 110 else: 111 print(blue("\n ****************************************************************")) 112 print(blue(" * Directory " + repr(configName) + " already exists ")) 270 113 271 114 tmp_ref = os.path.isdir(configName+'/tmp_reference') 272 115 ref = os.path.isdir(configName+'/reference') and not tmp_ref 116 117 if tmp_ref : 118 print(blue(" * Start comparing results with NON-validated references")) 119 path = configName+"/tmp_reference" 120 if ref : 121 print(blue(" * Start comparing results with validated references")) 122 path = configName+"reference" 123 124 OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ') 125 print(blue(" ****************************************************************\n")) 273 126 274 275 if ref: 276 print blue("\n =====================================") 277 print blue(" | Comparing results with references |") 278 print blue(" =====================================\n") 279 path = configName+'/reference/' 280 OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ') 281 elif tmp_ref: 282 print blue("\n ===================================================") 283 print blue(" | Comparing results with NON-Validated references |") 284 print blue(" ===================================================\n") 285 path = configName+'/tmp_reference/' 286 OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ') 287 288 f=open("checkfile.def", "r") 289 preport=open("../plain_report.txt", "a+") 290 127 f=open(checkfile, "r") 128 reportfile = open("../plain_report.txt", "a+") 291 129 for line in f: 292 293 line=line.strip() 294 line=line.rstrip() 295 296 if line.startswith("all") : 297 for thisfile in os.listdir(path): 298 if thisfile.endswith(".nc"): 299 cmd = 'cdo -W diffn '+ thisfile +' '+ path + thisfile + ' | tail -1 > diff.txt' 300 301 OSinfo(cmd) 302 303 if os.stat("diff.txt").st_size==0: 304 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(1)+" \n") 305 print green("\n ****************************************") 306 print green(" ** "+thisfile+" is valid !!! **") 307 print green("\n ****************************************") 308 309 else: 310 g=open("diff.txt", "r") 311 for gline in g: 312 if gline.strip().startswith("0") or ":" in gline : 313 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(1)+" \n") 314 print green("\n ****************************************") 315 print green(" ** "+thisfile+" is valid !!! **") 316 print green("\n ****************************************") 317 else: 318 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(-1)+" \n") 319 print red("\n **************************************************") 320 print red(" ** "+thisfile+" is NOT valid. Please debugging.. **") 321 print red("\n **************************************************") 322 g.close() 323 324 # generate job submission script 325 hostname=os.getenv('machine_name') 326 327 if hostname.startswith("jeanzay"): 328 write_jz_job(configName, NumberClients+NumberServers) 329 330 elif hostname.startswith("irene"): 331 write_irene_job(configName, NumberClients+NumberServers) 332 333 else : 334 print("machine unknown") 335 print >> sys.stderr, 1 336 sys.exit() 337 break 338 339 340 elif not line.startswith("#") and len(line) != 0: 341 cmd = 'cdo -W diffn '+ line +' '+ path + line + ' | tail -1 > diff.txt' 342 343 OSinfo(cmd) 130 line = line.strip() 131 line = line.rstrip() 132 if not line.startswith("#") and len(line) != 0: 133 OSinfo("cdo -W diffn "+line+" "+path+line+" | > diff.txt") 134 # this command should change according to cdo version 344 135 345 136 if os.stat("diff.txt").st_size==0: 346 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n") 347 print green("\n ****************************************") 348 print green(" ** "+line+" is valid !!! **") 349 print green("\n ****************************************") 350 351 else: 137 reportfile.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n") 138 print(green("\n ****************************************")) 139 print(green(" ** "+line+" is valid !!! **")) 140 print(green("\n ****************************************")) 141 else: 352 142 g=open("diff.txt", "r") 353 143 for gline in g: 354 if gline.strip().startswith("0") or ":" in gline 355 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n")356 print green("\n ****************************************")357 print green(" ** "+line+" is valid !!! **")358 print green("\n ****************************************")144 if gline.strip().startswith("0") or ":" in gline: 145 reportfile.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n") 146 print(green("\n ****************************************")) 147 print(green(" ** "+line+" is valid !!! **")) 148 print(green("\n ****************************************")) 359 149 else: 360 preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(-1)+" \n")361 print red("\n **************************************************")362 print red(" ** "+line+" is NOT valid. Please debugging.. **")363 print red("\n **************************************************")150 reportfile.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(-1)+" \n") 151 print(green("\n ****************************************")) 152 print(green(" ** "+line+" is NOT valid. Please debugging.. **")) 153 print(green("\n ****************************************")) 364 154 g.close() 155 f.close() 156 reportfile.close() 365 157 366 # generate job submission script 367 hostname=os.getenv('machine_name') 368 369 if hostname.startswith("jeanzay"): 370 write_jz_job(configName, NumberClients+NumberServers) 371 372 elif hostname.startswith("irene"): 373 write_irene_job(configName, NumberClients+NumberServers) 374 375 else : 376 print("machine unknown") 377 print >> sys.stderr, 1 378 sys.exit() 379 380 381 f.close() 382 preport.close() 383 384 if not os.path.exists('current_run'): 385 OSinfo('mkdir current_run') 386 387 if not os.path.exists('current_run/'+configName): 388 OSinfo('mkdir current_run/'+configName) 389 390 OSinfo('mv -f param.def current_run/'+configName) 391 OSinfo('mv -f all_param.def current_run/'+configName) 392 OSinfo('mv -f user_param.py current_run/'+configName) 393 OSinfo('mv -f iodef.xml current_run/'+configName) 394 OSinfo('rm -f current_run/'+configName+'/context_atm.xml') 395 OSinfo('cp -n context_atm.xml current_run/'+configName) 396 OSinfo('cp -f context_grid_dynamico.xml current_run/'+configName) 397 OSinfo('cp -f dynamico_grid.nc current_run/'+configName) 398 OSinfo('mv -f xios_*.out current_run/'+configName) 399 OSinfo('mv -f xios_*.err current_run/'+configName) 400 158 if os.path.exists("checkfile_new.def"): 159 OSinfo("rm -f checkfile_new.def") 401 160 402 161
Note: See TracChangeset
for help on using the changeset viewer.