Context Navigation

source: XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/check.py @ 1776

Last change on this file since 1776 was 1776, checked in by yushan, 4 years ago
dev_trunk_omp : update xios_report.html
File size: 17.4 KB

Line
1	import os
2	import sys
3	import subprocess
4	import socket
5	import os.path
6	from os import path
7
8
9	from default_param import *
10	from user_param import *
11
12	# load default_param first, variables defines in user_param will overwrite the default value
13
14
15
16	def OSinfo(runthis):
17	red = lambda text: '\033[0;31m' + text + '\033[0m'
18	osstdout = subprocess.Popen(runthis, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
19	theInfo = osstdout.communicate()[0].strip()
20	if osstdout.returncode!=0:
21	print(red(runthis+" FAILED"))
22	print >> sys.stderr, osstdout.returncode
23	sys.exit()
24	# else:
25	# print(runthis+" OK")
26
27
28	def write_jz_job(configName, nbproc=8):
29	red = lambda text: '\033[0;31m' + text + '\033[0m'
30	f=open("jz_job_"+configName+".sh", "w")
31	f.write("#!/bin/bash\n")
32	f.write("#SBATCH --ntasks="+repr(nbproc)+" # Nombre total de processus MPI\n")
33	f.write("#SBATCH --ntasks-per-node=16\n")
34	f.write("#SBATCH --hint=nomultithread\n")
35	f.write("#SBATCH -t 00:10:00 # Temps elapsed maximum d'execution\n")
36	f.write("#SBATCH -o single_config.out # Nom du fichier de sortie\n")
37	f.write("#SBATCH -e single_config.err # Nom du fichier d'erreur\n")
38	f.write("#SBATCH --account=psl@cpu\n")
39	f.write("cd ${SLURM_SUBMIT_DIR}\n")
40	f.write("module purge\n")
41	f.write("module load intel-all/19.0.4\n")
42	f.write("module load netcdf/4.7.0/intel-19.0.4-mpi\n")
43	f.write("module load netcdf-cxx/4.2/intel-19.0.4-mpi\n")
44	f.write("module load netcdf-fortran/4.4.5/intel-19.0.4-mpi\n")
45	f.write("module load gcc/6.5.0/gcc-4.8.5\n")
46	f.write("module load cmake\n")
47	f.write("module load cdo\n")
48	f.write("ulimit -c 0\n")
49	xios_root=os.getenv('xios_dir')
50	build_folder=os.getenv('build_dir')
51	f.write("export xios_dir="+xios_root+"\n")
52	f.write("export build_dir="+build_folder+"\n")
53	f.write("export machine_name=jeanzay\n")
54	f.write("cp ../setup.py ./\n")
55	f.write("cp ../run_sub_test.sh ./\n")
56	f.write("mv user_params.def user_params.def_bkp\n")
57	f.write("cp "+configName+"/setup/user_param.py user_params.def\n")
58	f.write("python setup.py\n")
59	f.write("cmake .\n")
60	f.write("ctest -V #enable all output\n")
61	f.write("rm -f setup.py\n")
62	f.write("rm -f run_sub_test.sh\n")
63	f.write("rm -f run_test_*.py\n")
64	f.write("rm -f CMakeLists.txt\n")
65	f.write("rm -f context_grid_dynamico.xml\n")
66	f.write("rm -f dynamico_grid.nc\n")
67	f.write("rm -f default_param.*\n")
68	f.write("rm -f user_param.*\n")
69	f.write("mv user_params.def_bkp user_params.def\n")
70	f.write("\n")
71	f.close
72
73	def write_irene_job(configName, nbproc=8):
74	red = lambda text: '\033[0;31m' + text + '\033[0m'
75	f=open("irene_job_"+configName+".sh", "w")
76	f.write("#!/bin/bash\n")
77	f.write("#MSUB -r XIOS\n")
78	f.write("#MSUB -o client_output.out # standard output\n")
79	f.write("#MSUB -e client_error.err #error output\n")
80	f.write("#MSUB -eo\n")
81	f.write("#MSUB -c 1\n")
82	f.write("#MSUB -n "+ repr(nbproc) + " # Number of MPI tasks (SPMD case) or cores (MPMD case)\n")
83	f.write("#MSUB -X \n")
84	f.write("#MSUB -x \n")
85	f.write("#MSUB -T 600 # Wall clock limit (seconds)\n")
86	f.write("#MSUB -q skylake # thin nodes\n")
87	f.write("#MSUB -A devcmip6\n")
88	f.write("#MSUB -Q test\n")
89	f.write("#MSUB -m work\n")
90	f.write("cd $BRIDGE_MSUB_PWD\n")
91	f.write("module unload netcdf-c netcdf-fortran hdf5 flavor perl hdf5 boost blitz mpi gnu\n")
92	f.write("module load gnu\n")
93	f.write("module load mpi/intelmpi/2017.0.6.256\n")
94	f.write("module load flavor/buildcompiler/intel/17\n")
95	f.write("module load flavor/hdf5/parallel\n")
96	f.write("module load netcdf-fortran/4.4.4\n")
97	f.write("module load hdf5/1.8.20\n")
98	f.write("module load boost\n")
99	f.write("module load blitz\n")
100	f.write("module load feature/bridge/heterogenous_mpmd\n")
101	f.write("module load nco\n")
102	f.write("module load cdo\n")
103	f.write("export KMP_STACKSIZE=3g\n")
104	f.write("export KMP_LIBRARY=turnaround\n")
105	f.write("export MKL_SERIAL=YES\n")
106	f.write("export OMP_NUM_THREADS=${BRIDGE_MSUB_NCORE}\n")
107	f.write("set -x \n")
108	xios_root=os.getenv('xios_dir')
109	build_folder=os.getenv('build_dir')
110	f.write("export xios_dir="+xios_root+"\n")
111	f.write("export build_dir="+build_folder+"\n")
112	f.write("export machine_name=irene\n")
113	f.write("cp ../setup.py ./\n")
114	f.write("cp ../run_sub_test.sh ./\n")
115	f.write("mv user_params.def user_params.def_bkp\n")
116	f.write("cp "+configName+"/setup/user_param.py user_params.def\n")
117	f.write("python setup.py\n")
118	f.write("cmake .\n")
119	f.write("ctest -V #enable all output\n")
120	f.write("rm -f setup.py\n")
121	f.write("rm -f run_sub_test.sh\n")
122	f.write("rm -f run_test_*.py\n")
123	f.write("rm -f CMakeLists.txt\n")
124	f.write("rm -f context_grid_dynamico.xml\n")
125	f.write("rm -f dynamico_grid.nc\n")
126	f.write("rm -f default_param.*\n")
127	f.write("rm -f user_param.*\n")
128	f.write("mv user_params.def_bkp user_params.def\n")
129	f.write("\n")
130	f.close
131
132
133
134	def main():
135
136	black = lambda text: '\033[0;30m' + text + '\033[0m'
137	red = lambda text: '\033[0;31m' + text + '\033[0m'
138	green = lambda text: '\033[0;32m' + text + '\033[0m'
139	yellow = lambda text: '\033[0;33m' + text + '\033[0m'
140	blue = lambda text: '\033[0;34m' + text + '\033[0m'
141	magenta = lambda text: '\033[0;35m' + text + '\033[0m'
142	cyan = lambda text: '\033[0;36m' + text + '\033[0m'
143	lgray = lambda text: '\033[0;37m' + text + '\033[0m'
144	dgray = lambda text: '\033[1;30m' + text + '\033[0m'
145	lred = lambda text: '\033[1;32m' + text + '\033[0m'
146	lgreen = lambda text: '\033[1;32m' + text + '\033[0m'
147	lyellow = lambda text: '\033[1;33m' + text + '\033[0m'
148	lblue = lambda text: '\033[1;34m' + text + '\033[0m'
149	lpurple = lambda text: '\033[1;35m' + text + '\033[0m'
150	lcyan = lambda text: '\033[1;36m' + text + '\033[0m'
151	white = lambda text: '\033[1;37m' + text + '\033[0m'
152
153	configNamePattern=[0]*7
154
155	f=open("user_param.py", "r")
156	for line in f:
157	line = line.strip()
158	if line.startswith("UsingServer2"):
159	configNamePattern[0] = '_UsingSrv2=' + UsingServer2
160	elif line.startswith("RatioServer2"):
161	configNamePattern[1] = '_RatioSrv2=' + repr(RatioServer2)
162	elif line.startswith("NumberPoolsServer2"):
163	configNamePattern[2] = '_NbPoolsSrv2=' + repr(NumberPoolsServer2)
164	elif line.startswith("NumberClients"):
165	configNamePattern[3] = '_NbClients=' + repr(NumberClients)
166	elif line.startswith("NumberServers"):
167	configNamePattern[4] = '_NbServers=' + repr(NumberServers)
168	elif line.startswith("Duration"):
169	configNamePattern[5] = '_Duration=' + Duration
170	elif line.startswith("ATMdomain"):
171	configNamePattern[5] = '_ATMdomain=' + ATMdomain
172	f.close()
173
174
175
176	configName = "config"
177	defaultConfig = True
178
179	for i in range(len(configNamePattern)):
180	if configNamePattern[i] != 0:
181	configName = configName + configNamePattern[i]
182	defaultConfig = False
183
184	if defaultConfig:
185	configName = "config_default"
186
187
188	if not os.path.exists(configName):
189
190
191
192
193	os.mkdir(configName)
194	print blue("\n ****************************************************************")
195	print blue(" * Directory " + repr(configName) + " Created ")
196	print blue(" * Copy results to \'not_validated_reference\' folder")
197	print blue(" ****************************************************************\n")
198
199	OSinfo("mkdir "+configName+'/setup')
200	OSinfo('mkdir '+configName+'/tmp_reference')
201	OSinfo('mkdir '+configName+'/xios_output')
202
203	OSinfo('mv param.def '+configName+'/setup/')
204	OSinfo('mv all_param.def '+configName+'/setup/')
205	OSinfo('mv user_param.py '+configName+'/setup/')
206	OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ')
207	OSinfo('mv iodef.xml '+configName+'/setup/')
208	OSinfo('cp context_atm.xml '+configName+'/setup/')
209	OSinfo('cp context_grid_dynamico.xml '+configName+'/setup/')
210	OSinfo('cp dynamico_grid.nc '+configName+'/setup/')
211	OSinfo('mv xios_*.out '+configName+'/xios_output/')
212	OSinfo('mv xios_*.err '+configName+'/xios_output/')
213
214
215	f=open("checkfile.def", "r")
216	h=open("report.txt", "a+")
217	l=open("report.html", "a+")
218	preport=open("../plein_report.txt", "a+")
219	h.write("Config : "+configName+"\n")
220	l.write("<p>Config : "+configName+"</p>")
221	for line in f:
222	line=line.strip()
223	line=line.rstrip()
224
225	if line.startswith("all"):
226	allfiles = os.listdir(os.getcwd())
227	for thisfile in allfiles:
228	if thisfile.endswith(".nc") and thisfile!="dynamico_grid.nc":
229	cmd = 'mv '+thisfile + ' '+configName+'/tmp_reference/'
230	OSinfo(cmd)
231
232	h.write("test for "+lpurple("all NC files")+" \t"+yellow('INITIALIZED')+"\n")
233	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"_all_nc_files"+" "+repr(0)+" \n")
234	l.write("<p>test for <font color=\"purple\">"+"all NC files"+"</font>        <font color=\"darkorange\">INITIALIZED</font></p>")
235	print yellow("\n ****************************************")
236	print yellow(" "+"all NC files"+" is stored as temporal reference !!! ")
237	print yellow("\n ****************************************")
238
239	break
240
241	if not line.startswith("#") and len(line) != 0:
242	cmd = 'mv '+line + ' '+configName+'/tmp_reference/'
243	OSinfo(cmd)
244
245	h.write("test for "+lpurple(line)+" \t"+yellow('INITIALIZED')+"\n")
246	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(0)+" \n")
247	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"darkorange\">INITIALIZED</font></p>")
248	print yellow("\n ****************************************")
249	print yellow(" "+line+" is stored as temporal reference !!! ")
250	print yellow("\n ****************************************")
251
252	f.close()
253	h.write("\n")
254	l.write("<br>")
255	h.close()
256	l.close()
257	preport.close()
258	#generate job scripts
259	hostname=os.getenv('machine_name')
260	if hostname.startswith("jeanzay"):
261	write_jz_job(configName, NumberClients+NumberServers)
262
263	elif hostname.startswith("irene"):
264	write_irene_job(configName, NumberClients+NumberServers)
265
266	else :
267	print("machine unknown")
268	print >> sys.stderr, 1
269	sys.exit()
270
271
272	else:
273
274
275
276	print blue("\n ********************************************************")
277	print blue(" * Directory " + repr(configName) + " already exists")
278	print blue(" * Start comparing results with references")
279	print blue(" ********************************************************\n")
280
281	tmp_ref = os.path.isdir(configName+'/tmp_reference')
282	ref = os.path.isdir(configName+'/reference') and not tmp_ref
283
284	if ref:
285	print blue("\n =====================================")
286	print blue(" \| Comparing results with references \|")
287	print blue(" =====================================\n")
288	path = configName+'/reference/'
289	elif tmp_ref:
290	print blue("\n ===================================================")
291	print blue(" \| Comparing results with NON-Validated references \|")
292	print blue(" ===================================================\n")
293	path = configName+'/tmp_reference/'
294
295	f=open("checkfile.def", "r")
296	h=open("report.txt", "a+")
297	l=open("report.html", "a+")
298	preport=open("../plein_report.txt", "a+")
299
300
301	h.write("Config : "+configName+"\n")
302	l.write("<p>Config : "+configName+"</p>")
303	for line in f:
304
305	line=line.strip()
306	line=line.rstrip()
307
308	if line.startswith("all") :
309	for thisfile in os.listdir(path):
310	if thisfile.endswith(".nc"):
311	cmd = 'cdo -W diffn '+ thisfile +' '+ path + thisfile + ' \| tail -1 > diff.txt'
312
313	OSinfo(cmd)
314
315	if os.stat("diff.txt").st_size==0:
316	h.write("test for "+lpurple(thisfile)+" \t"+green('PASSED')+"\n")
317	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(1)+" \n")
318	l.write("<p>test for <font color=\"purple\">"+thisfile+"</font>        <font color=\"green\">PASSED</font></p>")
319	print green("\n ****************************************")
320	print green(" "+thisfile+" is valid !!! ")
321	print green("\n ****************************************")
322
323	else:
324	g=open("diff.txt", "r")
325	for gline in g:
326	if gline.strip().startswith("0") or ":" in gline :
327	h.write("test for "+lpurple(thisfile)+" \t"+green('PASSED')+"\n")
328	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(1)+" \n")
329	l.write("<p>test for <font color=\"purple\">"+thisfile+"</font>        <font color=\"green\">PASSED</font></p>")
330	print green("\n ****************************************")
331	print green(" "+thisfile+" is valid !!! ")
332	print green("\n ****************************************")
333	else:
334	h.write("test for "+lpurple(thisfile)+" \t"+red('FAILED')+"\n")
335	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(-1)+" \n")
336	l.write("<p>test for <font color=\"purple\">"+thisfile+"</font>        <font color=\"red\">FAILED</font></p>")
337	print red("\n **************************************************")
338	print red(" "+thisfile+" is NOT valid. Please debugging.. ")
339	print red("\n **************************************************")
340	g.close()
341
342	# generate job submission script
343	hostname=os.getenv('machine_name')
344
345	if hostname.startswith("jeanzay"):
346	write_jz_job(configName, NumberClients+NumberServers)
347
348	elif hostname.startswith("irene"):
349	write_irene_job(configName, NumberClients+NumberServers)
350
351	else :
352	print("machine unknown")
353	print >> sys.stderr, 1
354	sys.exit()
355	break
356
357
358	elif not line.startswith("#") and len(line) != 0:
359	cmd = 'cdo -W diffn '+ line +' '+ path + line + ' \| tail -1 > diff.txt'
360
361	OSinfo(cmd)
362
363	if os.stat("diff.txt").st_size==0:
364	h.write("test for "+lpurple(line)+" \t"+green('PASSED')+"\n")
365	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n")
366	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"green\">PASSED</font></p>")
367	print green("\n ****************************************")
368	print green(" "+line+" is valid !!! ")
369	print green("\n ****************************************")
370
371	else:
372	g=open("diff.txt", "r")
373	for gline in g:
374	if gline.strip().startswith("0") or ":" in gline :
375	h.write("test for "+lpurple(line)+" \t"+green('PASSED')+"\n")
376	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n")
377	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"green\">PASSED</font></p>")
378	print green("\n ****************************************")
379	print green(" "+line+" is valid !!! ")
380	print green("\n ****************************************")
381	else:
382	h.write("test for "+lpurple(line)+" \t"+red('FAILED')+"\n")
383	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(-1)+" \n")
384	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"red\">FAILED</font></p>")
385	print red("\n **************************************************")
386	print red(" "+line+" is NOT valid. Please debugging.. ")
387	print red("\n **************************************************")
388	g.close()
389
390	# generate job submission script
391	hostname=os.getenv('machine_name')
392
393	if hostname.startswith("jeanzay"):
394	write_jz_job(configName, NumberClients+NumberServers)
395
396	elif hostname.startswith("irene"):
397	write_irene_job(configName, NumberClients+NumberServers)
398
399	else :
400	print("machine unknown")
401	print >> sys.stderr, 1
402	sys.exit()
403
404
405	f.close()
406	h.write("\n")
407	l.write("<br>")
408	h.close()
409	l.close()
410	preport.close()
411
412	if not os.path.exists('current_run'):
413	OSinfo('mkdir current_run')
414
415	if not os.path.exists('current_run/'+configName):
416	OSinfo('mkdir current_run/'+configName)
417
418	OSinfo('mv -f param.def current_run/'+configName)
419	OSinfo('mv -f all_param.def current_run/'+configName)
420	OSinfo('mv -f user_param.py current_run/'+configName)
421	OSinfo('mv -f iodef.xml current_run/'+configName)
422	OSinfo('rm -f current_run/'+configName+'/context_atm.xml')
423	OSinfo('cp -n context_atm.xml current_run/'+configName)
424	OSinfo('cp -f context_grid_dynamico.xml current_run/'+configName)
425	OSinfo('cp -f dynamico_grid.nc current_run/'+configName)
426	OSinfo('mv -f xios_*.out current_run/'+configName)
427	OSinfo('mv -f xios_*.err current_run/'+configName)
428
429
430
431
432	if __name__== "__main__":
433	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: