Context Navigation

source: XIOS/dev/dev_trunk_omp/GENERIC_TESTCASE/check.py @ 1776

Last change on this file since 1776 was 1776, checked in by yushan, 4 years ago
dev_trunk_omp : update xios_report.html
File size: 17.4 KB

Rev	Line
[1708]	1	import os
	2	import sys
[1734]	3	import subprocess
[1738]	4	import socket
	5	import os.path
	6	from os import path
[1708]	7
	8
[1738]	9	from default_param import *
	10	from user_param import *
[1771]	11
[1738]	12	# load default_param first, variables defines in user_param will overwrite the default value
[1708]	13
[1734]	14
[1738]	15
[1734]	16	def OSinfo(runthis):
[1737]	17	red = lambda text: '\033[0;31m' + text + '\033[0m'
[1734]	18	osstdout = subprocess.Popen(runthis, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
	19	theInfo = osstdout.communicate()[0].strip()
	20	if osstdout.returncode!=0:
[1737]	21	print(red(runthis+" FAILED"))
[1734]	22	print >> sys.stderr, osstdout.returncode
	23	sys.exit()
[1737]	24	# else:
	25	# print(runthis+" OK")
[1734]	26
	27
[1738]	28	def write_jz_job(configName, nbproc=8):
	29	red = lambda text: '\033[0;31m' + text + '\033[0m'
	30	f=open("jz_job_"+configName+".sh", "w")
	31	f.write("#!/bin/bash\n")
	32	f.write("#SBATCH --ntasks="+repr(nbproc)+" # Nombre total de processus MPI\n")
	33	f.write("#SBATCH --ntasks-per-node=16\n")
	34	f.write("#SBATCH --hint=nomultithread\n")
	35	f.write("#SBATCH -t 00:10:00 # Temps elapsed maximum d'execution\n")
	36	f.write("#SBATCH -o single_config.out # Nom du fichier de sortie\n")
	37	f.write("#SBATCH -e single_config.err # Nom du fichier d'erreur\n")
	38	f.write("#SBATCH --account=psl@cpu\n")
	39	f.write("cd ${SLURM_SUBMIT_DIR}\n")
	40	f.write("module purge\n")
	41	f.write("module load intel-all/19.0.4\n")
	42	f.write("module load netcdf/4.7.0/intel-19.0.4-mpi\n")
	43	f.write("module load netcdf-cxx/4.2/intel-19.0.4-mpi\n")
	44	f.write("module load netcdf-fortran/4.4.5/intel-19.0.4-mpi\n")
	45	f.write("module load gcc/6.5.0/gcc-4.8.5\n")
	46	f.write("module load cmake\n")
	47	f.write("module load cdo\n")
	48	f.write("ulimit -c 0\n")
[1743]	49	xios_root=os.getenv('xios_dir')
	50	build_folder=os.getenv('build_dir')
	51	f.write("export xios_dir="+xios_root+"\n")
	52	f.write("export build_dir="+build_folder+"\n")
[1738]	53	f.write("export machine_name=jeanzay\n")
	54	f.write("cp ../setup.py ./\n")
	55	f.write("cp ../run_sub_test.sh ./\n")
	56	f.write("mv user_params.def user_params.def_bkp\n")
	57	f.write("cp "+configName+"/setup/user_param.py user_params.def\n")
	58	f.write("python setup.py\n")
	59	f.write("cmake .\n")
	60	f.write("ctest -V #enable all output\n")
[1743]	61	f.write("rm -f setup.py\n")
	62	f.write("rm -f run_sub_test.sh\n")
[1738]	63	f.write("rm -f run_test_*.py\n")
[1743]	64	f.write("rm -f CMakeLists.txt\n")
	65	f.write("rm -f context_grid_dynamico.xml\n")
	66	f.write("rm -f dynamico_grid.nc\n")
	67	f.write("rm -f default_param.*\n")
	68	f.write("rm -f user_param.*\n")
[1738]	69	f.write("mv user_params.def_bkp user_params.def\n")
	70	f.write("\n")
	71	f.close
	72
	73	def write_irene_job(configName, nbproc=8):
	74	red = lambda text: '\033[0;31m' + text + '\033[0m'
	75	f=open("irene_job_"+configName+".sh", "w")
	76	f.write("#!/bin/bash\n")
	77	f.write("#MSUB -r XIOS\n")
	78	f.write("#MSUB -o client_output.out # standard output\n")
	79	f.write("#MSUB -e client_error.err #error output\n")
	80	f.write("#MSUB -eo\n")
	81	f.write("#MSUB -c 1\n")
	82	f.write("#MSUB -n "+ repr(nbproc) + " # Number of MPI tasks (SPMD case) or cores (MPMD case)\n")
	83	f.write("#MSUB -X \n")
	84	f.write("#MSUB -x \n")
	85	f.write("#MSUB -T 600 # Wall clock limit (seconds)\n")
	86	f.write("#MSUB -q skylake # thin nodes\n")
	87	f.write("#MSUB -A devcmip6\n")
	88	f.write("#MSUB -Q test\n")
	89	f.write("#MSUB -m work\n")
	90	f.write("cd $BRIDGE_MSUB_PWD\n")
	91	f.write("module unload netcdf-c netcdf-fortran hdf5 flavor perl hdf5 boost blitz mpi gnu\n")
	92	f.write("module load gnu\n")
	93	f.write("module load mpi/intelmpi/2017.0.6.256\n")
	94	f.write("module load flavor/buildcompiler/intel/17\n")
	95	f.write("module load flavor/hdf5/parallel\n")
	96	f.write("module load netcdf-fortran/4.4.4\n")
	97	f.write("module load hdf5/1.8.20\n")
	98	f.write("module load boost\n")
	99	f.write("module load blitz\n")
	100	f.write("module load feature/bridge/heterogenous_mpmd\n")
	101	f.write("module load nco\n")
	102	f.write("module load cdo\n")
	103	f.write("export KMP_STACKSIZE=3g\n")
	104	f.write("export KMP_LIBRARY=turnaround\n")
	105	f.write("export MKL_SERIAL=YES\n")
	106	f.write("export OMP_NUM_THREADS=${BRIDGE_MSUB_NCORE}\n")
	107	f.write("set -x \n")
[1743]	108	xios_root=os.getenv('xios_dir')
	109	build_folder=os.getenv('build_dir')
	110	f.write("export xios_dir="+xios_root+"\n")
	111	f.write("export build_dir="+build_folder+"\n")
[1738]	112	f.write("export machine_name=irene\n")
	113	f.write("cp ../setup.py ./\n")
	114	f.write("cp ../run_sub_test.sh ./\n")
	115	f.write("mv user_params.def user_params.def_bkp\n")
	116	f.write("cp "+configName+"/setup/user_param.py user_params.def\n")
	117	f.write("python setup.py\n")
	118	f.write("cmake .\n")
	119	f.write("ctest -V #enable all output\n")
[1743]	120	f.write("rm -f setup.py\n")
	121	f.write("rm -f run_sub_test.sh\n")
[1738]	122	f.write("rm -f run_test_*.py\n")
[1743]	123	f.write("rm -f CMakeLists.txt\n")
	124	f.write("rm -f context_grid_dynamico.xml\n")
	125	f.write("rm -f dynamico_grid.nc\n")
	126	f.write("rm -f default_param.*\n")
	127	f.write("rm -f user_param.*\n")
[1738]	128	f.write("mv user_params.def_bkp user_params.def\n")
	129	f.write("\n")
	130	f.close
	131
	132
	133
[1708]	134	def main():
[1720]	135
	136	black = lambda text: '\033[0;30m' + text + '\033[0m'
	137	red = lambda text: '\033[0;31m' + text + '\033[0m'
	138	green = lambda text: '\033[0;32m' + text + '\033[0m'
	139	yellow = lambda text: '\033[0;33m' + text + '\033[0m'
	140	blue = lambda text: '\033[0;34m' + text + '\033[0m'
	141	magenta = lambda text: '\033[0;35m' + text + '\033[0m'
	142	cyan = lambda text: '\033[0;36m' + text + '\033[0m'
[1721]	143	lgray = lambda text: '\033[0;37m' + text + '\033[0m'
	144	dgray = lambda text: '\033[1;30m' + text + '\033[0m'
	145	lred = lambda text: '\033[1;32m' + text + '\033[0m'
	146	lgreen = lambda text: '\033[1;32m' + text + '\033[0m'
	147	lyellow = lambda text: '\033[1;33m' + text + '\033[0m'
	148	lblue = lambda text: '\033[1;34m' + text + '\033[0m'
	149	lpurple = lambda text: '\033[1;35m' + text + '\033[0m'
	150	lcyan = lambda text: '\033[1;36m' + text + '\033[0m'
	151	white = lambda text: '\033[1;37m' + text + '\033[0m'
[1708]	152
[1712]	153	configNamePattern=[0]*7
[1708]	154
	155	f=open("user_param.py", "r")
	156	for line in f:
	157	line = line.strip()
	158	if line.startswith("UsingServer2"):
	159	configNamePattern[0] = '_UsingSrv2=' + UsingServer2
	160	elif line.startswith("RatioServer2"):
	161	configNamePattern[1] = '_RatioSrv2=' + repr(RatioServer2)
	162	elif line.startswith("NumberPoolsServer2"):
	163	configNamePattern[2] = '_NbPoolsSrv2=' + repr(NumberPoolsServer2)
	164	elif line.startswith("NumberClients"):
	165	configNamePattern[3] = '_NbClients=' + repr(NumberClients)
	166	elif line.startswith("NumberServers"):
	167	configNamePattern[4] = '_NbServers=' + repr(NumberServers)
	168	elif line.startswith("Duration"):
	169	configNamePattern[5] = '_Duration=' + Duration
[1712]	170	elif line.startswith("ATMdomain"):
	171	configNamePattern[5] = '_ATMdomain=' + ATMdomain
[1708]	172	f.close()
	173
	174
[1737]	175
[1708]	176	configName = "config"
	177	defaultConfig = True
	178
	179	for i in range(len(configNamePattern)):
	180	if configNamePattern[i] != 0:
	181	configName = configName + configNamePattern[i]
	182	defaultConfig = False
	183
	184	if defaultConfig:
	185	configName = "config_default"
	186
	187
[1771]	188	if not os.path.exists(configName):
[1734]	189
[1771]	190
	191
	192
[1708]	193	os.mkdir(configName)
[1720]	194	print blue("\n ****************************************************************")
	195	print blue(" * Directory " + repr(configName) + " Created ")
	196	print blue(" * Copy results to \'not_validated_reference\' folder")
	197	print blue(" ****************************************************************\n")
[1708]	198
[1737]	199	OSinfo("mkdir "+configName+'/setup')
	200	OSinfo('mkdir '+configName+'/tmp_reference')
	201	OSinfo('mkdir '+configName+'/xios_output')
[1708]	202
[1737]	203	OSinfo('mv param.def '+configName+'/setup/')
	204	OSinfo('mv all_param.def '+configName+'/setup/')
	205	OSinfo('mv user_param.py '+configName+'/setup/')
[1771]	206	OSinfo('cp '+configName+'/setup/user_param.py '+configName+'/setup/user_param.def ')
[1737]	207	OSinfo('mv iodef.xml '+configName+'/setup/')
	208	OSinfo('cp context_atm.xml '+configName+'/setup/')
	209	OSinfo('cp context_grid_dynamico.xml '+configName+'/setup/')
	210	OSinfo('cp dynamico_grid.nc '+configName+'/setup/')
	211	OSinfo('mv xios_*.out '+configName+'/xios_output/')
	212	OSinfo('mv xios_*.err '+configName+'/xios_output/')
[1708]	213
	214
[1714]	215	f=open("checkfile.def", "r")
	216	h=open("report.txt", "a+")
[1725]	217	l=open("report.html", "a+")
[1771]	218	preport=open("../plein_report.txt", "a+")
[1725]	219	h.write("Config : "+configName+"\n")
	220	l.write("<p>Config : "+configName+"</p>")
[1714]	221	for line in f:
	222	line=line.strip()
	223	line=line.rstrip()
[1734]	224
	225	if line.startswith("all"):
	226	allfiles = os.listdir(os.getcwd())
	227	for thisfile in allfiles:
	228	if thisfile.endswith(".nc") and thisfile!="dynamico_grid.nc":
	229	cmd = 'mv '+thisfile + ' '+configName+'/tmp_reference/'
[1737]	230	OSinfo(cmd)
[1734]	231
[1771]	232	h.write("test for "+lpurple("all NC files")+" \t"+yellow('INITIALIZED')+"\n")
	233	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"_all_nc_files"+" "+repr(0)+" \n")
[1734]	234	l.write("<p>test for <font color=\"purple\">"+"all NC files"+"</font>        <font color=\"darkorange\">INITIALIZED</font></p>")
	235	print yellow("\n ****************************************")
	236	print yellow(" "+"all NC files"+" is stored as temporal reference !!! ")
	237	print yellow("\n ****************************************")
	238
	239	break
	240
[1714]	241	if not line.startswith("#") and len(line) != 0:
[1726]	242	cmd = 'mv '+line + ' '+configName+'/tmp_reference/'
[1737]	243	OSinfo(cmd)
[1714]	244
[1725]	245	h.write("test for "+lpurple(line)+" \t"+yellow('INITIALIZED')+"\n")
[1771]	246	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(0)+" \n")
[1725]	247	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"darkorange\">INITIALIZED</font></p>")
[1720]	248	print yellow("\n ****************************************")
	249	print yellow(" "+line+" is stored as temporal reference !!! ")
	250	print yellow("\n ****************************************")
[1714]	251
	252	f.close()
[1725]	253	h.write("\n")
	254	l.write("<br>")
[1714]	255	h.close()
[1771]	256	l.close()
	257	preport.close()
[1743]	258	#generate job scripts
	259	hostname=os.getenv('machine_name')
	260	if hostname.startswith("jeanzay"):
	261	write_jz_job(configName, NumberClients+NumberServers)
	262
	263	elif hostname.startswith("irene"):
	264	write_irene_job(configName, NumberClients+NumberServers)
	265
	266	else :
	267	print("machine unknown")
	268	print >> sys.stderr, 1
	269	sys.exit()
[1714]	270
	271
[1708]	272	else:
[1771]	273
	274
	275
[1720]	276	print blue("\n ********************************************************")
	277	print blue(" * Directory " + repr(configName) + " already exists")
[1734]	278	print blue(" * Start comparing results with references")
[1720]	279	print blue(" ********************************************************\n")
[1708]	280
	281	tmp_ref = os.path.isdir(configName+'/tmp_reference')
[1734]	282	ref = os.path.isdir(configName+'/reference') and not tmp_ref
[1708]	283
	284	if ref:
[1720]	285	print blue("\n =====================================")
	286	print blue(" \| Comparing results with references \|")
	287	print blue(" =====================================\n")
[1708]	288	path = configName+'/reference/'
	289	elif tmp_ref:
[1720]	290	print blue("\n ===================================================")
	291	print blue(" \| Comparing results with NON-Validated references \|")
	292	print blue(" ===================================================\n")
[1734]	293	path = configName+'/tmp_reference/'
[1708]	294
	295	f=open("checkfile.def", "r")
	296	h=open("report.txt", "a+")
[1723]	297	l=open("report.html", "a+")
[1771]	298	preport=open("../plein_report.txt", "a+")
[1708]	299
[1714]	300
[1725]	301	h.write("Config : "+configName+"\n")
	302	l.write("<p>Config : "+configName+"</p>")
[1708]	303	for line in f:
[1714]	304
[1708]	305	line=line.strip()
	306	line=line.rstrip()
[1714]	307
[1734]	308	if line.startswith("all") :
	309	for thisfile in os.listdir(path):
	310	if thisfile.endswith(".nc"):
	311	cmd = 'cdo -W diffn '+ thisfile +' '+ path + thisfile + ' \| tail -1 > diff.txt'
	312
	313	OSinfo(cmd)
	314
	315	if os.stat("diff.txt").st_size==0:
	316	h.write("test for "+lpurple(thisfile)+" \t"+green('PASSED')+"\n")
[1771]	317	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(1)+" \n")
[1734]	318	l.write("<p>test for <font color=\"purple\">"+thisfile+"</font>        <font color=\"green\">PASSED</font></p>")
	319	print green("\n ****************************************")
	320	print green(" "+thisfile+" is valid !!! ")
	321	print green("\n ****************************************")
[1708]	322
[1734]	323	else:
	324	g=open("diff.txt", "r")
	325	for gline in g:
	326	if gline.strip().startswith("0") or ":" in gline :
[1771]	327	h.write("test for "+lpurple(thisfile)+" \t"+green('PASSED')+"\n")
	328	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(1)+" \n")
[1734]	329	l.write("<p>test for <font color=\"purple\">"+thisfile+"</font>        <font color=\"green\">PASSED</font></p>")
	330	print green("\n ****************************************")
	331	print green(" "+thisfile+" is valid !!! ")
	332	print green("\n ****************************************")
	333	else:
[1771]	334	h.write("test for "+lpurple(thisfile)+" \t"+red('FAILED')+"\n")
	335	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+thisfile+" "+repr(-1)+" \n")
[1734]	336	l.write("<p>test for <font color=\"purple\">"+thisfile+"</font>        <font color=\"red\">FAILED</font></p>")
	337	print red("\n **************************************************")
	338	print red(" "+thisfile+" is NOT valid. Please debugging.. ")
	339	print red("\n **************************************************")
	340	g.close()
[1738]	341
	342	# generate job submission script
	343	hostname=os.getenv('machine_name')
	344
	345	if hostname.startswith("jeanzay"):
	346	write_jz_job(configName, NumberClients+NumberServers)
	347
	348	elif hostname.startswith("irene"):
	349	write_irene_job(configName, NumberClients+NumberServers)
	350
	351	else :
	352	print("machine unknown")
	353	print >> sys.stderr, 1
	354	sys.exit()
[1734]	355	break
	356
	357
	358	elif not line.startswith("#") and len(line) != 0:
	359	cmd = 'cdo -W diffn '+ line +' '+ path + line + ' \| tail -1 > diff.txt'
	360
	361	OSinfo(cmd)
	362
[1719]	363	if os.stat("diff.txt").st_size==0:
[1771]	364	h.write("test for "+lpurple(line)+" \t"+green('PASSED')+"\n")
	365	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n")
[1725]	366	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"green\">PASSED</font></p>")
[1720]	367	print green("\n ****************************************")
	368	print green(" "+line+" is valid !!! ")
	369	print green("\n ****************************************")
[1719]	370
	371	else:
	372	g=open("diff.txt", "r")
	373	for gline in g:
	374	if gline.strip().startswith("0") or ":" in gline :
[1725]	375	h.write("test for "+lpurple(line)+" \t"+green('PASSED')+"\n")
[1771]	376	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(1)+" \n")
[1725]	377	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"green\">PASSED</font></p>")
[1720]	378	print green("\n ****************************************")
	379	print green(" "+line+" is valid !!! ")
	380	print green("\n ****************************************")
[1719]	381	else:
[1771]	382	h.write("test for "+lpurple(line)+" \t"+red('FAILED')+"\n")
	383	preport.write(os.path.basename(os.getcwd())+" "+os.path.basename(os.getcwd())[5:]+"@"+configName+" "+os.path.basename(os.getcwd())[5:]+"@"+configName[7:]+"@"+line+" "+repr(-1)+" \n")
[1725]	384	l.write("<p>test for <font color=\"purple\">"+line+"</font>        <font color=\"red\">FAILED</font></p>")
[1720]	385	print red("\n **************************************************")
	386	print red(" "+line+" is NOT valid. Please debugging.. ")
	387	print red("\n **************************************************")
[1719]	388	g.close()
[1738]	389
	390	# generate job submission script
	391	hostname=os.getenv('machine_name')
	392
	393	if hostname.startswith("jeanzay"):
	394	write_jz_job(configName, NumberClients+NumberServers)
	395
	396	elif hostname.startswith("irene"):
	397	write_irene_job(configName, NumberClients+NumberServers)
	398
	399	else :
	400	print("machine unknown")
	401	print >> sys.stderr, 1
	402	sys.exit()
	403
	404
[1708]	405	f.close()
[1725]	406	h.write("\n")
	407	l.write("<br>")
[1714]	408	h.close()
[1771]	409	l.close()
	410	preport.close()
[1708]	411
[1726]	412	if not os.path.exists('current_run'):
[1737]	413	OSinfo('mkdir current_run')
[1708]	414
[1726]	415	if not os.path.exists('current_run/'+configName):
[1737]	416	OSinfo('mkdir current_run/'+configName)
[1708]	417
[1737]	418	OSinfo('mv -f param.def current_run/'+configName)
	419	OSinfo('mv -f all_param.def current_run/'+configName)
	420	OSinfo('mv -f user_param.py current_run/'+configName)
	421	OSinfo('mv -f iodef.xml current_run/'+configName)
	422	OSinfo('rm -f current_run/'+configName+'/context_atm.xml')
	423	OSinfo('cp -n context_atm.xml current_run/'+configName)
	424	OSinfo('cp -f context_grid_dynamico.xml current_run/'+configName)
	425	OSinfo('cp -f dynamico_grid.nc current_run/'+configName)
	426	OSinfo('mv -f xios_*.out current_run/'+configName)
	427	OSinfo('mv -f xios_*.err current_run/'+configName)
[1708]	428
[1734]	429
	430
[1737]	431
[1708]	432	if __name__== "__main__":
[1718]	433	main()

Note: See TracBrowser for help on using the repository browser.

Download in other formats: