New URL for NEMO forge! http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.

sette_rpt.sh in utils/CI/sette – NEMO

Context Navigation

source: utils/CI/sette/sette_rpt.sh @ 9911

Last change on this file since 9911 was 9911, checked in by acc, 6 years ago
A couple of bug fixes in sette_rpt.sh
Property svn:executable set to ``*
File size: 14.1 KB

Rev	Line
[6124]	1	#!/bin/bash -f
[9019]	2	# set -vx
[6124]	3	# simple SETTE report generator.
	4	#
	5	# This version should be run in the SETTE directory.
	6	# The machine name will be picked up from the sette.sh script but the location of the
[9602]	7	# validation directory needs to be set here (currently assumed to reside in the ../cfgs directory)
[6124]	8	#
	9	#########################################################################################
	10	######################### Start of function definitions #################################
	11	##
[8468]	12
	13	function restfile() {
	14	# Rebuild ice restart for SAS CONFIG, and restartability checks. Expects LONG and SHORT run directories.
	15	# For Stand Alone Surface configuration ocean is not running, just run ice model; so no outputs ocean files.
	16	# Compares LONG rebuild restart ice file with equivalent entry from the SHORT rebuild restart ice file.
	17	#
[9019]	18	# check nco module loaded, and load it if not
	19	if [ ! $( echo $LOADEDMODULES \| grep cdo ) ]; then module load cdo >& /dev/null ; fi
	20	#
[8468]	21	vdir=$1
	22	nam=$2
	23	pass=$3
[9019]	24
[8468]	25	#
	26	if [ -d $vdir/$nam ]; then
	27	dorv=`ls -1rt $vdir/$nam/$mach/ \| tail -1l `
	28	dorv=`echo $dorv \| sed -e 's:.*/::'`
	29	rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ \| tail -2l \| head -1 `
	30	rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ \| tail -1l`
	31	cd ${SAS_RESTART_DIR}/LONG
	32	#SF add here compilation of rebuild_tools to rebuild restart files, and add comparison of restart files
	33	cd ${TOOLS_DIR}
	34	./maketools -n REBUILD_NEMO -m ${mach} > /dev/null 2>&1
	35	cd ${TOOLS_DIR}/REBUILD_NEMO
	36	#SF echo "REBUILD LONG restart SAS files, without standard output"
[9663]	37	./rebuild_nemo -t 4 ../../cfgs/ORCA2_SAS_ICE_ST/LONG/SAS_00000240_restart_ice $NPROC > /dev/null 2>&1
[8468]	38	#SF echo "REBUILD SHORT restart SAS files, without standard output"
[9663]	39	./rebuild_nemo -t 4 ../../cfgs/ORCA2_SAS_ICE_ST/SHORT/SAS_00000240_restart_ice $NPROC >&-
[8468]	40	cd ${SAS_RESTART_DIR}/LONG
	41	#SF echo "COPY rebuild restart files"
[8815]	42	cp SAS_00000240_restart_ice.nc $vdir/$nam/$mach/$dorv/LONG/.
	43	cp ../SHORT/SAS_00000240_restart_ice.nc $vdir/$nam/$mach/$dorv/SHORT/.
[8468]	44
[8815]	45	f1o=$vdir/$nam/$mach/$dorv/LONG/SAS_00000240_restart_ice.nc
	46	f2o=$vdir/$nam/$mach/$dorv/SHORT/SAS_00000240_restart_ice.nc
[8468]	47	if [ ! -f $f1o ] && [ ! -f $f2o ] ; then
[9525]	48	printf "%-27s %s\n" $nam " REBUILD SAS restart ice DOES NOT exists; incomplete test";
[8468]	49	return;
	50	fi
	51	#
	52	done_oce=0
	53	#
[9019]	54	if [ -f $f1o ] && [ -f $f2o ]; then
	55	## Compare the two netcdf files
	56	cdo diffn $f1o $f2o > cdo_diff.out 2> /dev/null
	57	## Identical if cdo_diff.out exists but has zero size
	58	if [ ! -s cdo_diff.out ]; then
	59	difi=0
	60	else
	61	## Identical if first character of $dif ==0
	62	dif=$( grep -om1 '[0-9]* of [0-9]* records differ' cdo_diff.out )
	63	# difi contains the first character of summary of cdo dif. if = 0, then 0 record differ between the 2 files
	64	if [ -n "$dif" ]; then
	65	difi=`echo $dif \| cut -c -1`
	66	fi
	67	fi
	68	\rm cdo_diff.out
	69
	70	if [ $difi == 0 ]; then
	71	if [ $pass == 0 ]; then
[9525]	72	printf "%-27s %s %s\n" $nam " ice restarts are IDENTICAL passed : " $dorv
[9019]	73	fi
	74	else
[9525]	75	printf "%-27s %s %s\n" $nam " ice restarts are DIFFERENT FAILED : " $dorv
[8468]	76	#
	77	# Offer view of differences on the second pass
	78	#
	79	if [ $pass == 1 ]; then
	80	echo "<return> to view restart_ice.nc differences"
	81	read y
	82	cdo -diffv $f1o $f2o
	83	done_oce=1
	84	#echo "<return> to continue"
	85	#read y
	86	fi
	87	fi
[9019]	88	else
[9525]	89	printf "%-27s %s\n" $nam " incomplete test";
[9019]	90	return;
	91	fi
[8468]	92	#
	93	fi
	94	}
	95
[6124]	96	function resttest() {
	97	#
	98	# Restartability checks. Expects LONG and SHORT run directories
	99	# Compares end of LONG stat files with equivalent entries from the SHORT stat files.
	100	#
	101	vdir=$1
	102	nam=$2
	103	pass=$3
	104	#
	105	if [ -d $vdir/$nam ]; then
	106	dorv=`ls -1rt $vdir/$nam/$mach/ \| tail -1l `
	107	dorv=`echo $dorv \| sed -e 's:.*/::'`
	108	f1o=$vdir/$nam/$mach/$dorv/LONG/ocean.output
[9019]	109	f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
[6124]	110	f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
	111	f2o=$vdir/$nam/$mach/$dorv/SHORT/ocean.output
[9019]	112	f2s=$vdir/$nam/$mach/$dorv/SHORT/run.stat
[6124]	113	f2t=$vdir/$nam/$mach/$dorv/SHORT/tracer.stat
	114
[6152]	115	if [ ! -f $f1s ] && [ ! -f $f1t ] ; then
[9525]	116	printf "%-27s %s\n" $nam " incomplete test";
[6124]	117	return;
	118	fi
[6152]	119	if [ ! -f $f2s ] && [ ! -f $f2t ] ; then
[9525]	120	printf "%-27s %s\n" $nam " incomplete test";
[6124]	121	return;
	122	fi
	123	#
	124	done_oce=0
	125
[6152]	126	if [ -f $f1s ] && [ -f $f2s ]; then
	127	nl=(`wc -l $f2s`)
	128	tail -${nl[0]} $f1s > f1.tmp$$
	129	cmp -s f1.tmp$$ $f2s
	130	if [ $? == 0 ]; then
	131	if [ $pass == 0 ]; then
[9525]	132	printf "%-27s %s %s\n" $nam " run.stat restartability passed : " $dorv
[6152]	133	fi
	134	else
[9525]	135	printf "%-27s %s %s\n" $nam " run.stat restartability FAILED : " $dorv
[6124]	136	#
	137	# Offer view of differences on the second pass
	138	#
[6152]	139	if [ $pass == 1 ]; then
[9019]	140	echo "<return> to view run.stat differences"
[6152]	141	read y
	142	sdiff f1.tmp$$ $f2s
	143	echo "<return> to view ocean.output differences"
	144	read y
	145	sdiff $f1o $f2o \| grep "\|"
	146	done_oce=1
	147	echo "<return> to continue"
	148	read y
	149	fi
[6124]	150	fi
	151	fi
	152	#
	153	# Check tracer.stat files (if they exist)
	154	#
[6152]	155	if [ -f $f1t ] && [ -f $f2t ]; then
[6124]	156	nl=(`wc -l $f2t`)
	157	tail -${nl[0]} $f1t > f1.tmp$$
	158	cmp -s f1.tmp$$ $f2t
	159	if [ $? == 0 ]; then
[6152]	160	if [ $pass == 0 ]; then
[9525]	161	printf "%-27s %s %s\n" $nam " tracer.stat restartability passed : " $dorv
[6152]	162	fi
[6124]	163	else
[9525]	164	printf "%-27s %s %s\n" $nam " tracer.stat restartability FAILED : " $dorv
[6124]	165	#
	166	# Offer view of differences on the second pass
	167	#
	168	if [ $pass == 1 ]; then
	169	echo "<return> to view tracer.stat differences"
	170	read y
	171	sdiff f1.tmp$$ $f2t
	172	#
	173	# Only offer ocean.output view if it has not been viewed previously
	174	#
	175	if [ $done_oce == 0 ]; then
	176	echo "<return> to view ocean.output differences"
	177	read y
	178	sdiff $f1o $f2o \| grep "\|"
	179	fi
	180	echo "<return> to continue"
	181	read y
	182	fi
	183	fi
	184	fi
	185	rm f1.tmp$$
	186	fi
	187	}
	188
	189	function reprotest(){
	190	#
	191	# Reproducibility checks. Expects REPRO_N_M and REPRO_I_J run directories
	192	# Compares end of stat files from each
	193	#
	194	vdir=$1
	195	nam=$2
	196	pass=$3
	197	#
	198	if [ -d $vdir/$nam ]; then
	199	dorv=`ls -1rt $vdir/$nam/$mach/ \| tail -1l `
	200	dorv=`echo $dorv \| sed -e 's:.*/::'`
[9019]	201	rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ \| grep REPRO \| tail -2l \| head -1 `
	202	rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ \| grep REPRO \| tail -1l`
[6124]	203	f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output
[9019]	204	f1s=$vdir/$nam/$mach/$dorv/$rep1/run.stat
[6124]	205	f1t=$vdir/$nam/$mach/$dorv/$rep1/tracer.stat
	206	f2o=$vdir/$nam/$mach/$dorv/$rep2/ocean.output
[9019]	207	f2s=$vdir/$nam/$mach/$dorv/$rep2/run.stat
[6124]	208	f2t=$vdir/$nam/$mach/$dorv/$rep2/tracer.stat
	209
[6152]	210	if [ ! -f $f1s ] && [ ! -f $f1t ] ; then
[9525]	211	printf "%-27s %s\n" $nam " incomplete test";
[6124]	212	return;
	213	fi
[6152]	214	if [ ! -f $f2s ] && [ ! -f $f2t ] ; then
[9525]	215	printf "%-27s %s\n" $nam " incomplete test";
[6124]	216	return;
	217	fi
	218	#
	219	done_oce=0
	220
[6152]	221	if [ -f $f1s ] && [ -f $f2s ] ; then
	222	cmp -s $f1s $f2s
	223	if [ $? == 0 ]; then
	224	if [ $pass == 0 ]; then
[9525]	225	printf "%-27s %s %s\n" $nam " run.stat reproducibility passed : " $dorv
[6152]	226	fi
	227	else
[9525]	228	printf "%-27s %s %s\n" $nam " run.stat reproducibility FAILED : " $dorv
[6124]	229	#
	230	# Offer view of differences on the second pass
	231	#
[6152]	232	if [ $pass == 1 ]; then
[9019]	233	echo "<return> to view run.stat differences"
[6152]	234	read y
[9911]	235	sdiff $f1s $f2s
[6152]	236	echo "<return> to view ocean.output differences"
	237	read y
	238	sdiff $f1o $f2o \| grep "\|"
	239	done_oce=1
	240	echo "<return> to continue"
	241	read y
	242	fi
[6124]	243	fi
	244	fi
	245	#
	246	# Check tracer.stat files (if they exist)
	247	#
[6152]	248	if [ -f $f1t ] && [ -f $f2t ] ; then
[6124]	249	cmp -s $f1t $f2t
	250	if [ $? == 0 ]; then
[9525]	251	if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam " tracer.stat reproducibility passed : " $dorv
[6152]	252	fi
[6124]	253	else
[9525]	254	printf "%-27s %s %s\n" $nam " tracer.stat reproducibility FAILED : " $dorv
[6124]	255	#
	256	# Offer view of differences on the second pass
	257	#
	258	if [ $pass == 1 ]; then
	259	echo "<return> to view tracer.stat differences"
	260	read y
	261	sdiff $f1t $f2t
	262	#
	263	# Only offer ocean.output view if it has not been viewed previously
	264	#
	265	if [ $done_oce == 0 ]; then
	266	echo "<return> to view ocean.output differences"
	267	read y
	268	sdiff $f1o $f2o \| grep "\|"
	269	fi
	270	echo "<return> to continue"
	271	read y
	272	fi
	273	fi
	274	fi
[9223]	275	fi
	276	}
	277
	278	function runtest(){
	279	#
	280	# Run checks.
	281	# Check presence of E R R O R in ocean.output from each
	282	#
	283	vdir=$1
	284	nam=$2
	285	pass=$3
	286	#
	287	if [ -d $vdir/$nam ]; then
	288	dorv=`ls -1rt $vdir/$nam/$mach/ \| tail -1l `
	289	dorv=`echo $dorv \| sed -e 's:.*/::'`
[9744]	290	rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ \| grep LONG \| head -1 `
[9911]	291	if [ -z $rep1 ] ; then
	292	# Not all configurations test restartability (LONG/SHORT);
	293	# check for a reproducibility directory instead (REPRO)
	294	rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ \| grep REPRO \| head -1 `
	295	fi
[9223]	296	f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output
	297	if [ ! -f $f1o ] ; then
[9733]	298	printf "%-27s %s %s\n" $nam " ocean.output is MISSING : " $dorv
[9223]	299	return;
	300	else
	301	nerr=`grep 'E R R O R' $f1o \| wc -l`
	302	if [[ $nerr > 0 ]]; then
[9525]	303	printf "%-27s %s %s\n" $nam " run FAILED : " $dorv
[9223]	304	if [ $pass == 1 ]; then
	305	echo "<return> to view end of ocean.output"
	306	read y
	307	tail -100 $f1o
	308	echo ''
	309	echo "full ocean.output available here: $f1o"
	310	fi
	311	fi
	312	fi
[9221]	313	else
[9733]	314	printf "%-27s %s %s\n" $nam " directory is MISSING : " $dorv
[6124]	315	fi
	316	}
[9019]	317
	318	function identictest(){
	319	#
	320	# checks AGRIF does not corrupe results with no AGRIF zoom. Expects ORCA2AGUL/AGRIFNOZ and ORCA2AGUL_NAGR/AGRIFNO run directories
	321	# Compares solver.stat files for each
	322	#
	323	vdir=$1
	324	dir1=$2
	325	dir2=$3
	326	pass=$4
	327	#
	328	if [ -d $vdir/$dir1 ] && [ -d $vdir/$dir2 ]; then
	329	dorv1=`ls -1rt $vdir/$dir1/$mach/ \| tail -1l `
	330	dorv1=`echo $dorv1 \| sed -e 's:.*/::'`
	331	dorv2=`ls -1rt $vdir/$dir2/$mach/ \| tail -1l `
	332	dorv2=`echo $dorv2 \| sed -e 's:.*/::'`
	333
[9518]	334	rep1=`ls -1rt $vdir/$dir1/$mach/$dorv1/ \| tail -1l`
[9525]	335	#clem rep2=`ls -1rt $vdir/$dir2/$mach/$dorv2/ \| tail -1l`
	336	rep2=`ls -1rt $vdir/$dir1/$mach/$dorv1/ \| tail -1l`
[9019]	337	f1s=$vdir/$dir1/$mach/$dorv1/$rep1/run.stat
	338	f2s=$vdir/$dir2/$mach/$dorv2/$rep2/run.stat
	339
	340	if [ ! -f $f1s ] && [ ! -f $f2s ] ; then
[9525]	341	printf "%-27s %s\n" $dir1 $dir2 " incomplete test";
[9019]	342	return;
	343	fi
	344	#
	345	done_oce=0
	346
	347	if [ -f $f1s ] && [ -f $f2s ] ; then
	348	cmp -s $f1s $f2s
	349	if [ $? == 0 ]; then
	350	if [ $pass == 0 ]; then
[9525]	351	printf "%-5s %s %-5s %s %s %s\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat unchanged - passed : " $dorv1 $dorv2
[9019]	352	fi
	353	else
[9525]	354	printf "%-5s %s %-5s %s %s %s\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat changed - FAILED : " $dorv1 $dorv2
[9019]	355	#
	356	# Offer view of differences on the second pass
	357	#
	358	if [ $pass == 1 ]; then
	359	echo "<return> to view run.stat differences"
	360	read y
	361	sdiff $f1s $f2s
	362	done_oce=1
	363	echo "<return> to continue"
	364	read y
	365	fi
	366	fi
	367	fi
	368	else
[9525]	369	printf "%-27s %s\n" $dir1 $dir2 " incomplete test";
[9019]	370	fi
	371	}
[6124]	372	########################### END of function definitions #################################
	373	## ##
	374	## Main script ##
	375	## ##
	376	#########################################################################################
	377	#
	378	mach=`grep "COMPILER=" ./sette.sh \| sed -e 's/COMPILER=//'`
	379	NEMO_VALID=`grep "NEMO_VALIDATION_DIR=" ./param.cfg \| sed -e 's/NEMO_VALIDATION_DIR=//'`
[9019]	380	NEMO_VALID=`eval "echo $NEMO_VALID"`
[6124]	381	#
	382	if [ ! -d $NEMO_VALID ]; then
	383	echo "$NEMO_VALID validation directory not found"
	384	exit
	385	fi
	386	#
[9019]	387	# Directory to run the tests
	388	SETTE_DIR=$(cd $(dirname "$0"); pwd)
[9685]	389	MAIN_DIR=${SETTE_DIR}/..
[9602]	390	CONFIG_DIR0=${MAIN_DIR}/cfgs
[9605]	391	TOOLS_DIR=${MAIN_DIR}/tools
[9019]	392	NPROC=32
[9663]	393	SAS_RESTART_DIR=${CONFIG_DIR0}/ORCA2_SAS_ICE_ST
[9019]	394	#
	395	# Show current revision tag and branch name
	396	#
	397	cmd="svn"
[9534]	398	[ ! -d "$MAIN_DIR/.svn" ] && cmd="git $cmd"
	399	echo $cmd
	400	echo "$MAIN_DIR/.svn"
	401	lastchange=`$cmd info ${MAIN_DIR} \| grep -i "Last Changed Rev:" \| sed -e "s/ //g" \| cut -d ":" -f 2`
[9019]	402	revision=`$cmd info \| grep Revision \| cut -d ":" -f 2 \| tr -d ' '`
	403	branchname=`$cmd info \| grep URL \| rev \| cut -d "/" -f 3 \| rev`
[9534]	404	echo "SETTE validation report : $branchname @ r$revision ( last change @ r$lastchange )"
[9019]	405	#
[6124]	406	# The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory
	407	#
	408	for pass in 0 1
	409	do
	410	#
[9525]	411	if [ $pass == 0 ]; then
	412	echo ""
	413	echo "!!---------------1st pass------------------!!"
	414	fi
	415	if [ $pass == 1 ]; then
	416	echo ""
	417	echo "!!---------------2nd pass------------------!!"
	418	fi
[6124]	419	#
[8468]	420
	421	# Rebuild and restartability test for SAS
[9518]	422	# clem: not needed anymore
[9663]	423	# for restart_file in WORCA2_SAS_ICE_ST
[9518]	424	# do
	425	# restfile $NEMO_VALID $restart_file $pass
	426	# done
[8468]	427	#
[6124]	428	# Restartability test
[9525]	429	echo ""
[9518]	430	echo " !----restart----! "
[9789]	431	for restart_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WOVERFLOW_ST WLOCK_EXCHANGE_ST WVORTEX_ST WICEDYN_ST
[6124]	432	do
	433	resttest $NEMO_VALID $restart_test $pass
[9733]	434	runtest $NEMO_VALID $restart_test $pass
[6124]	435	done
	436	#
	437	# Reproducibility tests
[9525]	438	echo ""
[9518]	439	echo " !----repro----! "
[9789]	440	for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WORCA2_ICE_OBS_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICEDYN_ST
[6124]	441	do
	442	reprotest $NEMO_VALID $repro_test $pass
[9223]	443	runtest $NEMO_VALID $repro_test $pass
[6124]	444	done
[9019]	445
[9518]	446	# AGRIF special check to ensure results are unchanged with and without key_agrif
[9525]	447	echo ""
[9518]	448	echo " !----agrif check----! "
[9776]	449	dir1=WAGRIF_DEMO_NOAGRIF_ST
	450	dir2=WAGRIF_DEMO_ST
[9019]	451	identictest $NEMO_VALID $dir1 $dir2 $pass
	452
	453	done
[6124]	454	#
[9019]	455
[6124]	456	exit

Note: See TracBrowser for help on using the repository browser.

Download in other formats: