Context Navigation

source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 869

Last change on this file since 869 was 869, checked in by sdipsl, 11 years ago
Enhance error message by providing the stack call tree when error has been detected No more fullstack file. Everything is in the stack file Log un encoded message to ease debuging
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 16.8 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Patrick Brockmann, Martial Mancip
5	# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#==================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#==================================================
19
20	#==================================================
21	# Add high level verbosity
22	typeset -i Verbosity=${Verbosity:=3}
23
24	#==================================================
25	# DEBUG_debug
26	# Add low level verbosity
27	typeset DEBUG_debug=${DEBUG_debug:=false}
28
29	# Where the stack file containing call tree will be stored.
30	typeset StackFileLocation=${StackFileLocation:=${PWD}}
31
32	if ( $DEBUG_debug ) ; then
33	if [ -f ${StackFileLocation}/stack ] ;
34	then
35	echo "Stack of an libIGCM job :" >> ${StackFileLocation}/stack
36	else
37	echo "Stack of an libIGCM job :" > ${StackFileLocation}/stack
38	fi
39	fi
40
41	#==================================================
42	# NULL_STR
43	# Default null string
44	typeset -r NULL_STR="_0_"
45
46	#==================================================
47	# libIGCM_CurrentTag
48	# Current libIGCM tag, check compatibilty with *.card
49	typeset -r libIGCM_CurrentTag="1.0"
50
51	#==================================================
52	# Exit Flag (internal debug)
53	# When true, end the master loop AFTER SAVES FILES
54	ExitFlag=false
55
56	#==================================================
57	# Declare a stack of functions calls
58
59	# insert last argument of the Stack
60	#set -A IGCM_debug_Stack ${NULL_STR}
61	#set -A IGCM_debug_StackArgs ${NULL_STR}
62	unset IGCM_debug_Stack
63	unset IGCM_debug_StackArgs
64	IGCM_debug_Stack[0]=${NULL_STR}
65	IGCM_debug_StackArgs[0]=${NULL_STR}
66	IGCM_debug_LenStack=0
67
68	#D-#==================================================================
69	#D-function IGCM_debug_CallStack
70	#D-* Purpose: Echo the Stack
71	#D-
72	function IGCM_debug_CallStack {
73	#echo
74	#echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
75	#echo "!! IGCM_debug_CallStack !!"
76	#echo "!------------------------!"
77	#echo
78	if ( $DEBUG_debug ) ; then
79	# La pile d'appels est affichée de la plus vieille à la plus récente
80	# (c'est donc l'inverse de la norme d'affichage).
81	typeset i decal
82	i=0
83	until [ $i -eq ${IGCM_debug_LenStack} ]; do
84	decal=0
85	until [ $decal -eq ${i} ]; do
86	printf -- ' '
87	(( decal = decal + 1 ))
88	done
89	echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
90	((i = i + 1))
91	done
92	#echo "!------------------------!"
93	fi
94	}
95
96	#D-#==================================================================
97	#D-function IGCM_debug_PushStack
98	#D-* Purpose: Push a function name in the stack
99	#D-
100	function IGCM_debug_PushStack {
101
102	if ( $DEBUG_debug ) ; then
103	typeset decal inputs
104	echo >> ${StackFileLocation}/stack
105	decal=0
106	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
107	printf ' ' >> ${StackFileLocation}/stack
108	(( decal = decal + 1 ))
109	done
110
111	# STORE input list in an indexed array
112	INPUTS=( $@ )
113	# We add function call name on beginning of the stack
114	set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
115
116	# We include the "null" Args in the beginning of the StackArgs
117	set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}
118	# Then, we shift StackArgs tabular
119	if [ $# -gt 1 ]; then
120	IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} \| sed -e "s/\ /,/g")
121	fi
122
123	# Fill the stack file
124	echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack
125
126	# Fill the rabbitMQ queue
127	if [ X${ActivateBigBro} = Xtrue ] ; then
128	# Only cosmetics
129	decal=0
130	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
131	printf ' ' >> ${StackFileLocation}/stack
132	(( decal = decal + 1 ))
133	done
134	# RabbitMQ message
135	code=2000
136	#
137	Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
138	encodedBody=$( echo "${Body}" \| base64 -w 0 )
139	#
140	#sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
141	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" >> ${StackFileLocation}/stack
142	echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
143	sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
144	status=$?
145	if [ ${status} -gt 0 ] ; then
146	IGCM_debug_Print 2 "IGCM_debug_PushStack : command sendAMQPMsg failed error code ${status}"
147	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
148	exit
149	fi
150	fi
151
152	# Increment LenStack
153	(( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
154
155	# If you want to print CallStack each time :
156	#IGCM_debug_CallStack
157	fi
158	}
159
160	#D-#==================================================================
161	#D-function IGCM_debug_PopStack
162	#D-* Purpose: Pop a function name in the stack
163	#D-
164	function IGCM_debug_PopStack {
165	if ( $DEBUG_debug ) ; then
166
167	typeset decal
168	if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
169	(( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
170	set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
171	set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
172	else
173	echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
174	IGCM_debug_Exit $@
175	fi
176	decal=0
177	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
178	printf ' ' >> ${StackFileLocation}/stack
179	(( decal = decal + 1 ))
180	done
181
182	if ( ${ExitFlag} ) ; then
183	# Inform the stack file
184	echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/stack
185	# Inform the rabbitMQ queue
186	if [ X${ActivateBigBro} = Xtrue ] ; then
187	# Only cosmetics
188	decal=0
189	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
190	printf ' ' >> ${StackFileLocation}/stack
191	(( decal = decal + 1 ))
192	done
193	# RabbitMQ message
194	code=9000
195	#
196	Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
197	encodedBody=$( echo "{Body}" \| base64 -w 0 )
198	#
199	#sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card -b ${encodedBody}
200	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" >> ${StackFileLocation}/stack
201	echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
202	sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
203	status=$?
204	if [ ${status} -gt 0 ] ; then
205	IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
206	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
207	exit
208	fi
209	fi
210	else
211	# Inform the stack file
212	echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack
213
214	# Inform the rabbitMQ queue
215	if [ X${ActivateBigBro} = Xtrue ] ; then
216	# Only cosmetics
217	decal=0
218	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
219	printf ' ' >> ${StackFileLocation}/stack
220	(( decal = decal + 1 ))
221	done
222	# RabbitMQ message
223	code=3000
224	#
225	Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
226	encodedBody=$( echo "{Body}" \| base64 -w 0 )
227	#
228	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" >> ${StackFileLocation}/stack
229	echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
230	sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
231	status=$?
232	if [ ${status} -gt 0 ] ; then
233	IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
234	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
235	exit
236	fi
237	fi
238	fi
239
240	if [ ${IGCM_debug_LenStack} = 0 ]; then
241	# Reset array only when necessary
242	#echo
243	#IGCM_debug_Print 3 "Clean stack array"
244	#echo
245	#set -A IGCM_debug_Stack ${NULL_STR}
246	#set -A IGCM_debug_StackArgs ${NULL_STR}
247	unset IGCM_debug_Stack
248	unset IGCM_debug_StackArgs
249	IGCM_debug_Stack[0]=${NULL_STR}
250	IGCM_debug_StackArgs[0]=${NULL_STR}
251	fi
252	fi
253	#IGCM_debug_CallStack
254	}
255
256	#D-#==================================================================
257	#D-function IGCM_debug_ActivateBigBro
258	#D-* Purpose: switch rabbitMQ on
259	#D-
260	function IGCM_debug_ActivateBigBro {
261	IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"
262
263	# Fill the rabbitMQ queue
264	if [ X${BigBrother} = Xtrue ] ; then
265	# ID to identify a simulation
266	simuid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE
267	# ID to identify a job. Several Jobs are needed to complete a simulation
268	jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE.${CumulPeriod}
269	# Only cosmetics
270	decal=0
271	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
272	printf ' ' >> ${StackFileLocation}/stack
273	(( decal = decal + 1 ))
274	done
275	# RabbitMQ message
276	if ( ${FirstInitialize} ) ; then
277	code=0000
278	else
279	code=1000
280	fi
281	Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
282	encodedBody=$( echo "{Body}" \| base64 -w 0 )
283	#
284	cat ${SUBMIT_DIR}/config.card \| base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
285	#
286	echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b "${Body}" >> ${StackFileLocation}/stack
287	echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
288	sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
289	status=$?
290	if [ ${status} -gt 0 ] ; then
291	IGCM_debug_Print 2 "IGCM_debug_ActivateBigBro : command failed error code ${status}"
292	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
293	IGCM_debug_Exit "IGCM_debug_ActivateBigBro"
294	fi
295	ActivateBigBro=true
296	fi
297	IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
298	}
299
300	#D-#==================================================================
301	#D-function IGCM_debug_Exit
302	#D-* Purpose: Print Call Stack and set ExitFlag to true
303	#D-
304	function IGCM_debug_Exit {
305	IGCM_debug_PushStack "IGCM_debug_Exit"
306	echo "IGCM_debug_Exit : " "${@}"
307	#IGCM_debug_CallStack
308	ExitFlag=true
309	IGCM_debug_PopStack "IGCM_debug_Exit"
310	}
311
312	#D-#==================================================
313	#D-function IGCM_debug_Verif_Exit
314	#D-* Purpose: exit with number 1 if ExitFlag is true
315	#D-
316	function IGCM_debug_Verif_Exit {
317	if ( ${ExitFlag} ) ; then
318	# Plan to send an email here with IGCM_sys_SendMail
319	if [ X${TaskType} != Xchecking ] ; then
320	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
321	echo "IGCM_debug_Verif_Exit : Something wrong happened."
322	echo " EXIT THE JOB."
323	echo
324	IGCM_debug_CallStack
325	fi
326	if ( $DEBUG_debug ) ; then
327	echo "Your files on ${R_OUT} :"
328	IGCM_sys_Tree ${R_SAVE}
329	echo
330	fi
331	if [ X${ActivateBigBro} = Xtrue ] ; then
332	# Only cosmetics
333	decal=0
334	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
335	printf ' ' >> ${StackFileLocation}/stack
336	(( decal = decal + 1 ))
337	done
338	# RabbitMQ message
339	code=9999
340	#
341	Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
342	encodedBody=$( echo "{Body}" \| base64 -w 0 )
343	#
344	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" >> ${StackFileLocation}/stack
345	echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
346	sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
347	status=$?
348	if [ ${status} -gt 0 ] ; then
349	IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
350	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
351	exit
352	fi
353	fi
354	# Mail notification
355	IGCM_sys_SendMail
356	# And Good Bye
357	date
358	exit 1
359	fi
360	}
361
362	#D-#==================================================
363	#D-function IGCM_debug_Verif_Exit_Post
364	#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
365	#D-
366	function IGCM_debug_Verif_Exit_Post {
367	if ( ${ExitFlag} ) ; then
368	echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
369	# If SpaceName is PROD then we stop if post_processing fails
370	# Plan to send an email here with IGCM_sys_SendMail
371	if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
372	echo " EXIT THE JOB."
373	echo
374	# Mail notification
375	#IGCM_sys_SendMailPost
376	# And Good Bye
377	date
378	exit 1
379	else
380	echo "Either inside config.card the variable SpaceName is not in PROD"
381	echo "or inside the main Job the variable JobType is not in RUN mode"
382	echo " SO WE DO NOT EXIT THE JOB."
383	echo
384	date
385	fi
386	fi
387	}
388
389	#D-#==================================================================
390	#D-function IGCM_debug_Print
391	#D-* Purpose: Print arguments according to a level of verbosity.
392	#D-
393	function IGCM_debug_Print
394	{
395	typeset level=$1
396	shift
397
398	if [ X"${1}" = X"-e" ]; then
399	typeset cmd_echo="echo -e"
400	shift
401	else
402	typeset cmd_echo="echo"
403	fi
404
405	if [ ${level} -le ${Verbosity} ] ; then
406	typeset i
407	case "${level}" in
408	1) for i in "$@" ; do
409	${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
410	done ;;
411	2) for i in "$@" ; do
412	${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
413	done ;;
414	3) for i in "$@" ; do
415	${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
416	done ;;
417	esac
418	fi
419	}
420
421	#D-#==================================================================
422	#D-function IGCM_debug_PrintVariables
423	#D-* Purpose: Print arguments when match a pattern
424	#D- according to a level of verbosity.
425	function IGCM_debug_PrintVariables
426	{
427	typeset level=$1
428	shift
429
430	list=$( set \| grep ^$1 \| sed -e "s/'//g" )
431
432	if [ "X${list}" != X ] ; then
433	IGCM_debug_Print ${level} ${list}
434	fi
435	}
436
437	#D-#==================================================================
438	#D-function IGCM_debug_Check
439	#D- * Purpose: Check the present file by comparison with a reference file
440	function IGCM_debug_Check
441	{
442	#---------------------
443	if [ ! -n "${libIGCM}" ] ; then
444	echo "Check libIGCM_debug ..........................................[ FAILED ]"
445	echo "--Error--> libIGCM variable is not defined"
446	exit 2
447	fi
448
449	#---------------------
450	if [ ! -n "${Verbosity}" ] ; then
451	echo "Check libIGCM_debug ..........................................[ FAILED ]"
452	echo "--Error--> Verbosity variable is not defined"
453	exit 3
454	fi
455
456	#---------------------
457	${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > IGCM_debug_Test.ref.failed 2>&1
458	sleep 2
459
460	# Remove date stamp.
461	sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" IGCM_debug_Test.ref.failed > IGCM_debug_Test.ref.failed.nodate
462	mv IGCM_debug_Test.ref.failed.nodate IGCM_debug_Test.ref.failed
463
464	if diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
465	echo "Check libIGCM_debug ..............................................[ OK ]"
466	rm -f IGCM_debug_Test.ref.failed
467	else
468	echo "Check libIGCM_debug ..........................................[ FAILED ]"
469	echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
470	echo " has produced the file IGCM_debug_Test.ref.failed"
471	echo " Please analyse differences with the reference file by typing:"
472	echo " diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
473	echo " Report errors to the author: Patrick.Brockmann@cea.fr"
474	exit 4
475	fi
476	#---------------------
477	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: