Context Navigation

source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 1076

Last change on this file since 1076 was 1076, checked in by sdipsl, 10 years ago
Have smaller message, except the very first one stack files will go in ${SUBMIT_DIR}/STACK directory. Just for debug... Comments and cosmetics
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 30.3 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Patrick Brockmann, Martial Mancip
5	# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#==================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#==================================================
19
20	#==================================================
21	# Add high level verbosity
22	typeset -i Verbosity=${Verbosity:=3}
23
24	#==================================================
25	# DEBUG_debug
26	# Add low level verbosity
27	typeset DEBUG_debug=${DEBUG_debug:=false}
28
29	#==================================================
30	# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
31	typeset RandomError=false
32
33	# Stack file containing call tree will be stored there.
34	if ( $DEBUG_debug ) ; then
35	if [ X${TaskType} = Xcomputing ]; then
36	typeset StackFileLocation=${StackFileLocation:=${PWD}/STACK}
37	typeset StackFileName=computing.stack.$$
38	[ ! -d ${StackFileLocation} ] && mkdir -p ${StackFileLocation}
39	elif [ X${TaskType} = Xpost-processing ]; then
40	typeset StackFileLocation=${POST_DIR:=${StackFileLocation}}
41	typeset StackFileName=post-processing.stack.$$
42	elif [ X${TaskType} = Xchecking ]; then
43	typeset StackFileLocation=${POST_DIR:=${PWD}}
44	typeset StackFileName=checking.stack.$$
45	else
46	typeset StackFileLocation=${POST_DIR:=${PWD}}
47	typeset StackFileName=checking.stack.$$
48	fi
49
50	if [ -f ${StackFileLocation}/${StackFileName} ] ;
51	then
52	echo "Stack of an libIGCM job :" >> ${StackFileLocation}/${StackFileName}
53	else
54	echo "Stack of an libIGCM job :" > ${StackFileLocation}/${StackFileName}
55	fi
56	fi
57
58	#==================================================
59	# NULL_STR
60	# Default null string
61	typeset -r NULL_STR="_0_"
62
63	#==================================================
64	# libIGCM_CurrentTag
65	# Current libIGCM tag, check compatibilty with *.card
66	typeset -r libIGCM_CurrentTag="1.0"
67
68	#==================================================
69	# Exit Flag (internal debug)
70	# When true, end the master loop AFTER SAVES FILES
71	ExitFlag=false
72
73	#==================================================
74	# Declare a stack of functions calls
75	unset IGCM_debug_Stack
76	unset IGCM_debug_StackArgs
77	unset IGCM_debug_StackTiming
78	IGCM_debug_Stack[0]=${NULL_STR}
79	IGCM_debug_StackArgs[0]=${NULL_STR}
80	IGCM_debug_StackTiming[0]=${NULL_STR}
81	IGCM_debug_LenStack=0
82
83	#D-#==================================================================
84	#D-function IGCM_debug_getDate_ms
85	#D- * Purpose: Give number of milliseconds since 01-jan-1970
86	function IGCM_debug_getDate_ms
87	{
88	typeset nanosecs ms
89	# nano secondes since 01-jan-1970
90	nanosecs=$( date +%s%N )
91
92	# truncate the last 6 digits to get milliseconds since 01-jan-1970
93	ms=${nanosecs:0:${#nanosecs}-6}
94
95	echo "$ms"
96	}
97
98	#D-#==================================================================
99	#D-function IGCM_debug_sizeOfTabContent
100	#D- * Purpose: Give sumed size of a list of files
101	#D- * Usage: IGCM_debug_sizeOfTabContent entityList destination
102	#D- * where entityList is a list of files or directory
103	#D- * where dest is either a directory or a file name
104	function IGCM_debug_sizeOfTabContent
105	{
106	typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo
107
108	eval set +A entityListe \${${1}}
109	destination=${2}
110	sumSizeKo=0
111
112	# Here we will try to compute size (file or directory size) from local path and not from archive.
113	for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do
114	if [ -f ${entityListe[$i]} ] ; then
115	# One file or a bunch of files has been copied without renaming from a visible filesystem
116	iEntity=${entityListe[$i]}
117	elif [ -f ${entityListe[$i]##/*/} ] ; then
118	# One file or a bunch of files has been copied without renaming from an non visible filesystem
119	# remove path /home/login/../ from entityListe elements
120	iEntity=${entityListe[$i]##/*/}
121	elif [ -f ${destination} ] ; then
122	# a file has been copied and renamed
123	iEntity=${destination}
124	elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then
125	# a copy in a directory but not in ${PWD}
126	iEntity=${destination}/${entityListe[$i]##/*/}
127	elif [ -d ${entityListe[$i]} ] ; then
128	# a directory has been copied from a non remote place
129	iEntity=${entityListe[$i]}
130	elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then
131	# a directory has been copied from a remote archive and not renamed
132	iEntity=${destination}/${entityListe[$i]##/*/}
133	elif [ -d ${destination} ] ; then
134	# a directory has been copied from a remote archive and renamed
135	iEntity=${destination}
136	fi
137	sizeKo=$( du --apparent-size -sk ${iEntity} \| gawk '{print $1}' )
138	sumSizeKo=$(( $sumSizeKo + $sizeKo ))
139	done
140	sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" \| bc )
141	echo "${sumSizeKo}\|${sumSizeMo}"
142	}
143
144	#D-#==================================================================
145	#D-function IGCM_debug_send_AMQP_msg__MAILTUNNEL
146	#D- * Purpose: Take over AMQP C client using mail as a message recipient
147	#D- * One argument : base64 encoded message
148	#D- * Attach encoded config.card when starting the simulation
149
150	function IGCM_debug_send_AMQP_msg__MAILTUNNEL {
151
152	typeset b64_encoded_msg mail_recipient
153	typeset buffer send_messages mail_frequency
154	typeset last_mail_date__file
155
156	b64_encoded_msg=$1
157
158	buffer=/tmp/buffer.$$
159	mail_recipient="superviseur@ipsl.jussieu.fr"
160	send_messages=0
161	mail_frequency=3600 # in seconds
162	# use to keep track when was last mail sent (maybe to be replaced with global variable)
163	last_mail_date__file=/tmp/stamp.$$
164
165	# init
166	if [ ! -f "${buffer}" ]; then
167	touch "${buffer}"
168	fi
169
170	# retrieve pending messages number
171	messages_count=$( wc -l ${buffer} \| gawk {'print $1'} )
172
173	if [ ! -f "${last_mail_date__file}" ]; then
174	# if we are here, it means no mail has been sent yet
175	# send the first mail
176	[ ${messages_count} -gt 0 ] && send_messages=1
177	else
178	# compute last time the file was changed (in seconds)
179	seconds_since_last_mail=$(( $(date +%s) - $(stat -c %Y ${last_mail_date__file}) ))
180	# send message when exceeding threshold
181	[ ${seconds_since_last_mail} -gt ${mail_frequency} ] && send_messages=1
182	fi
183
184	# queue messages in the buffer
185	echo ${b64_encoded_msg} >> ${buffer}
186
187	# send mail
188
189	if [ X${initBigBro} = Xtrue ] ; then
190	mailx -s "[TEMPORARY AMQP CHANNEL]" -a ${SUBMIT_DIR}/config.card.base64 ${mail_recipient} < ${buffer} # send buffer
191	rm -f $buffer ; touch ${buffer} # clear buffer
192	touch ${last_mail_date__file} # memorize last mail date
193	initBigBro=false
194	elif ( [ ${send_messages} -eq 1 ] \|\| [ X${FlushAMQP} = XTRUE ] ) ; then
195	mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer
196	rm -f $buffer ; touch ${buffer} # clear buffer
197	touch ${last_mail_date__file} # memorize last mail date
198	fi
199
200	# Allways all good for now.
201	return 0
202	}
203
204	#D-#==================================================================
205	#D-function IGCM_debug_SendAMQP
206	#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
207	function IGCM_debug_sendAMQP {
208
209	typeset decal first additionnalOption encodedBody
210
211	# Encode message Body
212	encodedBody=$( echo "${Body}" \| base64 -w 0 )
213
214	# Send config.card ?
215	if [ X${1} = Xactivate ] ; then
216	# Encode config.card
217	cat ${SUBMIT_DIR}/config.card \| base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
218	# Prepare additionnal option
219	additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
220	#
221	initBigBro=true
222	else
223	additionnalOption=
224	#
225	initBigBro=false
226	fi
227
228	# Only cosmetics : stack file
229	decal=0
230	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
231	printf ' ' >> ${StackFileLocation}/${StackFileName}
232	(( decal = decal + 1 ))
233	done
234	# Log to stack file using human readable format
235	echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b "${Body}" >> ${StackFileLocation}/${StackFileName}
236
237	# Log separately encoded AMQP message command for reuse in a mock up
238	echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> /tmp/send.AMQP.${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.${CumulPeriod}.history.txt
239
240	# Send the message
241	if [ X${BigBrotherChannel} = XMAIL ] ; then
242	IGCM_debug_send_AMQP_msg__MAILTUNNEL "${encodedBody}"
243	status=$?
244	else
245	sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
246	status=$?
247	fi
248
249	status=$?
250	if [ ${status} -gt 0 ] ; then
251	IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
252	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
253	exit
254	fi
255	}
256
257	#D-#==================================================================
258	#D-function IGCM_debug_CallStack
259	#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
260	#D-
261	function IGCM_debug_CallStack {
262	if ( $DEBUG_debug ) ; then
263	# Cosmetics
264	typeset i decal
265	i=0
266	until [ $i -eq ${IGCM_debug_LenStack} ]; do
267	decal=0
268	until [ $decal -eq ${i} ]; do
269	printf -- ' '
270	(( decal = decal + 1 ))
271	done
272	echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
273	((i = i + 1))
274	done
275	fi
276	}
277
278	#D-#==================================================================
279	#D-function IGCM_debug_PushStack
280	#D-* Purpose: Push a function name in the stack
281	#D-
282	function IGCM_debug_PushStack {
283	if ( $DEBUG_debug ) ; then
284	typeset decal inputs startTime_ms
285
286	# Only cosmetics : stack file
287	echo >> ${StackFileLocation}/${StackFileName}
288	decal=0
289	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
290	printf ' ' >> ${StackFileLocation}/${StackFileName}
291	(( decal = decal + 1 ))
292	done
293
294	# Fill the stack file
295	echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
296
297	# Save input list in an indexed array
298	INPUTS=( $@ )
299
300	# Get timing information
301	startTime_ms=$( IGCM_debug_getDate_ms )
302
303	# We add function call name on beginning of the stack
304	set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
305
306	# Save timing in milliseconds in an indexed array
307	set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}
308
309	# We include the "null" Args in the beginning of the StackArgs
310	set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}
311
312	# Then, we shift StackArgs tabular
313	# Replacing blank separated list by comma separated list of quoted elements (except the first and last element)
314	if [ $# -gt 1 ]; then
315	IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} \| sed -e "s/\ /\",\"/g" )
316	fi
317
318	if [ X${ActivateBigBro} = Xtrue ] ; then
319	# RabbitMQ message code "PUSHSTACK"
320	code=2000
321	# RabbitMQ message body
322	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${IGCM_debug_Stack[0]}\",\"arguments\":[\"${IGCM_debug_StackArgs[0]}\"],\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
323	# Fill the rabbitMQ queue
324	IGCM_debug_sendAMQP
325	fi
326
327	# Increment LenStack
328	(( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
329
330	#IGCM_debug_CallStack
331	fi
332	}
333
334	#D-#==================================================================
335	#D-function IGCM_debug_PopStack
336	#D-* Purpose: Pop a function name in the stack
337	#D-
338	function IGCM_debug_PopStack {
339	if ( $DEBUG_debug ) ; then
340	typeset i decal command arguments startTime_ms endTime_ms
341	typeset instrumentation dest prefix
342	# they are not typeset because they are send "by adress" to son functions
343	# we unset them to avoid "memory effect"
344	unset fileList source
345
346	# INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
347	# PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
348	# THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
349	if ( ${RandomError} ) ; then
350	if [ $((RANDOM%10000)) -le 10 ] ; then
351	IGCM_debug_Print 1 "Random error has been triggered"
352	echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName}
353	ExitFlag=true
354	fi
355	fi
356
357	if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
358	# Everything is cool
359
360	# Get timing information
361	endTime_ms=$( IGCM_debug_getDate_ms )
362
363	# Save Stack information before poping the stack
364	command=${IGCM_debug_Stack[0]}
365
366	# Go from comma separated (list) to space separated in an array
367	set -A arguments -- $( echo ${IGCM_debug_StackArgs[0]} \| sed -e "s/,/\ /g" )
368
369	# Save Stack information before poping the stack
370	startTime_ms=${IGCM_debug_StackTiming[0]}
371
372	# Pop the stack
373	(( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
374	set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
375	set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
376	set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
377	else
378	echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
379	IGCM_debug_Exit $@
380	fi
381
382	# Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
383	# We are interested in:
384	# 0. Which command performs the work
385	# 1. Size of entity we are working with
386	# 2. Where are we reading
387	# 3. Where are we writing
388	# 4. How long it took
389
390	instrumentation=false
391
392	case ${command} in
393	# Classical copy (only files are given to IGCM_sys_Cp as options)
394	IGCM_sys_Cp)
395	instrumentation=true
396	# All but the latest
397	fileList=${arguments[]:0:${#arguments[]}-1}
398	# just need the first file to get the directory
399	source=${arguments[0]}
400	# Nothing but the latest
401	dest=${arguments[${#arguments[*]}-1]}
402	# Size of file whose name are stored in a list
403	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
404	;;
405
406	# Copy from archive machine or from buffer
407	IGCM_sys_Get\|IGCM_sys_GetBuffer)
408	instrumentation=true
409	if [ ${#arguments[*]} -eq 2 ] ; then
410	source=${arguments[0]}
411	dest=${arguments[1]}
412	# Size of file whose name are stored in a variable
413	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
414	elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
415	# IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/
416	# Keep the array name hosting the all list
417	eval set +A fileList \${${arguments[1]}}
418	# just need the first file to get the directory
419	source=${fileList[0]}
420	dest=${arguments[2]}
421	# Size of file whose name are stored in a list
422	entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
423	elif [ [ ${#arguments[*]} -ge 3 ] ; then
424	# All but the latest
425	fileList=${arguments[]:0:${#arguments[]}-1}
426	# just need the first file to get the directory
427	source=${arguments[0]}
428	# Nothing but the latest
429	dest=${arguments[${#arguments[*]}-1]}
430	# Size of file whose name are stored in a list
431	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
432	fi
433	;;
434
435	# Copy from compute node or copy to archive/buffer
436	IGCM_sys_Get_Master\|IGCM_sys_Get_Dir\|IGCM_sys_Put_Out\|IGCM_sys_PutBuffer_Out)
437	instrumentation=true
438	source=${arguments[0]}
439	dest=${arguments[1]}
440	# Size of file whose name are stored in a variable
441	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
442	;;
443
444	# Rebuild command
445	IGCM_sys_rebuild\|IGCM_sys_rebuild_station)
446	instrumentation=true
447	# All but the first
448	fileList=${arguments[]:1:${#arguments[]}-1}
449	# just need a file to get the directory
450	source=${arguments[1]}
451	# Nothing but the first
452	dest=${arguments[0]}
453	# Size of file whose name are stored in a list
454	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
455	;;
456
457	# NCO commands
458	IGCM_sys_ncrcat\|IGCM_sys_ncecat\|IGCM_sys_ncra\|IGCM_sys_ncks\|IGCM_sys_cdo)
459	# Example of what we want to catch : only filenames in those command lines
460	# IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE}
461	# IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
462	# IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output}
463	# IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
464	instrumentation=true
465	keepGoing=true
466	prefix=.
467	i=0
468	while ( ${keepGoing} ) ; do
469	# the last one is not interesting
470	if [ ${i} -eq ${#arguments[*]}-1 ] ; then
471	keepGoing=false
472	# look after "-p" option. Path prefix is the following arguments
473	elif [ ${arguments[${i}]} = "-p" ] ; then
474	((i = i + 1))
475	prefix=${arguments[${i}]}
476	((i = i + 1))
477	elif [ ${i} -eq ${#arguments[*]}-1 ] ; then
478	keepGoing=false
479	# looking for files
480	elif [ -f ${prefix}/${arguments[${i}]} ] ; then
481	fileList="${fileList} ${prefix}/${arguments[${i}]}"
482	((i = i + 1))
483	# other options are not interesting
484	else
485	((i = i + 1))
486	fi
487	done
488
489	# i value is at least 1
490	# just need one file to get the directory
491	source=$( echo ${fileList} \| gawk '{print $1}' )
492	# Nothing but the latest
493	dest=${arguments[${#arguments[*]}-1]}
494	# Size of file whose name are stored in a list
495	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
496	;;
497	esac
498
499	# Print information related to instrumentation
500	( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}
501
502	# Only cosmetics : stack file
503	decal=0
504	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
505	printf ' ' >> ${StackFileLocation}/${StackFileName}
506	(( decal = decal + 1 ))
507	done
508
509	if ( ${ExitFlag} ) ; then
510	# Inform the stack file
511	echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName}
512
513	if [ X${ActivateBigBro} = Xtrue ] ; then
514	# RabbitMQ message code "ERROR HAS BEEN TRIGGERED"
515	code=9000
516	# RabbitMQ message body
517	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
518	# Fill the rabbitMQ queue
519	IGCM_debug_sendAMQP
520	fi
521	else
522	# Inform the stack file
523	echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
524	if [ X${ActivateBigBro} = Xtrue ] ; then
525	# RabbitMQ message code "POPSTACK"
526	code=3000
527	# RabbitMQ message body
528	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
529	# Fill the rabbitMQ queue
530	IGCM_debug_sendAMQP
531	fi
532	fi
533
534	# Reset array if necessary
535	if [ ${IGCM_debug_LenStack} = 0 ]; then
536	#echo
537	#IGCM_debug_Print 3 "Clean stack array"
538	#echo
539	unset IGCM_debug_Stack
540	unset IGCM_debug_StackArgs
541	unset IGCM_debug_StackTiming
542	IGCM_debug_Stack[0]=${NULL_STR}
543	IGCM_debug_StackArgs[0]=${NULL_STR}
544	IGCM_debug_StackTiming[0]=${NULL_STR}
545	fi
546	fi
547	#IGCM_debug_CallStack
548	}
549
550	#D-#==================================================================
551	#D-function IGCM_debug_ActivateBigBro
552	#D-* Purpose: switch rabbitMQ on
553	#D-
554	function IGCM_debug_ActivateBigBro {
555	IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"
556
557	# Message type standard fields:
558	# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
559
560	# Message type dictionnary and custom fields:
561	# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
562
563	if [ X${BigBrother} = Xtrue ] ; then
564	# create a unique ID for this specific job
565	jobuid=$(uuidgen)
566
567	if ( ${FirstInitialize} ) ; then
568	# RabbitMQ message code "BEGIN A SIMULATION"
569	code=0000
570	# create and persist a unique id for this simulation
571	simuid=$(uuidgen)
572	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid}
573
574	# Standard fields for the first message
575	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
576	else
577	# RabbitMQ message code "A NEW JOB IS RUNNING PART OF A SIMULATION"
578	code=1000
579	#
580	# retrieve this simulation's unique id
581	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
582	simuid=${config_Configuration_simuid}
583
584	# Using standard fields for message others than the first one. Still subject to change
585	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
586	fi
587
588	# RabbitMQ message body with specific fields associated message codes treated here
589	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
590
591	# Fill the rabbitMQ queue (the config.card in use will be sent)
592	IGCM_debug_sendAMQP activate
593
594	# NOT VERY NICE BUT ... IT WORKS
595	# Be sure that the genericSimulationID will be small from now on
596	# Using standard fields for messages others than the first one. Still subject to change
597	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
598
599	# Turn the flag on
600	ActivateBigBro=true
601	fi
602	IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
603	}
604
605	#D-#==================================================================
606	#D-function IGCM_debug_Exit
607	#D-* Purpose: Print Call Stack and set ExitFlag to true
608	#D-
609	function IGCM_debug_Exit {
610	IGCM_debug_PushStack "IGCM_debug_Exit"
611	echo "IGCM_debug_Exit : " "${@}"
612	echo
613	echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
614	echo "!! ERROR TRIGGERED !!"
615	echo "!! EXIT FLAG SET !!"
616	echo "!------------------------!"
617	echo
618	IGCM_debug_CallStack
619	ExitFlag=true
620	IGCM_debug_PopStack "IGCM_debug_Exit"
621	}
622
623	#D-#==================================================
624	#D-function IGCM_debug_Verif_Exit
625	#D-* Purpose: exit with number 1 if ExitFlag is true
626	#D-
627	function IGCM_debug_Verif_Exit {
628	if ( ${ExitFlag} ) ; then
629	# Plan to send an email here with IGCM_sys_SendMail
630	if [ X${TaskType} != Xchecking ] ; then
631	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
632	echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
633	echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
634	echo " EXIT THE JOB."
635	echo
636	IGCM_debug_CallStack
637	fi
638
639	if [ X${ActivateBigBro} = Xtrue ] ; then
640	# RabbitMQ message code "EXIT THE JOBS BECAUSE ERROR(S) HAS BEEN TRIGGERED"
641	code=9999
642	# RabbitMQ message body
643	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
644	# Fill the rabbitMQ queue
645	IGCM_debug_sendAMQP
646	fi
647
648	# Not sure about that one ...
649	if ( $DEBUG_debug ) ; then
650	echo "Your files on ${R_OUT} :"
651	IGCM_sys_Tree ${R_SAVE}
652	echo
653	fi
654
655	# Mail notification
656	IGCM_sys_SendMail
657
658	# And Good Bye
659	date
660	exit 1
661	fi
662	}
663
664	#D-#==================================================
665	#D-function IGCM_debug_Verif_Exit_Post
666	#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
667	#D-
668	function IGCM_debug_Verif_Exit_Post {
669	if ( ${ExitFlag} ) ; then
670	echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
671	# If SpaceName is PROD then we stop if post_processing fails
672	# Plan to send an email here with IGCM_sys_SendMail
673	if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
674	echo " EXIT THE JOB."
675	echo
676	# Mail notification
677	#IGCM_sys_SendMailPost
678	# And Good Bye
679	date
680	exit 1
681	else
682	echo "Either inside config.card the variable SpaceName is not in PROD"
683	echo "or inside the main Job the variable JobType is not in RUN mode"
684	echo " SO WE DO NOT EXIT THE JOB."
685	echo
686	date
687	fi
688	fi
689	}
690
691	#D-#==================================================================
692	#D-function IGCM_debug_Print
693	#D-* Purpose: Print arguments according to a level of verbosity.
694	#D-
695	function IGCM_debug_Print
696	{
697	typeset level=$1
698	shift
699
700	if [ X"${1}" = X"-e" ]; then
701	typeset cmd_echo="echo -e"
702	shift
703	else
704	typeset cmd_echo="echo"
705	fi
706
707	if [ ${level} -le ${Verbosity} ] ; then
708	typeset i
709	case "${level}" in
710	1) for i in "$@" ; do
711	${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
712	done ;;
713	2) for i in "$@" ; do
714	${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
715	done ;;
716	3) for i in "$@" ; do
717	${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
718	done ;;
719	esac
720	fi
721	}
722
723	#D-#==================================================================
724	#D-function IGCM_debug_PrintVariables
725	#D-* Purpose: Print arguments when match a pattern
726	#D- according to a level of verbosity.
727	function IGCM_debug_PrintVariables
728	{
729	typeset level=$1
730	shift
731
732	list=$( set \| grep ^$1 \| sed -e "s/'//g" )
733
734	if [ "X${list}" != X ] ; then
735	IGCM_debug_Print ${level} ${list}
736	fi
737	}
738
739	#D-#==================================================================
740	#D-function IGCM_debug_PrintInfosActions
741	#D-* Purpose: Print information related to instrumentation
742	function IGCM_debug_PrintInfosActions
743	{
744	typeset actionType=$1
745	typeset entitySize=$2
746	typeset start_ms=$3
747	typeset end_ms=$4
748
749	typeset dest=$5
750	typeset source=$6
751
752	typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
753	typeset name dirOut dirSource
754
755	name="doNotKnow"
756
757	diff_ms=$(( $end_ms - $start_ms ))
758	# echo "diff_ms=$diff_ms"
759
760	entitySizeKo=$( echo ${entitySize} \| gawk -F"\|" '{print $1}' )
761	# echo "entitySizeKo=$entitySizeKo"
762	entitySizeMo=$( echo ${entitySize} \| gawk -F"\|" '{print $2}' )
763
764	# flux en Ko / ms
765	flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" \| bc )
766	# echo "flux_Ko_ms=$flux_Ko_ms"
767
768	# flux en Ko / s
769	flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
770	# echo "flux_Ko_s=$flux_Ko_s"
771
772	# flux en Mo / s
773	flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" \| bc )
774	# echo "flux_Mo_s=$flux_Mo_s"
775
776	if [ -d $dest ] ; then
777	dirOut=$( readlink -f ${dest} )
778	else
779	dirOut=$( readlink -f $( dirname ${dest} ) )
780	fi
781
782	if [ -d $source ] ; then
783	dirSource=$( readlink -f ${source} )
784	else
785	dirSource=$( readlink -f $( dirname ${source} ) )
786	fi
787
788	instrumentationContent=$( echo "\"act\":\"${actionType}\",\"sz\":\"${entitySizeMo}\",\"ms\":\"${diff_ms}\",\"fx(ko)\":\"${flux_Ko_s}\",\"fx(mo)\":\"${flux_Mo_s}\",\"nm\":\"${name}\",\"dirSource\":\"${dirSource}\",\"dirOut\":\"${dirOut}\"" )
789
790	#echo "==>act:${actionType}\|sz:${entitySizeMo}\|ms:${diff_ms}\|fx(ko):${flux_Ko_s}\|fx(mo):${flux_Mo_s}\|nm:${name}\|dirSource:${dirSource}\|dirOut:${dirOut}"
791
792	echo ${instrumentationContent}
793
794	# Inform the rabbitMQ queue
795	if [ X${ActivateBigBro} = Xtrue ] ; then
796	# RabbitMQ message body
797	Body=$( echo "{${genericSimulationID},\"msgCode\":\"7000\",\"msgUID\":\"$(uuidgen)\",${instrumentationContent},\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
798	# Fill the rabbitMQ queue
799	IGCM_debug_sendAMQP
800	fi
801	}
802
803	#D-#==================================================================
804	#D-function IGCM_debug_Check
805	#D- * Purpose: Check the present file by comparison with a reference file
806	function IGCM_debug_Check
807	{
808	#---------------------
809	if [ ! -n "${libIGCM}" ] ; then
810	echo "Check libIGCM_debug ..........................................[ FAILED ]"
811	echo "--Error--> libIGCM variable is not defined"
812	exit 2
813	fi
814
815	#---------------------
816	if [ ! -n "${Verbosity}" ] ; then
817	echo "Check libIGCM_debug ..........................................[ FAILED ]"
818	echo "--Error--> Verbosity variable is not defined"
819	exit 3
820	fi
821
822	#---------------------
823	${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > /tmp/IGCM_debug_Test.$$.ref.failed 2>&1
824	sleep 2
825
826	# Remove date stamp.
827	sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" /tmp/IGCM_debug_Test.$$.ref.failed > /tmp/IGCM_debug_Test.$$.ref.failed.nodate
828	mv /tmp/IGCM_debug_Test.$$.ref.failed.nodate /tmp/IGCM_debug_Test.$$.ref.failed
829
830	if diff /tmp/IGCM_debug_Test.$$.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
831	echo "Check libIGCM_debug ..............................................[ OK ]"
832	rm -f /tmp/IGCM_debug_Test.$$.ref.failed
833	else
834	echo "Check libIGCM_debug ..........................................[ FAILED ]"
835	echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
836	echo " has produced the file IGCM_debug_Test.ref.failed"
837	echo " Please analyse differences with the reference file by typing:"
838	echo " diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
839	echo " Report errors to the author: Patrick.Brockmann@cea.fr"
840	cat /tmp/IGCM_debug_Test.$$.ref.failed
841	exit 4
842	fi
843	#---------------------
844	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: