Context Navigation

source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 1189

Last change on this file since 1189 was 1189, checked in by sdipsl, 9 years ago
Extend Big Brother scope to also watch post-processing jobs. Only the metrics LMDZ job for now. See #252 Some refactoring
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 33.8 KB

Line
1	#!/bin/ksh
2
3	#**************************************************************
4	# Author: Patrick Brockmann, Martial Mancip
5	# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
6	# $Revision:: $ Revision of last commit
7	# $Author:: $ Author of last commit
8	# $Date:: $ Date of last commit
9	# IPSL (2006)
10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11	#
12	#**************************************************************
13
14	#==================================================
15	# The documentation of this file can be automatically generated
16	# if you use the prefix #D- for comments to be extracted.
17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
18	#==================================================
19
20	#==================================================
21	# Add high level verbosity
22	typeset -i Verbosity=${Verbosity:=3}
23
24	#==================================================
25	# DEBUG_debug
26	# Add low level verbosity
27	DEBUG_debug=${DEBUG_debug:=false}
28
29	#==================================================
30	# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
31	typeset RandomError=false
32
33	#==================================================
34	# NULL_STR
35	# Default null string
36	typeset -r NULL_STR="_0_"
37
38	#==================================================
39	# libIGCM_CurrentTag
40	# Current libIGCM tag, check compatibilty with *.card
41	typeset -r libIGCM_CurrentTag="1.0"
42
43	#==================================================
44	# Exit Flag (internal debug)
45	# When true, end the master loop AFTER SAVES FILES
46	ExitFlag=false
47
48	#==================================================
49	# Declare a stack of functions calls
50	unset IGCM_debug_Stack
51	unset IGCM_debug_StackArgs
52	unset IGCM_debug_StackTiming
53	IGCM_debug_Stack[0]=${NULL_STR}
54	IGCM_debug_StackArgs[0]=${NULL_STR}
55	IGCM_debug_StackTiming[0]=${NULL_STR}
56	IGCM_debug_LenStack=0
57
58	#D-#==================================================================
59	#D-function IGCM_debug_getDate_ms
60	#D- * Purpose: Give number of milliseconds since 01-jan-1970
61	function IGCM_debug_getDate_ms
62	{
63	typeset nanosecs ms
64	# nano secondes since 01-jan-1970
65	nanosecs=$( date +%s%N )
66
67	# truncate the last 6 digits to get milliseconds since 01-jan-1970
68	ms=${nanosecs:0:${#nanosecs}-6}
69
70	echo "$ms"
71	}
72
73	#D-#==================================================================
74	#D-function IGCM_debug_sizeOfTabContent
75	#D- * Purpose: Give sumed size of a list of files
76	#D- * Usage: IGCM_debug_sizeOfTabContent entityList destination
77	#D- * where entityList is a list of files or directory
78	#D- * where dest is either a directory or a file name
79	function IGCM_debug_sizeOfTabContent
80	{
81	typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo
82
83	eval set +A entityListe \${${1}}
84	destination=${2}
85	sumSizeKo=0
86
87	# Here we will try to compute size (file or directory size) from local path and not from archive.
88	for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do
89	if [ -f ${entityListe[$i]} ] ; then
90	# One file or a bunch of files has been copied without renaming from a visible filesystem
91	iEntity=${entityListe[$i]}
92	elif [ -f ${entityListe[$i]##/*/} ] ; then
93	# One file or a bunch of files has been copied without renaming from an non visible filesystem
94	# remove path /home/login/../ from entityListe elements
95	iEntity=${entityListe[$i]##/*/}
96	elif [ -f ${destination} ] ; then
97	# a file has been copied and renamed
98	iEntity=${destination}
99	elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then
100	# a copy in a directory but not in ${PWD}
101	iEntity=${destination}/${entityListe[$i]##/*/}
102	elif [ -d ${entityListe[$i]} ] ; then
103	# a directory has been copied from a non remote place
104	iEntity=${entityListe[$i]}
105	elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then
106	# a directory has been copied from a remote archive and not renamed
107	iEntity=${destination}/${entityListe[$i]##/*/}
108	elif [ -d ${destination} ] ; then
109	# a directory has been copied from a remote archive and renamed
110	iEntity=${destination}
111	fi
112	sizeKo=$( du --apparent-size -skL ${iEntity} \| gawk '{print $1}' )
113	sumSizeKo=$(( $sumSizeKo + $sizeKo ))
114	done
115	sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" \| bc )
116	echo "${sumSizeKo}\|${sumSizeMo}"
117	}
118
119	#D-#==================================================================
120	#D-function IGCM_debug_send_AMQP_msg__MAILTUNNEL
121	#D- * Purpose: Take over AMQP C client using mail as a message recipient
122	#D- * One argument : base64 encoded message
123	#D- * Attach encoded config.card when starting the simulation
124
125	function IGCM_debug_send_AMQP_msg__MAILTUNNEL {
126
127	typeset b64_encoded_msg mail_recipient
128	typeset buffer send_messages mail_frequency
129	typeset last_mail_date__file
130
131	b64_encoded_msg=$1
132
133	mail_recipient="superviseur@ipsl.jussieu.fr"
134	send_messages=0
135	mail_frequency=3600 # in seconds
136	# use to keep track when was last mail sent (maybe to be replaced with global variable)
137	last_mail_date__file=${R_BUF}/.stamp.${config_UserChoices_TagName}.${config_UserChoices_JobName}
138	# use to accumulate messages before sending them
139	buffer=${R_BUF}/.buffer.${config_UserChoices_TagName}.${config_UserChoices_JobName}
140
141	# init
142	if [ ! -f "${buffer}" ]; then
143	touch ${buffer}
144	fi
145
146	if [ ! -f "${last_mail_date__file}" ]; then
147	touch ${last_mail_date__file}
148	else
149	# compute last time the file was changed (in seconds)
150	seconds_since_last_mail=$(( $(date +%s) - $(stat -c %Y ${last_mail_date__file}) ))
151	# send message when exceeding threshold
152	[ ${seconds_since_last_mail} -gt ${mail_frequency} ] && send_messages=1
153	fi
154
155	# queue messages in the buffer
156	echo ${b64_encoded_msg} >> ${buffer}
157
158	# send mail
159
160	if [ X${initBigBro} = Xtrue ] ; then
161	#echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") > ${SUBMIT_DIR}/mail.txt
162	mailx -s "[TEMPORARY AMQP CHANNEL]" -a ${SUBMIT_DIR}/config.card.base64 ${mail_recipient} < ${buffer} # send buffer
163	rm -f $buffer ; touch ${buffer} # clear buffer
164	touch ${last_mail_date__file} # memorize last mail date
165	initBigBro=false
166	elif [ ${send_messages} -eq 1 ] ; then
167	#echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") >> ${SUBMIT_DIR}/mail.txt
168	mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer
169	rm -f ${buffer} ; touch ${buffer} # flush the buffer
170	touch ${last_mail_date__file} # memorize last mail date
171	fi
172
173	if ( ${FlushAMQP} ) ; then
174	mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer
175	rm -f ${buffer} # cleaning behind us
176	rm -f ${last_mail_date__file} # cleaning behind us
177	fi
178
179	# Allways all good for now.
180	return 0
181	}
182
183	#D-#==================================================================
184	#D-function IGCM_debug_sendAMQP_Metrics
185	#D- * Purpose: Take over AMQP C client using mail as a message recipient
186	#D- * One argument : Directory where metrics.json files can be found
187	#D- * Attach encoded metrics.json files.
188
189	function IGCM_debug_sendAMQP_Metrics {
190
191	typeset mail_recipient encodedBody
192	if [ X${ActivateBigBro} = Xtrue ] ; then
193	mail_recipient="superviseur@ipsl.jussieu.fr"
194	# Metrics tag on server side
195	code=7100
196	# Usual AMQP message to route messages on server side
197	encodedBody=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" \| base64 -w 0 )
198	# send mail
199	attachmentsOptions=""
200	for metricsFile in $( ls $1/*json ) ; do
201	attachmentsOptions="-a ${metricsFile} ${attachmentsOptions}"
202	done
203	echo ${encodedBody}\|mailx -s "[TEMPORARY AMQP CHANNEL]" ${attachmentsOptions} ${mail_recipient}
204	fi
205
206	# Allways all good for now.
207	return 0
208	}
209
210	#D-#==================================================================
211	#D-function IGCM_debug_SendAMQP
212	#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
213	function IGCM_debug_sendAMQP {
214
215	typeset decal first additionnalOption encodedBody
216
217	# Encode message Body
218	encodedBody=$( echo "${Body}" \| base64 -w 0 )
219
220	# Send config.card ?
221	if [ X${1} = Xactivate ] ; then
222	# Encode config.card
223	cat ${SUBMIT_DIR}/config.card \| base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
224	# Prepare additionnal option
225	additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
226	#
227	initBigBro=true
228	else
229	additionnalOption=
230	#
231	initBigBro=false
232	fi
233
234	# Only cosmetics : stack file
235	if [ X${ActivateStackFilling} = Xtrue ] ; then
236	decal=0
237	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
238	printf ' ' >> ${StackFileLocation}/${StackFileName}
239	(( decal = decal + 1 ))
240	done
241	# Log to stack file using human readable format
242	echo "${Body}" >> ${StackFileLocation}/${StackFileName}
243	fi
244
245	# Log separately encoded AMQP message command for reuse in a mock up
246	#echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> ${RUN_DIR_PATH}/send.AMQP.${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.${CumulPeriod}.history.txt
247
248	# Send the message
249	if [ X${BigBrotherChannel} = XMAIL ] ; then
250	IGCM_debug_send_AMQP_msg__MAILTUNNEL "${encodedBody}"
251	status=$?
252	else
253	sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
254	status=$?
255	fi
256
257	if [ ${status} -gt 0 ] ; then
258	IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
259	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
260	exit 1
261	fi
262	}
263
264	#D-#==================================================================
265	#D-function IGCM_debug_CallStack
266	#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
267	#D-
268	function IGCM_debug_CallStack {
269	if ( $DEBUG_debug ) ; then
270	# Cosmetics
271	typeset i decal
272	i=0
273	until [ $i -eq ${IGCM_debug_LenStack} ]; do
274	decal=0
275	until [ $decal -eq ${i} ]; do
276	printf -- ' '
277	(( decal = decal + 1 ))
278	done
279	echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
280	((i = i + 1))
281	done
282	fi
283	}
284
285	#D-#==================================================================
286	#D-function IGCM_debug_PushStack
287	#D-* Purpose: Push a function name in the stack
288	#D-
289	function IGCM_debug_PushStack {
290	if ( $DEBUG_debug ) ; then
291	typeset decal inputs startTime_ms
292
293	# Only cosmetics : stack file
294	if [ X${ActivateStackFilling} = Xtrue ] ; then
295	echo >> ${StackFileLocation}/${StackFileName}
296	decal=0
297	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
298	printf ' ' >> ${StackFileLocation}/${StackFileName}
299	(( decal = decal + 1 ))
300	done
301
302	# Fill the stack file
303	echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
304	fi
305
306	# Save input list in an indexed array
307	INPUTS=( $@ )
308
309	# Get timing information
310	startTime_ms=$( IGCM_debug_getDate_ms )
311
312	# We add function call name on beginning of the stack
313	set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
314
315	# Save timing in milliseconds in an indexed array
316	set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}
317
318	# We include the "null" Args in the beginning of the StackArgs
319	set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}
320
321	# Then, we shift StackArgs tabular
322	# Replacing blank separated list by comma separated list of quoted elements (except the first and last element)
323	if [ $# -gt 1 ]; then
324	IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} \| sed -e "s/\ /\",\"/g" )
325	fi
326
327	# Unplugged message 4000 handling for now. To ease downstream treatment.
328	#if [ X${ActivateBigBro} = Xtrue ] ; then
329	# # RabbitMQ message code "PUSHSTACK"
330	# code=4000
331	# # RabbitMQ message body
332	# Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${IGCM_debug_Stack[0]}\",\"arguments\":[\"${IGCM_debug_StackArgs[0]}\"],\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
333	# # Fill the rabbitMQ queue
334	# IGCM_debug_sendAMQP
335	#fi
336
337	# Increment LenStack
338	(( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
339
340	#IGCM_debug_CallStack
341	fi
342	}
343
344	#D-#==================================================================
345	#D-function IGCM_debug_PopStack
346	#D-* Purpose: Pop a function name in the stack
347	#D-
348	function IGCM_debug_PopStack {
349	if ( $DEBUG_debug ) ; then
350	typeset i decal command arguments startTime_ms endTime_ms
351	typeset instrumentation dest prefix
352	# they are not typeset because they are send "by adress" to son functions
353	# we unset them to avoid "memory effect"
354	unset fileList source
355
356	# INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
357	# PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
358	# THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
359	if ( ${RandomError} ) ; then
360	if [ $((RANDOM%10000)) -le 10 ] ; then
361	IGCM_debug_Print 1 "Random error has been triggered"
362	if [ X${ActivateStackFilling} = Xtrue ] ; then
363	echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName}
364	fi
365	ExitFlag=true
366	fi
367	fi
368
369	if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
370	# Everything is cool
371
372	# Get timing information
373	endTime_ms=$( IGCM_debug_getDate_ms )
374
375	# Save Stack information before poping the stack
376	command=${IGCM_debug_Stack[0]}
377
378	# Go from comma separated list of quoted elements (except the first and the last element)
379	# to unquoted space separated elements in an array
380	set -A arguments -- $( echo ${IGCM_debug_StackArgs[0]} \| sed -e "s/\",\"/\ /g" )
381
382	# Save Stack information before poping the stack
383	startTime_ms=${IGCM_debug_StackTiming[0]}
384
385	# Pop the stack
386	(( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
387	set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
388	set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
389	set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
390	else
391	echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
392	IGCM_debug_Exit $@
393	fi
394
395	# Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
396	# We are interested in:
397	# 0. Which command performs the work
398	# 1. Size of entity we are working with
399	# 2. Where are we reading
400	# 3. Where are we writing
401	# 4. How long it took
402
403	instrumentation=false
404
405	case ${command} in
406	# Classical copy (only files are given to IGCM_sys_Cp as options)
407	IGCM_sys_Cp)
408	instrumentation=true
409	# All but the latest
410	fileList=${arguments[]:0:${#arguments[]}-1}
411	# just need the first file to get the directory
412	source=${arguments[0]}
413	# Nothing but the latest
414	dest=${arguments[${#arguments[*]}-1]}
415	# Size of file whose name are stored in a list
416	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
417	;;
418
419	# Copy from archive machine or from buffer
420	IGCM_sys_Get\|IGCM_sys_GetBuffer)
421	instrumentation=true
422	if [ ${#arguments[*]} -eq 2 ] ; then
423	source=${arguments[0]}
424	dest=${arguments[1]}
425	# Size of file whose name are stored in a variable
426	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
427	elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
428	# IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/
429	# Keep the array name hosting the all list
430	eval set +A fileList \${${arguments[1]}}
431	# just need the first file to get the directory
432	source=${fileList[0]}
433	dest=${arguments[2]}
434	# Size of file whose name are stored in a list
435	entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
436	elif [ [ ${#arguments[*]} -ge 3 ] ; then
437	# All but the latest
438	fileList=${arguments[]:0:${#arguments[]}-1}
439	# just need the first file to get the directory
440	source=${arguments[0]}
441	# Nothing but the latest
442	dest=${arguments[${#arguments[*]}-1]}
443	# Size of file whose name are stored in a list
444	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
445	fi
446	;;
447
448	# Copy from compute node or copy to archive/buffer
449	IGCM_sys_Get_Master\|IGCM_sys_Get_Dir\|IGCM_sys_Put_Out\|IGCM_sys_PutBuffer_Out)
450	instrumentation=true
451	source=${arguments[0]}
452	dest=${arguments[1]}
453	# Size of file whose name are stored in a variable
454	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
455	;;
456
457	# Rebuild command
458	IGCM_sys_rebuild\|IGCM_sys_rebuild_station)
459	instrumentation=true
460	# All but the first
461	fileList=${arguments[]:1:${#arguments[]}-1}
462	# just need a file to get the directory
463	source=${arguments[1]}
464	# Nothing but the first
465	dest=${arguments[0]}
466	# Size of file whose name are stored in a list
467	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
468	;;
469
470	# NCO commands
471	IGCM_sys_ncrcat\|IGCM_sys_ncecat\|IGCM_sys_ncra\|IGCM_sys_ncks\|IGCM_sys_cdo)
472	# Example of what we want to catch : only filenames in those command lines
473	# IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE}
474	# IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
475	# IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output}
476	# IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
477	instrumentation=true
478	keepGoing=true
479	prefix=.
480	i=0
481	while ( ${keepGoing} ) ; do
482	# the last one is not interesting
483	if [ ${i} -eq ${#arguments[*]}-1 ] ; then
484	keepGoing=false
485	# look after "-p" option. Path prefix is the following arguments
486	elif [ ${arguments[${i}]} = "-p" ] ; then
487	((i = i + 1))
488	prefix=${arguments[${i}]}
489	((i = i + 1))
490	elif [ ${i} -eq ${#arguments[*]}-1 ] ; then
491	keepGoing=false
492	# looking for files
493	elif [ -f ${prefix}/${arguments[${i}]} ] ; then
494	fileList="${fileList} ${prefix}/${arguments[${i}]}"
495	((i = i + 1))
496	# other options are not interesting
497	else
498	((i = i + 1))
499	fi
500	done
501
502	# i value is at least 1
503	# just need one file to get the directory
504	source=$( echo ${fileList} \| gawk '{print $1}' )
505	# Nothing but the latest
506	dest=${arguments[${#arguments[*]}-1]}
507	# Size of file whose name are stored in a list
508	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
509	;;
510	esac
511
512	# Print information related to instrumentation
513	( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}
514
515	# Only cosmetics : stack file
516	if [ X${ActivateStackFilling} = Xtrue ] ; then
517	decal=0
518	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
519	printf ' ' >> ${StackFileLocation}/${StackFileName}
520	(( decal = decal + 1 ))
521	done
522	fi
523
524	if ( ${ExitFlag} ) ; then
525	# Inform the stack file
526	if [ X${ActivateStackFilling} = Xtrue ] ; then
527	echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName}
528	fi
529
530	if [ X${ActivateBigBro} = Xtrue ] ; then
531	# RabbitMQ message code "ERROR HAS BEEN TRIGGERED"
532	code=4900
533	# RabbitMQ message body
534	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
535	# Fill the rabbitMQ queue
536	IGCM_debug_sendAMQP
537	fi
538	else
539	# Inform the stack file
540	if [ X${ActivateStackFilling} = Xtrue ] ; then
541	echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
542	fi
543
544	# Unplugged message 4100 handling for now. To ease downstream treatment.
545	#if [ X${ActivateBigBro} = Xtrue ] ; then
546	# # RabbitMQ message code "POPSTACK"
547	# code=4100
548	# # RabbitMQ message body
549	# Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
550	# # Fill the rabbitMQ queue
551	# IGCM_debug_sendAMQP
552	#fi
553	fi
554
555	# Reset array if necessary
556	if [ ${IGCM_debug_LenStack} = 0 ]; then
557	#echo
558	#IGCM_debug_Print 3 "Clean stack array"
559	#echo
560	unset IGCM_debug_Stack
561	unset IGCM_debug_StackArgs
562	unset IGCM_debug_StackTiming
563	IGCM_debug_Stack[0]=${NULL_STR}
564	IGCM_debug_StackArgs[0]=${NULL_STR}
565	IGCM_debug_StackTiming[0]=${NULL_STR}
566	fi
567	fi
568	#IGCM_debug_CallStack
569	}
570
571	#D-#==================================================================
572	#D-function IGCM_debug_BigBro_Initialize
573	#D-* Purpose: switch rabbitMQ on
574	#D-
575	function IGCM_debug_BigBro_Initialize {
576	IGCM_debug_PushStack "IGCM_debug_BigBro_Initialize"
577
578	# Message type standard fields:
579	# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
580
581	# Message type dictionnary and custom fields:
582	# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
583
584	if [ X${BigBrother} = Xtrue ] ; then
585	# create a unique ID for this specific job
586	jobuid=$(uuidgen)
587
588	if [ X${TaskType} = Xcomputing ]; then
589	if ( ${FirstInitialize} ) ; then
590	# RabbitMQ message code "BEGIN A SIMULATION"
591	code=0000
592	# create and persist a unique id for this simulation
593	simuid=$(uuidgen)
594	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid}
595	# Standard fields for the first message
596	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
597	# RabbitMQ message body with specific fields associated message codes treated here
598	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
599	# Fill the rabbitMQ queue (the config.card in use will be sent)
600	IGCM_debug_sendAMQP activate
601	else
602	# RabbitMQ message code "A NEW COMPUTING JOB IS RUNNING PART OF A SIMULATION"
603	code=1000
604	# retrieve this simulation's unique id
605	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
606	simuid=${run_Configuration_simuid}
607	# Using standard fields for message others than the first one. Still subject to change
608	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
609	# RabbitMQ message body with specific fields associated message codes treated here
610	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
611	# Fill the rabbitMQ queue
612	IGCM_debug_sendAMQP
613	fi
614
615	# NOT VERY NICE BUT ... IT WORKS
616	# Be sure that the genericSimulationID will be small from now on
617	# Using standard fields for messages others than the first one. Still subject to change
618	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
619
620	elif [ X${TaskType} = Xpost-processing ]; then
621	# RabbitMQ message code "A NEW POST-PROCESSING JOB IS RUNNING PART OF A SIMULATION"
622	code=2000
623	# retrieve this simulation's unique id
624	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
625	simuid=${run_Configuration_simuid}
626	# Using standard fields for message others than the first one. Still subject to change
627	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
628	# RabbitMQ message body with specific fields associated message codes treated here
629	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
630	# Fill the rabbitMQ queue
631	IGCM_debug_sendAMQP
632	fi
633	# Turn the flag on
634	ActivateBigBro=true
635	# Dont flush by default
636	FlushAMQP=false
637	fi
638	IGCM_debug_PopStack "IGCM_debug_BigBro_Initialize"
639	}
640
641	#D-#==================================================================
642	#D-function IGCM_debug_BigBro_Finalize
643	#D-* Purpose: Finalize rabbitMQ messages exchanges
644	#D-
645	function IGCM_debug_BigBro_Finalize {
646	IGCM_debug_PushStack "IGCM_debug_BigBro_Finalize"
647
648	# Message type standard fields:
649	# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
650
651	# Message type dictionnary and custom fields:
652	# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
653
654	if ( $DEBUG_debug ) ; then
655	if [ X${ActivateBigBro} = Xtrue ] ; then
656	if [ X${TaskType} = Xcomputing ]; then
657	if ( ${simulationIsOver} ) ; then
658	# RabbitMQ message code "SIMULATION ENDS"
659	code=0100
660	FlushAMQP=true
661	else
662	# RabbitMQ message code "COMPUTING JOB ENDS"
663	code=1100
664	fi
665	elif [ X${TaskType} = Xpost-processing ]; then
666	# RabbitMQ message code "POST-PROCESSING JOB ENDS"
667	code=2100
668	FlushAMQP=true
669	fi
670	# RabbitMQ message body
671	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
672	# Fill the rabbitMQ queue
673	IGCM_debug_sendAMQP
674	fi
675	fi
676
677	IGCM_debug_PopStack "IGCM_debug_BigBro_Finalize"
678	}
679
680	#D-#==================================================================
681	#D-function IGCM_debug_Exit
682	#D-* Purpose: Print Call Stack and set ExitFlag to true
683	#D-
684	function IGCM_debug_Exit {
685	IGCM_debug_PushStack "IGCM_debug_Exit"
686	echo "IGCM_debug_Exit : " "${@}"
687	echo
688	echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
689	echo "!! ERROR TRIGGERED !!"
690	echo "!! EXIT FLAG SET !!"
691	echo "!------------------------!"
692	echo
693	IGCM_debug_CallStack
694	ExitFlag=true
695	IGCM_debug_PopStack "IGCM_debug_Exit"
696	}
697
698	#D-#==================================================
699	#D-function IGCM_debug_Verif_Exit
700	#D-* Purpose: exit with number 1 if ExitFlag is true
701	#D-
702	function IGCM_debug_Verif_Exit {
703	if ( ${ExitFlag} ) ; then
704	# Plan to send an email here with IGCM_sys_SendMail
705	if [ X${TaskType} != Xchecking ] ; then
706	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
707	echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
708	echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
709	echo " EXIT THE JOB."
710	echo
711	IGCM_debug_CallStack
712	fi
713
714	if [ X${ActivateBigBro} = Xtrue ] ; then
715	# RabbitMQ message code "EXIT THE JOBS BECAUSE ERROR(S) HAS BEEN TRIGGERED"
716	code=9999
717	# RabbitMQ message body
718	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
719	# To be sure we flush message buffer and clean up behind us before the end of the simulation
720	FlushAMQP=true
721	# Fill the rabbitMQ queue
722	IGCM_debug_sendAMQP
723	fi
724
725	# Mail notification
726	IGCM_sys_SendMail
727
728	# And Good Bye
729	date
730	exit 1
731	fi
732	}
733
734	#D-#==================================================
735	#D-function IGCM_debug_Verif_Exit_Post
736	#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
737	#D-
738	function IGCM_debug_Verif_Exit_Post {
739	if ( ${ExitFlag} ) ; then
740	echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
741	# If SpaceName is PROD then we stop if post_processing fails
742	# Plan to send an email here with IGCM_sys_SendMail
743	if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
744	echo " EXIT THE JOB."
745	echo
746	# Mail notification
747	#IGCM_sys_SendMailPost
748	# And Good Bye
749	date
750	exit 1
751	else
752	echo "Either inside config.card the variable SpaceName is not in PROD"
753	echo "or inside the main Job the variable JobType is not in RUN mode"
754	echo " SO WE DO NOT EXIT THE JOB."
755	echo
756	date
757	fi
758	fi
759	}
760
761	#D-#==================================================================
762	#D-function IGCM_debug_Print
763	#D-* Purpose: Print arguments according to a level of verbosity.
764	#D-
765	function IGCM_debug_Print
766	{
767	typeset level=$1
768	shift
769
770	if [ X"${1}" = X"-e" ]; then
771	typeset cmd_echo="echo -e"
772	shift
773	else
774	typeset cmd_echo="echo"
775	fi
776
777	if [ ${level} -le ${Verbosity} ] ; then
778	typeset i
779	case "${level}" in
780	1) for i in "$@" ; do
781	${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
782	done ;;
783	2) for i in "$@" ; do
784	${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
785	done ;;
786	3) for i in "$@" ; do
787	${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
788	done ;;
789	esac
790	fi
791	}
792
793	#D-#==================================================================
794	#D-function IGCM_debug_PrintVariables
795	#D-* Purpose: Print arguments when match a pattern
796	#D- according to a level of verbosity.
797	function IGCM_debug_PrintVariables
798	{
799	typeset level=$1
800	shift
801
802	list=$( set \| grep ^$1 \| sed -e "s/'//g" )
803
804	if [ "X${list}" != X ] ; then
805	IGCM_debug_Print ${level} ${list}
806	fi
807	}
808
809	#D-#==================================================================
810	#D-function IGCM_debug_PrintInfosActions
811	#D-* Purpose: Print information related to instrumentation
812	function IGCM_debug_PrintInfosActions
813	{
814	typeset actionType=$1
815	typeset entitySize=$2
816	typeset start_ms=$3
817	typeset end_ms=$4
818
819	typeset dest=$5
820	typeset source=$6
821
822	typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
823	typeset dirFrom dirTo
824
825	diff_ms=$(( $end_ms - $start_ms ))
826	# echo "diff_ms=$diff_ms"
827
828	entitySizeKo=$( echo ${entitySize} \| gawk -F"\|" '{print $1}' )
829	# echo "entitySizeKo=$entitySizeKo"
830	entitySizeMo=$( echo ${entitySize} \| gawk -F"\|" '{print $2}' )
831
832	# flux en Ko / ms
833	flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" \| bc )
834	# echo "flux_Ko_ms=$flux_Ko_ms"
835
836	# flux en Ko / s
837	flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
838	# echo "flux_Ko_s=$flux_Ko_s"
839
840	# flux en Mo / s
841	flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" \| bc )
842	# echo "flux_Mo_s=$flux_Mo_s"
843
844	if [ -d $dest ] ; then
845	dirTo=$( readlink -f ${dest} )
846	else
847	dirTo=$( readlink -f $( dirname ${dest} ) )
848	fi
849
850	if [ -d $source ] ; then
851	dirFrom=$( readlink -f ${source} )
852	else
853	dirFrom=$( readlink -f $( dirname ${source} ) )
854	fi
855
856	instrumentationContent=$( echo "\"actionName\":\"${actionType}\",\"size_Mo\":\"${entitySizeMo}\",\"duration_ms\":\"${diff_ms}\",\"throughput_Mo_s\":\"${flux_Mo_s}\",\"dirFrom\":\"${dirFrom}\",\"dirTo\":\"${dirTo}\"" )
857
858	if [ X${ActivateStackFilling} = Xtrue ] ; then
859	echo "{${instrumentationContent}}" >> ${StackFileLocation}/${StackFileName}
860	fi
861
862	# Inform the rabbitMQ queue
863	if [ X${ActivateBigBro} = Xtrue ] ; then
864	# RabbitMQ message body
865	Body=$( echo "{${genericSimulationID},\"msgCode\":\"7000\",\"msgUID\":\"$(uuidgen)\",${instrumentationContent},\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
866	# Fill the rabbitMQ queue
867	IGCM_debug_sendAMQP
868	fi
869	}
870
871	#D-#==================================================================
872	#D-function IGCM_debug_Check
873	#D- * Purpose: Check the present file by comparison with a reference file
874	function IGCM_debug_Check
875	{
876	#---------------------
877	if [ ! -n "${libIGCM}" ] ; then
878	echo "Check libIGCM_debug ..........................................[ FAILED ]"
879	echo "--Error--> libIGCM variable is not defined"
880	exit 2
881	fi
882
883	#---------------------
884	if [ ! -n "${Verbosity}" ] ; then
885	echo "Check libIGCM_debug ..........................................[ FAILED ]"
886	echo "--Error--> Verbosity variable is not defined"
887	exit 3
888	fi
889
890	#---------------------
891	# Need to remove timestamps here
892	diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh \| sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g") > /dev/null 2>&1
893	status=$?
894
895	if [ ${status} -eq 0 ] ; then
896	echo "Check libIGCM_debug ..............................................[ OK ]"
897	else
898	echo "Check libIGCM_debug ..........................................[ FAILED ]"
899	echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
900	echo " has produced the file IGCM_debug_Test.ref.failed"
901	echo " Please analyse differences with the reference file by typing:"
902	echo " diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
903	echo " Report errors to the author: Patrick.Brockmann@cea.fr"
904	diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh \| sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g")
905	exit 4
906	fi
907	#---------------------
908	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: