Context Navigation

source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 1418

Last change on this file since 1418 was 1418, checked in by sdipsl, 6 years ago
correction to the supervisor adress. Old server will be deprecated.
Property licence set to The following licence information concerns ONLY the libIGCM tools ================================================================== Copyright © Centre National de la Recherche Scientifique CNRS Commissariat à l'Énergie Atomique CEA libIGCM : Library for Portable Models Computation of IGCM Group. IGCM Group is the french IPSL Global Climate Model Group. This library is a set of shell scripts and functions whose purpose is the management of the initialization, the launch, the transfer of output files, the post-processing and the monitoring of datas produce by any numerical program on any plateforme. This software is governed by the CeCILL license under French law and abiding by the rules of distribution of free software. You can use, modify and/ or redistribute the software under the terms of the CeCILL license as circulated by CEA, CNRS and INRIA at the following URL "http://www.cecill.info". As a counterpart to the access to the source code and rights to copy, modify and redistribute granted by the license, users are provided only with a limited warranty and the software's author, the holder of the economic rights, and the successive licensors have only limited liability. In this respect, the user's attention is drawn to the risks associated with loading, using, modifying and/or developing or reproducing the software by the user in light of its specific status of free software, that may mean that it is complicated to manipulate, and that also therefore means that it is reserved for developers and experienced professionals having in-depth computer knowledge. Users are therefore encouraged to load and test the software's suitability as regards their requirements in conditions enabling the security of their systems and/or data to be ensured and, more generally, to use and operate it in the same conditions as regards security. The fact that you are presently reading this means that you have had knowledge of the CeCILL license and that you accept its terms. Property svn:keywords set to `Revision Author Date`
File size: 40.8 KB

Rev	Line
[913]	1	#!/bin/ksh
[2]	2
	3	#**************************************************************
	4	# Author: Patrick Brockmann, Martial Mancip
[373]	5	# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
	6	# $Revision:: $ Revision of last commit
	7	# $Author:: $ Author of last commit
	8	# $Date:: $ Date of last commit
[2]	9	# IPSL (2006)
	10	# This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
	11	#
	12	#**************************************************************
	13
	14	#==================================================
	15	# The documentation of this file can be automatically generated
[913]	16	# if you use the prefix #D- for comments to be extracted.
[2]	17	# Extract with command: cat lib* \| grep "^#D-" \| cut -c "4-"
	18	#==================================================
	19
	20	#==================================================
	21	# Add high level verbosity
	22	typeset -i Verbosity=${Verbosity:=3}
	23
	24	#==================================================
	25	# DEBUG_debug
	26	# Add low level verbosity
[1083]	27	DEBUG_debug=${DEBUG_debug:=false}
[2]	28
[1290]	29	#D- postProcessingStopLevel (0,1,2,3)
	30	#D- 3 stop if any post-processing went wrong
	31	#D- 2 dont stop if atlas, monitoring or metrics failed
	32	#D- 1 dont stop if atlas, monitoring, metrics, create_ts or create_se failed
	33	#D- 0 dont stop if atlas, monitoring, metrics, create_ts, create_se, rebuild or pack_* failed
	34	postProcessingStopLevel=${postProcessingStopLevel:=0}
	35
[872]	36	#==================================================
	37	# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
[1244]	38	typeset -r RandomError=false
[872]	39
[2]	40	#==================================================
	41	# NULL_STR
	42	# Default null string
[913]	43	typeset -r NULL_STR="_0_"
[2]	44
	45	#==================================================
	46	# libIGCM_CurrentTag
	47	# Current libIGCM tag, check compatibilty with *.card
[1404]	48	typeset -r libIGCMVersion="2.8.3"
[2]	49
	50	#==================================================
	51	# Exit Flag (internal debug)
	52	# When true, end the master loop AFTER SAVES FILES
	53	ExitFlag=false
	54
	55	#==================================================
[1243]	56	# When we start to run the simulation is not finished
	57	simulationIsOver=false
	58
	59	#==================================================
[1268]	60	# When we start to run we dont flush AMQP messages
	61	FlushAMQP=false
	62
	63	#==================================================
[2]	64	# Declare a stack of functions calls
[59]	65	unset IGCM_debug_Stack
	66	unset IGCM_debug_StackArgs
[913]	67	unset IGCM_debug_StackTiming
[54]	68	IGCM_debug_Stack[0]=${NULL_STR}
	69	IGCM_debug_StackArgs[0]=${NULL_STR}
[913]	70	IGCM_debug_StackTiming[0]=${NULL_STR}
[2]	71	IGCM_debug_LenStack=0
	72
	73	#D-#==================================================================
[913]	74	#D-function IGCM_debug_getDate_ms
	75	#D- * Purpose: Give number of milliseconds since 01-jan-1970
	76	function IGCM_debug_getDate_ms
	77	{
[926]	78	typeset nanosecs ms
[913]	79	# nano secondes since 01-jan-1970
	80	nanosecs=$( date +%s%N )
	81
	82	# truncate the last 6 digits to get milliseconds since 01-jan-1970
	83	ms=${nanosecs:0:${#nanosecs}-6}
	84
	85	echo "$ms"
	86	}
	87
	88	#D-#==================================================================
	89	#D-function IGCM_debug_sizeOfTabContent
	90	#D- * Purpose: Give sumed size of a list of files
[924]	91	#D- * Usage: IGCM_debug_sizeOfTabContent entityList destination
	92	#D- * where entityList is a list of files or directory
	93	#D- * where dest is either a directory or a file name
[913]	94	function IGCM_debug_sizeOfTabContent
	95	{
[924]	96	typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo
	97
	98	eval set +A entityListe \${${1}}
[941]	99	destination=${2}
[924]	100	sumSizeKo=0
	101
	102	# Here we will try to compute size (file or directory size) from local path and not from archive.
[941]	103	for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do
	104	if [ -f ${entityListe[$i]} ] ; then
	105	# One file or a bunch of files has been copied without renaming from a visible filesystem
	106	iEntity=${entityListe[$i]}
	107	elif [ -f ${entityListe[$i]##/*/} ] ; then
	108	# One file or a bunch of files has been copied without renaming from an non visible filesystem
[924]	109	# remove path /home/login/../ from entityListe elements
	110	iEntity=${entityListe[$i]##/*/}
	111	elif [ -f ${destination} ] ; then
[941]	112	# a file has been copied and renamed
[924]	113	iEntity=${destination}
	114	elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then
	115	# a copy in a directory but not in ${PWD}
	116	iEntity=${destination}/${entityListe[$i]##/*/}
	117	elif [ -d ${entityListe[$i]} ] ; then
[941]	118	# a directory has been copied from a non remote place
[924]	119	iEntity=${entityListe[$i]}
	120	elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then
[941]	121	# a directory has been copied from a remote archive and not renamed
[924]	122	iEntity=${destination}/${entityListe[$i]##/*/}
	123	elif [ -d ${destination} ] ; then
[941]	124	# a directory has been copied from a remote archive and renamed
[924]	125	iEntity=${destination}
[917]	126	fi
[1083]	127	sizeKo=$( du --apparent-size -skL ${iEntity} \| gawk '{print $1}' )
[924]	128	sumSizeKo=$(( $sumSizeKo + $sizeKo ))
[913]	129	done
[924]	130	sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" \| bc )
	131	echo "${sumSizeKo}\|${sumSizeMo}"
[913]	132	}
	133
	134	#D-#==================================================================
[983]	135	#D-function IGCM_debug_send_AMQP_msg__MAILTUNNEL
	136	#D- * Purpose: Take over AMQP C client using mail as a message recipient
	137	#D- * One argument : base64 encoded message
[1051]	138	#D- * Attach encoded config.card when starting the simulation
	139
[983]	140	function IGCM_debug_send_AMQP_msg__MAILTUNNEL {
	141
[987]	142	typeset b64_encoded_msg mail_recipient
[1076]	143	typeset buffer send_messages mail_frequency
[987]	144	typeset last_mail_date__file
[1234]	145	typeset secondsBetweenRefAndLastMail secondsSinceLastMail
[987]	146
[983]	147	b64_encoded_msg=$1
	148
[1398]	149	mail_recipient="superviseur@mail.ipsl.upmc.fr"
[983]	150	send_messages=0
	151	mail_frequency=3600 # in seconds
	152	# use to keep track when was last mail sent (maybe to be replaced with global variable)
[1150]	153	last_mail_date__file=${R_BUF}/.stamp.${config_UserChoices_TagName}.${config_UserChoices_JobName}
	154	# use to accumulate messages before sending them
	155	buffer=${R_BUF}/.buffer.${config_UserChoices_TagName}.${config_UserChoices_JobName}
[983]	156
	157	# init
	158	if [ ! -f "${buffer}" ]; then
[1150]	159	touch ${buffer}
[983]	160	fi
	161
	162	if [ ! -f "${last_mail_date__file}" ]; then
[1150]	163	touch ${last_mail_date__file}
[983]	164	else
[1150]	165	# compute last time the file was changed (in seconds)
[1234]	166	secondsBetweenRefAndLastMail=$(stat -c %Y ${last_mail_date__file})
	167	status=$?
	168	#
[1275]	169	# Only execute this block when the stat command succeeded.
	170	# The stat command might fail in some circumstance but we consider it is ok to continue anyway.
	171	if [ ${status} -eq 0 ] ; then
	172	secondsSinceLastMail=$(( $(date +%s) - ${secondsBetweenRefAndLastMail} ))
	173	# send message when exceeding threshold
	174	[ ${secondsSinceLastMail} -gt ${mail_frequency} ] && send_messages=1
[1234]	175	fi
[983]	176	fi
	177
[997]	178	# queue messages in the buffer
	179	echo ${b64_encoded_msg} >> ${buffer}
	180
	181	# send mail
[1051]	182
[1053]	183	if [ X${initBigBro} = Xtrue ] ; then
[1087]	184	#echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") > ${SUBMIT_DIR}/mail.txt
[1051]	185	mailx -s "[TEMPORARY AMQP CHANNEL]" -a ${SUBMIT_DIR}/config.card.base64 ${mail_recipient} < ${buffer} # send buffer
	186	rm -f $buffer ; touch ${buffer} # clear buffer
	187	touch ${last_mail_date__file} # memorize last mail date
[1293]	188	rm -f ${SUBMIT_DIR}/config.card.base64 # remove temporary file
[1051]	189	initBigBro=false
[1150]	190	elif [ ${send_messages} -eq 1 ] ; then
[1087]	191	#echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") >> ${SUBMIT_DIR}/mail.txt
[1051]	192	mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer
[1150]	193	rm -f ${buffer} ; touch ${buffer} # flush the buffer
[1051]	194	touch ${last_mail_date__file} # memorize last mail date
[983]	195	fi
[987]	196
[1189]	197	if ( ${FlushAMQP} ) ; then
[1150]	198	mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer
	199	rm -f ${buffer} # cleaning behind us
	200	rm -f ${last_mail_date__file} # cleaning behind us
	201	fi
	202
[983]	203	# Allways all good for now.
	204	return 0
	205	}
	206
	207	#D-#==================================================================
[1355]	208	#D-function IGCM_debug_sendAMQP_Purge
	209	#D- * Purpose: Take over AMQP C client using mail as a message recipient
	210	#D- * No argument : send a message indicating a simulation has been purged
	211
	212	function IGCM_debug_sendAMQP_Purge {
	213
	214	typeset mail_recipient encodedBody
	215	if [ X${ActivateBigBro} = Xtrue ] ; then
[1418]	216	mail_recipient="superviseur@mail.ipsl.upmc.fr"
[1355]	217	# Clea nup tag on server side
	218	code=8888
	219	# Usual AMQP message to route messages on server side
	220	encodedBody=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" \| base64 -w 0 )
	221	# send mail
	222
	223	IGCM_debug_Print 2 "IGCM_debug_sendAMQP_Purge"
	224	echo ${encodedBody}\|mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient}
	225	fi
	226
	227	# Allways all good for now.
	228	return 0
	229	}
	230
	231	#D-#==================================================================
[1162]	232	#D-function IGCM_debug_sendAMQP_Metrics
	233	#D- * Purpose: Take over AMQP C client using mail as a message recipient
[1202]	234	#D- * Two arguments : - Directory where metrics.json files can be found
	235	#D- * - Metrics Group Name. metrics will be added to this group
[1162]	236	#D- * Attach encoded metrics.json files.
	237
	238	function IGCM_debug_sendAMQP_Metrics {
	239
	240	typeset mail_recipient encodedBody
	241	if [ X${ActivateBigBro} = Xtrue ] ; then
[1418]	242	mail_recipient="superviseur@mail.ipsl.upmc.fr"
[1162]	243	# Metrics tag on server side
	244	code=7100
	245	# Usual AMQP message to route messages on server side
[1202]	246	encodedBody=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"metricsGroupName\":\"${2}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" \| base64 -w 0 )
[1162]	247	# send mail
	248	attachmentsOptions=""
	249	for metricsFile in $( ls $1/*json ) ; do
	250	attachmentsOptions="-a ${metricsFile} ${attachmentsOptions}"
	251	done
[1355]	252	IGCM_debug_Print 2 "IGCM_debug_sendAMQP_Metrics"
[1162]	253	echo ${encodedBody}\|mailx -s "[TEMPORARY AMQP CHANNEL]" ${attachmentsOptions} ${mail_recipient}
	254	fi
	255
	256	# Allways all good for now.
	257	return 0
	258	}
	259
	260	#D-#==================================================================
[1285]	261	#D-function IGCM_debug_sendAMQP_projectAccounting
	262	#D- * Purpose: Take over AMQP C client using mail as a message recipient
	263	#D- * One argument : - File name where project accounting details are stored
	264	#D- * Attach encoded accounting file.
	265
	266	function IGCM_debug_sendAMQP_projectAccounting {
	267
	268	typeset mail_recipient encodedBody
	269	if [ X${ActivateBigBro} = Xtrue ] ; then
[1418]	270	mail_recipient="superviseur@mail.ipsl.upmc.fr"
[1285]	271	# Metrics tag on server side
	272	code=7010
	273	# Usual AMQP message to route messages on server side
	274	encodedBody=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"accountingProject\":\"${PROJECT}\",\"centre\":\"${CENTER}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" \| base64 -w 0 )
	275	# send mail
	276	attachmentsOptions="-a ${1}"
	277	IGCM_debug_Print 2 "IGCM_debug_sendAMQP_projectAccounting"
	278	echo ${encodedBody}\|mailx -s "[TEMPORARY AMQP CHANNEL]" ${attachmentsOptions} ${mail_recipient}
[1290]	279	# clean behind
	280	rm -f $1
[1285]	281	fi
	282
	283	# Allways all good for now.
	284	return 0
	285	}
	286
	287	#D-#==================================================================
[913]	288	#D-function IGCM_debug_SendAMQP
	289	#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
[1053]	290	function IGCM_debug_sendAMQP {
	291
[913]	292	typeset decal first additionnalOption encodedBody
	293
	294	# Encode message Body
	295	encodedBody=$( echo "${Body}" \| base64 -w 0 )
	296
	297	# Send config.card ?
	298	if [ X${1} = Xactivate ] ; then
	299	# Encode config.card
	300	cat ${SUBMIT_DIR}/config.card \| base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
	301	# Prepare additionnal option
	302	additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
[1076]	303	#
[1051]	304	initBigBro=true
[913]	305	else
	306	additionnalOption=
[1051]	307	#
	308	initBigBro=false
[913]	309	fi
	310
	311	# Only cosmetics : stack file
[1115]	312	if [ X${ActivateStackFilling} = Xtrue ] ; then
	313	decal=0
	314	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
	315	printf ' ' >> ${StackFileLocation}/${StackFileName}
	316	(( decal = decal + 1 ))
	317	done
	318	# Log to stack file using human readable format
	319	echo "${Body}" >> ${StackFileLocation}/${StackFileName}
	320	fi
[913]	321
[983]	322	# Log separately encoded AMQP message command for reuse in a mock up
[1120]	323	#echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> ${RUN_DIR_PATH}/send.AMQP.${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.${CumulPeriod}.history.txt
[913]	324
	325	# Send the message
[983]	326	if [ X${BigBrotherChannel} = XMAIL ] ; then
	327	IGCM_debug_send_AMQP_msg__MAILTUNNEL "${encodedBody}"
	328	status=$?
	329	else
	330	sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
	331	status=$?
	332	fi
	333
[913]	334	if [ ${status} -gt 0 ] ; then
	335	IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
[1051]	336	echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
[1090]	337	exit 1
[913]	338	fi
	339	}
	340
	341	#D-#==================================================================
[2]	342	#D-function IGCM_debug_CallStack
[913]	343	#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
[2]	344	#D-
	345	function IGCM_debug_CallStack {
[544]	346	if ( $DEBUG_debug ) ; then
[913]	347	# Cosmetics
[544]	348	typeset i decal
[823]	349	i=0
[544]	350	until [ $i -eq ${IGCM_debug_LenStack} ]; do
	351	decal=0
	352	until [ $decal -eq ${i} ]; do
[869]	353	printf -- ' '
[823]	354	(( decal = decal + 1 ))
[544]	355	done
[869]	356	echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
[823]	357	((i = i + 1))
[544]	358	done
	359	fi
[2]	360	}
	361
	362	#D-#==================================================================
	363	#D-function IGCM_debug_PushStack
	364	#D-* Purpose: Push a function name in the stack
	365	#D-
	366	function IGCM_debug_PushStack {
[544]	367	if ( $DEBUG_debug ) ; then
[913]	368	typeset decal inputs startTime_ms
	369
	370	# Only cosmetics : stack file
[1115]	371	if [ X${ActivateStackFilling} = Xtrue ] ; then
	372	echo >> ${StackFileLocation}/${StackFileName}
	373	decal=0
	374	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
	375	printf ' ' >> ${StackFileLocation}/${StackFileName}
	376	(( decal = decal + 1 ))
	377	done
[2]	378
[1115]	379	# Fill the stack file
	380	echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
	381	fi
[926]	382
[913]	383	# Save input list in an indexed array
[823]	384	INPUTS=( $@ )
[913]	385
	386	# Get timing information
	387	startTime_ms=$( IGCM_debug_getDate_ms )
	388
[544]	389	# We add function call name on beginning of the stack
	390	set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
[2]	391
[913]	392	# Save timing in milliseconds in an indexed array
	393	set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}
	394
[544]	395	# We include the "null" Args in the beginning of the StackArgs
[913]	396	set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}
	397
[544]	398	# Then, we shift StackArgs tabular
[1065]	399	# Replacing blank separated list by comma separated list of quoted elements (except the first and last element)
[913]	400	if [ $# -gt 1 ]; then
[1065]	401	IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} \| sed -e "s/\ /\",\"/g" )
[544]	402	fi
[855]	403
	404	# Increment LenStack
[544]	405	(( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
[2]	406
[869]	407	#IGCM_debug_CallStack
[544]	408	fi
[2]	409	}
	410
	411	#D-#==================================================================
	412	#D-function IGCM_debug_PopStack
	413	#D-* Purpose: Pop a function name in the stack
	414	#D-
	415	function IGCM_debug_PopStack {
[544]	416	if ( $DEBUG_debug ) ; then
[926]	417	typeset i decal command arguments startTime_ms endTime_ms
[941]	418	typeset instrumentation dest prefix
[926]	419	# they are not typeset because they are send "by adress" to son functions
	420	# we unset them to avoid "memory effect"
	421	unset fileList source
[913]	422
	423	# INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
	424	# PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
	425	# THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
[1277]	426	# ONLY WHEN TaskType is "computing".
	427	if [ X${ActivateBigBro} = Xtrue ] ; then
	428	if [ X${TaskType} = Xcomputing ]; then
	429	if ( ${RandomError} ) ; then
	430	if [ $((RANDOM%10000)) -le 10 ] ; then
	431	IGCM_debug_Print 1 "Random error has been triggered"
	432	if [ X${ActivateStackFilling} = Xtrue ] ; then
	433	echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName}
	434	fi
	435	ExitFlag=true
	436	fi
[1115]	437	fi
[913]	438	fi
	439	fi
	440
[544]	441	if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
[913]	442	# Everything is cool
	443
	444	# Get timing information
	445	endTime_ms=$( IGCM_debug_getDate_ms )
	446
	447	# Save Stack information before poping the stack
	448	command=${IGCM_debug_Stack[0]}
	449
[1084]	450	# Go from comma separated list of quoted elements (except the first and the last element)
	451	# to unquoted space separated elements in an array
[1083]	452	set -A arguments -- $( echo ${IGCM_debug_StackArgs[0]} \| sed -e "s/\",\"/\ /g" )
[913]	453
	454	# Save Stack information before poping the stack
	455	startTime_ms=${IGCM_debug_StackTiming[0]}
	456
	457	# Pop the stack
[823]	458	(( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
	459	set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
	460	set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
[913]	461	set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
[544]	462	else
	463	echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
	464	IGCM_debug_Exit $@
	465	fi
[913]	466
[914]	467	# Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
[913]	468	# We are interested in:
	469	# 0. Which command performs the work
	470	# 1. Size of entity we are working with
	471	# 2. Where are we reading
	472	# 3. Where are we writing
	473	# 4. How long it took
	474
[915]	475	instrumentation=false
	476
[913]	477	case ${command} in
[925]	478	# Classical copy (only files are given to IGCM_sys_Cp as options)
	479	IGCM_sys_Cp)
	480	instrumentation=true
	481	# All but the latest
	482	fileList=${arguments[]:0:${#arguments[]}-1}
	483	# just need the first file to get the directory
	484	source=${arguments[0]}
	485	# Nothing but the latest
	486	dest=${arguments[${#arguments[*]}-1]}
	487	# Size of file whose name are stored in a list
	488	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
	489	;;
	490
[913]	491	# Copy from archive machine or from buffer
	492	IGCM_sys_Get\|IGCM_sys_GetBuffer)
[915]	493	instrumentation=true
[913]	494	if [ ${#arguments[*]} -eq 2 ] ; then
	495	source=${arguments[0]}
	496	dest=${arguments[1]}
	497	# Size of file whose name are stored in a variable
[917]	498	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
[913]	499	elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
[936]	500	# IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/
[913]	501	# Keep the array name hosting the all list
[936]	502	eval set +A fileList \${${arguments[1]}}
[913]	503	# just need the first file to get the directory
[936]	504	source=${fileList[0]}
[931]	505	dest=${arguments[2]}
[934]	506	# Size of file whose name are stored in a list
[936]	507	entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
[913]	508	elif [ [ ${#arguments[*]} -ge 3 ] ; then
	509	# All but the latest
[916]	510	fileList=${arguments[]:0:${#arguments[]}-1}
[913]	511	# just need the first file to get the directory
	512	source=${arguments[0]}
	513	# Nothing but the latest
	514	dest=${arguments[${#arguments[*]}-1]}
	515	# Size of file whose name are stored in a list
[917]	516	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
[913]	517	fi
	518	;;
	519
[925]	520	# Copy from compute node or copy to archive/buffer
	521	IGCM_sys_Get_Master\|IGCM_sys_Get_Dir\|IGCM_sys_Put_Out\|IGCM_sys_PutBuffer_Out)
[924]	522	instrumentation=true
[916]	523	source=${arguments[0]}
[924]	524	dest=${arguments[1]}
	525	# Size of file whose name are stored in a variable
	526	entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
[913]	527	;;
	528
	529	# Rebuild command
	530	IGCM_sys_rebuild\|IGCM_sys_rebuild_station)
[915]	531	instrumentation=true
[913]	532	# All but the first
	533	fileList=${arguments[]:1:${#arguments[]}-1}
	534	# just need a file to get the directory
	535	source=${arguments[1]}
	536	# Nothing but the first
	537	dest=${arguments[0]}
	538	# Size of file whose name are stored in a list
[917]	539	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
[913]	540	;;
[941]	541
[926]	542	# NCO commands
	543	IGCM_sys_ncrcat\|IGCM_sys_ncecat\|IGCM_sys_ncra\|IGCM_sys_ncks\|IGCM_sys_cdo)
	544	# Example of what we want to catch : only filenames in those command lines
	545	# IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE}
	546	# IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
	547	# IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output}
	548	# IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
	549	instrumentation=true
[941]	550	keepGoing=true
	551	prefix=.
[926]	552	i=0
	553	while ( ${keepGoing} ) ; do
[941]	554	# the last one is not interesting
	555	if [ ${i} -eq ${#arguments[*]}-1 ] ; then
	556	keepGoing=false
	557	# look after "-p" option. Path prefix is the following arguments
	558	elif [ ${arguments[${i}]} = "-p" ] ; then
[926]	559	((i = i + 1))
[941]	560	prefix=${arguments[${i}]}
	561	((i = i + 1))
	562	elif [ ${i} -eq ${#arguments[*]}-1 ] ; then
[926]	563	keepGoing=false
[941]	564	# looking for files
	565	elif [ -f ${prefix}/${arguments[${i}]} ] ; then
	566	fileList="${fileList} ${prefix}/${arguments[${i}]}"
	567	((i = i + 1))
	568	# other options are not interesting
[926]	569	else
	570	((i = i + 1))
	571	fi
	572	done
[941]	573
[926]	574	# i value is at least 1
	575	# just need one file to get the directory
[941]	576	source=$( echo ${fileList} \| gawk '{print $1}' )
[926]	577	# Nothing but the latest
	578	dest=${arguments[${#arguments[*]}-1]}
	579	# Size of file whose name are stored in a list
	580	entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
	581	;;
[913]	582	esac
	583
	584	# Print information related to instrumentation
[915]	585	( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}
[913]	586
	587	# Only cosmetics : stack file
[1115]	588	if [ X${ActivateStackFilling} = Xtrue ] ; then
	589	decal=0
	590	while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
	591	printf ' ' >> ${StackFileLocation}/${StackFileName}
	592	(( decal = decal + 1 ))
	593	done
	594	fi
[2]	595
[855]	596	if ( ${ExitFlag} ) ; then
	597	# Inform the stack file
[1115]	598	if [ X${ActivateStackFilling} = Xtrue ] ; then
	599	echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName}
	600	fi
[874]	601
[1216]	602	# Unplugged message 4900 handling for now. To ease downstream treatment.
[1244]	603	if [ X${ActivateBigBro} = Xtrue ] ; then
	604	if [ X${TaskType} = Xcomputing ]; then
	605	# RabbitMQ message code "COMPUTING JOBs COMMAND FAILURE"
	606	code=1900
	607	elif [ X${TaskType} = Xpost-processing ]; then
	608	# RabbitMQ message code "POST-PROCESSING JOBs COMMAND FAILURE"
	609	code=2900
	610	elif [ X${TaskType} = Xchecking ]; then
	611	# RabbitMQ message code "POST-PROCESSING FROM CHECKER JOBs COMMAND FAILURE"
	612	code=3900
	613	fi
	614	# RabbitMQ message body
	615	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
	616
	617	# Fill the rabbitMQ queue
	618	IGCM_debug_sendAMQP
	619	fi
[855]	620	else
	621	# Inform the stack file
[1115]	622	if [ X${ActivateStackFilling} = Xtrue ] ; then
	623	echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
	624	fi
[855]	625	fi
	626
[913]	627	# Reset array if necessary
[544]	628	if [ ${IGCM_debug_LenStack} = 0 ]; then
	629	#echo
	630	#IGCM_debug_Print 3 "Clean stack array"
	631	#echo
	632	unset IGCM_debug_Stack
	633	unset IGCM_debug_StackArgs
[913]	634	unset IGCM_debug_StackTiming
[544]	635	IGCM_debug_Stack[0]=${NULL_STR}
	636	IGCM_debug_StackArgs[0]=${NULL_STR}
[913]	637	IGCM_debug_StackTiming[0]=${NULL_STR}
[2]	638	fi
[544]	639	fi
[869]	640	#IGCM_debug_CallStack
[2]	641	}
	642
	643	#D-#==================================================================
[1189]	644	#D-function IGCM_debug_BigBro_Initialize
[855]	645	#D-* Purpose: switch rabbitMQ on
	646	#D-
[1189]	647	function IGCM_debug_BigBro_Initialize {
	648	IGCM_debug_PushStack "IGCM_debug_BigBro_Initialize"
[855]	649
[1347]	650	typeset postProcessingIDLength postProcessingName postProcessingDate postProcessingDimn postProcessingComp postProcessingFile accountingFile
[1229]	651
[1051]	652	# Message type standard fields:
	653	# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
	654
	655	# Message type dictionnary and custom fields:
	656	# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
	657
[868]	658	if [ X${BigBrother} = Xtrue ] ; then
[1051]	659	# create a unique ID for this specific job
	660	jobuid=$(uuidgen)
[913]	661
[1244]	662	# get the assigned id by the scheduler for that job
	663	IGCM_sys_getJobSchedulerID jobSchedulerID
	664
[1189]	665	if [ X${TaskType} = Xcomputing ]; then
	666	if ( ${FirstInitialize} ) ; then
	667	# RabbitMQ message code "BEGIN A SIMULATION"
	668	code=0000
	669	# create and persist a unique id for this simulation
	670	simuid=$(uuidgen)
	671	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid}
	672	# Standard fields for the first message
[1244]	673	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"msgProducerVersion\":\"${libIGCMVersion}\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
[1189]	674	# RabbitMQ message body with specific fields associated message codes treated here
[1353]	675	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"jobSchedulerID\":\"${jobSchedulerID}\",\"jobSubmissionPath\":\"${SUBMIT_DIR}\",\"archivePath\":\"${R_SAVE}\",\"storagePath\":\"${R_BUFR}\",\"storageSmallPath\":\"${R_FIGR}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[1189]	676	# Fill the rabbitMQ queue (the config.card in use will be sent)
	677	IGCM_debug_sendAMQP activate
	678	else
	679	# RabbitMQ message code "A NEW COMPUTING JOB IS RUNNING PART OF A SIMULATION"
	680	code=1000
	681	# retrieve this simulation's unique id
	682	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
	683	simuid=${run_Configuration_simuid}
	684	# Using standard fields for message others than the first one. Still subject to change
[1244]	685	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"msgProducerVersion\":\"${libIGCMVersion}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
[1189]	686	# RabbitMQ message body with specific fields associated message codes treated here
[1353]	687	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"jobSchedulerID\":\"${jobSchedulerID}\",\"jobSubmissionPath\":\"${SUBMIT_DIR}\",\"archivePath\":\"${R_SAVE}\",\"storagePath\":\"${R_BUFR}\",\"storageSmallPath\":\"${R_FIGR}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[1189]	688	# Fill the rabbitMQ queue
	689	IGCM_debug_sendAMQP
	690	fi
	691
	692	# NOT VERY NICE BUT ... IT WORKS
	693	# Be sure that the genericSimulationID will be small from now on
	694	# Using standard fields for messages others than the first one. Still subject to change
[1244]	695	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"msgProducerVersion\":\"${libIGCMVersion}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
[1189]	696
	697	elif [ X${TaskType} = Xpost-processing ]; then
	698	# RabbitMQ message code "A NEW POST-PROCESSING JOB IS RUNNING PART OF A SIMULATION"
	699	code=2000
[1076]	700	# retrieve this simulation's unique id
[1051]	701	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
[1086]	702	simuid=${run_Configuration_simuid}
[1076]	703	# Using standard fields for message others than the first one. Still subject to change
[1244]	704	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"msgProducerVersion\":\"${libIGCMVersion}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
[1229]	705
	706	# Specify the post-processing task we are dealing with
[1244]	707	postProcessingIDLength=$( echo "${Script_Post_Output}" \| tr -d -c "\." \| wc -c )
[1229]	708	postProcessingName=$( echo "${Script_Post_Output}" \| gawk -F. '{print $1}' )
	709	postProcessingDate=$( echo "${Script_Post_Output}" \| gawk -F. '{print $2}' )
[1231]	710	postProcessingDimn="null"
	711	postProcessingComp="null"
	712	postProcessingFile="null"
[1254]	713	if [ ${postProcessingIDLength} -eq 2 ] ; then
[1231]	714	postProcessingDimn=$( echo "${Script_Post_Output}" \| gawk -F. '{print $3}' )
[1254]	715	elif [ ${postProcessingIDLength} -eq 4 ] ; then
[1229]	716	postProcessingComp=$( echo "${Script_Post_Output}" \| gawk -F. '{print $4}' )
	717	postProcessingFile=$( echo "${Script_Post_Output}" \| gawk -F. '{print $5}' )
	718	fi
	719
[1087]	720	# RabbitMQ message body with specific fields associated message codes treated here
[1244]	721	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"jobSchedulerID\":\"${jobSchedulerID}\",\"jobSubmissionPath\":\"${SUBMIT_DIR}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\",\"postProcessingName\":\"${postProcessingName}\",\"postProcessingDate\":\"${postProcessingDate}\",\"postProcessingDimn\":\"${postProcessingDimn}\",\"postProcessingComp\":\"${postProcessingComp}\",\"postProcessingFile\":\"${postProcessingFile}\"}" )
[1087]	722	# Fill the rabbitMQ queue
	723	IGCM_debug_sendAMQP
[1355]	724
	725	elif [ X${TaskType} = Xchecking ]; then
	726	# retrieve this simulation's unique id
	727	IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
	728	simuid=${run_Configuration_simuid}
	729	# Using standard fields for message others than the first one. Still subject to change
	730	genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"msgProducerVersion\":\"${libIGCMVersion}\",\"simuid\":\"${simuid}\",\"jobuid\":\"N\/A\"" )
[855]	731	fi
[1285]	732
[1295]	733	# Turn the flag on
	734	ActivateBigBro=true
	735
[1354]	736	if [ X${TaskType} = Xcomputing ]; then
	737	# Save project accounting details in a file
	738	accountingFile=cpt_${CENTER}_${PROJECT}_$( date +"%Y%m%d_%H%M" ).dat
	739	IGCM_sys_projectAccounting ${accountingFile}
[1285]	740
[1354]	741	# And send it
	742	IGCM_debug_sendAMQP_projectAccounting ${accountingFile}
	743	fi
[855]	744	fi
[1189]	745	IGCM_debug_PopStack "IGCM_debug_BigBro_Initialize"
[855]	746	}
	747
	748	#D-#==================================================================
[1332]	749	#D-function IGCM_debug_BigBro_Update
	750	#D-* Purpose: Update rabbitMQ messages exchanges during the run
	751	#D-
	752	function IGCM_debug_BigBro_Update {
	753	IGCM_debug_PushStack "IGCM_debug_BigBro_Update"
	754
	755	# Message type standard fields:
	756	# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
	757
	758	# Message type dictionnary and custom fields:
	759	# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
	760
	761	if [ X${ActivateBigBro} = Xtrue ] ; then
[1354]	762	if [ X${TaskType} = Xcomputing ]; then
	763	# Send ${PeriodDateBegin}, ${PeriodDateEnd} and ${CumulPeriod}
	764	# RabbitMQ message body
	765	Body=$( echo "{${genericSimulationID},\"msgCode\":\"1001\",\"PeriodDateBegin\":\"${PeriodDateBegin}\",\"PeriodDateEnd\":\"${PeriodDateEnd}\",\"CumulPeriod\":\"${CumulPeriod}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
	766	# Fill the rabbitMQ queue
	767	IGCM_debug_sendAMQP
	768	fi
[1332]	769	fi
	770	IGCM_debug_PopStack "IGCM_debug_BigBro_Update"
	771	}
	772
	773	#D-#==================================================================
[1189]	774	#D-function IGCM_debug_BigBro_Finalize
	775	#D-* Purpose: Finalize rabbitMQ messages exchanges
	776	#D-
	777	function IGCM_debug_BigBro_Finalize {
	778	IGCM_debug_PushStack "IGCM_debug_BigBro_Finalize"
	779
	780	# Message type standard fields:
	781	# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
	782
	783	# Message type dictionnary and custom fields:
	784	# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
	785
	786	if ( $DEBUG_debug ) ; then
	787	if [ X${ActivateBigBro} = Xtrue ] ; then
	788	if [ X${TaskType} = Xcomputing ]; then
	789	if ( ${simulationIsOver} ) ; then
	790	# RabbitMQ message code "SIMULATION ENDS"
	791	code=0100
	792	FlushAMQP=true
[1207]	793	elif ( ${ExitFlag} ) ; then
	794	# RabbitMQ message code "EXIT THE JOBS BECAUSE ERROR(S) HAS BEEN TRIGGERED"
[1244]	795	code=1999
[1207]	796	FlushAMQP=true
[1189]	797	else
	798	# RabbitMQ message code "COMPUTING JOB ENDS"
	799	code=1100
	800	fi
	801	elif [ X${TaskType} = Xpost-processing ]; then
[1207]	802	if ( ${ExitFlag} ) ; then
	803	# RabbitMQ message code "POST-PROCESSING JOB FAILS"
[1244]	804	code=2999
[1207]	805	FlushAMQP=true
[1244]	806	else
[1207]	807	# RabbitMQ message code "POST-PROCESSING JOB ENDS"
	808	code=2100
	809	FlushAMQP=true
	810	fi
[1244]	811	elif [ X${TaskType} = Xchecking ]; then
	812	if ( ${ExitFlag} ) ; then
	813	# RabbitMQ message code "POST-PROCESSING JOB FAILS"
	814	code=3999
	815	FlushAMQP=true
	816	else
	817	# RabbitMQ message code "POST-PROCESSING JOB ENDS"
	818	code=3100
	819	FlushAMQP=true
	820	fi
[1189]	821	fi
	822	# RabbitMQ message body
	823	Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
	824	# Fill the rabbitMQ queue
	825	IGCM_debug_sendAMQP
	826	fi
	827	fi
	828
	829	IGCM_debug_PopStack "IGCM_debug_BigBro_Finalize"
	830	}
	831
	832	#D-#==================================================================
[2]	833	#D-function IGCM_debug_Exit
	834	#D-* Purpose: Print Call Stack and set ExitFlag to true
	835	#D-
	836	function IGCM_debug_Exit {
[544]	837	IGCM_debug_PushStack "IGCM_debug_Exit"
	838	echo "IGCM_debug_Exit : " "${@}"
[913]	839	echo
[894]	840	echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
[913]	841	echo "!! ERROR TRIGGERED !!"
	842	echo "!! EXIT FLAG SET !!"
	843	echo "!------------------------!"
	844	echo
[894]	845	IGCM_debug_CallStack
[544]	846	ExitFlag=true
	847	IGCM_debug_PopStack "IGCM_debug_Exit"
[2]	848	}
	849
	850	#D-#==================================================
	851	#D-function IGCM_debug_Verif_Exit
	852	#D-* Purpose: exit with number 1 if ExitFlag is true
	853	#D-
	854	function IGCM_debug_Verif_Exit {
[544]	855	if ( ${ExitFlag} ) ; then
[1206]	856	echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
[1207]	857	echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
[1206]	858	# Only computing TaskType stops the job for now.
	859	if [ X${TaskType} = Xcomputing ] ; then
[775]	860	IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
	861	echo " EXIT THE JOB."
	862	echo
[869]	863	IGCM_debug_CallStack
[874]	864
[1206]	865	# Mail notification
	866	IGCM_sys_SendMail
[913]	867
[1207]	868	# Inform the rabbitMQ queue
	869	IGCM_debug_BigBro_Finalize
	870
[544]	871	# And Good Bye
	872	date
	873	exit 1
[1206]	874
	875	elif [ X${TaskType} = Xpost-processing ] ; then
[1302]	876	StopAll=false
[1282]	877	# Test if we need to stop the computing job
	878	case ${postProcessingName} in
	879	atlas\|monitoring\|metrics*)
	880	[ ${postProcessingStopLevel} -gt 2 ] && StopAll=true ;;
	881	create_*)
	882	[ ${postProcessingStopLevel} -gt 1 ] && StopAll=true ;;
	883	rebuild\|pack_)
	884	[ ${postProcessingStopLevel} -gt 0 ] && StopAll=true ;;
	885	esac
	886	# Notify the computing job that something wrong happened and stop it.
[1302]	887	( ${StopAll} ) && IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal.${Script_Post_Output}"
[1282]	888
	889	# If SpaceName is PROD we stop when post_processing failed
[1206]	890	if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
[1220]	891	echo " EXIT THE POST-PROCESSING JOB."
[1206]	892	echo
[1207]	893	IGCM_debug_CallStack
	894
	895	# Inform the rabbitMQ queue
	896	IGCM_debug_BigBro_Finalize
	897
[1206]	898	# And Good Bye
	899	date
	900	exit 1
	901	else
[1207]	902	echo "In config.card the variable SpaceName is not in PROD"
[1206]	903	echo " SO WE DO NOT EXIT THE JOB."
	904	echo
	905	date
	906	fi
	907	elif [ X${TaskType} = Xchecking ] ; then
[1207]	908	echo "Nothing will happen for now"
[2]	909	fi
[544]	910	fi
[2]	911	}
	912
	913	#D-#==================================================================
	914	#D-function IGCM_debug_Print
	915	#D-* Purpose: Print arguments according to a level of verbosity.
	916	#D-
	917	function IGCM_debug_Print
	918	{
[544]	919	typeset level=$1
	920	shift
	921
	922	if [ X"${1}" = X"-e" ]; then
	923	typeset cmd_echo="echo -e"
[2]	924	shift
[544]	925	else
	926	typeset cmd_echo="echo"
	927	fi
[2]	928
[544]	929	if [ ${level} -le ${Verbosity} ] ; then
	930	typeset i
	931	case "${level}" in
	932	1) for i in "$@" ; do
[734]	933	${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
[913]	934	done ;;
[544]	935	2) for i in "$@" ; do
[734]	936	${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
[913]	937	done ;;
[544]	938	3) for i in "$@" ; do
[734]	939	${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
[913]	940	done ;;
[544]	941	esac
	942	fi
[2]	943	}
	944
	945	#D-#==================================================================
	946	#D-function IGCM_debug_PrintVariables
	947	#D-* Purpose: Print arguments when match a pattern
	948	#D- according to a level of verbosity.
	949	function IGCM_debug_PrintVariables
	950	{
[544]	951	typeset level=$1
	952	shift
[2]	953
[830]	954	list=$( set \| grep ^$1 \| sed -e "s/'//g" )
[54]	955
[544]	956	if [ "X${list}" != X ] ; then
	957	IGCM_debug_Print ${level} ${list}
	958	fi
[2]	959	}
	960
	961	#D-#==================================================================
[914]	962	#D-function IGCM_debug_PrintInfosActions
[913]	963	#D-* Purpose: Print information related to instrumentation
	964	function IGCM_debug_PrintInfosActions
	965	{
	966	typeset actionType=$1
	967	typeset entitySize=$2
	968	typeset start_ms=$3
	969	typeset end_ms=$4
	970
	971	typeset dest=$5
	972	typeset source=$6
	973
	974	typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
[1090]	975	typeset dirFrom dirTo
[913]	976
	977	diff_ms=$(( $end_ms - $start_ms ))
	978	# echo "diff_ms=$diff_ms"
	979
	980	entitySizeKo=$( echo ${entitySize} \| gawk -F"\|" '{print $1}' )
	981	# echo "entitySizeKo=$entitySizeKo"
	982	entitySizeMo=$( echo ${entitySize} \| gawk -F"\|" '{print $2}' )
	983
	984	# flux en Ko / ms
	985	flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" \| bc )
	986	# echo "flux_Ko_ms=$flux_Ko_ms"
	987
	988	# flux en Ko / s
	989	flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
	990	# echo "flux_Ko_s=$flux_Ko_s"
	991
	992	# flux en Mo / s
	993	flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" \| bc )
	994	# echo "flux_Mo_s=$flux_Mo_s"
	995
	996	if [ -d $dest ] ; then
[1090]	997	dirTo=$( readlink -f ${dest} )
[913]	998	else
[1090]	999	dirTo=$( readlink -f $( dirname ${dest} ) )
[913]	1000	fi
	1001
	1002	if [ -d $source ] ; then
[1090]	1003	dirFrom=$( readlink -f ${source} )
[913]	1004	else
[1090]	1005	dirFrom=$( readlink -f $( dirname ${source} ) )
[913]	1006	fi
	1007
[1094]	1008	instrumentationContent=$( echo "\"actionName\":\"${actionType}\",\"size_Mo\":\"${entitySizeMo}\",\"duration_ms\":\"${diff_ms}\",\"throughput_Mo_s\":\"${flux_Mo_s}\",\"dirFrom\":\"${dirFrom}\",\"dirTo\":\"${dirTo}\"" )
[1050]	1009
[1115]	1010	if [ X${ActivateStackFilling} = Xtrue ] ; then
	1011	echo "{${instrumentationContent}}" >> ${StackFileLocation}/${StackFileName}
	1012	fi
[1050]	1013
	1014	# Inform the rabbitMQ queue
	1015	if [ X${ActivateBigBro} = Xtrue ] ; then
	1016	# RabbitMQ message body
[1065]	1017	Body=$( echo "{${genericSimulationID},\"msgCode\":\"7000\",\"msgUID\":\"$(uuidgen)\",${instrumentationContent},\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[1050]	1018	# Fill the rabbitMQ queue
	1019	IGCM_debug_sendAMQP
	1020	fi
[913]	1021	}
	1022
	1023	#D-#==================================================================
[2]	1024	#D-function IGCM_debug_Check
	1025	#D- * Purpose: Check the present file by comparison with a reference file
	1026	function IGCM_debug_Check
	1027	{
[544]	1028	#---------------------
	1029	if [ ! -n "${libIGCM}" ] ; then
	1030	echo "Check libIGCM_debug ..........................................[ FAILED ]"
	1031	echo "--Error--> libIGCM variable is not defined"
	1032	exit 2
	1033	fi
[2]	1034
[544]	1035	#---------------------
	1036	if [ ! -n "${Verbosity}" ] ; then
	1037	echo "Check libIGCM_debug ..........................................[ FAILED ]"
	1038	echo "--Error--> Verbosity variable is not defined"
	1039	exit 3
	1040	fi
[2]	1041
[544]	1042	#---------------------
[1118]	1043	# Need to remove timestamps here
	1044	diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh \| sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g") > /dev/null 2>&1
[1106]	1045	status=$?
[2]	1046
[1118]	1047	if [ ${status} -eq 0 ] ; then
[544]	1048	echo "Check libIGCM_debug ..............................................[ OK ]"
	1049	else
	1050	echo "Check libIGCM_debug ..........................................[ FAILED ]"
	1051	echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
	1052	echo " has produced the file IGCM_debug_Test.ref.failed"
	1053	echo " Please analyse differences with the reference file by typing:"
	1054	echo " diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
	1055	echo " Report errors to the author: Patrick.Brockmann@cea.fr"
[1118]	1056	diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh \| sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g")
[544]	1057	exit 4
	1058	fi
	1059	#---------------------
[2]	1060	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: