#!/bin/ksh #************************************************************** # Author: Patrick Brockmann, Martial Mancip # Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr # $Revision:: $ Revision of last commit # $Author:: $ Author of last commit # $Date:: $ Date of last commit # IPSL (2006) # This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC # #************************************************************** #================================================== # The documentation of this file can be automatically generated # if you use the prefix #D- for comments to be extracted. # Extract with command: cat lib* | grep "^#D-" | cut -c "4-" #================================================== #================================================== # Add high level verbosity typeset -i Verbosity=${Verbosity:=3} #================================================== # DEBUG_debug # Add low level verbosity DEBUG_debug=${DEBUG_debug:=false} #================================================== # GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} ) typeset RandomError=false #================================================== # NULL_STR # Default null string typeset -r NULL_STR="_0_" #================================================== # libIGCM_CurrentTag # Current libIGCM tag, check compatibilty with *.card typeset -r libIGCM_CurrentTag="1.0" #================================================== # Exit Flag (internal debug) # When true, end the master loop AFTER SAVES FILES ExitFlag=false #================================================== # Declare a stack of functions calls unset IGCM_debug_Stack unset IGCM_debug_StackArgs unset IGCM_debug_StackTiming IGCM_debug_Stack[0]=${NULL_STR} IGCM_debug_StackArgs[0]=${NULL_STR} IGCM_debug_StackTiming[0]=${NULL_STR} IGCM_debug_LenStack=0 #D-#================================================================== #D-function IGCM_debug_getDate_ms #D- * Purpose: Give number of milliseconds since 01-jan-1970 function IGCM_debug_getDate_ms { typeset nanosecs ms # nano secondes since 01-jan-1970 nanosecs=$( date +%s%N ) # truncate the last 6 digits to get milliseconds since 01-jan-1970 ms=${nanosecs:0:${#nanosecs}-6} echo "$ms" } #D-#================================================================== #D-function IGCM_debug_sizeOfTabContent #D- * Purpose: Give sumed size of a list of files #D- * Usage: IGCM_debug_sizeOfTabContent entityList destination #D- * where entityList is a list of files or directory #D- * where dest is either a directory or a file name function IGCM_debug_sizeOfTabContent { typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo eval set +A entityListe \${${1}} destination=${2} sumSizeKo=0 # Here we will try to compute size (file or directory size) from local path and not from archive. for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do if [ -f ${entityListe[$i]} ] ; then # One file or a bunch of files has been copied without renaming from a visible filesystem iEntity=${entityListe[$i]} elif [ -f ${entityListe[$i]##/*/} ] ; then # One file or a bunch of files has been copied without renaming from an non visible filesystem # remove path /home/login/../ from entityListe elements iEntity=${entityListe[$i]##/*/} elif [ -f ${destination} ] ; then # a file has been copied and renamed iEntity=${destination} elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then # a copy in a directory but not in ${PWD} iEntity=${destination}/${entityListe[$i]##/*/} elif [ -d ${entityListe[$i]} ] ; then # a directory has been copied from a non remote place iEntity=${entityListe[$i]} elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then # a directory has been copied from a remote archive and not renamed iEntity=${destination}/${entityListe[$i]##/*/} elif [ -d ${destination} ] ; then # a directory has been copied from a remote archive and renamed iEntity=${destination} fi sizeKo=$( du --apparent-size -skL ${iEntity} | gawk '{print $1}' ) sumSizeKo=$(( $sumSizeKo + $sizeKo )) done sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" | bc ) echo "${sumSizeKo}|${sumSizeMo}" } #D-#================================================================== #D-function IGCM_debug_send_AMQP_msg__MAILTUNNEL #D- * Purpose: Take over AMQP C client using mail as a message recipient #D- * One argument : base64 encoded message #D- * Attach encoded config.card when starting the simulation function IGCM_debug_send_AMQP_msg__MAILTUNNEL { typeset b64_encoded_msg mail_recipient typeset buffer send_messages mail_frequency typeset last_mail_date__file b64_encoded_msg=$1 mail_recipient="superviseur@ipsl.jussieu.fr" send_messages=0 mail_frequency=3600 # in seconds # use to keep track when was last mail sent (maybe to be replaced with global variable) last_mail_date__file=${R_BUF}/.stamp.${config_UserChoices_TagName}.${config_UserChoices_JobName} # use to accumulate messages before sending them buffer=${R_BUF}/.buffer.${config_UserChoices_TagName}.${config_UserChoices_JobName} # init if [ ! -f "${buffer}" ]; then touch ${buffer} fi if [ ! -f "${last_mail_date__file}" ]; then touch ${last_mail_date__file} else # compute last time the file was changed (in seconds) seconds_since_last_mail=$(( $(date +%s) - $(stat -c %Y ${last_mail_date__file}) )) # send message when exceeding threshold [ ${seconds_since_last_mail} -gt ${mail_frequency} ] && send_messages=1 fi # queue messages in the buffer echo ${b64_encoded_msg} >> ${buffer} # send mail if [ X${initBigBro} = Xtrue ] ; then #echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") > ${SUBMIT_DIR}/mail.txt mailx -s "[TEMPORARY AMQP CHANNEL]" -a ${SUBMIT_DIR}/config.card.base64 ${mail_recipient} < ${buffer} # send buffer rm -f $buffer ; touch ${buffer} # clear buffer touch ${last_mail_date__file} # memorize last mail date initBigBro=false elif [ ${send_messages} -eq 1 ] ; then #echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") >> ${SUBMIT_DIR}/mail.txt mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer rm -f ${buffer} ; touch ${buffer} # flush the buffer touch ${last_mail_date__file} # memorize last mail date fi if ( ${FlushAMQP} ) ; then mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient} < ${buffer} # send buffer rm -f ${buffer} # cleaning behind us rm -f ${last_mail_date__file} # cleaning behind us fi # Allways all good for now. return 0 } #D-#================================================================== #D-function IGCM_debug_sendAMQP_Metrics #D- * Purpose: Take over AMQP C client using mail as a message recipient #D- * Two arguments : - Directory where metrics.json files can be found #D- * - Metrics Group Name. metrics will be added to this group #D- * Attach encoded metrics.json files. function IGCM_debug_sendAMQP_Metrics { typeset mail_recipient encodedBody if [ X${ActivateBigBro} = Xtrue ] ; then mail_recipient="superviseur@ipsl.jussieu.fr" # Metrics tag on server side code=7100 # Usual AMQP message to route messages on server side encodedBody=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"metricsGroupName\":\"${2}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" | base64 -w 0 ) # send mail attachmentsOptions="" for metricsFile in $( ls $1/*json ) ; do attachmentsOptions="-a ${metricsFile} ${attachmentsOptions}" done IGCM_debug_Print 2 "IGCM_debug_sendAMQP_Metrics " echo ${encodedBody}|mailx -s "[TEMPORARY AMQP CHANNEL]" ${attachmentsOptions} ${mail_recipient} fi # Allways all good for now. return 0 } #D-#================================================================== #D-function IGCM_debug_SendAMQP #D- * Purpose: Send body; encoded body and config.card to rabbitMQ function IGCM_debug_sendAMQP { typeset decal first additionnalOption encodedBody # Encode message Body encodedBody=$( echo "${Body}" | base64 -w 0 ) # Send config.card ? if [ X${1} = Xactivate ] ; then # Encode config.card cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64 # Prepare additionnal option additionnalOption="-f ${SUBMIT_DIR}/config.card.base64" # initBigBro=true else additionnalOption= # initBigBro=false fi # Only cosmetics : stack file if [ X${ActivateStackFilling} = Xtrue ] ; then decal=0 while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do printf ' ' >> ${StackFileLocation}/${StackFileName} (( decal = decal + 1 )) done # Log to stack file using human readable format echo "${Body}" >> ${StackFileLocation}/${StackFileName} fi # Log separately encoded AMQP message command for reuse in a mock up #echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> ${RUN_DIR_PATH}/send.AMQP.${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.${CumulPeriod}.history.txt # Send the message if [ X${BigBrotherChannel} = XMAIL ] ; then IGCM_debug_send_AMQP_msg__MAILTUNNEL "${encodedBody}" status=$? else sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} status=$? fi if [ ${status} -gt 0 ] ; then IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}" echo sendAMQPMsg -h localhost -p 5672 -b "${Body}" exit 1 fi } #D-#================================================================== #D-function IGCM_debug_CallStack #D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard) #D- function IGCM_debug_CallStack { if ( $DEBUG_debug ) ; then # Cosmetics typeset i decal i=0 until [ $i -eq ${IGCM_debug_LenStack} ]; do decal=0 until [ $decal -eq ${i} ]; do printf -- ' ' (( decal = decal + 1 )) done echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})" ((i = i + 1)) done fi } #D-#================================================================== #D-function IGCM_debug_PushStack #D-* Purpose: Push a function name in the stack #D- function IGCM_debug_PushStack { if ( $DEBUG_debug ) ; then typeset decal inputs startTime_ms # Only cosmetics : stack file if [ X${ActivateStackFilling} = Xtrue ] ; then echo >> ${StackFileLocation}/${StackFileName} decal=0 while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do printf ' ' >> ${StackFileLocation}/${StackFileName} (( decal = decal + 1 )) done # Fill the stack file echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName} fi # Save input list in an indexed array INPUTS=( $@ ) # Get timing information startTime_ms=$( IGCM_debug_getDate_ms ) # We add function call name on beginning of the stack set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]} # Save timing in milliseconds in an indexed array set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]} # We include the "null" Args in the beginning of the StackArgs set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]} # Then, we shift StackArgs tabular # Replacing blank separated list by comma separated list of quoted elements (except the first and last element) if [ $# -gt 1 ]; then IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /\",\"/g" ) fi # Unplugged message 4000 handling for now. To ease downstream treatment. #if [ X${ActivateBigBro} = Xtrue ] ; then # # RabbitMQ message code "PUSHSTACK" # code=4000 # # RabbitMQ message body # Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${IGCM_debug_Stack[0]}\",\"arguments\":[\"${IGCM_debug_StackArgs[0]}\"],\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) # # Fill the rabbitMQ queue # IGCM_debug_sendAMQP #fi # Increment LenStack (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 )) #IGCM_debug_CallStack fi } #D-#================================================================== #D-function IGCM_debug_PopStack #D-* Purpose: Pop a function name in the stack #D- function IGCM_debug_PopStack { if ( $DEBUG_debug ) ; then typeset i decal command arguments startTime_ms endTime_ms typeset instrumentation dest prefix # they are not typeset because they are send "by adress" to son functions # we unset them to avoid "memory effect" unset fileList source # INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR # PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL # THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD if ( ${RandomError} ) ; then if [ $((RANDOM%10000)) -le 10 ] ; then IGCM_debug_Print 1 "Random error has been triggered" if [ X${ActivateStackFilling} = Xtrue ] ; then echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName} fi ExitFlag=true fi fi if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then # Everything is cool # Get timing information endTime_ms=$( IGCM_debug_getDate_ms ) # Save Stack information before poping the stack command=${IGCM_debug_Stack[0]} # Go from comma separated list of quoted elements (except the first and the last element) # to unquoted space separated elements in an array set -A arguments -- $( echo ${IGCM_debug_StackArgs[0]} | sed -e "s/\",\"/\ /g" ) # Save Stack information before poping the stack startTime_ms=${IGCM_debug_StackTiming[0]} # Pop the stack (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 )) set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1} set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1} set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1} else echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack} IGCM_debug_Exit $@ fi # Special actions depending on command to prepare IGCM_debug_PrintInfosActions call # We are interested in: # 0. Which command performs the work # 1. Size of entity we are working with # 2. Where are we reading # 3. Where are we writing # 4. How long it took instrumentation=false case ${command} in # Classical copy (only files are given to IGCM_sys_Cp as options) IGCM_sys_Cp) instrumentation=true # All but the latest fileList=${arguments[*]:0:${#arguments[*]}-1} # just need the first file to get the directory source=${arguments[0]} # Nothing but the latest dest=${arguments[${#arguments[*]}-1]} # Size of file whose name are stored in a list entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} ) ;; # Copy from archive machine or from buffer IGCM_sys_Get|IGCM_sys_GetBuffer) instrumentation=true if [ ${#arguments[*]} -eq 2 ] ; then source=${arguments[0]} dest=${arguments[1]} # Size of file whose name are stored in a variable entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} ) elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then # IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/ # Keep the array name hosting the all list eval set +A fileList \${${arguments[1]}} # just need the first file to get the directory source=${fileList[0]} dest=${arguments[2]} # Size of file whose name are stored in a list entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} ) elif [ [ ${#arguments[*]} -ge 3 ] ; then # All but the latest fileList=${arguments[*]:0:${#arguments[*]}-1} # just need the first file to get the directory source=${arguments[0]} # Nothing but the latest dest=${arguments[${#arguments[*]}-1]} # Size of file whose name are stored in a list entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} ) fi ;; # Copy from compute node or copy to archive/buffer IGCM_sys_Get_Master|IGCM_sys_Get_Dir|IGCM_sys_Put_Out|IGCM_sys_PutBuffer_Out) instrumentation=true source=${arguments[0]} dest=${arguments[1]} # Size of file whose name are stored in a variable entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} ) ;; # Rebuild command IGCM_sys_rebuild|IGCM_sys_rebuild_station) instrumentation=true # All but the first fileList=${arguments[*]:1:${#arguments[*]}-1} # just need a file to get the directory source=${arguments[1]} # Nothing but the first dest=${arguments[0]} # Size of file whose name are stored in a list entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} ) ;; # NCO commands IGCM_sys_ncrcat|IGCM_sys_ncecat|IGCM_sys_ncra|IGCM_sys_ncks|IGCM_sys_cdo) # Example of what we want to catch : only filenames in those command lines # IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE} # IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out} # IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output} # IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output} instrumentation=true keepGoing=true prefix=. i=0 while ( ${keepGoing} ) ; do # the last one is not interesting if [ ${i} -eq ${#arguments[*]}-1 ] ; then keepGoing=false # look after "-p" option. Path prefix is the following arguments elif [ ${arguments[${i}]} = "-p" ] ; then ((i = i + 1)) prefix=${arguments[${i}]} ((i = i + 1)) elif [ ${i} -eq ${#arguments[*]}-1 ] ; then keepGoing=false # looking for files elif [ -f ${prefix}/${arguments[${i}]} ] ; then fileList="${fileList} ${prefix}/${arguments[${i}]}" ((i = i + 1)) # other options are not interesting else ((i = i + 1)) fi done # i value is at least 1 # just need one file to get the directory source=$( echo ${fileList} | gawk '{print $1}' ) # Nothing but the latest dest=${arguments[${#arguments[*]}-1]} # Size of file whose name are stored in a list entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} ) ;; esac # Print information related to instrumentation ( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source} # Only cosmetics : stack file if [ X${ActivateStackFilling} = Xtrue ] ; then decal=0 while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do printf ' ' >> ${StackFileLocation}/${StackFileName} (( decal = decal + 1 )) done fi if ( ${ExitFlag} ) ; then # Inform the stack file if [ X${ActivateStackFilling} = Xtrue ] ; then echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName} fi if [ X${ActivateBigBro} = Xtrue ] ; then # RabbitMQ message code "ERROR HAS BEEN TRIGGERED" code=4900 # RabbitMQ message body Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) # Fill the rabbitMQ queue IGCM_debug_sendAMQP fi else # Inform the stack file if [ X${ActivateStackFilling} = Xtrue ] ; then echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName} fi # Unplugged message 4100 handling for now. To ease downstream treatment. #if [ X${ActivateBigBro} = Xtrue ] ; then # # RabbitMQ message code "POPSTACK" # code=4100 # # RabbitMQ message body # Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) # # Fill the rabbitMQ queue # IGCM_debug_sendAMQP #fi fi # Reset array if necessary if [ ${IGCM_debug_LenStack} = 0 ]; then #echo #IGCM_debug_Print 3 "Clean stack array" #echo unset IGCM_debug_Stack unset IGCM_debug_StackArgs unset IGCM_debug_StackTiming IGCM_debug_Stack[0]=${NULL_STR} IGCM_debug_StackArgs[0]=${NULL_STR} IGCM_debug_StackTiming[0]=${NULL_STR} fi fi #IGCM_debug_CallStack } #D-#================================================================== #D-function IGCM_debug_BigBro_Initialize #D-* Purpose: switch rabbitMQ on #D- function IGCM_debug_BigBro_Initialize { IGCM_debug_PushStack "IGCM_debug_BigBro_Initialize" # Message type standard fields: # https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields # Message type dictionnary and custom fields: # https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary if [ X${BigBrother} = Xtrue ] ; then # create a unique ID for this specific job jobuid=$(uuidgen) if [ X${TaskType} = Xcomputing ]; then if ( ${FirstInitialize} ) ; then # RabbitMQ message code "BEGIN A SIMULATION" code=0000 # create and persist a unique id for this simulation simuid=$(uuidgen) IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid} # Standard fields for the first message genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) # RabbitMQ message body with specific fields associated message codes treated here Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) # Fill the rabbitMQ queue (the config.card in use will be sent) IGCM_debug_sendAMQP activate else # RabbitMQ message code "A NEW COMPUTING JOB IS RUNNING PART OF A SIMULATION" code=1000 # retrieve this simulation's unique id IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid simuid=${run_Configuration_simuid} # Using standard fields for message others than the first one. Still subject to change genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) # RabbitMQ message body with specific fields associated message codes treated here Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) # Fill the rabbitMQ queue IGCM_debug_sendAMQP fi # NOT VERY NICE BUT ... IT WORKS # Be sure that the genericSimulationID will be small from now on # Using standard fields for messages others than the first one. Still subject to change genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) elif [ X${TaskType} = Xpost-processing ]; then # RabbitMQ message code "A NEW POST-PROCESSING JOB IS RUNNING PART OF A SIMULATION" code=2000 # retrieve this simulation's unique id IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid simuid=${run_Configuration_simuid} # Using standard fields for message others than the first one. Still subject to change genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" ) # RabbitMQ message body with specific fields associated message codes treated here Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"accountingProject\":\"${PROJECT}\",\"jobWarningDelay\":\"${jobWarningDelay}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) # Fill the rabbitMQ queue IGCM_debug_sendAMQP fi # Turn the flag on ActivateBigBro=true # Dont flush by default FlushAMQP=false fi IGCM_debug_PopStack "IGCM_debug_BigBro_Initialize" } #D-#================================================================== #D-function IGCM_debug_BigBro_Finalize #D-* Purpose: Finalize rabbitMQ messages exchanges #D- function IGCM_debug_BigBro_Finalize { IGCM_debug_PushStack "IGCM_debug_BigBro_Finalize" # Message type standard fields: # https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields # Message type dictionnary and custom fields: # https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary if ( $DEBUG_debug ) ; then if [ X${ActivateBigBro} = Xtrue ] ; then if [ X${TaskType} = Xcomputing ]; then if ( ${simulationIsOver} ) ; then # RabbitMQ message code "SIMULATION ENDS" code=0100 FlushAMQP=true else # RabbitMQ message code "COMPUTING JOB ENDS" code=1100 fi elif [ X${TaskType} = Xpost-processing ]; then # RabbitMQ message code "POST-PROCESSING JOB ENDS" code=2100 FlushAMQP=true fi # RabbitMQ message body Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) # Fill the rabbitMQ queue IGCM_debug_sendAMQP fi fi IGCM_debug_PopStack "IGCM_debug_BigBro_Finalize" } #D-#================================================================== #D-function IGCM_debug_Exit #D-* Purpose: Print Call Stack and set ExitFlag to true #D- function IGCM_debug_Exit { IGCM_debug_PushStack "IGCM_debug_Exit" echo "IGCM_debug_Exit : " "${@}" echo echo "!!!!!!!!!!!!!!!!!!!!!!!!!!" echo "!! ERROR TRIGGERED !!" echo "!! EXIT FLAG SET !!" echo "!------------------------!" echo IGCM_debug_CallStack ExitFlag=true IGCM_debug_PopStack "IGCM_debug_Exit" } #D-#================================================== #D-function IGCM_debug_Verif_Exit #D-* Purpose: exit with number 1 if ExitFlag is true #D- function IGCM_debug_Verif_Exit { if ( ${ExitFlag} ) ; then # Plan to send an email here with IGCM_sys_SendMail if [ X${TaskType} != Xchecking ] ; then IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal" echo "IGCM_debug_Verif_Exit : Something wrong happened previously." echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where." echo " EXIT THE JOB." echo IGCM_debug_CallStack fi if [ X${ActivateBigBro} = Xtrue ] ; then # RabbitMQ message code "EXIT THE JOBS BECAUSE ERROR(S) HAS BEEN TRIGGERED" code=9999 # RabbitMQ message body Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) # To be sure we flush message buffer and clean up behind us before the end of the simulation FlushAMQP=true # Fill the rabbitMQ queue IGCM_debug_sendAMQP fi # Mail notification IGCM_sys_SendMail # And Good Bye date exit 1 fi } #D-#================================================== #D-function IGCM_debug_Verif_Exit_Post #D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment #D- function IGCM_debug_Verif_Exit_Post { if ( ${ExitFlag} ) ; then echo "IGCM_debug_Verif_Exit_Post : Something wrong happened." # If SpaceName is PROD then we stop if post_processing fails # Plan to send an email here with IGCM_sys_SendMail if [ X${config_UserChoices_SpaceName} = XPROD ] ; then echo " EXIT THE JOB." echo # Mail notification #IGCM_sys_SendMailPost # And Good Bye date exit 1 else echo "Either inside config.card the variable SpaceName is not in PROD" echo "or inside the main Job the variable JobType is not in RUN mode" echo " SO WE DO NOT EXIT THE JOB." echo date fi fi } #D-#================================================================== #D-function IGCM_debug_Print #D-* Purpose: Print arguments according to a level of verbosity. #D- function IGCM_debug_Print { typeset level=$1 shift if [ X"${1}" = X"-e" ]; then typeset cmd_echo="echo -e" shift else typeset cmd_echo="echo" fi if [ ${level} -le ${Verbosity} ] ; then typeset i case "${level}" in 1) for i in "$@" ; do ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i} done ;; 2) for i in "$@" ; do ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i} done ;; 3) for i in "$@" ; do ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i} done ;; esac fi } #D-#================================================================== #D-function IGCM_debug_PrintVariables #D-* Purpose: Print arguments when match a pattern #D- according to a level of verbosity. function IGCM_debug_PrintVariables { typeset level=$1 shift list=$( set | grep ^$1 | sed -e "s/'//g" ) if [ "X${list}" != X ] ; then IGCM_debug_Print ${level} ${list} fi } #D-#================================================================== #D-function IGCM_debug_PrintInfosActions #D-* Purpose: Print information related to instrumentation function IGCM_debug_PrintInfosActions { typeset actionType=$1 typeset entitySize=$2 typeset start_ms=$3 typeset end_ms=$4 typeset dest=$5 typeset source=$6 typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s typeset dirFrom dirTo diff_ms=$(( $end_ms - $start_ms )) # echo "diff_ms=$diff_ms" entitySizeKo=$( echo ${entitySize} | gawk -F"|" '{print $1}' ) # echo "entitySizeKo=$entitySizeKo" entitySizeMo=$( echo ${entitySize} | gawk -F"|" '{print $2}' ) # flux en Ko / ms flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" | bc ) # echo "flux_Ko_ms=$flux_Ko_ms" # flux en Ko / s flux_Ko_s=$(( $flux_Ko_ms * 1000 )) # echo "flux_Ko_s=$flux_Ko_s" # flux en Mo / s flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" | bc ) # echo "flux_Mo_s=$flux_Mo_s" if [ -d $dest ] ; then dirTo=$( readlink -f ${dest} ) else dirTo=$( readlink -f $( dirname ${dest} ) ) fi if [ -d $source ] ; then dirFrom=$( readlink -f ${source} ) else dirFrom=$( readlink -f $( dirname ${source} ) ) fi instrumentationContent=$( echo "\"actionName\":\"${actionType}\",\"size_Mo\":\"${entitySizeMo}\",\"duration_ms\":\"${diff_ms}\",\"throughput_Mo_s\":\"${flux_Mo_s}\",\"dirFrom\":\"${dirFrom}\",\"dirTo\":\"${dirTo}\"" ) if [ X${ActivateStackFilling} = Xtrue ] ; then echo "{${instrumentationContent}}" >> ${StackFileLocation}/${StackFileName} fi # Inform the rabbitMQ queue if [ X${ActivateBigBro} = Xtrue ] ; then # RabbitMQ message body Body=$( echo "{${genericSimulationID},\"msgCode\":\"7000\",\"msgUID\":\"$(uuidgen)\",${instrumentationContent},\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" ) # Fill the rabbitMQ queue IGCM_debug_sendAMQP fi } #D-#================================================================== #D-function IGCM_debug_Check #D- * Purpose: Check the present file by comparison with a reference file function IGCM_debug_Check { #--------------------- if [ ! -n "${libIGCM}" ] ; then echo "Check libIGCM_debug ..........................................[ FAILED ]" echo "--Error--> libIGCM variable is not defined" exit 2 fi #--------------------- if [ ! -n "${Verbosity}" ] ; then echo "Check libIGCM_debug ..........................................[ FAILED ]" echo "--Error--> Verbosity variable is not defined" exit 3 fi #--------------------- # Need to remove timestamps here diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh | sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g") > /dev/null 2>&1 status=$? if [ ${status} -eq 0 ] ; then echo "Check libIGCM_debug ..............................................[ OK ]" else echo "Check libIGCM_debug ..........................................[ FAILED ]" echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh" echo " has produced the file IGCM_debug_Test.ref.failed" echo " Please analyse differences with the reference file by typing:" echo " diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref" echo " Report errors to the author: Patrick.Brockmann@cea.fr" diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh | sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g") exit 4 fi #--------------------- }