source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 1094

#!/bin/ksh

#**************************************************************
# Author: Patrick Brockmann, Martial Mancip
# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
# $Revision::                                          $ Revision of last commit
# $Author::                                            $ Author of last commit
# $Date::                                              $ Date of last commit
# IPSL (2006)
#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
#
#**************************************************************

#==================================================
# The documentation of this file can be automatically generated
# if you use the prefix #D- for comments to be extracted.
# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
#==================================================

#==================================================
# Add high level verbosity
typeset -i Verbosity=${Verbosity:=3}

#==================================================
# DEBUG_debug
# Add low level verbosity
DEBUG_debug=${DEBUG_debug:=false}

#==================================================
# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
typeset RandomError=false

# Stack file containing call tree will be stored there.
if ( $DEBUG_debug ) ; then
  if [ X${TaskType} = Xcomputing ]; then
    typeset StackFileLocation=${StackFileLocation:=${PWD}/STACK}
    typeset StackFileName=computing.stack.$$
    [ ! -d ${StackFileLocation} ] && mkdir -p ${StackFileLocation}
  elif [ X${TaskType} = Xpost-processing ]; then
    typeset StackFileLocation=${POST_DIR:=${StackFileLocation}}
    typeset StackFileName=post-processing.stack.$$
  elif [ X${TaskType} = Xchecking ]; then
    typeset StackFileLocation=${POST_DIR:=${PWD}}
    typeset StackFileName=checking.stack.$$
  else
    typeset StackFileLocation=${POST_DIR:=${PWD}}
    typeset StackFileName=checking.stack.$$
  fi

  if [ -f ${StackFileLocation}/${StackFileName} ] ;
  then
    echo "Stack of an libIGCM job :" >> ${StackFileLocation}/${StackFileName}
  else
    echo "Stack of an libIGCM job :" >  ${StackFileLocation}/${StackFileName}
  fi
fi

#==================================================
# NULL_STR
# Default null string
typeset -r NULL_STR="_0_"

#==================================================
# libIGCM_CurrentTag
# Current libIGCM tag, check compatibilty with *.card
typeset -r libIGCM_CurrentTag="1.0"

#==================================================
# Exit Flag (internal debug)
# When true, end the master loop AFTER SAVES FILES
ExitFlag=false

#==================================================
# Declare a stack of functions calls
unset IGCM_debug_Stack
unset IGCM_debug_StackArgs
unset IGCM_debug_StackTiming
IGCM_debug_Stack[0]=${NULL_STR}
IGCM_debug_StackArgs[0]=${NULL_STR}
IGCM_debug_StackTiming[0]=${NULL_STR}
IGCM_debug_LenStack=0

#D-#==================================================================
#D-function IGCM_debug_getDate_ms
#D- * Purpose: Give number of milliseconds since 01-jan-1970
function IGCM_debug_getDate_ms
{
  typeset nanosecs ms
  # nano secondes since 01-jan-1970
  nanosecs=$( date +%s%N )

  # truncate the last 6 digits to get milliseconds since 01-jan-1970
  ms=${nanosecs:0:${#nanosecs}-6}

  echo "$ms"
}

#D-#==================================================================
#D-function IGCM_debug_sizeOfTabContent
#D- * Purpose: Give sumed size of a list of files
#D- * Usage: IGCM_debug_sizeOfTabContent entityList destination
#D- *        where entityList is a list of files or directory
#D- *        where dest is either a directory or a file name
function IGCM_debug_sizeOfTabContent
{
  typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo

  eval set +A entityListe \${${1}}
  destination=${2}
  sumSizeKo=0

  # Here we will try to compute size (file or directory size) from local path and not from archive.
  for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do
    if [ -f ${entityListe[$i]} ] ; then
      # One file or a bunch of files has been copied without renaming from a visible filesystem
      iEntity=${entityListe[$i]}
    elif [ -f ${entityListe[$i]##/*/} ] ; then
      # One file or a bunch of files has been copied without renaming from an non visible filesystem
      # remove path /home/login/../ from entityListe elements
      iEntity=${entityListe[$i]##/*/}
    elif [ -f ${destination} ] ; then
      # a file has been copied and renamed
      iEntity=${destination}
    elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then
      # a copy in a directory but not in ${PWD}
      iEntity=${destination}/${entityListe[$i]##/*/}
    elif [ -d ${entityListe[$i]} ] ; then
      # a directory has been copied from a non remote place
      iEntity=${entityListe[$i]}
    elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then
      # a directory has been copied from a remote archive and not renamed
      iEntity=${destination}/${entityListe[$i]##/*/}
    elif [ -d ${destination} ] ; then
      # a directory has been copied from a remote archive and renamed
      iEntity=${destination}
    fi
    sizeKo=$( du --apparent-size -skL ${iEntity} | gawk '{print $1}' )
    sumSizeKo=$(( $sumSizeKo + $sizeKo ))
  done
  sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" | bc )
  echo "${sumSizeKo}|${sumSizeMo}"
}

#D-#==================================================================
#D-function IGCM_debug_send_AMQP_msg__MAILTUNNEL
#D- * Purpose: Take over AMQP C client using mail as a message recipient
#D- * One argument : base64 encoded message
#D- * Attach encoded config.card when starting the simulation

function IGCM_debug_send_AMQP_msg__MAILTUNNEL {

  typeset b64_encoded_msg mail_recipient
  typeset buffer send_messages mail_frequency
  typeset last_mail_date__file

  b64_encoded_msg=$1

  buffer=/tmp/buffer.$$
  mail_recipient="superviseur@ipsl.jussieu.fr"
  send_messages=0
  mail_frequency=3600 # in seconds
  # use to keep track when was last mail sent (maybe to be replaced with global variable)
  last_mail_date__file=/tmp/stamp.$$

  # init
  if [ ! -f "${buffer}" ]; then
    touch "${buffer}"
  fi

  # retrieve pending messages number
  messages_count=$( wc -l ${buffer} | gawk {'print $1'} )

  if [ ! -f "${last_mail_date__file}" ]; then
    # if we are here, it means no mail has been sent yet
    # send the first mail
    [ ${messages_count} -gt 0 ] && send_messages=1
  else
     # compute last time the file was changed (in seconds)
    seconds_since_last_mail=$(( $(date +%s) - $(stat -c %Y ${last_mail_date__file}) ))
    # send message when exceeding threshold
    [ ${seconds_since_last_mail} -gt ${mail_frequency} ] && send_messages=1
  fi

  # queue messages in the buffer
  echo ${b64_encoded_msg} >> ${buffer}

  # send mail

  if [ X${initBigBro} = Xtrue ] ; then
    #echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") > ${SUBMIT_DIR}/mail.txt
    mailx -s "[TEMPORARY AMQP CHANNEL]" -a ${SUBMIT_DIR}/config.card.base64 ${mail_recipient} < ${buffer} # send buffer
    rm -f $buffer ; touch ${buffer}                                    # clear buffer
    touch ${last_mail_date__file}                                      # memorize last mail date
    initBigBro=false
  elif ( [ ${send_messages} -eq 1 ] || [ X${FlushAMQP} = XTRUE ] ) ; then
    #echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") >> ${SUBMIT_DIR}/mail.txt
    mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient}  < ${buffer} # send buffer
    rm -f $buffer ; touch ${buffer}                                    # clear buffer
    touch ${last_mail_date__file}                                      # memorize last mail date
  fi

  # Allways all good for now.
  return 0
}

#D-#==================================================================
#D-function IGCM_debug_SendAMQP
#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
function IGCM_debug_sendAMQP {

  typeset decal first additionnalOption encodedBody

  # Encode message Body
  encodedBody=$( echo "${Body}" | base64 -w 0 )

  # Send config.card ?
  if [ X${1} = Xactivate ] ; then
    # Encode config.card
    cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
    # Prepare additionnal option
    additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
    #
    initBigBro=true
  else
    additionnalOption=
    #
    initBigBro=false
  fi

  # Only cosmetics : stack file
  decal=0
  while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
    printf ' ' >> ${StackFileLocation}/${StackFileName}
    (( decal = decal + 1 ))
  done
  # Log to stack file using human readable format
  echo "${Body}" >> ${StackFileLocation}/${StackFileName}

  # Log separately encoded AMQP message command for reuse in a mock up
  #echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> /tmp/send.AMQP.${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.${CumulPeriod}.history.txt

  # Send the message
  if [ X${BigBrotherChannel} = XMAIL ] ; then
    IGCM_debug_send_AMQP_msg__MAILTUNNEL "${encodedBody}"
    status=$?
  else
    sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
    status=$?
  fi

  if [ ${status} -gt 0 ] ; then
    IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
    echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
    exit 1
  fi
}

#D-#==================================================================
#D-function IGCM_debug_CallStack
#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
#D-
function IGCM_debug_CallStack {
  if ( $DEBUG_debug ) ; then
    # Cosmetics
    typeset i decal
    i=0
    until [ $i -eq ${IGCM_debug_LenStack} ]; do
      decal=0
      until [ $decal -eq ${i} ]; do
        printf -- ' '
        (( decal = decal + 1 ))
      done
      echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
      ((i = i + 1))
    done
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PushStack
#D-* Purpose: Push a function name in the stack
#D-
function IGCM_debug_PushStack {
  if ( $DEBUG_debug ) ; then
    typeset decal inputs startTime_ms

    # Only cosmetics : stack file
    echo >> ${StackFileLocation}/${StackFileName}
    decal=0
    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
      printf ' ' >> ${StackFileLocation}/${StackFileName}
      (( decal = decal + 1 ))
    done

    # Fill the stack file
    echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}

    # Save input list in an indexed array
    INPUTS=( $@ )

    # Get timing information
    startTime_ms=$( IGCM_debug_getDate_ms )

    # We add function call name on beginning of the stack
    set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}

    # Save timing in milliseconds in an indexed array
    set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}

    # We include the "null" Args in the beginning of the StackArgs
    set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}

    # Then, we shift StackArgs tabular
    # Replacing blank separated list by comma separated list of quoted elements (except the first and last element)
    if [ $# -gt 1 ]; then
      IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /\",\"/g" )
    fi

    if [ X${ActivateBigBro} = Xtrue ] ; then
      # RabbitMQ message code "PUSHSTACK"
      code=2000
      # RabbitMQ message body
      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${IGCM_debug_Stack[0]}\",\"arguments\":[\"${IGCM_debug_StackArgs[0]}\"],\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
      # Fill the rabbitMQ queue
      IGCM_debug_sendAMQP
    fi

    # Increment LenStack
    (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))

    #IGCM_debug_CallStack
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PopStack
#D-* Purpose: Pop a function name in the stack
#D-
function IGCM_debug_PopStack {
  if ( $DEBUG_debug ) ; then
    typeset i decal command arguments startTime_ms endTime_ms
    typeset instrumentation dest prefix
    # they are not typeset because they are send "by adress" to son functions
    # we unset them to avoid "memory effect"
    unset fileList source

    # INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
    # PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
    # THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
    if ( ${RandomError} ) ; then
      if [ $((RANDOM%10000)) -le 10 ] ; then
        IGCM_debug_Print 1 "Random error has been triggered"
        echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName}
        ExitFlag=true
      fi
    fi

    if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
      # Everything is cool

      # Get timing information
      endTime_ms=$( IGCM_debug_getDate_ms )

      # Save Stack information before poping the stack
      command=${IGCM_debug_Stack[0]}

      # Go from comma separated list of quoted elements (except the first and the last element)
      # to unquoted space separated elements in an array
      set -A arguments -- $( echo ${IGCM_debug_StackArgs[0]} | sed -e "s/\",\"/\ /g" )

      # Save Stack information before poping the stack
      startTime_ms=${IGCM_debug_StackTiming[0]}

      # Pop the stack
      (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
      set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
      set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
      set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
    else
      echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
      IGCM_debug_Exit $@
    fi

    # Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
    # We are interested in:
    #  0. Which command performs the work
    #  1. Size of entity we are working with
    #  2. Where are we reading
    #  3. Where are we writing
    #  4. How long it took

    instrumentation=false

    case ${command} in
    # Classical copy (only files are given to IGCM_sys_Cp as options)
    IGCM_sys_Cp)
      instrumentation=true
      # All but the latest
      fileList=${arguments[*]:0:${#arguments[*]}-1}
      # just need the first file to get the directory
      source=${arguments[0]}
      # Nothing but the latest
      dest=${arguments[${#arguments[*]}-1]}
      # Size of file whose name are stored in a list
      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
      ;;

    # Copy from archive machine or from buffer
    IGCM_sys_Get|IGCM_sys_GetBuffer)
      instrumentation=true
      if [ ${#arguments[*]} -eq 2 ] ; then
        source=${arguments[0]}
        dest=${arguments[1]}
        # Size of file whose name are stored in a variable
        entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
      elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
        # IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/
        # Keep the array name hosting the all list
        eval set +A fileList \${${arguments[1]}}
        # just need the first file to get the directory
        source=${fileList[0]}
        dest=${arguments[2]}
        # Size of file whose name are stored in a list
        entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
      elif [ [ ${#arguments[*]} -ge 3 ] ; then
       # All but the latest
        fileList=${arguments[*]:0:${#arguments[*]}-1}
        # just need the first file to get the directory
        source=${arguments[0]}
        # Nothing but the latest
        dest=${arguments[${#arguments[*]}-1]}
        # Size of file whose name are stored in a list
        entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
      fi
      ;;

    # Copy from compute node or copy to archive/buffer
    IGCM_sys_Get_Master|IGCM_sys_Get_Dir|IGCM_sys_Put_Out|IGCM_sys_PutBuffer_Out)
      instrumentation=true
      source=${arguments[0]}
      dest=${arguments[1]}
      # Size of file whose name are stored in a variable
      entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
      ;;

    # Rebuild command
    IGCM_sys_rebuild|IGCM_sys_rebuild_station)
      instrumentation=true
      # All but the first
      fileList=${arguments[*]:1:${#arguments[*]}-1}
      # just need a file to get the directory
      source=${arguments[1]}
      # Nothing but the first
      dest=${arguments[0]}
      # Size of file whose name are stored in a list
      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
      ;;

    # NCO commands
    IGCM_sys_ncrcat|IGCM_sys_ncecat|IGCM_sys_ncra|IGCM_sys_ncks|IGCM_sys_cdo)
      # Example of what we want to catch : only filenames in those command lines
      # IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE}
      # IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
      # IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output}
      # IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
      instrumentation=true
      keepGoing=true
      prefix=.
      i=0
      while ( ${keepGoing} ) ; do
        # the last one is not interesting
        if [ ${i} -eq ${#arguments[*]}-1 ] ; then
          keepGoing=false
        # look after "-p" option. Path prefix is the following arguments
        elif [ ${arguments[${i}]} = "-p" ] ; then
          ((i = i + 1))
          prefix=${arguments[${i}]}
          ((i = i + 1))
        elif [ ${i} -eq ${#arguments[*]}-1 ] ; then
          keepGoing=false
        # looking for files
        elif [ -f ${prefix}/${arguments[${i}]} ] ; then
          fileList="${fileList} ${prefix}/${arguments[${i}]}"
          ((i = i + 1))
        # other options are not interesting
        else
          ((i = i + 1))
        fi
      done

      # i value is at least 1
      # just need one file to get the directory
      source=$( echo ${fileList} | gawk '{print $1}' )
      # Nothing but the latest
      dest=${arguments[${#arguments[*]}-1]}
      # Size of file whose name are stored in a list
      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
      ;;
    esac

    # Print information related to instrumentation
    ( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}

    # Only cosmetics : stack file
    decal=0
    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
      printf ' ' >> ${StackFileLocation}/${StackFileName}
      (( decal = decal + 1 ))
    done

    if ( ${ExitFlag} ) ; then
      # Inform the stack file
      echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName}

      if [ X${ActivateBigBro} = Xtrue ] ; then
        # RabbitMQ message code "ERROR HAS BEEN TRIGGERED"
        code=9000
        # RabbitMQ message body
        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
        # Fill the rabbitMQ queue
        IGCM_debug_sendAMQP
      fi
    else
      # Inform the stack file
      echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
      if [ X${ActivateBigBro} = Xtrue ] ; then
        # RabbitMQ message code "POPSTACK"
        code=3000
        # RabbitMQ message body
        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
        # Fill the rabbitMQ queue
        IGCM_debug_sendAMQP
      fi
    fi

    # Reset array if necessary
    if [ ${IGCM_debug_LenStack} = 0 ]; then
      #echo
      #IGCM_debug_Print 3 "Clean stack array"
      #echo
      unset IGCM_debug_Stack
      unset IGCM_debug_StackArgs
      unset IGCM_debug_StackTiming
      IGCM_debug_Stack[0]=${NULL_STR}
      IGCM_debug_StackArgs[0]=${NULL_STR}
      IGCM_debug_StackTiming[0]=${NULL_STR}
    fi
  fi
  #IGCM_debug_CallStack
}

#D-#==================================================================
#D-function IGCM_debug_ActivateBigBro
#D-* Purpose: switch rabbitMQ on
#D-
function IGCM_debug_ActivateBigBro {
  IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"

# Message type standard fields:
# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields

# Message type dictionnary and custom fields:
# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary

  if [ X${BigBrother} = Xtrue ] ; then
    # create a unique ID for this specific job
    jobuid=$(uuidgen)

    if ( ${FirstInitialize} ) ; then
      # RabbitMQ message code "BEGIN A SIMULATION"
      code=0000
      # create and persist a unique id for this simulation
      simuid=$(uuidgen)
      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid}
      # Standard fields for the first message
      genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
      # RabbitMQ message body with specific fields associated message codes treated here
      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
      # Fill the rabbitMQ queue (the config.card in use will be sent)
      IGCM_debug_sendAMQP activate
    else
      # RabbitMQ message code "A NEW JOB IS RUNNING PART OF A SIMULATION"
      code=1000
      # retrieve this simulation's unique id
      IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
      simuid=${run_Configuration_simuid}
      # Using standard fields for message others than the first one. Still subject to change
      genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
      # RabbitMQ message body with specific fields associated message codes treated here
      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
      # Fill the rabbitMQ queue
      IGCM_debug_sendAMQP
    fi

    # NOT VERY NICE BUT ... IT WORKS
    # Be sure that the genericSimulationID will be small from now on
    # Using standard fields for messages others than the first one. Still subject to change
    genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )

    # Turn the flag on
    ActivateBigBro=true
  fi
  IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
}

#D-#==================================================================
#D-function IGCM_debug_Exit
#D-* Purpose: Print Call Stack and set ExitFlag to true
#D-
function IGCM_debug_Exit {
  IGCM_debug_PushStack "IGCM_debug_Exit"
  echo "IGCM_debug_Exit : " "${@}"
  echo
  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
  echo "!!   ERROR TRIGGERED    !!"
  echo "!!   EXIT FLAG SET      !!"
  echo "!------------------------!"
  echo
  IGCM_debug_CallStack
  ExitFlag=true
  IGCM_debug_PopStack "IGCM_debug_Exit"
}

#D-#==================================================
#D-function IGCM_debug_Verif_Exit
#D-* Purpose: exit with number 1 if ExitFlag is true
#D-
function IGCM_debug_Verif_Exit {
  if ( ${ExitFlag} ) ; then
    # Plan to send an email here with IGCM_sys_SendMail
    if [ X${TaskType} != Xchecking ] ; then
      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
      echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
      echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
      echo "                        EXIT THE JOB."
      echo
      IGCM_debug_CallStack
    fi

    if [ X${ActivateBigBro} = Xtrue ] ; then
      # RabbitMQ message code "EXIT THE JOBS BECAUSE ERROR(S) HAS BEEN TRIGGERED"
      code=9999
      # RabbitMQ message body
      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
      # Fill the rabbitMQ queue
      IGCM_debug_sendAMQP
    fi

    # Not sure about that one ...
    if ( $DEBUG_debug ) ; then
      echo "Your files on ${R_OUT} :"
      IGCM_sys_Tree ${R_SAVE}
      echo
    fi

    # Mail notification
    IGCM_sys_SendMail

    # And Good Bye
    date
    exit 1
  fi
}

#D-#==================================================
#D-function IGCM_debug_Verif_Exit_Post
#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
#D-
function IGCM_debug_Verif_Exit_Post {
  if ( ${ExitFlag} ) ; then
    echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
    # If SpaceName is PROD then we stop if post_processing fails
    # Plan to send an email here with IGCM_sys_SendMail
    if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
      echo "                        EXIT THE JOB."
      echo
      # Mail notification
      #IGCM_sys_SendMailPost
      # And Good Bye
      date
      exit 1
    else
      echo "Either inside config.card the variable SpaceName is not in PROD"
      echo "or inside the main Job the variable JobType is not in RUN mode"
      echo "              SO WE DO NOT EXIT THE JOB."
      echo
      date
    fi
  fi
}

#D-#==================================================================
#D-function IGCM_debug_Print
#D-* Purpose: Print arguments according to a level of verbosity.
#D-
function IGCM_debug_Print
{
  typeset level=$1
  shift

  if [ X"${1}" = X"-e" ]; then
    typeset cmd_echo="echo -e"
    shift
  else
    typeset cmd_echo="echo"
  fi

  if [ ${level} -le ${Verbosity} ] ; then
    typeset i
    case "${level}" in
    1) for i in "$@" ; do
      ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
      done ;;
    2) for i in "$@" ; do
      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
      done ;;
    3) for i in "$@" ; do
      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
      done ;;
    esac
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PrintVariables
#D-* Purpose: Print arguments when match a pattern
#D-           according to a level of verbosity.
function IGCM_debug_PrintVariables
{
  typeset level=$1
  shift

  list=$( set | grep ^$1 | sed -e "s/'//g" )

  if [ "X${list}" != X ]  ; then
    IGCM_debug_Print ${level} ${list}
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PrintInfosActions
#D-* Purpose: Print information related to instrumentation
function IGCM_debug_PrintInfosActions
{
  typeset actionType=$1
  typeset entitySize=$2
  typeset start_ms=$3
  typeset end_ms=$4

  typeset dest=$5
  typeset source=$6

  typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
  typeset dirFrom dirTo

  diff_ms=$(( $end_ms - $start_ms ))
  # echo "diff_ms=$diff_ms"

  entitySizeKo=$( echo ${entitySize} | gawk -F"|" '{print $1}' )
  # echo "entitySizeKo=$entitySizeKo"
  entitySizeMo=$( echo ${entitySize} | gawk -F"|" '{print $2}' )

  # flux en Ko / ms
  flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" | bc )
  # echo "flux_Ko_ms=$flux_Ko_ms"

  # flux en Ko / s
  flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
  # echo "flux_Ko_s=$flux_Ko_s"

  # flux en Mo / s
  flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" | bc )
  # echo "flux_Mo_s=$flux_Mo_s"

  if [ -d $dest ] ; then
    dirTo=$( readlink -f ${dest} )
  else
    dirTo=$( readlink -f $( dirname ${dest} ) )
  fi

  if [ -d $source ] ; then
    dirFrom=$( readlink -f ${source} )
  else
    dirFrom=$( readlink -f $( dirname ${source} ) )
  fi

  instrumentationContent=$( echo "\"actionName\":\"${actionType}\",\"size_Mo\":\"${entitySizeMo}\",\"duration_ms\":\"${diff_ms}\",\"throughput_Mo_s\":\"${flux_Mo_s}\",\"dirFrom\":\"${dirFrom}\",\"dirTo\":\"${dirTo}\"" )

  echo "{${instrumentationContent}}"

  # Inform the rabbitMQ queue
  if [ X${ActivateBigBro} = Xtrue ] ; then
    # RabbitMQ message body
    Body=$( echo "{${genericSimulationID},\"msgCode\":\"7000\",\"msgUID\":\"$(uuidgen)\",${instrumentationContent},\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
    # Fill the rabbitMQ queue
    IGCM_debug_sendAMQP
  fi
}

#D-#==================================================================
#D-function IGCM_debug_Check
#D- * Purpose: Check the present file by comparison with a reference file
function IGCM_debug_Check
{
  #---------------------
  if [ ! -n "${libIGCM}" ] ; then
    echo "Check libIGCM_debug ..........................................[ FAILED ]"
    echo "--Error--> libIGCM variable is not defined"
    exit 2
  fi

  #---------------------
  if [ ! -n "${Verbosity}" ] ; then
    echo "Check libIGCM_debug ..........................................[ FAILED ]"
    echo "--Error--> Verbosity variable is not defined"
    exit 3
  fi

  #---------------------
  ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > /tmp/IGCM_debug_Test.$$.ref.failed 2>&1
  sleep 2

  # Remove date stamp.
  sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" /tmp/IGCM_debug_Test.$$.ref.failed > /tmp/IGCM_debug_Test.$$.ref.failed.nodate
  mv /tmp/IGCM_debug_Test.$$.ref.failed.nodate /tmp/IGCM_debug_Test.$$.ref.failed

  if diff /tmp/IGCM_debug_Test.$$.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
    echo "Check libIGCM_debug ..............................................[ OK ]"
    rm -f /tmp/IGCM_debug_Test.$$.ref.failed
  else
    echo "Check libIGCM_debug ..........................................[ FAILED ]"
    echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
    echo "           has produced the file IGCM_debug_Test.ref.failed"
    echo "           Please analyse differences with the reference file by typing:"
    echo "           diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
    echo "           Report errors to the author: Patrick.Brockmann@cea.fr"
    cat /tmp/IGCM_debug_Test.$$.ref.failed
    exit 4
  fi
  #---------------------
}