source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 855

#!/bin/ksh 

#**************************************************************
# Author: Patrick Brockmann, Martial Mancip
# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
# $Revision::                                          $ Revision of last commit
# $Author::                                            $ Author of last commit
# $Date::                                              $ Date of last commit
# IPSL (2006)
#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
#
#**************************************************************

#==================================================
# The documentation of this file can be automatically generated
# if you use the prefix #D- for comments to be extracted. 
# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
#==================================================

#==================================================
# Add high level verbosity
typeset -i Verbosity=${Verbosity:=3}

#==================================================
# DEBUG_debug
# Add low level verbosity
typeset DEBUG_debug=${DEBUG_debug:=false}

# Where the stack file containing call tree will be stored.
typeset StackFileLocation=${StackFileLocation:=${PWD}}

if ( $DEBUG_debug ) ; then
  if [ -f ${StackFileLocation}/stack ] ;
  then
    echo "Stack of an libIGCM job :" >> ${StackFileLocation}/stack
  else
    echo "Stack of an libIGCM job :" >  ${StackFileLocation}/stack
  fi

  if [ -f ${StackFileLocation}/fullstack ] ;
  then
    echo "Full stack of an libIGCM job :" >> ${StackFileLocation}/fullstack
  else
    echo "Full stack of an libIGCM job :" >  ${StackFileLocation}/fullstack
  fi

fi

#==================================================
# NULL_STR
# Default null string
typeset -r NULL_STR="_0_" 

#==================================================
# libIGCM_CurrentTag
# Current libIGCM tag, check compatibilty with *.card
typeset -r libIGCM_CurrentTag="1.0" 

#==================================================
# Exit Flag (internal debug)
# When true, end the master loop AFTER SAVES FILES
ExitFlag=false

#==================================================
# Declare a stack of functions calls

# insert last argument of the Stack
#set -A IGCM_debug_Stack ${NULL_STR}
#set -A IGCM_debug_StackArgs ${NULL_STR}
unset IGCM_debug_Stack
unset IGCM_debug_StackArgs
IGCM_debug_Stack[0]=${NULL_STR}
IGCM_debug_StackArgs[0]=${NULL_STR}
IGCM_debug_LenStack=0

#D-#==================================================================
#D-function IGCM_debug_CallStack
#D-* Purpose: Echo the Stack
#D-
function IGCM_debug_CallStack {
  #echo
  #echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
  #echo "!! IGCM_debug_CallStack !!"
  #echo "!------------------------!"
  #echo
  if ( $DEBUG_debug ) ; then
    # La pile d'appels est affichée de la plus vieille à la plus récente 
    # (c'est donc l'inverse de la norme d'affichage).
    typeset i decal
    i=0
    until [ $i -eq ${IGCM_debug_LenStack} ]; do
      decal=0
      until [ $decal -eq ${i} ]; do
        printf -- ' ' >> ${StackFileLocation}/fullstack
        (( decal = decal + 1 ))
      done
      echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}"\
           "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})" >> ${StackFileLocation}/fullstack
      ((i = i + 1))
    done
    #echo "!------------------------!"
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PushStack
#D-* Purpose: Push a function name in the stack
#D-
function IGCM_debug_PushStack {

  if ( $DEBUG_debug ) ; then
    typeset decal inputs
    echo >> ${StackFileLocation}/stack
    decal=0
    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
      printf ' ' >> ${StackFileLocation}/stack
      (( decal = decal + 1 ))
    done

    # STORE input list in an indexed array
    INPUTS=( $@ )
    # We add function call name on beginning of the stack
    set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}

    # We include the "null" Args in the beginning of the StackArgs
    set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]} 
    # Then, we shift StackArgs tabular
    if [ $# -gt 1 ]; then 
      IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /,/g")
    fi

    # Fill the stack file
    echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack

    # Fill the rabbitMQ queue
    if [ X${ActivateBigBro} = Xtrue ] ; then
      code=2000
      #
      encodedBody=$( echo "{\"code\":\"${code}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" | base64 -w 0 )
      #
      #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
      echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/sendAMQP.history.txt
      sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
      status=$?
      if [ ${status} -gt 0 ] ; then
        IGCM_debug_Print 2 "IGCM_debug_PushStack : command sendAMQPMsg failed error code ${status}"
        echo  sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
        exit
      fi
    fi

    # Increment LenStack
    (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))

    # If you want to print CallStack each time :
    IGCM_debug_CallStack
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PopStack
#D-* Purpose: Pop a function name in the stack
#D-
function IGCM_debug_PopStack {
  if ( $DEBUG_debug ) ; then

    typeset decal
    if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
      (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
      set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
      set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
    else
      echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
      IGCM_debug_Exit $@
    fi
    decal=0
    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
      printf ' ' >> ${StackFileLocation}/stack
      (( decal = decal + 1 ))
    done

    if ( ${ExitFlag} ) ; then
      # Inform the stack file
      echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/stack
      # Inform the rabbitMQ queue
      if [ X${ActivateBigBro} = Xtrue ] ; then
        code=9000
        #
        encodedBody=$( echo "{\"code\":\"${code}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" | base64 -w 0 )
        #
        #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card -b ${encodedBody}
        echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/sendAMQP.history.txt
        sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
        status=$?
        if [ ${status} -gt 0 ] ; then
          IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
          echo  sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
          exit
        fi
      fi
    else
      # Inform the stack file
      echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack

      # Inform the rabbitMQ queue
      if [ X${ActivateBigBro} = Xtrue ] ; then
        code=3000
        #
        encodedBody=$( echo "{\"code\":\"${code}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" | base64 -w 0 )

        #
        #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
        echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/sendAMQP.history.txt
        sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
        status=$?
        if [ ${status} -gt 0 ] ; then
          IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
          echo  sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
          exit
        fi
      fi
    fi

    if [ ${IGCM_debug_LenStack} = 0 ]; then
      # Reset array only when necessary
      #echo
      #IGCM_debug_Print 3 "Clean stack array"
      #echo
      #set -A IGCM_debug_Stack ${NULL_STR}
      #set -A IGCM_debug_StackArgs ${NULL_STR}
      unset IGCM_debug_Stack
      unset IGCM_debug_StackArgs
      IGCM_debug_Stack[0]=${NULL_STR}
      IGCM_debug_StackArgs[0]=${NULL_STR}
    fi
  fi
  IGCM_debug_CallStack
}

#D-#==================================================================
#D-function IGCM_debug_ActivateBigBro
#D-* Purpose: switch rabbitMQ on
#D-
function IGCM_debug_ActivateBigBro {
  IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"

  set -vx

  # Fill the rabbitMQ queue
  if [ X${BigBrother}=Xtrue ] ; then
    jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE
    if ( ${FirstInitialize} ) ; then
      code=0000
    else
      code=1000
    fi
    encodedBody=$( echo "{\"code\":\"${code}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" | base64 -w 0 )
    #
    cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
    #
    #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
    echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} >> /tmp/sendAMQP.history.txt
    sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
    status=$?
    if [ ${status} -gt 0 ] ; then
      IGCM_debug_Print 2 "IGCM_debug_ActivateBigBro : command failed error code ${status}"
      IGCM_debug_Exit "IGCM_debug_ActivateBigBro"
    fi
    ActivateBigBro=true
  fi
  IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
}

#D-#==================================================================
#D-function IGCM_debug_Exit
#D-* Purpose: Print Call Stack and set ExitFlag to true
#D-
function IGCM_debug_Exit {
  IGCM_debug_PushStack "IGCM_debug_Exit"
  echo "IGCM_debug_Exit : " "${@}"
  #IGCM_debug_CallStack
  ExitFlag=true
  IGCM_debug_PopStack "IGCM_debug_Exit"
}

#D-#==================================================
#D-function IGCM_debug_Verif_Exit
#D-* Purpose: exit with number 1 if ExitFlag is true
#D-
function IGCM_debug_Verif_Exit {
  if ( ${ExitFlag} ) ; then
    # Plan to send an email here with IGCM_sys_SendMail
    if [ X${TaskType} != Xchecking ] ; then
      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
      echo "IGCM_debug_Verif_Exit : Something wrong happened."
      echo "                        EXIT THE JOB."
      echo
    fi
    if ( $DEBUG_debug ) ; then
      echo "Your files on ${R_OUT} :"
      IGCM_sys_Tree ${R_SAVE}
      echo
    fi
    if [ X${ActivateBigBro} = Xtrue ] ; then
      code=9999
      #
      encodedBody=$( echo "{\"code\":\"${code}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" | base64 -w 0 )
      #
      #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card -b ${encodedBody}
      echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/sendAMQP.history.txt
      sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
      status=$?
      if [ ${status} -gt 0 ] ; then
        IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
        echo  sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
        exit
      fi
    fi
    # Mail notification
    IGCM_sys_SendMail
    # And Good Bye
    date
    exit 1
  fi
}

#D-#==================================================
#D-function IGCM_debug_Verif_Exit_Post
#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
#D-
function IGCM_debug_Verif_Exit_Post {
  if ( ${ExitFlag} ) ; then
    echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
    # If SpaceName is PROD then we stop if post_processing fails
    # Plan to send an email here with IGCM_sys_SendMail
    if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
      echo "                        EXIT THE JOB."
      echo
      # Mail notification
      #IGCM_sys_SendMailPost
      # And Good Bye
      date
      exit 1
    else
      echo "Either inside config.card the variable SpaceName is not in PROD"
      echo "or inside the main Job the variable JobType is not in RUN mode"
      echo "              SO WE DO NOT EXIT THE JOB."
      echo
      date
    fi
  fi
}

#D-#==================================================================
#D-function IGCM_debug_Print
#D-* Purpose: Print arguments according to a level of verbosity.
#D-
function IGCM_debug_Print
{
  typeset level=$1
  shift

  if [ X"${1}" = X"-e" ]; then
    typeset cmd_echo="echo -e"
    shift
  else
    typeset cmd_echo="echo"
  fi

  if [ ${level} -le ${Verbosity} ] ; then
    typeset i
    case "${level}" in
    1) for i in "$@" ; do
      ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
      done ;; 
    2) for i in "$@" ; do
      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
      done ;; 
    3) for i in "$@" ; do
      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
      done ;; 
    esac
  fi
}

#D-#==================================================================
#D-function IGCM_debug_PrintVariables
#D-* Purpose: Print arguments when match a pattern
#D-           according to a level of verbosity.
function IGCM_debug_PrintVariables
{
  typeset level=$1
  shift

  list=$( set | grep ^$1 | sed -e "s/'//g" )

  if [ "X${list}" != X ]  ; then
    IGCM_debug_Print ${level} ${list}
  fi
}

#D-#==================================================================
#D-function IGCM_debug_Check
#D- * Purpose: Check the present file by comparison with a reference file
function IGCM_debug_Check
{
  #---------------------
  if [ ! -n "${libIGCM}" ] ; then
    echo "Check libIGCM_debug ..........................................[ FAILED ]"
    echo "--Error--> libIGCM variable is not defined"
    exit 2
  fi

  #---------------------
  if [ ! -n "${Verbosity}" ] ; then
    echo "Check libIGCM_debug ..........................................[ FAILED ]"
    echo "--Error--> Verbosity variable is not defined"
    exit 3
  fi

  #---------------------
  ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > IGCM_debug_Test.ref.failed 2>&1
  sleep 2

  # Remove date stamp.
  sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" IGCM_debug_Test.ref.failed > IGCM_debug_Test.ref.failed.nodate
  mv IGCM_debug_Test.ref.failed.nodate IGCM_debug_Test.ref.failed

  if diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
    echo "Check libIGCM_debug ..............................................[ OK ]"
    rm -f IGCM_debug_Test.ref.failed
  else
    echo "Check libIGCM_debug ..........................................[ FAILED ]"
    echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
    echo "           has produced the file IGCM_debug_Test.ref.failed"
    echo "           Please analyse differences with the reference file by typing:"
    echo "           diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
    echo "           Report errors to the author: Patrick.Brockmann@cea.fr"
    exit 4
  fi
  #---------------------
}