Ignore:
Timestamp:
06/11/13 09:18:12 (11 years ago)
Author:
sdipsl
Message:
  • Enhance error message by providing the stack call tree when error has been detected
  • No more fullstack file. Everything is in the stack file
  • Log un encoded message to ease debuging
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh

    r868 r869  
    3737    echo "Stack of an libIGCM job :" >  ${StackFileLocation}/stack 
    3838  fi 
    39  
    40   if [ -f ${StackFileLocation}/fullstack ] ; 
    41   then 
    42     echo "Full stack of an libIGCM job :" >> ${StackFileLocation}/fullstack 
    43   else 
    44     echo "Full stack of an libIGCM job :" >  ${StackFileLocation}/fullstack 
    45   fi 
    46  
    4739fi 
    4840 
    … …  
    9284      decal=0 
    9385      until [ $decal -eq ${i} ]; do 
    94         printf -- ' ' >> ${StackFileLocation}/fullstack 
     86        printf -- ' ' 
    9587        (( decal = decal + 1 )) 
    9688      done 
    97       echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}"\ 
    98            "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})" >> ${StackFileLocation}/fullstack 
     89      echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})" 
    9990      ((i = i + 1)) 
    10091    done 
    … …  
    135126    # Fill the rabbitMQ queue 
    136127    if [ X${ActivateBigBro} = Xtrue ] ; then 
     128      # Only cosmetics 
     129      decal=0 
     130      while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do 
     131        printf ' ' >> ${StackFileLocation}/stack 
     132        (( decal = decal + 1 )) 
     133      done 
     134      # RabbitMQ message 
    137135      code=2000 
    138136      # 
    139       encodedBody=$( echo "{\"code\":\"${code}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" | base64 -w 0 ) 
     137      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" ) 
     138      encodedBody=$( echo "${Body}" | base64 -w 0 ) 
    140139      # 
    141140      #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} 
     141      echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack 
    142142      echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt 
    143143      sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} 
    … …  
    145145      if [ ${status} -gt 0 ] ; then 
    146146        IGCM_debug_Print 2 "IGCM_debug_PushStack : command sendAMQPMsg failed error code ${status}" 
    147         echo  sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} 
     147        echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}" 
    148148        exit 
    149149      fi 
    … …  
    154154 
    155155    # If you want to print CallStack each time : 
    156     IGCM_debug_CallStack 
     156    #IGCM_debug_CallStack 
    157157  fi 
    158158} 
    … …  
    185185      # Inform the rabbitMQ queue 
    186186      if [ X${ActivateBigBro} = Xtrue ] ; then 
     187        # Only cosmetics 
     188        decal=0 
     189        while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do 
     190          printf ' ' >> ${StackFileLocation}/stack 
     191          (( decal = decal + 1 )) 
     192        done 
     193        # RabbitMQ message 
    187194        code=9000 
    188195        # 
    189         encodedBody=$( echo "{\"code\":\"${code}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" | base64 -w 0 ) 
     196        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" ) 
     197        encodedBody=$( echo "{Body}" | base64 -w 0 ) 
    190198        # 
    191199        #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card -b ${encodedBody} 
     200        echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack 
    192201        echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt 
    193202        sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} 
    … …  
    195204        if [ ${status} -gt 0 ] ; then 
    196205          IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}" 
    197           echo  sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} 
     206          echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}" 
    198207          exit 
    199208        fi 
    … …  
    205214      # Inform the rabbitMQ queue 
    206215      if [ X${ActivateBigBro} = Xtrue ] ; then 
     216        # Only cosmetics 
     217        decal=0 
     218        while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do 
     219          printf ' ' >> ${StackFileLocation}/stack 
     220          (( decal = decal + 1 )) 
     221        done 
     222        # RabbitMQ message 
    207223        code=3000 
    208224        # 
    209         encodedBody=$( echo "{\"code\":\"${code}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" | base64 -w 0 ) 
    210  
     225        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" ) 
     226        encodedBody=$( echo "{Body}" | base64 -w 0 ) 
    211227        # 
    212         #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} 
     228        echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack 
    213229        echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt 
    214230        sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} 
    … …  
    216232        if [ ${status} -gt 0 ] ; then 
    217233          IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}" 
    218           echo  sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} 
     234          echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}" 
    219235          exit 
    220236        fi 
    … …  
    235251    fi 
    236252  fi 
    237   IGCM_debug_CallStack 
     253  #IGCM_debug_CallStack 
    238254} 
    239255 
    … …  
    247263  # Fill the rabbitMQ queue 
    248264  if [ X${BigBrother} = Xtrue ] ; then 
    249     jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE 
     265    # ID to identify a simulation 
     266    simuid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE 
     267    # ID to identify a job. Several Jobs are needed to complete a simulation 
     268    jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE.${CumulPeriod} 
     269    # Only cosmetics 
     270    decal=0 
     271    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do 
     272      printf ' ' >> ${StackFileLocation}/stack 
     273      (( decal = decal + 1 )) 
     274    done 
     275    # RabbitMQ message 
    250276    if ( ${FirstInitialize} ) ; then 
    251277      code=0000 
    … …  
    253279      code=1000 
    254280    fi 
    255     encodedBody=$( echo "{\"code\":\"${code}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" | base64 -w 0 ) 
     281    Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" ) 
     282    encodedBody=$( echo "{Body}" | base64 -w 0 ) 
    256283    # 
    257284    cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64 
    258285    # 
    259     #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} 
     286    echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b "${Body}"      >> ${StackFileLocation}/stack 
    260287    echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt 
    261288    sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} 
    … …  
    263290    if [ ${status} -gt 0 ] ; then 
    264291      IGCM_debug_Print 2 "IGCM_debug_ActivateBigBro : command failed error code ${status}" 
     292      echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}" 
    265293      IGCM_debug_Exit "IGCM_debug_ActivateBigBro" 
    266294    fi 
    … …  
    294322      echo "                        EXIT THE JOB." 
    295323      echo 
     324      IGCM_debug_CallStack 
    296325    fi 
    297326    if ( $DEBUG_debug ) ; then 
    … …  
    301330    fi 
    302331    if [ X${ActivateBigBro} = Xtrue ] ; then 
     332      # Only cosmetics 
     333      decal=0 
     334      while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do 
     335        printf ' ' >> ${StackFileLocation}/stack 
     336        (( decal = decal + 1 )) 
     337      done 
     338      # RabbitMQ message 
    303339      code=9999 
    304340      # 
    305       encodedBody=$( echo "{\"code\":\"${code}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" | base64 -w 0 ) 
     341      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" ) 
     342      encodedBody=$( echo "{Body}" | base64 -w 0 ) 
    306343      # 
    307       #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card -b ${encodedBody} 
     344      echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack 
    308345      echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt 
    309346      sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} 
    … …  
    311348      if [ ${status} -gt 0 ] ; then 
    312349        IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}" 
    313         echo  sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} 
     350        echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}" 
    314351        exit 
    315352      fi 
Note: See TracChangeset for help on using the changeset viewer.