source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 872

Last change on this file since 872 was 872, checked in by sdipsl, 11 years ago
  • introduce a random error mechanism
  • correct a typo
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 17.4 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Patrick Brockmann, Martial Mancip
5# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#==================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#==================================================
19
20#==================================================
21# Add high level verbosity
22typeset -i Verbosity=${Verbosity:=3}
23
24#==================================================
25# DEBUG_debug
26# Add low level verbosity
27typeset DEBUG_debug=${DEBUG_debug:=false}
28
29#==================================================
30# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
31typeset RandomError=false
32
33# Where the stack file containing call tree will be stored.
34typeset StackFileLocation=${StackFileLocation:=${PWD}}
35
36if ( $DEBUG_debug ) ; then
37  if [ -f ${StackFileLocation}/stack ] ;
38  then
39    echo "Stack of an libIGCM job :" >> ${StackFileLocation}/stack
40  else
41    echo "Stack of an libIGCM job :" >  ${StackFileLocation}/stack
42  fi
43fi
44
45#==================================================
46# NULL_STR
47# Default null string
48typeset -r NULL_STR="_0_" 
49
50#==================================================
51# libIGCM_CurrentTag
52# Current libIGCM tag, check compatibilty with *.card
53typeset -r libIGCM_CurrentTag="1.0" 
54
55#==================================================
56# Exit Flag (internal debug)
57# When true, end the master loop AFTER SAVES FILES
58ExitFlag=false
59
60#==================================================
61# Declare a stack of functions calls
62
63# insert last argument of the Stack
64#set -A IGCM_debug_Stack ${NULL_STR}
65#set -A IGCM_debug_StackArgs ${NULL_STR}
66unset IGCM_debug_Stack
67unset IGCM_debug_StackArgs
68IGCM_debug_Stack[0]=${NULL_STR}
69IGCM_debug_StackArgs[0]=${NULL_STR}
70IGCM_debug_LenStack=0
71
72#D-#==================================================================
73#D-function IGCM_debug_CallStack
74#D-* Purpose: Echo the Stack
75#D-
76function IGCM_debug_CallStack {
77  #echo
78  #echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
79  #echo "!! IGCM_debug_CallStack !!"
80  #echo "!------------------------!"
81  #echo
82  if ( $DEBUG_debug ) ; then
83    # La pile d'appels est affichée de la plus vieille à la plus récente
84    # (c'est donc l'inverse de la norme d'affichage).
85    typeset i decal
86    i=0
87    until [ $i -eq ${IGCM_debug_LenStack} ]; do
88      decal=0
89      until [ $decal -eq ${i} ]; do
90        printf -- ' '
91        (( decal = decal + 1 ))
92      done
93      echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
94      ((i = i + 1))
95    done
96    #echo "!------------------------!"
97  fi
98}
99
100#D-#==================================================================
101#D-function IGCM_debug_PushStack
102#D-* Purpose: Push a function name in the stack
103#D-
104function IGCM_debug_PushStack {
105
106  if ( $DEBUG_debug ) ; then
107    typeset decal inputs
108    echo >> ${StackFileLocation}/stack
109    decal=0
110    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
111      printf ' ' >> ${StackFileLocation}/stack
112      (( decal = decal + 1 ))
113    done
114
115    # STORE input list in an indexed array
116    INPUTS=( $@ )
117    # We add function call name on beginning of the stack
118    set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
119
120    # We include the "null" Args in the beginning of the StackArgs
121    set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]} 
122    # Then, we shift StackArgs tabular
123    if [ $# -gt 1 ]; then
124      IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /,/g")
125    fi
126
127    # Fill the stack file
128    echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack
129
130    # Fill the rabbitMQ queue
131    if [ X${ActivateBigBro} = Xtrue ] ; then
132      # Only cosmetics
133      decal=0
134      while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
135        printf ' ' >> ${StackFileLocation}/stack
136        (( decal = decal + 1 ))
137      done
138      # RabbitMQ message
139      code=2000
140      #
141      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
142      encodedBody=$( echo "${Body}" | base64 -w 0 )
143      #
144      #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
145      echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
146      echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
147      sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
148      status=$?
149      if [ ${status} -gt 0 ] ; then
150        IGCM_debug_Print 2 "IGCM_debug_PushStack : command sendAMQPMsg failed error code ${status}"
151        echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
152        exit
153      fi
154    fi
155
156    # Increment LenStack
157    (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
158
159    # If you want to print CallStack each time :
160    #IGCM_debug_CallStack
161  fi
162}
163
164#D-#==================================================================
165#D-function IGCM_debug_PopStack
166#D-* Purpose: Pop a function name in the stack
167#D-
168function IGCM_debug_PopStack {
169  if ( $DEBUG_debug ) ; then
170    typeset decal
171    if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
172      (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
173      set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
174      set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
175    else
176      echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
177      IGCM_debug_Exit $@
178    fi
179    decal=0
180    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
181      printf ' ' >> ${StackFileLocation}/stack
182      (( decal = decal + 1 ))
183    done
184
185    # INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
186    # PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
187    # THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
188    #
189    if ( ${RandomError} ) ; then
190      if [ $((RANDOM%10000)) -le 0 ] ; then
191        IGCM_debug_Print 1 "A random error has been triggered"
192        echo "RANDOM ERROR" >> ${StackFileLocation}/stack
193        ExitFlag=true
194      fi
195    fi
196
197    if ( ${ExitFlag} ) ; then
198      # Inform the stack file
199      echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/stack
200      # Inform the rabbitMQ queue
201      if [ X${ActivateBigBro} = Xtrue ] ; then
202        # Only cosmetics
203        decal=0
204        while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
205          printf ' ' >> ${StackFileLocation}/stack
206          (( decal = decal + 1 ))
207        done
208        # RabbitMQ message
209        code=9000
210        #
211        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
212        encodedBody=$( echo "${Body}" | base64 -w 0 )
213        #
214        #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card -b ${encodedBody}
215        echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
216        echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
217        sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
218        status=$?
219        if [ ${status} -gt 0 ] ; then
220          IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
221          echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
222          exit
223        fi
224      fi
225    else
226      # Inform the stack file
227      echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack
228
229      # Inform the rabbitMQ queue
230      if [ X${ActivateBigBro} = Xtrue ] ; then
231        # Only cosmetics
232        decal=0
233        while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
234          printf ' ' >> ${StackFileLocation}/stack
235          (( decal = decal + 1 ))
236        done
237        # RabbitMQ message
238        code=3000
239        #
240        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
241        encodedBody=$( echo "${Body}" | base64 -w 0 )
242        #
243        echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
244        echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
245        sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
246        status=$?
247        if [ ${status} -gt 0 ] ; then
248          IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
249          echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
250          exit
251        fi
252      fi
253    fi
254
255    if [ ${IGCM_debug_LenStack} = 0 ]; then
256      # Reset array only when necessary
257      #echo
258      #IGCM_debug_Print 3 "Clean stack array"
259      #echo
260      #set -A IGCM_debug_Stack ${NULL_STR}
261      #set -A IGCM_debug_StackArgs ${NULL_STR}
262      unset IGCM_debug_Stack
263      unset IGCM_debug_StackArgs
264      IGCM_debug_Stack[0]=${NULL_STR}
265      IGCM_debug_StackArgs[0]=${NULL_STR}
266    fi
267  fi
268  #IGCM_debug_CallStack
269}
270
271#D-#==================================================================
272#D-function IGCM_debug_ActivateBigBro
273#D-* Purpose: switch rabbitMQ on
274#D-
275function IGCM_debug_ActivateBigBro {
276  IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"
277
278  # Fill the rabbitMQ queue
279  if [ X${BigBrother} = Xtrue ] ; then
280    # ID to identify a simulation
281    simuid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE
282    # ID to identify a job. Several Jobs are needed to complete a simulation
283    jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE.${CumulPeriod}
284    # Only cosmetics
285    decal=0
286    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
287      printf ' ' >> ${StackFileLocation}/stack
288      (( decal = decal + 1 ))
289    done
290    # RabbitMQ message
291    if ( ${FirstInitialize} ) ; then
292      code=0000
293    else
294      code=1000
295    fi
296    Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
297    encodedBody=$( echo "${Body}" | base64 -w 0 )
298    #
299    cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
300    #
301    echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b "${Body}"      >> ${StackFileLocation}/stack
302    echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
303    sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
304    status=$?
305    if [ ${status} -gt 0 ] ; then
306      IGCM_debug_Print 2 "IGCM_debug_ActivateBigBro : command failed error code ${status}"
307      echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
308      IGCM_debug_Exit "IGCM_debug_ActivateBigBro"
309    fi
310    ActivateBigBro=true
311  fi
312  IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
313}
314
315#D-#==================================================================
316#D-function IGCM_debug_Exit
317#D-* Purpose: Print Call Stack and set ExitFlag to true
318#D-
319function IGCM_debug_Exit {
320  IGCM_debug_PushStack "IGCM_debug_Exit"
321  echo "IGCM_debug_Exit : " "${@}"
322  #IGCM_debug_CallStack
323  ExitFlag=true
324  IGCM_debug_PopStack "IGCM_debug_Exit"
325}
326
327#D-#==================================================
328#D-function IGCM_debug_Verif_Exit
329#D-* Purpose: exit with number 1 if ExitFlag is true
330#D-
331function IGCM_debug_Verif_Exit {
332  if ( ${ExitFlag} ) ; then
333    # Plan to send an email here with IGCM_sys_SendMail
334    if [ X${TaskType} != Xchecking ] ; then
335      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
336      echo "IGCM_debug_Verif_Exit : Something wrong happened."
337      echo "                        EXIT THE JOB."
338      echo
339      IGCM_debug_CallStack
340    fi
341    if ( $DEBUG_debug ) ; then
342      echo "Your files on ${R_OUT} :"
343      IGCM_sys_Tree ${R_SAVE}
344      echo
345    fi
346    if [ X${ActivateBigBro} = Xtrue ] ; then
347      # Only cosmetics
348      decal=0
349      while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
350        printf ' ' >> ${StackFileLocation}/stack
351        (( decal = decal + 1 ))
352      done
353      # RabbitMQ message
354      code=9999
355      #
356      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
357      encodedBody=$( echo "${Body}" | base64 -w 0 )
358      #
359      echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
360      echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
361      sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
362      status=$?
363      if [ ${status} -gt 0 ] ; then
364        IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
365        echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
366        exit
367      fi
368    fi
369    # Mail notification
370    IGCM_sys_SendMail
371    # And Good Bye
372    date
373    exit 1
374  fi
375}
376
377#D-#==================================================
378#D-function IGCM_debug_Verif_Exit_Post
379#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
380#D-
381function IGCM_debug_Verif_Exit_Post {
382  if ( ${ExitFlag} ) ; then
383    echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
384    # If SpaceName is PROD then we stop if post_processing fails
385    # Plan to send an email here with IGCM_sys_SendMail
386    if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
387      echo "                        EXIT THE JOB."
388      echo
389      # Mail notification
390      #IGCM_sys_SendMailPost
391      # And Good Bye
392      date
393      exit 1
394    else
395      echo "Either inside config.card the variable SpaceName is not in PROD"
396      echo "or inside the main Job the variable JobType is not in RUN mode"
397      echo "              SO WE DO NOT EXIT THE JOB."
398      echo
399      date
400    fi
401  fi
402}
403
404#D-#==================================================================
405#D-function IGCM_debug_Print
406#D-* Purpose: Print arguments according to a level of verbosity.
407#D-
408function IGCM_debug_Print
409{
410  typeset level=$1
411  shift
412
413  if [ X"${1}" = X"-e" ]; then
414    typeset cmd_echo="echo -e"
415    shift
416  else
417    typeset cmd_echo="echo"
418  fi
419
420  if [ ${level} -le ${Verbosity} ] ; then
421    typeset i
422    case "${level}" in
423    1) for i in "$@" ; do
424      ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
425      done ;;
426    2) for i in "$@" ; do
427      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
428      done ;;
429    3) for i in "$@" ; do
430      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
431      done ;;
432    esac
433  fi
434}
435
436#D-#==================================================================
437#D-function IGCM_debug_PrintVariables
438#D-* Purpose: Print arguments when match a pattern
439#D-           according to a level of verbosity.
440function IGCM_debug_PrintVariables
441{
442  typeset level=$1
443  shift
444
445  list=$( set | grep ^$1 | sed -e "s/'//g" )
446
447  if [ "X${list}" != X ]  ; then
448    IGCM_debug_Print ${level} ${list}
449  fi
450}
451
452#D-#==================================================================
453#D-function IGCM_debug_Check
454#D- * Purpose: Check the present file by comparison with a reference file
455function IGCM_debug_Check
456{
457  #---------------------
458  if [ ! -n "${libIGCM}" ] ; then
459    echo "Check libIGCM_debug ..........................................[ FAILED ]"
460    echo "--Error--> libIGCM variable is not defined"
461    exit 2
462  fi
463
464  #---------------------
465  if [ ! -n "${Verbosity}" ] ; then
466    echo "Check libIGCM_debug ..........................................[ FAILED ]"
467    echo "--Error--> Verbosity variable is not defined"
468    exit 3
469  fi
470
471  #---------------------
472  ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > IGCM_debug_Test.ref.failed 2>&1
473  sleep 2
474
475  # Remove date stamp.
476  sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" IGCM_debug_Test.ref.failed > IGCM_debug_Test.ref.failed.nodate
477  mv IGCM_debug_Test.ref.failed.nodate IGCM_debug_Test.ref.failed
478
479  if diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
480    echo "Check libIGCM_debug ..............................................[ OK ]"
481    rm -f IGCM_debug_Test.ref.failed
482  else
483    echo "Check libIGCM_debug ..........................................[ FAILED ]"
484    echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
485    echo "           has produced the file IGCM_debug_Test.ref.failed"
486    echo "           Please analyse differences with the reference file by typing:"
487    echo "           diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
488    echo "           Report errors to the author: Patrick.Brockmann@cea.fr"
489    exit 4
490  fi
491  #---------------------
492}
Note: See TracBrowser for help on using the repository browser.