source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 874

Last change on this file since 874 was 874, checked in by sdipsl, 11 years ago
  • Only comments
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 17.6 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Patrick Brockmann, Martial Mancip
5# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#==================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#==================================================
19
20#==================================================
21# Add high level verbosity
22typeset -i Verbosity=${Verbosity:=3}
23
24#==================================================
25# DEBUG_debug
26# Add low level verbosity
27typeset DEBUG_debug=${DEBUG_debug:=false}
28
29#==================================================
30# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
31typeset RandomError=false
32
33# Where the stack file containing call tree will be stored.
34typeset StackFileLocation=${StackFileLocation:=${PWD}}
35
36if ( $DEBUG_debug ) ; then
37  if [ -f ${StackFileLocation}/stack ] ;
38  then
39    echo "Stack of an libIGCM job :" >> ${StackFileLocation}/stack
40  else
41    echo "Stack of an libIGCM job :" >  ${StackFileLocation}/stack
42  fi
43fi
44
45#==================================================
46# NULL_STR
47# Default null string
48typeset -r NULL_STR="_0_" 
49
50#==================================================
51# libIGCM_CurrentTag
52# Current libIGCM tag, check compatibilty with *.card
53typeset -r libIGCM_CurrentTag="1.0" 
54
55#==================================================
56# Exit Flag (internal debug)
57# When true, end the master loop AFTER SAVES FILES
58ExitFlag=false
59
60#==================================================
61# Declare a stack of functions calls
62
63# insert last argument of the Stack
64#set -A IGCM_debug_Stack ${NULL_STR}
65#set -A IGCM_debug_StackArgs ${NULL_STR}
66unset IGCM_debug_Stack
67unset IGCM_debug_StackArgs
68IGCM_debug_Stack[0]=${NULL_STR}
69IGCM_debug_StackArgs[0]=${NULL_STR}
70IGCM_debug_LenStack=0
71
72#D-#==================================================================
73#D-function IGCM_debug_CallStack
74#D-* Purpose: Echo the Stack
75#D-
76function IGCM_debug_CallStack {
77  #echo
78  #echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
79  #echo "!! IGCM_debug_CallStack !!"
80  #echo "!------------------------!"
81  #echo
82  if ( $DEBUG_debug ) ; then
83    # La pile d'appels est affichée de la plus vieille à la plus récente
84    # (c'est donc l'inverse de la norme d'affichage).
85    typeset i decal
86    i=0
87    until [ $i -eq ${IGCM_debug_LenStack} ]; do
88      decal=0
89      until [ $decal -eq ${i} ]; do
90        printf -- ' '
91        (( decal = decal + 1 ))
92      done
93      echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
94      ((i = i + 1))
95    done
96    #echo "!------------------------!"
97  fi
98}
99
100#D-#==================================================================
101#D-function IGCM_debug_PushStack
102#D-* Purpose: Push a function name in the stack
103#D-
104function IGCM_debug_PushStack {
105
106  if ( $DEBUG_debug ) ; then
107    typeset decal inputs
108    echo >> ${StackFileLocation}/stack
109    decal=0
110    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
111      printf ' ' >> ${StackFileLocation}/stack
112      (( decal = decal + 1 ))
113    done
114
115    # STORE input list in an indexed array
116    INPUTS=( $@ )
117    # We add function call name on beginning of the stack
118    set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
119
120    # We include the "null" Args in the beginning of the StackArgs
121    set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]} 
122    # Then, we shift StackArgs tabular
123    if [ $# -gt 1 ]; then
124      IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /,/g")
125    fi
126
127    # Fill the stack file
128    echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack
129
130    # Fill the rabbitMQ queue
131
132    # TO BE A FUNCTION BEGIN #
133
134    if [ X${ActivateBigBro} = Xtrue ] ; then
135      # Only cosmetics
136      decal=0
137      while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
138        printf ' ' >> ${StackFileLocation}/stack
139        (( decal = decal + 1 ))
140      done
141      # RabbitMQ message
142      code=2000
143      #
144      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
145      encodedBody=$( echo "${Body}" | base64 -w 0 )
146      #
147      #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
148      echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
149      echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
150      sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
151      status=$?
152      if [ ${status} -gt 0 ] ; then
153        IGCM_debug_Print 2 "IGCM_debug_PushStack : command sendAMQPMsg failed error code ${status}"
154        echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
155        exit
156      fi
157    fi
158
159    # TO BE A FUNCTION END #
160
161    # Increment LenStack
162    (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
163
164    # If you want to print CallStack each time :
165    #IGCM_debug_CallStack
166  fi
167}
168
169#D-#==================================================================
170#D-function IGCM_debug_PopStack
171#D-* Purpose: Pop a function name in the stack
172#D-
173function IGCM_debug_PopStack {
174  if ( $DEBUG_debug ) ; then
175    typeset decal
176    if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
177      (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
178      set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
179      set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
180    else
181      echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
182      IGCM_debug_Exit $@
183    fi
184    decal=0
185    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
186      printf ' ' >> ${StackFileLocation}/stack
187      (( decal = decal + 1 ))
188    done
189
190    # INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
191    # PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
192    # THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
193    #
194    if ( ${RandomError} ) ; then
195      if [ $((RANDOM%10000)) -le 0 ] ; then
196        IGCM_debug_Print 1 "A random error has been triggered"
197        echo "RANDOM ERROR" >> ${StackFileLocation}/stack
198        ExitFlag=true
199      fi
200    fi
201
202    if ( ${ExitFlag} ) ; then
203      # Inform the stack file
204      echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/stack
205      # Inform the rabbitMQ queue
206
207      # TO BE A FUNCTION BEGIN #
208
209      if [ X${ActivateBigBro} = Xtrue ] ; then
210        # Only cosmetics
211        decal=0
212        while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
213          printf ' ' >> ${StackFileLocation}/stack
214          (( decal = decal + 1 ))
215        done
216        # RabbitMQ message
217        code=9000
218        #
219        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"NOK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
220        encodedBody=$( echo "${Body}" | base64 -w 0 )
221        #
222        #sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card -b ${encodedBody}
223        echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
224        echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
225        sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
226        status=$?
227        if [ ${status} -gt 0 ] ; then
228          IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
229          echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
230          exit
231        fi
232      fi
233
234      # TO BE A FUNCTION END #
235
236    else
237      # Inform the stack file
238      echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/stack
239
240      # Inform the rabbitMQ queue
241
242      # TO BE A FUNCTION BEGIN #
243
244      if [ X${ActivateBigBro} = Xtrue ] ; then
245        # Only cosmetics
246        decal=0
247        while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
248          printf ' ' >> ${StackFileLocation}/stack
249          (( decal = decal + 1 ))
250        done
251        # RabbitMQ message
252        code=3000
253        #
254        Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"true\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${INPUTS[*]}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
255        encodedBody=$( echo "${Body}" | base64 -w 0 )
256        #
257        echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
258        echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
259        sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
260        status=$?
261        if [ ${status} -gt 0 ] ; then
262          IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
263          echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
264          exit
265        fi
266      fi
267
268      # TO BE A FUNCTION END #
269
270    fi
271
272    if [ ${IGCM_debug_LenStack} = 0 ]; then
273      # Reset array only when necessary
274      #echo
275      #IGCM_debug_Print 3 "Clean stack array"
276      #echo
277      #set -A IGCM_debug_Stack ${NULL_STR}
278      #set -A IGCM_debug_StackArgs ${NULL_STR}
279      unset IGCM_debug_Stack
280      unset IGCM_debug_StackArgs
281      IGCM_debug_Stack[0]=${NULL_STR}
282      IGCM_debug_StackArgs[0]=${NULL_STR}
283    fi
284  fi
285  #IGCM_debug_CallStack
286}
287
288#D-#==================================================================
289#D-function IGCM_debug_ActivateBigBro
290#D-* Purpose: switch rabbitMQ on
291#D-
292function IGCM_debug_ActivateBigBro {
293  IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"
294
295  # Fill the rabbitMQ queue
296  if [ X${BigBrother} = Xtrue ] ; then
297    # ID to identify a simulation
298    simuid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE
299    # ID to identify a job. Several Jobs are needed to complete a simulation
300    jobid=${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.p86denv.TGCC.CURIE.${CumulPeriod}
301    # Only cosmetics
302    decal=0
303    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
304      printf ' ' >> ${StackFileLocation}/stack
305      (( decal = decal + 1 ))
306    done
307    # RabbitMQ message
308    if ( ${FirstInitialize} ) ; then
309      code=0000
310    else
311      code=1000
312    fi
313    Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"OK\",\"out\":\"false\",\"nesting\":\"${IGCM_debug_LenStack}\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
314    encodedBody=$( echo "${Body}" | base64 -w 0 )
315    #
316    cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
317    #
318    echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b "${Body}"      >> ${StackFileLocation}/stack
319    echo sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
320    sendAMQPMsg -h localhost -p 5672 -f ${SUBMIT_DIR}/config.card.base64 -b ${encodedBody}
321    status=$?
322    if [ ${status} -gt 0 ] ; then
323      IGCM_debug_Print 2 "IGCM_debug_ActivateBigBro : command failed error code ${status}"
324      echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
325      IGCM_debug_Exit "IGCM_debug_ActivateBigBro"
326    fi
327    ActivateBigBro=true
328  fi
329  IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
330}
331
332#D-#==================================================================
333#D-function IGCM_debug_Exit
334#D-* Purpose: Print Call Stack and set ExitFlag to true
335#D-
336function IGCM_debug_Exit {
337  IGCM_debug_PushStack "IGCM_debug_Exit"
338  echo "IGCM_debug_Exit : " "${@}"
339  #IGCM_debug_CallStack
340  ExitFlag=true
341  IGCM_debug_PopStack "IGCM_debug_Exit"
342}
343
344#D-#==================================================
345#D-function IGCM_debug_Verif_Exit
346#D-* Purpose: exit with number 1 if ExitFlag is true
347#D-
348function IGCM_debug_Verif_Exit {
349  if ( ${ExitFlag} ) ; then
350    # Plan to send an email here with IGCM_sys_SendMail
351    if [ X${TaskType} != Xchecking ] ; then
352      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
353      echo "IGCM_debug_Verif_Exit : Something wrong happened."
354      echo "                        EXIT THE JOB."
355      echo
356      IGCM_debug_CallStack
357    fi
358    if ( $DEBUG_debug ) ; then
359      echo "Your files on ${R_OUT} :"
360      IGCM_sys_Tree ${R_SAVE}
361      echo
362    fi
363
364    # TO BE A FUNCTION BEGIN #
365
366    if [ X${ActivateBigBro} = Xtrue ] ; then
367      # Only cosmetics
368      decal=0
369      while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
370        printf ' ' >> ${StackFileLocation}/stack
371        (( decal = decal + 1 ))
372      done
373      # RabbitMQ message
374      code=9999
375      #
376      Body=$( echo "{\"code\":\"${code}\",\"simuid\":\"${simuid}\",\"jobid\":\"${jobid}\",\"status\":\"FATAL\",\"timestamp\":\"$( date +"%Y-%m-%d-%T" )\"}" )
377      encodedBody=$( echo "${Body}" | base64 -w 0 )
378      #
379      echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"      >> ${StackFileLocation}/stack
380      echo sendAMQPMsg -h localhost -p 5672 -b ${encodedBody} >> /tmp/send.AMQP.${jobid}.history.txt
381      sendAMQPMsg -h localhost -p 5672 -b ${encodedBody}
382      status=$?
383      if [ ${status} -gt 0 ] ; then
384        IGCM_debug_Print 2 "IGCM_debug_PopStack : command sendAMQPMsg failed error code ${status}"
385        echo  sendAMQPMsg -h localhost -p 5672 -b "${Body}"
386        exit
387      fi
388    fi
389
390    # TO BE A FUNCTION END #
391
392    # Mail notification
393    IGCM_sys_SendMail
394    # And Good Bye
395    date
396    exit 1
397  fi
398}
399
400#D-#==================================================
401#D-function IGCM_debug_Verif_Exit_Post
402#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
403#D-
404function IGCM_debug_Verif_Exit_Post {
405  if ( ${ExitFlag} ) ; then
406    echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
407    # If SpaceName is PROD then we stop if post_processing fails
408    # Plan to send an email here with IGCM_sys_SendMail
409    if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
410      echo "                        EXIT THE JOB."
411      echo
412      # Mail notification
413      #IGCM_sys_SendMailPost
414      # And Good Bye
415      date
416      exit 1
417    else
418      echo "Either inside config.card the variable SpaceName is not in PROD"
419      echo "or inside the main Job the variable JobType is not in RUN mode"
420      echo "              SO WE DO NOT EXIT THE JOB."
421      echo
422      date
423    fi
424  fi
425}
426
427#D-#==================================================================
428#D-function IGCM_debug_Print
429#D-* Purpose: Print arguments according to a level of verbosity.
430#D-
431function IGCM_debug_Print
432{
433  typeset level=$1
434  shift
435
436  if [ X"${1}" = X"-e" ]; then
437    typeset cmd_echo="echo -e"
438    shift
439  else
440    typeset cmd_echo="echo"
441  fi
442
443  if [ ${level} -le ${Verbosity} ] ; then
444    typeset i
445    case "${level}" in
446    1) for i in "$@" ; do
447      ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
448      done ;;
449    2) for i in "$@" ; do
450      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
451      done ;;
452    3) for i in "$@" ; do
453      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
454      done ;;
455    esac
456  fi
457}
458
459#D-#==================================================================
460#D-function IGCM_debug_PrintVariables
461#D-* Purpose: Print arguments when match a pattern
462#D-           according to a level of verbosity.
463function IGCM_debug_PrintVariables
464{
465  typeset level=$1
466  shift
467
468  list=$( set | grep ^$1 | sed -e "s/'//g" )
469
470  if [ "X${list}" != X ]  ; then
471    IGCM_debug_Print ${level} ${list}
472  fi
473}
474
475#D-#==================================================================
476#D-function IGCM_debug_Check
477#D- * Purpose: Check the present file by comparison with a reference file
478function IGCM_debug_Check
479{
480  #---------------------
481  if [ ! -n "${libIGCM}" ] ; then
482    echo "Check libIGCM_debug ..........................................[ FAILED ]"
483    echo "--Error--> libIGCM variable is not defined"
484    exit 2
485  fi
486
487  #---------------------
488  if [ ! -n "${Verbosity}" ] ; then
489    echo "Check libIGCM_debug ..........................................[ FAILED ]"
490    echo "--Error--> Verbosity variable is not defined"
491    exit 3
492  fi
493
494  #---------------------
495  ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > IGCM_debug_Test.ref.failed 2>&1
496  sleep 2
497
498  # Remove date stamp.
499  sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" IGCM_debug_Test.ref.failed > IGCM_debug_Test.ref.failed.nodate
500  mv IGCM_debug_Test.ref.failed.nodate IGCM_debug_Test.ref.failed
501
502  if diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
503    echo "Check libIGCM_debug ..............................................[ OK ]"
504    rm -f IGCM_debug_Test.ref.failed
505  else
506    echo "Check libIGCM_debug ..........................................[ FAILED ]"
507    echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
508    echo "           has produced the file IGCM_debug_Test.ref.failed"
509    echo "           Please analyse differences with the reference file by typing:"
510    echo "           diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
511    echo "           Report errors to the author: Patrick.Brockmann@cea.fr"
512    exit 4
513  fi
514  #---------------------
515}
Note: See TracBrowser for help on using the repository browser.