source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 1086

Last change on this file since 1086 was 1086, checked in by sdipsl, 10 years ago
  • bugfix. Due to a typo simuid were not read back after resubmission...
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 30.4 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Patrick Brockmann, Martial Mancip
5# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#==================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#==================================================
19
20#==================================================
21# Add high level verbosity
22typeset -i Verbosity=${Verbosity:=3}
23
24#==================================================
25# DEBUG_debug
26# Add low level verbosity
27DEBUG_debug=${DEBUG_debug:=false}
28
29#==================================================
30# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
31typeset RandomError=false
32
33# Stack file containing call tree will be stored there.
34if ( $DEBUG_debug ) ; then
35  if [ X${TaskType} = Xcomputing ]; then
36    typeset StackFileLocation=${StackFileLocation:=${PWD}/STACK}
37    typeset StackFileName=computing.stack.$$
38    [ ! -d ${StackFileLocation} ] && mkdir -p ${StackFileLocation}
39  elif [ X${TaskType} = Xpost-processing ]; then
40    typeset StackFileLocation=${POST_DIR:=${StackFileLocation}}
41    typeset StackFileName=post-processing.stack.$$
42  elif [ X${TaskType} = Xchecking ]; then
43    typeset StackFileLocation=${POST_DIR:=${PWD}}
44    typeset StackFileName=checking.stack.$$
45  else
46    typeset StackFileLocation=${POST_DIR:=${PWD}}
47    typeset StackFileName=checking.stack.$$
48  fi
49
50  if [ -f ${StackFileLocation}/${StackFileName} ] ;
51  then
52    echo "Stack of an libIGCM job :" >> ${StackFileLocation}/${StackFileName}
53  else
54    echo "Stack of an libIGCM job :" >  ${StackFileLocation}/${StackFileName}
55  fi
56fi
57
58#==================================================
59# NULL_STR
60# Default null string
61typeset -r NULL_STR="_0_"
62
63#==================================================
64# libIGCM_CurrentTag
65# Current libIGCM tag, check compatibilty with *.card
66typeset -r libIGCM_CurrentTag="1.0"
67
68#==================================================
69# Exit Flag (internal debug)
70# When true, end the master loop AFTER SAVES FILES
71ExitFlag=false
72
73#==================================================
74# Declare a stack of functions calls
75unset IGCM_debug_Stack
76unset IGCM_debug_StackArgs
77unset IGCM_debug_StackTiming
78IGCM_debug_Stack[0]=${NULL_STR}
79IGCM_debug_StackArgs[0]=${NULL_STR}
80IGCM_debug_StackTiming[0]=${NULL_STR}
81IGCM_debug_LenStack=0
82
83#D-#==================================================================
84#D-function IGCM_debug_getDate_ms
85#D- * Purpose: Give number of milliseconds since 01-jan-1970
86function IGCM_debug_getDate_ms
87{
88  typeset nanosecs ms
89  # nano secondes since 01-jan-1970
90  nanosecs=$( date +%s%N )
91
92  # truncate the last 6 digits to get milliseconds since 01-jan-1970
93  ms=${nanosecs:0:${#nanosecs}-6}
94
95  echo "$ms"
96}
97
98#D-#==================================================================
99#D-function IGCM_debug_sizeOfTabContent
100#D- * Purpose: Give sumed size of a list of files
101#D- * Usage: IGCM_debug_sizeOfTabContent entityList destination
102#D- *        where entityList is a list of files or directory
103#D- *        where dest is either a directory or a file name
104function IGCM_debug_sizeOfTabContent
105{
106  typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo
107
108  eval set +A entityListe \${${1}}
109  destination=${2}
110  sumSizeKo=0
111
112  # Here we will try to compute size (file or directory size) from local path and not from archive.
113  for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do
114    if [ -f ${entityListe[$i]} ] ; then
115      # One file or a bunch of files has been copied without renaming from a visible filesystem
116      iEntity=${entityListe[$i]}
117    elif [ -f ${entityListe[$i]##/*/} ] ; then
118      # One file or a bunch of files has been copied without renaming from an non visible filesystem
119      # remove path /home/login/../ from entityListe elements
120      iEntity=${entityListe[$i]##/*/}
121    elif [ -f ${destination} ] ; then
122      # a file has been copied and renamed
123      iEntity=${destination}
124    elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then
125      # a copy in a directory but not in ${PWD}
126      iEntity=${destination}/${entityListe[$i]##/*/}
127    elif [ -d ${entityListe[$i]} ] ; then
128      # a directory has been copied from a non remote place
129      iEntity=${entityListe[$i]}
130    elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then
131      # a directory has been copied from a remote archive and not renamed
132      iEntity=${destination}/${entityListe[$i]##/*/}
133    elif [ -d ${destination} ] ; then
134      # a directory has been copied from a remote archive and renamed
135      iEntity=${destination}
136    fi
137    sizeKo=$( du --apparent-size -skL ${iEntity} | gawk '{print $1}' )
138    sumSizeKo=$(( $sumSizeKo + $sizeKo ))
139  done
140  sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" | bc )
141  echo "${sumSizeKo}|${sumSizeMo}"
142}
143
144#D-#==================================================================
145#D-function IGCM_debug_send_AMQP_msg__MAILTUNNEL
146#D- * Purpose: Take over AMQP C client using mail as a message recipient
147#D- * One argument : base64 encoded message
148#D- * Attach encoded config.card when starting the simulation
149
150function IGCM_debug_send_AMQP_msg__MAILTUNNEL {
151
152  typeset b64_encoded_msg mail_recipient
153  typeset buffer send_messages mail_frequency
154  typeset last_mail_date__file
155
156  b64_encoded_msg=$1
157
158  buffer=/tmp/buffer.$$
159  mail_recipient="superviseur@ipsl.jussieu.fr"
160  send_messages=0
161  mail_frequency=3600 # in seconds
162  # use to keep track when was last mail sent (maybe to be replaced with global variable)
163  last_mail_date__file=/tmp/stamp.$$
164
165  # init
166  if [ ! -f "${buffer}" ]; then
167    touch "${buffer}"
168  fi
169
170  # retrieve pending messages number
171  messages_count=$( wc -l ${buffer} | gawk {'print $1'} )
172
173  if [ ! -f "${last_mail_date__file}" ]; then
174    # if we are here, it means no mail has been sent yet
175    # send the first mail
176    [ ${messages_count} -gt 0 ] && send_messages=1
177  else
178     # compute last time the file was changed (in seconds)
179    seconds_since_last_mail=$(( $(date +%s) - $(stat -c %Y ${last_mail_date__file}) ))
180    # send message when exceeding threshold
181    [ ${seconds_since_last_mail} -gt ${mail_frequency} ] && send_messages=1
182  fi
183
184  # queue messages in the buffer
185  echo ${b64_encoded_msg} >> ${buffer}
186
187  # send mail
188
189  if [ X${initBigBro} = Xtrue ] ; then
190    mailx -s "[TEMPORARY AMQP CHANNEL]" -a ${SUBMIT_DIR}/config.card.base64 ${mail_recipient} < ${buffer} # send buffer
191    rm -f $buffer ; touch ${buffer}                                    # clear buffer
192    touch ${last_mail_date__file}                                      # memorize last mail date
193    initBigBro=false
194  elif ( [ ${send_messages} -eq 1 ] || [ X${FlushAMQP} = XTRUE ] ) ; then
195    mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient}  < ${buffer} # send buffer
196    rm -f $buffer ; touch ${buffer}                                    # clear buffer
197    touch ${last_mail_date__file}                                      # memorize last mail date
198  fi
199
200  # Allways all good for now.
201  return 0
202}
203
204#D-#==================================================================
205#D-function IGCM_debug_SendAMQP
206#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
207function IGCM_debug_sendAMQP {
208
209  typeset decal first additionnalOption encodedBody
210
211  # Encode message Body
212  encodedBody=$( echo "${Body}" | base64 -w 0 )
213
214  # Send config.card ?
215  if [ X${1} = Xactivate ] ; then
216    # Encode config.card
217    cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
218    # Prepare additionnal option
219    additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
220    #
221    initBigBro=true
222  else
223    additionnalOption=
224    #
225    initBigBro=false
226  fi
227
228  # Only cosmetics : stack file
229  decal=0
230  while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
231    printf ' ' >> ${StackFileLocation}/${StackFileName}
232    (( decal = decal + 1 ))
233  done
234  # Log to stack file using human readable format
235  echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b "${Body}"      >> ${StackFileLocation}/${StackFileName}
236
237  # Log separately encoded AMQP message command for reuse in a mock up
238  echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> /tmp/send.AMQP.${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.${CumulPeriod}.history.txt
239
240  # Send the message
241  if [ X${BigBrotherChannel} = XMAIL ] ; then
242    IGCM_debug_send_AMQP_msg__MAILTUNNEL "${encodedBody}"
243    status=$?
244  else
245    sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
246    status=$?
247  fi
248
249  status=$?
250  if [ ${status} -gt 0 ] ; then
251    IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
252    echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
253    exit
254  fi
255}
256
257#D-#==================================================================
258#D-function IGCM_debug_CallStack
259#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
260#D-
261function IGCM_debug_CallStack {
262  if ( $DEBUG_debug ) ; then
263    # Cosmetics
264    typeset i decal
265    i=0
266    until [ $i -eq ${IGCM_debug_LenStack} ]; do
267      decal=0
268      until [ $decal -eq ${i} ]; do
269        printf -- ' '
270        (( decal = decal + 1 ))
271      done
272      echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
273      ((i = i + 1))
274    done
275  fi
276}
277
278#D-#==================================================================
279#D-function IGCM_debug_PushStack
280#D-* Purpose: Push a function name in the stack
281#D-
282function IGCM_debug_PushStack {
283  if ( $DEBUG_debug ) ; then
284    typeset decal inputs startTime_ms
285
286    # Only cosmetics : stack file
287    echo >> ${StackFileLocation}/${StackFileName}
288    decal=0
289    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
290      printf ' ' >> ${StackFileLocation}/${StackFileName}
291      (( decal = decal + 1 ))
292    done
293
294    # Fill the stack file
295    echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
296
297    # Save input list in an indexed array
298    INPUTS=( $@ )
299
300    # Get timing information
301    startTime_ms=$( IGCM_debug_getDate_ms )
302
303    # We add function call name on beginning of the stack
304    set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
305
306    # Save timing in milliseconds in an indexed array
307    set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}
308
309    # We include the "null" Args in the beginning of the StackArgs
310    set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}
311
312    # Then, we shift StackArgs tabular
313    # Replacing blank separated list by comma separated list of quoted elements (except the first and last element)
314    if [ $# -gt 1 ]; then
315      IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /\",\"/g" )
316    fi
317
318    if [ X${ActivateBigBro} = Xtrue ] ; then
319      # RabbitMQ message code "PUSHSTACK"
320      code=2000
321      # RabbitMQ message body
322      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${IGCM_debug_Stack[0]}\",\"arguments\":[\"${IGCM_debug_StackArgs[0]}\"],\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
323      # Fill the rabbitMQ queue
324      IGCM_debug_sendAMQP
325    fi
326
327    # Increment LenStack
328    (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
329
330    #IGCM_debug_CallStack
331  fi
332}
333
334#D-#==================================================================
335#D-function IGCM_debug_PopStack
336#D-* Purpose: Pop a function name in the stack
337#D-
338function IGCM_debug_PopStack {
339  if ( $DEBUG_debug ) ; then
340    typeset i decal command arguments startTime_ms endTime_ms
341    typeset instrumentation dest prefix
342    # they are not typeset because they are send "by adress" to son functions
343    # we unset them to avoid "memory effect"
344    unset fileList source
345
346    # INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
347    # PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
348    # THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
349    if ( ${RandomError} ) ; then
350      if [ $((RANDOM%10000)) -le 10 ] ; then
351        IGCM_debug_Print 1 "Random error has been triggered"
352        echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName}
353        ExitFlag=true
354      fi
355    fi
356
357    if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
358      # Everything is cool
359
360      # Get timing information
361      endTime_ms=$( IGCM_debug_getDate_ms )
362
363      # Save Stack information before poping the stack
364      command=${IGCM_debug_Stack[0]}
365
366      # Go from comma separated list of quoted elements (except the first and the last element)
367      # to unquoted space separated elements in an array
368      set -A arguments -- $( echo ${IGCM_debug_StackArgs[0]} | sed -e "s/\",\"/\ /g" )
369
370      # Save Stack information before poping the stack
371      startTime_ms=${IGCM_debug_StackTiming[0]}
372
373      # Pop the stack
374      (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
375      set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
376      set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
377      set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
378    else
379      echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
380      IGCM_debug_Exit $@
381    fi
382
383    # Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
384    # We are interested in:
385    #  0. Which command performs the work
386    #  1. Size of entity we are working with
387    #  2. Where are we reading
388    #  3. Where are we writing
389    #  4. How long it took
390
391    instrumentation=false
392
393    case ${command} in
394    # Classical copy (only files are given to IGCM_sys_Cp as options)
395    IGCM_sys_Cp)
396      instrumentation=true
397      # All but the latest
398      fileList=${arguments[*]:0:${#arguments[*]}-1}
399      # just need the first file to get the directory
400      source=${arguments[0]}
401      # Nothing but the latest
402      dest=${arguments[${#arguments[*]}-1]}
403      # Size of file whose name are stored in a list
404      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
405      ;;
406
407    # Copy from archive machine or from buffer
408    IGCM_sys_Get|IGCM_sys_GetBuffer)
409      instrumentation=true
410      if [ ${#arguments[*]} -eq 2 ] ; then
411        source=${arguments[0]}
412        dest=${arguments[1]}
413        # Size of file whose name are stored in a variable
414        entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
415      elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
416        # IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/
417        # Keep the array name hosting the all list
418        eval set +A fileList \${${arguments[1]}}
419        # just need the first file to get the directory
420        source=${fileList[0]}
421        dest=${arguments[2]}
422        # Size of file whose name are stored in a list
423        entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
424      elif [ [ ${#arguments[*]} -ge 3 ] ; then
425       # All but the latest
426        fileList=${arguments[*]:0:${#arguments[*]}-1}
427        # just need the first file to get the directory
428        source=${arguments[0]}
429        # Nothing but the latest
430        dest=${arguments[${#arguments[*]}-1]}
431        # Size of file whose name are stored in a list
432        entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
433      fi
434      ;;
435
436    # Copy from compute node or copy to archive/buffer
437    IGCM_sys_Get_Master|IGCM_sys_Get_Dir|IGCM_sys_Put_Out|IGCM_sys_PutBuffer_Out)
438      instrumentation=true
439      source=${arguments[0]}
440      dest=${arguments[1]}
441      # Size of file whose name are stored in a variable
442      entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
443      ;;
444
445    # Rebuild command
446    IGCM_sys_rebuild|IGCM_sys_rebuild_station)
447      instrumentation=true
448      # All but the first
449      fileList=${arguments[*]:1:${#arguments[*]}-1}
450      # just need a file to get the directory
451      source=${arguments[1]}
452      # Nothing but the first
453      dest=${arguments[0]}
454      # Size of file whose name are stored in a list
455      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
456      ;;
457
458    # NCO commands
459    IGCM_sys_ncrcat|IGCM_sys_ncecat|IGCM_sys_ncra|IGCM_sys_ncks|IGCM_sys_cdo)
460      # Example of what we want to catch : only filenames in those command lines
461      # IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE}
462      # IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
463      # IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output}
464      # IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
465      instrumentation=true
466      keepGoing=true
467      prefix=.
468      i=0
469      while ( ${keepGoing} ) ; do
470        # the last one is not interesting
471        if [ ${i} -eq ${#arguments[*]}-1 ] ; then
472          keepGoing=false
473        # look after "-p" option. Path prefix is the following arguments
474        elif [ ${arguments[${i}]} = "-p" ] ; then
475          ((i = i + 1))
476          prefix=${arguments[${i}]}
477          ((i = i + 1))
478        elif [ ${i} -eq ${#arguments[*]}-1 ] ; then
479          keepGoing=false
480        # looking for files
481        elif [ -f ${prefix}/${arguments[${i}]} ] ; then
482          fileList="${fileList} ${prefix}/${arguments[${i}]}"
483          ((i = i + 1))
484        # other options are not interesting
485        else
486          ((i = i + 1))
487        fi
488      done
489
490      # i value is at least 1
491      # just need one file to get the directory
492      source=$( echo ${fileList} | gawk '{print $1}' )
493      # Nothing but the latest
494      dest=${arguments[${#arguments[*]}-1]}
495      # Size of file whose name are stored in a list
496      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
497      ;;
498    esac
499
500    # Print information related to instrumentation
501    ( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}
502
503    # Only cosmetics : stack file
504    decal=0
505    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
506      printf ' ' >> ${StackFileLocation}/${StackFileName}
507      (( decal = decal + 1 ))
508    done
509
510    if ( ${ExitFlag} ) ; then
511      # Inform the stack file
512      echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName}
513
514      if [ X${ActivateBigBro} = Xtrue ] ; then
515        # RabbitMQ message code "ERROR HAS BEEN TRIGGERED"
516        code=9000
517        # RabbitMQ message body
518        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
519        # Fill the rabbitMQ queue
520        IGCM_debug_sendAMQP
521      fi
522    else
523      # Inform the stack file
524      echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
525      if [ X${ActivateBigBro} = Xtrue ] ; then
526        # RabbitMQ message code "POPSTACK"
527        code=3000
528        # RabbitMQ message body
529        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
530        # Fill the rabbitMQ queue
531        IGCM_debug_sendAMQP
532      fi
533    fi
534
535    # Reset array if necessary
536    if [ ${IGCM_debug_LenStack} = 0 ]; then
537      #echo
538      #IGCM_debug_Print 3 "Clean stack array"
539      #echo
540      unset IGCM_debug_Stack
541      unset IGCM_debug_StackArgs
542      unset IGCM_debug_StackTiming
543      IGCM_debug_Stack[0]=${NULL_STR}
544      IGCM_debug_StackArgs[0]=${NULL_STR}
545      IGCM_debug_StackTiming[0]=${NULL_STR}
546    fi
547  fi
548  #IGCM_debug_CallStack
549}
550
551#D-#==================================================================
552#D-function IGCM_debug_ActivateBigBro
553#D-* Purpose: switch rabbitMQ on
554#D-
555function IGCM_debug_ActivateBigBro {
556  IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"
557
558# Message type standard fields:
559# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
560
561# Message type dictionnary and custom fields:
562# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
563
564  if [ X${BigBrother} = Xtrue ] ; then
565    # create a unique ID for this specific job
566    jobuid=$(uuidgen)
567
568    if ( ${FirstInitialize} ) ; then
569      # RabbitMQ message code "BEGIN A SIMULATION"
570      code=0000
571      # create and persist a unique id for this simulation
572      simuid=$(uuidgen)
573      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid}
574
575      # Standard fields for the first message
576      genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
577    else
578      # RabbitMQ message code "A NEW JOB IS RUNNING PART OF A SIMULATION"
579      code=1000
580      #
581      # retrieve this simulation's unique id
582      IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
583      simuid=${run_Configuration_simuid}
584
585      # Using standard fields for message others than the first one. Still subject to change
586      genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
587    fi
588
589    # RabbitMQ message body with specific fields associated message codes treated here
590    Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
591
592    # Fill the rabbitMQ queue (the config.card in use will be sent)
593    IGCM_debug_sendAMQP activate
594
595    # NOT VERY NICE BUT ... IT WORKS
596    # Be sure that the genericSimulationID will be small from now on
597    # Using standard fields for messages others than the first one. Still subject to change
598    genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
599
600    # Turn the flag on
601    ActivateBigBro=true
602  fi
603  IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
604}
605
606#D-#==================================================================
607#D-function IGCM_debug_Exit
608#D-* Purpose: Print Call Stack and set ExitFlag to true
609#D-
610function IGCM_debug_Exit {
611  IGCM_debug_PushStack "IGCM_debug_Exit"
612  echo "IGCM_debug_Exit : " "${@}"
613  echo
614  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
615  echo "!!   ERROR TRIGGERED    !!"
616  echo "!!   EXIT FLAG SET      !!"
617  echo "!------------------------!"
618  echo
619  IGCM_debug_CallStack
620  ExitFlag=true
621  IGCM_debug_PopStack "IGCM_debug_Exit"
622}
623
624#D-#==================================================
625#D-function IGCM_debug_Verif_Exit
626#D-* Purpose: exit with number 1 if ExitFlag is true
627#D-
628function IGCM_debug_Verif_Exit {
629  if ( ${ExitFlag} ) ; then
630    # Plan to send an email here with IGCM_sys_SendMail
631    if [ X${TaskType} != Xchecking ] ; then
632      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
633      echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
634      echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
635      echo "                        EXIT THE JOB."
636      echo
637      IGCM_debug_CallStack
638    fi
639
640    if [ X${ActivateBigBro} = Xtrue ] ; then
641      # RabbitMQ message code "EXIT THE JOBS BECAUSE ERROR(S) HAS BEEN TRIGGERED"
642      code=9999
643      # RabbitMQ message body
644      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
645      # Fill the rabbitMQ queue
646      IGCM_debug_sendAMQP
647    fi
648
649    # Not sure about that one ...
650    if ( $DEBUG_debug ) ; then
651      echo "Your files on ${R_OUT} :"
652      IGCM_sys_Tree ${R_SAVE}
653      echo
654    fi
655
656    # Mail notification
657    IGCM_sys_SendMail
658
659    # And Good Bye
660    date
661    exit 1
662  fi
663}
664
665#D-#==================================================
666#D-function IGCM_debug_Verif_Exit_Post
667#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
668#D-
669function IGCM_debug_Verif_Exit_Post {
670  if ( ${ExitFlag} ) ; then
671    echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
672    # If SpaceName is PROD then we stop if post_processing fails
673    # Plan to send an email here with IGCM_sys_SendMail
674    if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
675      echo "                        EXIT THE JOB."
676      echo
677      # Mail notification
678      #IGCM_sys_SendMailPost
679      # And Good Bye
680      date
681      exit 1
682    else
683      echo "Either inside config.card the variable SpaceName is not in PROD"
684      echo "or inside the main Job the variable JobType is not in RUN mode"
685      echo "              SO WE DO NOT EXIT THE JOB."
686      echo
687      date
688    fi
689  fi
690}
691
692#D-#==================================================================
693#D-function IGCM_debug_Print
694#D-* Purpose: Print arguments according to a level of verbosity.
695#D-
696function IGCM_debug_Print
697{
698  typeset level=$1
699  shift
700
701  if [ X"${1}" = X"-e" ]; then
702    typeset cmd_echo="echo -e"
703    shift
704  else
705    typeset cmd_echo="echo"
706  fi
707
708  if [ ${level} -le ${Verbosity} ] ; then
709    typeset i
710    case "${level}" in
711    1) for i in "$@" ; do
712      ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
713      done ;;
714    2) for i in "$@" ; do
715      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
716      done ;;
717    3) for i in "$@" ; do
718      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
719      done ;;
720    esac
721  fi
722}
723
724#D-#==================================================================
725#D-function IGCM_debug_PrintVariables
726#D-* Purpose: Print arguments when match a pattern
727#D-           according to a level of verbosity.
728function IGCM_debug_PrintVariables
729{
730  typeset level=$1
731  shift
732
733  list=$( set | grep ^$1 | sed -e "s/'//g" )
734
735  if [ "X${list}" != X ]  ; then
736    IGCM_debug_Print ${level} ${list}
737  fi
738}
739
740#D-#==================================================================
741#D-function IGCM_debug_PrintInfosActions
742#D-* Purpose: Print information related to instrumentation
743function IGCM_debug_PrintInfosActions
744{
745  typeset actionType=$1
746  typeset entitySize=$2
747  typeset start_ms=$3
748  typeset end_ms=$4
749
750  typeset dest=$5
751  typeset source=$6
752
753  typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
754  typeset name dirOut dirSource
755
756  name="doNotKnow"
757
758  diff_ms=$(( $end_ms - $start_ms ))
759  # echo "diff_ms=$diff_ms"
760
761  entitySizeKo=$( echo ${entitySize} | gawk -F"|" '{print $1}' )
762  # echo "entitySizeKo=$entitySizeKo"
763  entitySizeMo=$( echo ${entitySize} | gawk -F"|" '{print $2}' )
764
765  # flux en Ko / ms
766  flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" | bc )
767  # echo "flux_Ko_ms=$flux_Ko_ms"
768
769  # flux en Ko / s
770  flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
771  # echo "flux_Ko_s=$flux_Ko_s"
772
773  # flux en Mo / s
774  flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" | bc )
775  # echo "flux_Mo_s=$flux_Mo_s"
776
777  if [ -d $dest ] ; then
778    dirOut=$( readlink -f ${dest} )
779  else
780    dirOut=$( readlink -f $( dirname ${dest} ) )
781  fi
782
783  if [ -d $source ] ; then
784    dirSource=$( readlink -f ${source} )
785  else
786    dirSource=$( readlink -f $( dirname ${source} ) )
787  fi
788
789  instrumentationContent=$( echo "\"act\":\"${actionType}\",\"sz\":\"${entitySizeMo}\",\"ms\":\"${diff_ms}\",\"fx(ko)\":\"${flux_Ko_s}\",\"fx(mo)\":\"${flux_Mo_s}\",\"nm\":\"${name}\",\"dirSource\":\"${dirSource}\",\"dirOut\":\"${dirOut}\"" )
790
791  #echo "==>act:${actionType}|sz:${entitySizeMo}|ms:${diff_ms}|fx(ko):${flux_Ko_s}|fx(mo):${flux_Mo_s}|nm:${name}|dirSource:${dirSource}|dirOut:${dirOut}"
792
793  echo ${instrumentationContent}
794
795  # Inform the rabbitMQ queue
796  if [ X${ActivateBigBro} = Xtrue ] ; then
797    # RabbitMQ message body
798    Body=$( echo "{${genericSimulationID},\"msgCode\":\"7000\",\"msgUID\":\"$(uuidgen)\",${instrumentationContent},\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
799    # Fill the rabbitMQ queue
800    IGCM_debug_sendAMQP
801  fi
802}
803
804#D-#==================================================================
805#D-function IGCM_debug_Check
806#D- * Purpose: Check the present file by comparison with a reference file
807function IGCM_debug_Check
808{
809  #---------------------
810  if [ ! -n "${libIGCM}" ] ; then
811    echo "Check libIGCM_debug ..........................................[ FAILED ]"
812    echo "--Error--> libIGCM variable is not defined"
813    exit 2
814  fi
815
816  #---------------------
817  if [ ! -n "${Verbosity}" ] ; then
818    echo "Check libIGCM_debug ..........................................[ FAILED ]"
819    echo "--Error--> Verbosity variable is not defined"
820    exit 3
821  fi
822
823  #---------------------
824  ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh > /tmp/IGCM_debug_Test.$$.ref.failed 2>&1
825  sleep 2
826
827  # Remove date stamp.
828  sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g" /tmp/IGCM_debug_Test.$$.ref.failed > /tmp/IGCM_debug_Test.$$.ref.failed.nodate
829  mv /tmp/IGCM_debug_Test.$$.ref.failed.nodate /tmp/IGCM_debug_Test.$$.ref.failed
830
831  if diff /tmp/IGCM_debug_Test.$$.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref > /dev/null 2>&1 ; then
832    echo "Check libIGCM_debug ..............................................[ OK ]"
833    rm -f /tmp/IGCM_debug_Test.$$.ref.failed
834  else
835    echo "Check libIGCM_debug ..........................................[ FAILED ]"
836    echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
837    echo "           has produced the file IGCM_debug_Test.ref.failed"
838    echo "           Please analyse differences with the reference file by typing:"
839    echo "           diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
840    echo "           Report errors to the author: Patrick.Brockmann@cea.fr"
841    cat /tmp/IGCM_debug_Test.$$.ref.failed
842    exit 4
843  fi
844  #---------------------
845}
Note: See TracBrowser for help on using the repository browser.