source: trunk/libIGCM/libIGCM_debug/libIGCM_debug.ksh @ 1162

Last change on this file since 1162 was 1162, checked in by sdipsl, 9 years ago

Send metrics results to a central place

  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 31.4 KB
RevLine 
[913]1#!/bin/ksh
[2]2
3#**************************************************************
4# Author: Patrick Brockmann, Martial Mancip
[373]5# Contact: Patrick.Brockmann__at__cea.fr Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
[2]9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#==================================================
15# The documentation of this file can be automatically generated
[913]16# if you use the prefix #D- for comments to be extracted.
[2]17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#==================================================
19
20#==================================================
21# Add high level verbosity
22typeset -i Verbosity=${Verbosity:=3}
23
24#==================================================
25# DEBUG_debug
26# Add low level verbosity
[1083]27DEBUG_debug=${DEBUG_debug:=false}
[2]28
[872]29#==================================================
30# GENERATE RANDOM ERROR ; only apply if ( ${DEBUG_debug} )
31typeset RandomError=false
32
[2]33#==================================================
34# NULL_STR
35# Default null string
[913]36typeset -r NULL_STR="_0_"
[2]37
38#==================================================
39# libIGCM_CurrentTag
40# Current libIGCM tag, check compatibilty with *.card
[915]41typeset -r libIGCM_CurrentTag="1.0"
[2]42
43#==================================================
44# Exit Flag (internal debug)
45# When true, end the master loop AFTER SAVES FILES
46ExitFlag=false
47
48#==================================================
49# Declare a stack of functions calls
[59]50unset IGCM_debug_Stack
51unset IGCM_debug_StackArgs
[913]52unset IGCM_debug_StackTiming
[54]53IGCM_debug_Stack[0]=${NULL_STR}
54IGCM_debug_StackArgs[0]=${NULL_STR}
[913]55IGCM_debug_StackTiming[0]=${NULL_STR}
[2]56IGCM_debug_LenStack=0
57
58#D-#==================================================================
[913]59#D-function IGCM_debug_getDate_ms
60#D- * Purpose: Give number of milliseconds since 01-jan-1970
61function IGCM_debug_getDate_ms
62{
[926]63  typeset nanosecs ms
[913]64  # nano secondes since 01-jan-1970
65  nanosecs=$( date +%s%N )
66
67  # truncate the last 6 digits to get milliseconds since 01-jan-1970
68  ms=${nanosecs:0:${#nanosecs}-6}
69
70  echo "$ms"
71}
72
73#D-#==================================================================
74#D-function IGCM_debug_sizeOfTabContent
75#D- * Purpose: Give sumed size of a list of files
[924]76#D- * Usage: IGCM_debug_sizeOfTabContent entityList destination
77#D- *        where entityList is a list of files or directory
78#D- *        where dest is either a directory or a file name
[913]79function IGCM_debug_sizeOfTabContent
80{
[924]81  typeset entityListe destination iEntity sizeKo sumSizeKo sumSizeMo
82
83  eval set +A entityListe \${${1}}
[941]84  destination=${2}
[924]85  sumSizeKo=0
86
87  # Here we will try to compute size (file or directory size) from local path and not from archive.
[941]88  for ((i = 0; i < ${#entityListe[*]}; i += 1)) ; do
89    if [ -f ${entityListe[$i]} ] ; then
90      # One file or a bunch of files has been copied without renaming from a visible filesystem
91      iEntity=${entityListe[$i]}
92    elif [ -f ${entityListe[$i]##/*/} ] ; then
93      # One file or a bunch of files has been copied without renaming from an non visible filesystem
[924]94      # remove path /home/login/../ from entityListe elements
95      iEntity=${entityListe[$i]##/*/}
96    elif [ -f ${destination} ] ; then
[941]97      # a file has been copied and renamed
[924]98      iEntity=${destination}
99    elif [ -f ${destination}/${entityListe[$i]##/*/} ] ; then
100      # a copy in a directory but not in ${PWD}
101      iEntity=${destination}/${entityListe[$i]##/*/}
102    elif [ -d ${entityListe[$i]} ] ; then
[941]103      # a directory has been copied from a non remote place
[924]104      iEntity=${entityListe[$i]}
105    elif [ -d ${destination}/${entityListe[$i]##/*/} ] ; then
[941]106      # a directory has been copied from a remote archive and not renamed
[924]107      iEntity=${destination}/${entityListe[$i]##/*/}
108    elif [ -d ${destination} ] ; then
[941]109      # a directory has been copied from a remote archive and renamed
[924]110      iEntity=${destination}
[917]111    fi
[1083]112    sizeKo=$( du --apparent-size -skL ${iEntity} | gawk '{print $1}' )
[924]113    sumSizeKo=$(( $sumSizeKo + $sizeKo ))
[913]114  done
[924]115  sumSizeMo=$( echo "scale=6;${sumSizeKo}/1024" | bc )
116  echo "${sumSizeKo}|${sumSizeMo}"
[913]117}
118
119#D-#==================================================================
[983]120#D-function IGCM_debug_send_AMQP_msg__MAILTUNNEL
121#D- * Purpose: Take over AMQP C client using mail as a message recipient
122#D- * One argument : base64 encoded message
[1051]123#D- * Attach encoded config.card when starting the simulation
124
[983]125function IGCM_debug_send_AMQP_msg__MAILTUNNEL {
126
[987]127  typeset b64_encoded_msg mail_recipient
[1076]128  typeset buffer send_messages mail_frequency
[987]129  typeset last_mail_date__file
130
[983]131  b64_encoded_msg=$1
132
[1072]133  mail_recipient="superviseur@ipsl.jussieu.fr"
[983]134  send_messages=0
135  mail_frequency=3600 # in seconds
136  # use to keep track when was last mail sent (maybe to be replaced with global variable)
[1150]137  last_mail_date__file=${R_BUF}/.stamp.${config_UserChoices_TagName}.${config_UserChoices_JobName}
138  # use to accumulate messages before sending them
139  buffer=${R_BUF}/.buffer.${config_UserChoices_TagName}.${config_UserChoices_JobName}
[983]140
141  # init
142  if [ ! -f "${buffer}" ]; then
[1150]143    touch ${buffer}
[983]144  fi
145
146  if [ ! -f "${last_mail_date__file}" ]; then
[1150]147    touch ${last_mail_date__file}
[983]148  else
[1150]149    # compute last time the file was changed (in seconds)
[983]150    seconds_since_last_mail=$(( $(date +%s) - $(stat -c %Y ${last_mail_date__file}) ))
[997]151    # send message when exceeding threshold
152    [ ${seconds_since_last_mail} -gt ${mail_frequency} ] && send_messages=1
[983]153  fi
154
[997]155  # queue messages in the buffer
156  echo ${b64_encoded_msg} >> ${buffer}
157
158  # send mail
[1051]159
[1053]160  if [ X${initBigBro} = Xtrue ] ; then
[1087]161    #echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") > ${SUBMIT_DIR}/mail.txt
[1051]162    mailx -s "[TEMPORARY AMQP CHANNEL]" -a ${SUBMIT_DIR}/config.card.base64 ${mail_recipient} < ${buffer} # send buffer
163    rm -f $buffer ; touch ${buffer}                                    # clear buffer
164    touch ${last_mail_date__file}                                      # memorize last mail date
165    initBigBro=false
[1150]166  elif [ ${send_messages} -eq 1 ] ; then
[1087]167    #echo $(date +"%Y-%m-%dT%H:%M:%S.%N%z") >> ${SUBMIT_DIR}/mail.txt
[1051]168    mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient}  < ${buffer} # send buffer
[1150]169    rm -f ${buffer} ; touch ${buffer}                                  # flush the buffer
[1051]170    touch ${last_mail_date__file}                                      # memorize last mail date
[983]171  fi
[987]172
[1150]173  if [ X${FlushAMQP} = XTRUE ] ; then
174    mailx -s "[TEMPORARY AMQP CHANNEL]" ${mail_recipient}  < ${buffer} # send buffer
175    rm -f ${buffer}                                                    # cleaning behind us
176    rm -f ${last_mail_date__file}                                      # cleaning behind us
177  fi
178
[983]179  # Allways all good for now.
180  return 0
181}
182
183#D-#==================================================================
[1162]184#D-function IGCM_debug_sendAMQP_Metrics
185#D- * Purpose: Take over AMQP C client using mail as a message recipient
186#D- * One argument : Directory where metrics.json files can be found
187#D- * Attach encoded metrics.json files.
188
189function IGCM_debug_sendAMQP_Metrics {
190
191  typeset mail_recipient encodedBody
192  if [ X${ActivateBigBro} = Xtrue ] ; then
193    mail_recipient="superviseur@ipsl.jussieu.fr"
194    # Metrics tag on server side
195    code=7100
196    # Usual AMQP message to route messages on server side
197    encodedBody=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" |  base64 -w 0 )
198    # send mail
199    attachmentsOptions=""
200    for metricsFile in $( ls $1/*json ) ; do
201      attachmentsOptions="-a ${metricsFile} ${attachmentsOptions}"
202    done
203    echo ${encodedBody}|mailx -s "[TEMPORARY AMQP CHANNEL]" ${attachmentsOptions} ${mail_recipient}
204  fi
205
206  # Allways all good for now.
207  return 0
208}
209
210#D-#==================================================================
[913]211#D-function IGCM_debug_SendAMQP
212#D- * Purpose: Send body; encoded body and config.card to rabbitMQ
[1053]213function IGCM_debug_sendAMQP {
214
[913]215  typeset decal first additionnalOption encodedBody
216
217  # Encode message Body
218  encodedBody=$( echo "${Body}" | base64 -w 0 )
219
220  # Send config.card ?
221  if [ X${1} = Xactivate ] ; then
222    # Encode config.card
223    cat ${SUBMIT_DIR}/config.card | base64 -w 0 > ${SUBMIT_DIR}/config.card.base64
224    # Prepare additionnal option
225    additionnalOption="-f ${SUBMIT_DIR}/config.card.base64"
[1076]226    #
[1051]227    initBigBro=true
[913]228  else
229    additionnalOption=
[1051]230    #
231    initBigBro=false
[913]232  fi
233
234  # Only cosmetics : stack file
[1115]235  if [ X${ActivateStackFilling} = Xtrue ] ; then
236    decal=0
237    while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
238      printf ' ' >> ${StackFileLocation}/${StackFileName}
239      (( decal = decal + 1 ))
240    done
241    # Log to stack file using human readable format
242    echo "${Body}" >> ${StackFileLocation}/${StackFileName}
243  fi
[913]244
[983]245  # Log separately encoded AMQP message command for reuse in a mock up
[1120]246  #echo sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody} >> ${RUN_DIR_PATH}/send.AMQP.${config_UserChoices_JobName}.${config_UserChoices_ExperimentName}.${config_UserChoices_SpaceName}.${config_UserChoices_TagName}.${CumulPeriod}.history.txt
[913]247
248  # Send the message
[983]249  if [ X${BigBrotherChannel} = XMAIL ] ; then
250    IGCM_debug_send_AMQP_msg__MAILTUNNEL "${encodedBody}"
251    status=$?
252  else
253    sendAMQPMsg -h localhost -p 5672 ${additionnalOption} -b ${encodedBody}
254    status=$?
255  fi
256
[913]257  if [ ${status} -gt 0 ] ; then
258    IGCM_debug_Print 2 "IGCM_debug_Push/PopStack/ActivateBigBro : command sendAMQPMsg failed error code ${status}"
[1051]259    echo sendAMQPMsg -h localhost -p 5672 -b "${Body}"
[1090]260    exit 1
[913]261  fi
262}
263
264#D-#==================================================================
[2]265#D-function IGCM_debug_CallStack
[913]266#D-* Purpose: Print the call stack tree from the oldest to the youngest (opposite of the display standard)
[2]267#D-
268function IGCM_debug_CallStack {
[544]269  if ( $DEBUG_debug ) ; then
[913]270    # Cosmetics
[544]271    typeset i decal
[823]272    i=0
[544]273    until [ $i -eq ${IGCM_debug_LenStack} ]; do
274      decal=0
275      until [ $decal -eq ${i} ]; do
[869]276        printf -- ' '
[823]277        (( decal = decal + 1 ))
[544]278      done
[869]279      echo "$i - ${IGCM_debug_Stack[$(( $IGCM_debug_LenStack-$i-1 ))]}" "(${IGCM_debug_StackArgs[$(( $IGCM_debug_LenStack-$i-1 ))]})"
[823]280      ((i = i + 1))
[544]281    done
282  fi
[2]283}
284
285#D-#==================================================================
286#D-function IGCM_debug_PushStack
287#D-* Purpose: Push a function name in the stack
288#D-
289function IGCM_debug_PushStack {
[544]290  if ( $DEBUG_debug ) ; then
[913]291    typeset decal inputs startTime_ms
292
293    # Only cosmetics : stack file
[1115]294    if [ X${ActivateStackFilling} = Xtrue ] ; then
295      echo >> ${StackFileLocation}/${StackFileName}
296      decal=0
297      while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
298        printf ' ' >> ${StackFileLocation}/${StackFileName}
299        (( decal = decal + 1 ))
300      done
[2]301
[1115]302      # Fill the stack file
303      echo "> ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
304    fi
[926]305
[913]306    # Save input list in an indexed array
[823]307    INPUTS=( $@ )
[913]308
309    # Get timing information
310    startTime_ms=$( IGCM_debug_getDate_ms )
311
[544]312    # We add function call name on beginning of the stack
313    set +A IGCM_debug_Stack -- ${1} ${IGCM_debug_Stack[*]}
[2]314
[913]315    # Save timing in milliseconds in an indexed array
316    set +A IGCM_debug_StackTiming -- ${startTime_ms} ${IGCM_debug_StackTiming[*]}
317
[544]318    # We include the "null" Args in the beginning of the StackArgs
[913]319    set +A IGCM_debug_StackArgs ${NULL_STR} ${IGCM_debug_StackArgs[*]}
320
[544]321    # Then, we shift StackArgs tabular
[1065]322    # Replacing blank separated list by comma separated list of quoted elements (except the first and last element)
[913]323    if [ $# -gt 1 ]; then
[1065]324      IGCM_debug_StackArgs[0]=$(echo ${INPUTS[*]:1} | sed -e "s/\ /\",\"/g" )
[544]325    fi
[855]326
[1150]327    # Unplugged message 2000 handling for now. To ease downstream treatment.
328    #if [ X${ActivateBigBro} = Xtrue ] ; then
329    #  # RabbitMQ message code "PUSHSTACK"
330    #  code=2000
331    #  # RabbitMQ message body
332    #  Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${IGCM_debug_Stack[0]}\",\"arguments\":[\"${IGCM_debug_StackArgs[0]}\"],\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
333    #  # Fill the rabbitMQ queue
334    #  IGCM_debug_sendAMQP
335    #fi
[855]336
337    # Increment LenStack
[544]338    (( IGCM_debug_LenStack = IGCM_debug_LenStack + 1 ))
[2]339
[869]340    #IGCM_debug_CallStack
[544]341  fi
[2]342}
343
344#D-#==================================================================
345#D-function IGCM_debug_PopStack
346#D-* Purpose: Pop a function name in the stack
347#D-
348function IGCM_debug_PopStack {
[544]349  if ( $DEBUG_debug ) ; then
[926]350    typeset i decal command arguments startTime_ms endTime_ms
[941]351    typeset instrumentation dest prefix
[926]352    # they are not typeset because they are send "by adress" to son functions
353    # we unset them to avoid "memory effect"
354    unset fileList source
[913]355
356    # INTRODUCE SIMPLE ERROR GENERATOR TO TEST SUPERVISOR
357    # PROBABILITY ERROR IS 0.0001 PER COMMAND OR FUNCTION CALL
358    # THERE ARE ~500 COMMAND OR FUNCTION CALL PER PERIOD
359    if ( ${RandomError} ) ; then
360      if [ $((RANDOM%10000)) -le 10 ] ; then
361        IGCM_debug_Print 1 "Random error has been triggered"
[1115]362        if [ X${ActivateStackFilling} = Xtrue ] ; then
363          echo "RANDOM ERROR" >> ${StackFileLocation}/${StackFileName}
364        fi
[913]365        ExitFlag=true
366      fi
367    fi
368
[544]369    if [ "${IGCM_debug_Stack[0]}" = "${1}" ]; then
[913]370      # Everything is cool
371
372      # Get timing information
373      endTime_ms=$( IGCM_debug_getDate_ms )
374
375      # Save Stack information before poping the stack
376      command=${IGCM_debug_Stack[0]}
377
[1084]378      # Go from comma separated list of quoted elements (except the first and the last element)
379      # to unquoted space separated elements in an array
[1083]380      set -A arguments -- $( echo ${IGCM_debug_StackArgs[0]} | sed -e "s/\",\"/\ /g" )
[913]381
382      # Save Stack information before poping the stack
383      startTime_ms=${IGCM_debug_StackTiming[0]}
384
385      # Pop the stack
[823]386      (( IGCM_debug_LenStack = IGCM_debug_LenStack - 1 ))
387      set -A IGCM_debug_Stack -- ${IGCM_debug_Stack[*]:1}
388      set -A IGCM_debug_StackArgs -- ${IGCM_debug_StackArgs[*]:1}
[913]389      set -A IGCM_debug_StackTiming -- ${IGCM_debug_StackTiming[*]:1}
[544]390    else
391      echo 'IGCM_debug_Exit : stack is corrupted ! LenStack =' ${IGCM_debug_LenStack}
392      IGCM_debug_Exit $@
393    fi
[913]394
[914]395    # Special actions depending on command to prepare IGCM_debug_PrintInfosActions call
[913]396    # We are interested in:
397    #  0. Which command performs the work
398    #  1. Size of entity we are working with
399    #  2. Where are we reading
400    #  3. Where are we writing
401    #  4. How long it took
402
[915]403    instrumentation=false
404
[913]405    case ${command} in
[925]406    # Classical copy (only files are given to IGCM_sys_Cp as options)
407    IGCM_sys_Cp)
408      instrumentation=true
409      # All but the latest
410      fileList=${arguments[*]:0:${#arguments[*]}-1}
411      # just need the first file to get the directory
412      source=${arguments[0]}
413      # Nothing but the latest
414      dest=${arguments[${#arguments[*]}-1]}
415      # Size of file whose name are stored in a list
416      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
417      ;;
418
[913]419    # Copy from archive machine or from buffer
420    IGCM_sys_Get|IGCM_sys_GetBuffer)
[915]421      instrumentation=true
[913]422      if [ ${#arguments[*]} -eq 2 ] ; then
423        source=${arguments[0]}
424        dest=${arguments[1]}
425        # Size of file whose name are stored in a variable
[917]426        entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
[913]427      elif ( [ ${#arguments[*]} -eq 3 ] && [ ${arguments[0]} = '/l' ] ) ; then
[936]428        # IGCM_sys_Get /l liste_file[*] /ccc/scratch/cont003/dsm/p86denv/RUN_DIR/985998_14754/
[913]429        # Keep the array name hosting the all list
[936]430        eval set +A fileList \${${arguments[1]}}
[913]431        # just need the first file to get the directory
[936]432        source=${fileList[0]}
[931]433        dest=${arguments[2]}
[934]434        # Size of file whose name are stored in a list
[936]435        entitySize=$( IGCM_debug_sizeOfTabContent fileList[*] ${dest} )
[913]436      elif [ [ ${#arguments[*]} -ge 3 ] ; then
437       # All but the latest
[916]438        fileList=${arguments[*]:0:${#arguments[*]}-1}
[913]439        # just need the first file to get the directory
440        source=${arguments[0]}
441        # Nothing but the latest
442        dest=${arguments[${#arguments[*]}-1]}
443        # Size of file whose name are stored in a list
[917]444        entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
[913]445      fi
446      ;;
447
[925]448    # Copy from compute node or copy to archive/buffer
449    IGCM_sys_Get_Master|IGCM_sys_Get_Dir|IGCM_sys_Put_Out|IGCM_sys_PutBuffer_Out)
[924]450      instrumentation=true
[916]451      source=${arguments[0]}
[924]452      dest=${arguments[1]}
453      # Size of file whose name are stored in a variable
454      entitySize=$( IGCM_debug_sizeOfTabContent source ${dest} )
[913]455      ;;
456
457    # Rebuild command
458    IGCM_sys_rebuild|IGCM_sys_rebuild_station)
[915]459      instrumentation=true
[913]460      # All but the first
461      fileList=${arguments[*]:1:${#arguments[*]}-1}
462      # just need a file to get the directory
463      source=${arguments[1]}
464      # Nothing but the first
465      dest=${arguments[0]}
466      # Size of file whose name are stored in a list
[917]467      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
[913]468      ;;
[941]469
[926]470    # NCO commands
471    IGCM_sys_ncrcat|IGCM_sys_ncecat|IGCM_sys_ncra|IGCM_sys_ncks|IGCM_sys_cdo)
472      # Example of what we want to catch : only filenames in those command lines
473      # IGCM_sys_ncrcat -O -v ${list_var_final_ncrcat} ${OUT_SE[*]} ${RESULT_SE}
474      # IGCM_sys_ncrcat --hst -v ${liste_coord}${var} ${file1} ${liste_file_tmp[*]} ${file_out}
475      # IGCM_sys_ncrcat -p ${dir} ${liste_file_tmp} --output ${output}
476      # IGCM_sys_ncrcat -x -v ${list_var} -p ${dir} ${liste_file_tmp} --output ${output}
477      instrumentation=true
[941]478      keepGoing=true
479      prefix=.
[926]480      i=0
481      while ( ${keepGoing} ) ; do
[941]482        # the last one is not interesting
483        if [ ${i} -eq ${#arguments[*]}-1 ] ; then
484          keepGoing=false
485        # look after "-p" option. Path prefix is the following arguments
486        elif [ ${arguments[${i}]} = "-p" ] ; then
[926]487          ((i = i + 1))
[941]488          prefix=${arguments[${i}]}
489          ((i = i + 1))
490        elif [ ${i} -eq ${#arguments[*]}-1 ] ; then
[926]491          keepGoing=false
[941]492        # looking for files
493        elif [ -f ${prefix}/${arguments[${i}]} ] ; then
494          fileList="${fileList} ${prefix}/${arguments[${i}]}"
495          ((i = i + 1))
496        # other options are not interesting
[926]497        else
498          ((i = i + 1))
499        fi
500      done
[941]501
[926]502      # i value is at least 1
503      # just need one file to get the directory
[941]504      source=$( echo ${fileList} | gawk '{print $1}' )
[926]505      # Nothing but the latest
506      dest=${arguments[${#arguments[*]}-1]}
507      # Size of file whose name are stored in a list
508      entitySize=$( IGCM_debug_sizeOfTabContent fileList ${dest} )
509      ;;
[913]510    esac
511
512    # Print information related to instrumentation
[915]513    ( ${instrumentation} ) && IGCM_debug_PrintInfosActions ${command} ${entitySize} ${startTime_ms} ${endTime_ms} ${dest} ${source}
[913]514
515    # Only cosmetics : stack file
[1115]516    if [ X${ActivateStackFilling} = Xtrue ] ; then
517      decal=0
518      while [ ${decal} -lt ${IGCM_debug_LenStack} ]; do
519        printf ' ' >> ${StackFileLocation}/${StackFileName}
520        (( decal = decal + 1 ))
521      done
522    fi
[2]523
[855]524    if ( ${ExitFlag} ) ; then
525      # Inform the stack file
[1115]526      if [ X${ActivateStackFilling} = Xtrue ] ; then
527        echo '!!! ExitFlag has been activated !!!' >> ${StackFileLocation}/${StackFileName}
528      fi
[874]529
[855]530      if [ X${ActivateBigBro} = Xtrue ] ; then
[1076]531        # RabbitMQ message code "ERROR HAS BEEN TRIGGERED"
[855]532        code=9000
[913]533        # RabbitMQ message body
[1065]534        Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[913]535        # Fill the rabbitMQ queue
536        IGCM_debug_sendAMQP
[855]537      fi
538    else
539      # Inform the stack file
[1115]540      if [ X${ActivateStackFilling} = Xtrue ] ; then
541        echo "< ${IGCM_debug_LenStack} : ${@}" >> ${StackFileLocation}/${StackFileName}
542      fi
543
[1150]544      # Unplugged message 3000 handling for now. To ease downstream treatment.
545      #if [ X${ActivateBigBro} = Xtrue ] ; then
546      #  # RabbitMQ message code "POPSTACK"
547      #  code=3000
548      #  # RabbitMQ message body
549      #  Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"nesting\":\"${IGCM_debug_LenStack}\",\"command\":\"${command}\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
550      #  # Fill the rabbitMQ queue
551      #  IGCM_debug_sendAMQP
552      #fi
[855]553    fi
554
[913]555    # Reset array if necessary
[544]556    if [ ${IGCM_debug_LenStack} = 0 ]; then
557      #echo
558      #IGCM_debug_Print 3 "Clean stack array"
559      #echo
560      unset IGCM_debug_Stack
561      unset IGCM_debug_StackArgs
[913]562      unset IGCM_debug_StackTiming
[544]563      IGCM_debug_Stack[0]=${NULL_STR}
564      IGCM_debug_StackArgs[0]=${NULL_STR}
[913]565      IGCM_debug_StackTiming[0]=${NULL_STR}
[2]566    fi
[544]567  fi
[869]568  #IGCM_debug_CallStack
[2]569}
570
571#D-#==================================================================
[855]572#D-function IGCM_debug_ActivateBigBro
573#D-* Purpose: switch rabbitMQ on
574#D-
575function IGCM_debug_ActivateBigBro {
576  IGCM_debug_PushStack "IGCM_debug_ActivateBigBro"
577
[1051]578# Message type standard fields:
579# https://github.com/Prodiguer/prodiguer-docs/wiki/MQ-Standard-Message-Fields
580
581# Message type dictionnary and custom fields:
582# https://github.com/Prodiguer/prodiguer-docs/wiki/Monitoring-Message-Dictionary
583
[868]584  if [ X${BigBrother} = Xtrue ] ; then
[1051]585    # create a unique ID for this specific job
586    jobuid=$(uuidgen)
[913]587
[855]588    if ( ${FirstInitialize} ) ; then
[1076]589      # RabbitMQ message code "BEGIN A SIMULATION"
[855]590      code=0000
[1051]591      # create and persist a unique id for this simulation
592      simuid=$(uuidgen)
593      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration simuid ${simuid}
594      # Standard fields for the first message
595      genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"activity\":\"IPSL\",\"name\":\"${config_UserChoices_JobName}\",\"cumulPeriod\":\"${CumulPeriod}\",\"experiment\":\"${config_UserChoices_ExperimentName}\",\"space\":\"${config_UserChoices_SpaceName}\",\"model\":\"${config_UserChoices_TagName}\",\"startDate\":\"${config_UserChoices_DateBegin}\",\"endDate\":\"${config_UserChoices_DateEnd}\",\"login\":\"${LOGIN}\",\"centre\":\"${CENTER}\",\"machine\":\"${MASTER}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
[1087]596      # RabbitMQ message body with specific fields associated message codes treated here
597      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
598      # Fill the rabbitMQ queue (the config.card in use will be sent)
599      IGCM_debug_sendAMQP activate
[855]600    else
[1076]601      # RabbitMQ message code "A NEW JOB IS RUNNING PART OF A SIMULATION"
[855]602      code=1000
[1076]603      # retrieve this simulation's unique id
[1051]604      IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration simuid
[1086]605      simuid=${run_Configuration_simuid}
[1076]606      # Using standard fields for message others than the first one. Still subject to change
[1051]607      genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
[1087]608      # RabbitMQ message body with specific fields associated message codes treated here
609      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
610      # Fill the rabbitMQ queue
611      IGCM_debug_sendAMQP
[855]612    fi
[1051]613
[1076]614    # NOT VERY NICE BUT ... IT WORKS
615    # Be sure that the genericSimulationID will be small from now on
616    # Using standard fields for messages others than the first one. Still subject to change
617    genericSimulationID=$( echo "\"msgApplication\":\"monitoring\",\"msgProducer\":\"libigcm\",\"cumulPeriod\":\"${CumulPeriod}\",\"simuid\":\"${simuid}\",\"jobuid\":\"${jobuid}\"" )
618
[913]619    # Turn the flag on
[855]620    ActivateBigBro=true
621  fi
622  IGCM_debug_PopStack "IGCM_debug_ActivateBigBro"
623}
624
625#D-#==================================================================
[2]626#D-function IGCM_debug_Exit
627#D-* Purpose: Print Call Stack and set ExitFlag to true
628#D-
629function IGCM_debug_Exit {
[544]630  IGCM_debug_PushStack "IGCM_debug_Exit"
631  echo "IGCM_debug_Exit : " "${@}"
[913]632  echo
[894]633  echo "!!!!!!!!!!!!!!!!!!!!!!!!!!"
[913]634  echo "!!   ERROR TRIGGERED    !!"
635  echo "!!   EXIT FLAG SET      !!"
636  echo "!------------------------!"
637  echo
[894]638  IGCM_debug_CallStack
[544]639  ExitFlag=true
640  IGCM_debug_PopStack "IGCM_debug_Exit"
[2]641}
642
643#D-#==================================================
644#D-function IGCM_debug_Verif_Exit
645#D-* Purpose: exit with number 1 if ExitFlag is true
646#D-
647function IGCM_debug_Verif_Exit {
[544]648  if ( ${ExitFlag} ) ; then
649    # Plan to send an email here with IGCM_sys_SendMail
[775]650    if [ X${TaskType} != Xchecking ] ; then
651      IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "Fatal"
[894]652      echo "IGCM_debug_Verif_Exit : Something wrong happened previously."
653      echo "IGCM_debug_Verif_Exit : ERROR and EXIT keyword will help find out where."
[775]654      echo "                        EXIT THE JOB."
655      echo
[869]656      IGCM_debug_CallStack
[775]657    fi
[874]658
[855]659    if [ X${ActivateBigBro} = Xtrue ] ; then
[1076]660      # RabbitMQ message code "EXIT THE JOBS BECAUSE ERROR(S) HAS BEEN TRIGGERED"
[855]661      code=9999
[913]662      # RabbitMQ message body
[1056]663      Body=$( echo "{${genericSimulationID},\"msgCode\":\"${code}\",\"msgUID\":\"$(uuidgen)\",\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[1153]664      # To be sure we flush message buffer and clean up behind us before the end of the simulation
665      FlushAMQP=TRUE
[913]666      # Fill the rabbitMQ queue
667      IGCM_debug_sendAMQP
[855]668    fi
[874]669
[544]670    # Mail notification
671    IGCM_sys_SendMail
[913]672
[544]673    # And Good Bye
674    date
675    exit 1
676  fi
[2]677}
678
679#D-#==================================================
680#D-function IGCM_debug_Verif_Exit_Post
681#D-* Purpose: exit with number 1 if ExitFlag is true for Post-treatment
682#D-
683function IGCM_debug_Verif_Exit_Post {
[544]684  if ( ${ExitFlag} ) ; then
[765]685    echo "IGCM_debug_Verif_Exit_Post : Something wrong happened."
[544]686    # If SpaceName is PROD then we stop if post_processing fails
687    # Plan to send an email here with IGCM_sys_SendMail
[706]688    if [ X${config_UserChoices_SpaceName} = XPROD ] ; then
[544]689      echo "                        EXIT THE JOB."
690      echo
691      # Mail notification
692      #IGCM_sys_SendMailPost
693      # And Good Bye
694      date
695      exit 1
696    else
697      echo "Either inside config.card the variable SpaceName is not in PROD"
[765]698      echo "or inside the main Job the variable JobType is not in RUN mode"
[544]699      echo "              SO WE DO NOT EXIT THE JOB."
700      echo
701      date
[2]702    fi
[544]703  fi
[2]704}
705
706#D-#==================================================================
707#D-function IGCM_debug_Print
708#D-* Purpose: Print arguments according to a level of verbosity.
709#D-
710function IGCM_debug_Print
711{
[544]712  typeset level=$1
713  shift
714
715  if [ X"${1}" = X"-e" ]; then
716    typeset cmd_echo="echo -e"
[2]717    shift
[544]718  else
719    typeset cmd_echo="echo"
720  fi
[2]721
[544]722  if [ ${level} -le ${Verbosity} ] ; then
723    typeset i
724    case "${level}" in
725    1) for i in "$@" ; do
[734]726      ${cmd_echo} $(date +"%Y-%m-%d %T") "--Debug1-->" ${i}
[913]727      done ;;
[544]728    2) for i in "$@" ; do
[734]729      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------Debug2-->" ${i}
[913]730      done ;;
[544]731    3) for i in "$@" ; do
[734]732      ${cmd_echo} $(date +"%Y-%m-%d %T") "--------------Debug3-->" ${i}
[913]733      done ;;
[544]734    esac
735  fi
[2]736}
737
738#D-#==================================================================
739#D-function IGCM_debug_PrintVariables
740#D-* Purpose: Print arguments when match a pattern
741#D-           according to a level of verbosity.
742function IGCM_debug_PrintVariables
743{
[544]744  typeset level=$1
745  shift
[2]746
[830]747  list=$( set | grep ^$1 | sed -e "s/'//g" )
[54]748
[544]749  if [ "X${list}" != X ]  ; then
750    IGCM_debug_Print ${level} ${list}
751  fi
[2]752}
753
754#D-#==================================================================
[914]755#D-function IGCM_debug_PrintInfosActions
[913]756#D-* Purpose: Print information related to instrumentation
757function IGCM_debug_PrintInfosActions
758{
759  typeset actionType=$1
760  typeset entitySize=$2
761  typeset start_ms=$3
762  typeset end_ms=$4
763
764  typeset dest=$5
765  typeset source=$6
766
767  typeset diff_ms entitySizeKo entitySizeMo flux_Ko_ms flux_Ko_s flux_Mo_s
[1090]768  typeset dirFrom dirTo
[913]769
770  diff_ms=$(( $end_ms - $start_ms ))
771  # echo "diff_ms=$diff_ms"
772
773  entitySizeKo=$( echo ${entitySize} | gawk -F"|" '{print $1}' )
774  # echo "entitySizeKo=$entitySizeKo"
775  entitySizeMo=$( echo ${entitySize} | gawk -F"|" '{print $2}' )
776
777  # flux en Ko / ms
778  flux_Ko_ms=$( echo "scale=6;${entitySizeKo}/${diff_ms}" | bc )
779  # echo "flux_Ko_ms=$flux_Ko_ms"
780
781  # flux en Ko / s
782  flux_Ko_s=$(( $flux_Ko_ms * 1000 ))
783  # echo "flux_Ko_s=$flux_Ko_s"
784
785  # flux en Mo / s
786  flux_Mo_s=$( echo "scale=6;${flux_Ko_s}/1024" | bc )
787  # echo "flux_Mo_s=$flux_Mo_s"
788
789  if [ -d $dest ] ; then
[1090]790    dirTo=$( readlink -f ${dest} )
[913]791  else
[1090]792    dirTo=$( readlink -f $( dirname ${dest} ) )
[913]793  fi
794
795  if [ -d $source ] ; then
[1090]796    dirFrom=$( readlink -f ${source} )
[913]797  else
[1090]798    dirFrom=$( readlink -f $( dirname ${source} ) )
[913]799  fi
800
[1094]801  instrumentationContent=$( echo "\"actionName\":\"${actionType}\",\"size_Mo\":\"${entitySizeMo}\",\"duration_ms\":\"${diff_ms}\",\"throughput_Mo_s\":\"${flux_Mo_s}\",\"dirFrom\":\"${dirFrom}\",\"dirTo\":\"${dirTo}\"" )
[1050]802
[1115]803  if [ X${ActivateStackFilling} = Xtrue ] ; then
804    echo "{${instrumentationContent}}" >> ${StackFileLocation}/${StackFileName}
805  fi
[1050]806
807  # Inform the rabbitMQ queue
808  if [ X${ActivateBigBro} = Xtrue ] ; then
809    # RabbitMQ message body
[1065]810    Body=$( echo "{${genericSimulationID},\"msgCode\":\"7000\",\"msgUID\":\"$(uuidgen)\",${instrumentationContent},\"msgTimestamp\":\"$( date +"%Y-%m-%dT%H:%M:%S.%N%z" )\"}" )
[1050]811    # Fill the rabbitMQ queue
812    IGCM_debug_sendAMQP
813  fi
[913]814}
815
816#D-#==================================================================
[2]817#D-function IGCM_debug_Check
818#D- * Purpose: Check the present file by comparison with a reference file
819function IGCM_debug_Check
820{
[544]821  #---------------------
822  if [ ! -n "${libIGCM}" ] ; then
823    echo "Check libIGCM_debug ..........................................[ FAILED ]"
824    echo "--Error--> libIGCM variable is not defined"
825    exit 2
826  fi
[2]827
[544]828  #---------------------
829  if [ ! -n "${Verbosity}" ] ; then
830    echo "Check libIGCM_debug ..........................................[ FAILED ]"
831    echo "--Error--> Verbosity variable is not defined"
832    exit 3
833  fi
[2]834
[544]835  #---------------------
[1118]836  # Need to remove timestamps here
837  diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh | sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g") > /dev/null 2>&1
[1106]838  status=$?
[2]839
[1118]840  if [ ${status} -eq 0 ] ; then
[544]841    echo "Check libIGCM_debug ..............................................[ OK ]"
842  else
843    echo "Check libIGCM_debug ..........................................[ FAILED ]"
844    echo "--Error--> Execution of ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh"
845    echo "           has produced the file IGCM_debug_Test.ref.failed"
846    echo "           Please analyse differences with the reference file by typing:"
847    echo "           diff IGCM_debug_Test.ref.failed ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref"
848    echo "           Report errors to the author: Patrick.Brockmann@cea.fr"
[1118]849    diff ${libIGCM}/libIGCM_debug/IGCM_debug_Test.ref <(${libIGCM}/libIGCM_debug/IGCM_debug_Test.ksh | sed -e "s:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]\:[0-9][0-9]\:[0-9][0-9] ::g")
[544]850    exit 4
851  fi
852  #---------------------
[2]853}
Note: See TracBrowser for help on using the repository browser.