source: trunk/libIGCM/AA_RunChecker @ 754

Last change on this file since 754 was 754, checked in by labetoulle, 12 years ago

Print the last 10 (default value, can be changed with -j option) rebuild, pack_* and atlas jobs instead of only the last one of each.
And use real boolean variables when needed.
  • Property svn:executable set to *
  • Property svn:keywords set to Revision Author Date
File size: 12.9 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sonia Labetoulle
5# Contact: sonia.labetoulle__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2012)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#======================================================================#
15# Display a report on how a simulation is faring.
16
17
18function ChangeUsr {
19
20#  echo ">$1<"
21
22  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
23                -e "s/${CurrentGrp}/${TargetGrp}/"
24
25}
26
27
28function SearchCatalog {
29
30  typeset num
31  unset SUBMIT_DIR
32
33  NbOcc=$( awk -v JobName=${JobName} \
34           'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
35
36  if [ ${NbOcc} -eq 0 ] ; then
37    if ( ${fg_new} ) ; then
38      echo "${JobName} not in Catalog, we'll try to find it in ${WORKDIR}"
39     
40      set -A FileList
41      FileList=$( find ${WORKDIR}/ -name "Job_${JobName}" )
42      echo $FileList
43      (( NbOcc = ${#FileList[@]} ))
44    else
45      echo "${JobName} not in Catalog."
46      echo "Add job_name, user_name, host_name and submit_dir in ${SimuCatalog}"
47      exit
48    fi
49  fi
50
51
52  if [ ${NbOcc} -eq 0 ] ; then
53    echo "${JobName} not found."
54    exit
55  else
56    if [ ${NbOcc} -eq 1 ] ; then
57
58      if ( [ ${TargetUsr} == $( awk -v JobName=${JobName} '$1 ~ JobName {print $2}' ${SimuCatalog} ) ] && [ ${HostName} == $( awk -v JobName=${JobName} '$1 ~ JobName {print $3}' ${SimuCatalog} ) ] ) ; then
59        JobName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
60        TargetUsr=$( awk -v JobName=${JobName} '$1 ~ JobName {print $2}' ${SimuCatalog} )
61        HostName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $3}' ${SimuCatalog} )
62        SUBMIT_DIR=$( awk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
63      else
64        echo "${JobName} not in Catalog."
65        exit
66      fi
67    fi
68  fi
69
70}
71
72
73function AffichResult {
74
75  fg_first=false
76  fg_last=false
77  fg_job=false
78
79  while [ $# -ne 0 ] ; do
80    case ${1} in
81      -f|--first)
82        fg_first=true
83        shift 1 ;;
84      -l|--last)
85        fg_last=true
86        shift 1 ;;
87      -j|--job)
88        fg_job=true
89        shift 1 ;;
90      -*)
91        echo "usage: ${0}"
92        echo "       options = -f; -l"
93        exit ;;
94      *)
95        break ;;
96    esac
97  done
98
99
100  # Define colors
101  # =============
102  ColEsc="\033["
103  ColNon="${ColEsc}0m"       # Return to normal
104  ColExp="${ColEsc}1m"       # Blanc - gras
105  ColFat="${ColEsc}1;31m"    # Fatal
106  ColCpl="${ColEsc}1;32m"    # Completed
107  ColAtt="${ColEsc}1;30m"    # Waiting
108  ColDef="${ColEsc}1;34m"    # Default
109  ColRbl="${ColEsc}31m"      # Rebuild
110
111
112 
113  # Print header
114  # ============
115  if ( ${fg_first} ) ; then
116    Dum=""
117    (( len = 67 - ${#JobName} ))
118    echo "|===========================================================================================================|"
119    printf "| JobName = ${ColExp}%-${#JobName}s${ColNon}" ${JobName}
120    printf "%-${len}s" ${Dum} 
121    printf "run.card : ${ColExp}%-17s${ColNon}|\n" "${LastWrite} "
122    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
123    echo "|                         |             |                         |             |     Pending Rebuilds      |"
124    echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
125    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
126
127    printf "| %-10s - %-10s | " \
128           $DateBegin $DateEnd 
129
130    case $PeriodState in
131      Fatal)
132        Color=${ColFat}
133        break ;;
134      Completed)
135        Color=${ColCpl}
136        break ;;
137      Waiting|OnQueue)
138        Color=${ColAtt}
139        break ;;
140      *)
141        Color=${ColDef}
142        break ;;
143    esac
144    printf "${Color}%-11s${ColNon} | " $PeriodState
145
146    printf "%-10s - %-10s | %11s | " \
147           $PeriodDateBegin $PeriodDateEnd $CumulPeriod
148
149    if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
150      printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
151             $NbRebuild $FirstRebuild $LastRebuild
152    else
153      printf "%3s : %-8s : %-8s |\n" \
154             $NbRebuild $FirstRebuild $LastRebuild
155    fi
156
157    printf "|-----------------------------------------------------------------------------------------------------------|\n"
158    printf "|                                                      Last                                                 |\n"
159    printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
160    printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
161
162    return
163  fi
164
165  # Print Post-processing job status
166  # ================================
167  if ( ${fg_job} ) ; then
168    printf "|"
169
170    # Print rebuild and pack jobs
171    # ---------------------------
172    for JobType in ${JobType_list[*]} ; do
173      eval Date=\${${JobType}_Date[${ind}]}
174      eval Status=\${${JobType}_Status[${ind}]}
175
176      if [ X${Status} == XOK  ] ; then
177        Color=${ColCpl}
178      else
179        Color=${ColFat}
180      fi
181      printf "     ${Color}%-8s${ColNon}     |" ${Date}
182    done
183
184    Color=${ColExp}
185
186    # Print monitoring jobs
187    # ---------------------
188    JobType=monitoring
189    if [ $ind -eq 0 ] ; then
190      eval Date=\${${JobType}_Date}
191    else
192      Date=""
193    fi
194    printf "     ${Color}%-4s${ColNon}     |" ${Date}
195
196    # Print atlas jobs
197    # ----------------
198    JobType=atlas
199    eval Date=\${${JobType}_Date[${ind}]}
200    printf "  ${Color}%-12s${ColNon}  |" ${Date}
201
202    printf "\n"
203
204    return
205  fi
206
207  # Print footer
208  # ============
209  if ( ${fg_last} ) ; then
210    printf "|===========================================================================================================|\n"
211    date +"${DateFormat}"
212    return
213  fi
214
215}
216
217#======================================================================#
218
219#D- Task type (computing or post-processing)
220TaskType=post-processing
221typeset -i Verbosity=0
222
223CurrentUsr=$( whoami )
224CurrentGrp=$( groups $CurrentUsr | awk '{print $3}' )
225
226if ( [ $# -eq 0 ] ) ; then
227  $0 -h
228  exit
229fi
230
231# Arguments
232# =========
233# Default argument values
234# -----------------------
235TargetUsr=${CurrentUsr}
236HostName=$( hostname | sed -e "s/[0-9].*//" )
237
238fg_color=true
239fg_new=false
240fg_quiet=false
241NbHisto=10
242
243# Get arguments from command line
244# -------------------------------
245while [ $# -ne 0 ] ; do
246  case $1 in
247    -h|--help|-help)
248      echo "usage: $0 [-u TargetUsr] JobName"
249      exit ;;
250    -b|-nocolor)
251      fg_color=false
252      shift 1 ;;
253    -n|-new)
254      fg_new=true
255      shift 1 ;;
256    -q|-quiet)
257      fg_quiet=true
258      shift 1 ;;
259    -p|-config-path)
260      ConfigPath="$2"
261      shift 2 ;;
262    -j|-job-number)
263      NbHisto="$2"
264      shift 2 ;;
265    -u|-user)
266      TargetUsr="$2"
267      shift 2 ;;
268    -*)
269      $0 -h
270      exit ;;
271    *)
272      break ;;
273  esac
274done
275
276if [ $# -lt 1 ] ; then
277  $0 -h
278  exit
279fi
280
281
282# Load libIGCM library
283# ====================
284libIGCM=${libIGCM:=::modipsl::/libIGCM}
285
286. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
287     ( ${DEBUG_debug} ) && IGCM_debug_Check
288. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
289     ( ${DEBUG_debug} ) && IGCM_card_Check
290. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
291     ( ${DEBUG_debug} ) && IGCM_date_Check
292#-------
293. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
294. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
295
296
297JobName=$1
298
299if ( ${fg_quiet} ) ; then
300  NbHisto=1
301fi
302
303echo "TargetUsr = ${TargetUsr}"
304echo "HostName = ${HostName}"
305
306TargetGrp=$( groups $TargetUsr | awk '{print $3}' )
307
308# Define the catalog in which the known simulations are stored
309SimuCatalog="$( ccc_home )/.simucatalog.dat"
310if [ ! -s ${SimuCatalog} ] ; then
311  touch ${SimuCatalog}
312fi
313
314# Date format
315DateFormat="%d/%m/%y %R:%S"
316
317# Find SUBMIT_DIR in catalog
318# ==========================
319SearchCatalog
320
321if [ ! X${SUBMIT_DIR} == X ] ; then
322
323  echo "Submit:  >${SUBMIT_DIR}<"
324  cd $SUBMIT_DIR
325
326
327  # Extract usefull information from run.card and config.card
328  # =========================================================
329
330  RunFile="${SUBMIT_DIR}/run.card"
331  ConfFile="${SUBMIT_DIR}/config.card"
332
333  IGCM_config_CommonConfiguration ${SUBMIT_DIR}/config.card
334
335
336  if [ -s ${RunFile} ] ; then
337    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodState
338    PeriodState=${run_Configuration_PeriodState}
339  else
340    PeriodState="Waiting"
341  fi
342
343  if ( [ X${PeriodState} == XRunning ] || [ X${PeriodState} == XOnQueue ] ) ; then
344    NbRun=$( ccc_mstat -f | grep -c ${JobName} )
345
346    if [ ${NbRun} -eq 0 ] ; then
347      PeriodState="Fatal"
348    fi
349  fi
350
351  DateBegin=${config_UserChoices_DateBegin}
352  DateEnd=${config_UserChoices_DateEnd}
353  TagName=${config_UserChoices_TagName}
354  ExperimentName=${config_UserChoices_ExperimentName}
355  SpaceName=${config_UserChoices_SpaceName}
356
357  IGCM_card_DefineVariableFromOption ${RunFile} Configuration CumulPeriod
358  CumulPeriod=${run_Configuration_CumulPeriod}
359
360  if ( [ ! X${PeriodState} == XWaiting ] && [ ! X${PeriodState} == XCompleted ] ) ; then
361    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateBegin
362    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateEnd
363    PeriodDateBegin=${run_Configuration_PeriodDateBegin}
364    PeriodDateEnd=${run_Configuration_PeriodDateEnd}
365  else
366    PeriodDateBegin="."
367    PeriodDateEnd="."
368  fi
369
370  DATA_DIR=${R_SAVE}
371  POST_DIR=${R_BUFR}/Out
372  CWORK_DIR=${R_FIGR}
373  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
374    REBUILD_DIR=${R_SAVE}/TMP
375    RebuildJob="rebuild_fromArchive"
376  else
377    REBUILD_DIR=${BIG_DIR}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
378    RebuildJob="rebuild_fromWorkdir"
379  fi
380
381  if [ ${TargetUsr} != ${CurrentUsr} ] ; then
382    DATA_DIR=$( ChangeUsr ${DATA_DIR})
383    POST_DIR=$( ChangeUsr ${POST_DIR} )
384    CWORK_DIR=$( ChangeUsr ${CWORK_DIR} )
385    REBUILD_DIR=$( ChangeUsr ${REBUILD_DIR} )
386  fi
387
388  echo "Data:    >${DATA_DIR}<"
389  echo "Rebuild: >${REBUILD_DIR}<"
390  echo "Post:    >${POST_DIR}<"
391  echo "Work:    >${CWORK_DIR}<"
392
393  if [ $PeriodState != "Waiting" ] ; then
394
395    # Check pending rebuilds
396    # ======================
397
398    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
399    if [ ${#RebuildList[*]} -gt 0 ] ; then
400      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )
401
402      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
403      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
404    else
405      NbRebuild="."
406
407      FirstRebuild="."
408      LastRebuild="."
409    fi
410  else
411    NbRebuild="."
412
413    FirstRebuild="."
414    LastRebuild="."
415  fi
416
417
418  # Check last REBUILD and PACK* jobs
419  # =================================
420  # Define input parameters
421  # -----------------------
422  set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug"
423
424  for JobType in ${JobType_list[*]} ; do
425    typeset    name1="${JobType}_String"
426    typeset    name2="${JobType}_Field"
427    if [ X${JobType} == X${RebuildJob} ] ; then
428      eval ${name1}=IGCM_sys_PutBuffer_Out
429      eval ${name2}=4
430    else
431      eval ${name1}=IGCM_sys_Put_Out
432      eval ${name2}=3
433    fi
434  done
435
436  # Check jobs
437  # ----------
438  for JobType in ${JobType_list[*]} ; do
439    eval String=\${${JobType}_String}
440    eval Field=\${${JobType}_Field}
441
442    FileList=$( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} )
443
444    (( ind = 0 ))
445    for FileName in ${FileList} ; do
446      LastDate=$( basename ${FileName} | awk -F"." '{ print $(NF-1) }' )
447
448      dum=$( basename $( grep ^${String} ${POST_DIR}/${JobType}.${LastDate}.out | \
449               tail -1 | awk -v Field=${Field} '{ print $( Field ) }' ) | \
450               awk -F"_" '{ print $3 }' )
451
452      if [[ X${dum} == X${LastDate} ]] ; then
453        Status=OK
454      else
455        Status=KO
456      fi
457
458      eval ${JobType}_Date[$ind]=${LastDate}
459      eval ${JobType}_Status[$ind]=${Status}
460     
461      (( ind = ind + 1 ))
462    done
463  done
464
465  # Check last MONITORING jobs
466  # ==========================
467  JobType=monitoring
468  LastDate=$( cdo showyear ${CWORK_DIR}/MONITORING/files/ATM_bils_global_ave.nc 2> /dev/null | \
469                  awk '{ print $NF }' )
470  eval ${JobType}_Date=${LastDate}
471
472  # Check last ATLAS jobs
473  # =====================
474  JobType=atlas
475  FileList=$( ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto} )
476
477  (( ind = 0 ))
478  for FileName in ${FileList} ; do
479    eval ${JobType}_Date[$ind]=${FileName}
480    (( ind = ind + 1 ))
481  done
482
483
484  # Time of last write on run.card
485  # ==============================
486  LastWrite=$( ls -l --time-style=+"${DateFormat}" ${SUBMIT_DIR}/run.card | awk '{print $6 " " $7}' )
487   
488
489  # Print results
490  # =============
491  AffichResult -f
492  ind=0
493  while [ $ind -lt $NbHisto ] ; do
494    AffichResult -j
495    (( ind = ind + 1 ))
496  done
497  AffichResult -l
498
499fi
500
Note: See TracBrowser for help on using the repository browser.