source: tags/libIGCM_v2.0_rc3/AA_RunChecker @ 1137

Last change on this file since 1137 was 785, checked in by labetoulle, 11 years ago

Cosmetics :

  • Convert tabs into space for better lisibility with different editors.
  • Property svn:executable set to *
  • Property svn:keywords set to Revision Author Date
File size: 15.9 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sonia Labetoulle
5# Contact: sonia.labetoulle__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2012)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#======================================================================#
15# Display a report on how a simulation is faring.
16
17
18function ChangeUsr {
19
20  CurrentGrp=$( groups $CurrentUsr | gawk '{print $3}' )
21  TargetGrp=$( groups $TargetUsr | gawk '{print $3}' )
22
23  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
24                -e "s/${CurrentGrp}/${TargetGrp}/"
25
26}
27
28
29function SearchCatalog {
30
31  typeset num
32  unset SUBMIT_DIR
33
34  fg_new=false
35
36  if [ ! X${JobName} == X ] ; then
37    NbOcc=$( gawk -v JobName=${JobName} \
38             'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
39  else
40    NbOcc=0
41  fi
42
43  if ( [ ${NbOcc} -eq 0 ] && ( $fg_path ) ) ; then
44    set -A FileList $( ls ${ConfigPath}/Job_* )
45    if [ X$FileList == X ] ; then
46      NbOcc=0
47    else
48      NbOcc=${#FileList[@]}
49      fg_new=true
50    fi
51  fi
52
53  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_search} ) ) ; then
54    SEARCH_DIR=${WORKDIR}
55    if [ ${TargetUsr} != ${CurrentUsr} ] ; then
56      SEARCH_DIR=$( ChangeUsr ${SEARCH_DIR})
57    fi
58    echo "${JobName} not in Catalog, we'll try to find it in ${SEARCH_DIR}"
59
60    set -A FileList $( find ${SEARCH_DIR}/ \
61                            -path ${SEARCH_DIR}/IGCM_OUT -prune -o \
62                            -name Job_${JobName} -print )
63    if [ X$FileList == X ] ; then
64      NbOcc=0
65    else
66      NbOcc=${#FileList[@]}
67      fg_new=true
68    fi
69  fi
70
71  if [ ${NbOcc} -gt 1 ] ; then
72    echo "More than one job"
73    ind=0
74    while [ ${ind} -lt ${NbOcc} ] ; do
75      printf '%2i) %-30s\n' ${ind} ${FileList[${ind}]}
76      (( ind = ind + 1 ))
77    done
78    echo "Give your choice number or 'q' to quit : "
79    read Choice
80    if [ X${Choice} == Xq ] ; then
81      exit
82    else
83      fg_new=true
84      FileList=${FileList[${Choice}]}
85      NbOcc=1
86    fi
87  fi
88
89  case ${NbOcc} in
90    0)
91      echo "${JobName} not found."
92      echo "You can try : *) '-s' option to automatically search your \$WORKDIR, "
93      echo "              *) '-p' option to provide the config.card path, "
94      echo "              *)  manually editing your ${SimuCatalog}"
95      exit ;;
96    1)
97      if ( ${fg_new} ) ; then
98        JobName=${JobName:=$( basename ${FileList} | gawk -F"_" '{ print $2 }' )}
99        SUBMIT_DIR=$( dirname ${FileList} )
100        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}"
101        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" >> ${SimuCatalog}
102        sort -u ${SimuCatalog} > ${SimuCatalog}.tmp
103        mv ${SimuCatalog}.tmp ${SimuCatalog}
104      elif ( [ ${TargetUsr} == $( gawk -v JobName=${JobName} \
105                                      '$1 ~ JobName {print $2}' \
106                                      ${SimuCatalog} ) ] \
107          && [ ${HostName}  == $( gawk -v JobName=${JobName} \
108                                      '$1 ~ JobName {print $3}' \
109                                      ${SimuCatalog} ) ] ) ; then
110        JobName=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
111        SUBMIT_DIR=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
112      else
113        echo "${JobName} not in Catalog."
114        exit
115      fi
116      break ;;
117    *)
118      break ;;
119  esac
120
121  return
122
123}
124
125
126function AffichResult {
127
128  fg_first=false
129  fg_last=false
130  fg_job=false
131
132  while [ $# -ne 0 ] ; do
133    case ${1} in
134      -f|--first)
135        fg_first=true
136        shift 1 ;;
137      -l|--last)
138        fg_last=true
139        shift 1 ;;
140      -j|--job)
141        fg_job=true
142        shift 1 ;;
143      -*)
144        echo "usage: ${0}"
145        echo "       options = -f; -l"
146        exit ;;
147      *)
148        break ;;
149    esac
150  done
151
152
153  # Define colors
154  # =============
155  ColEsc="\033["
156  ColNon="${ColEsc}0m"       # Return to normal
157  ColExp="${ColEsc}1m"       # Blanc - gras
158  ColFat="${ColEsc}1;31m"    # Fatal
159  ColCpl="${ColEsc}1;32m"    # Completed
160  ColAtt="${ColEsc}1;30m"    # Waiting
161  ColDef="${ColEsc}1;34m"    # Default
162  ColRbl="${ColEsc}31m"      # Rebuild
163
164
165 
166  # Print header
167  # ============
168  if ( ${fg_first} ) ; then
169    Dum=""
170    (( len = 67 - ${#JobName} ))
171    echo "|===========================================================================================================|"
172    printf "| JobName = ${ColExp}%-${#JobName}s${ColNon}" ${JobName}
173    printf "%-${len}s" ${Dum} 
174    printf "run.card : ${ColExp}%-18s${ColNon}|\n" "${LastWrite}"
175    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
176    echo "|                         |             |                         |             |     Pending Rebuilds      |"
177    echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
178    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
179
180    printf "| %-10s - %-10s | " \
181           $DateBegin $DateEnd 
182
183    case $PeriodState in
184      Fatal)
185        Color=${ColFat}
186        break ;;
187      Completed)
188        Color=${ColCpl}
189        break ;;
190      Waiting|OnQueue)
191        Color=${ColAtt}
192        break ;;
193      *)
194        Color=${ColDef}
195        break ;;
196    esac
197    printf "${Color}%-11s${ColNon} | " $PeriodState
198
199    printf "%-10s - %-10s | %11s | " \
200           $PeriodDateBegin $PeriodDateEnd $CumulPeriod
201
202    if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
203      printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
204             $NbRebuild $FirstRebuild $LastRebuild
205    else
206      printf "%3s : %-8s : %-8s |\n" \
207             $NbRebuild $FirstRebuild $LastRebuild
208    fi
209
210    if [ ${NbLines} -gt 0 ] ; then
211      printf "|-----------------------------------------------------------------------------------------------------------|\n"
212      printf "|                                                      Last                                                 |\n"
213      printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
214      printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
215    fi
216
217    return
218  fi
219
220  # Print Post-processing job status
221  # ================================
222  if ( ${fg_job} ) ; then
223    printf "|"
224
225    # Print rebuild and pack jobs
226    # ---------------------------
227    for JobType in ${JobType_list[*]} ; do
228      eval Date=\${${JobType}_Date[${ind}]}
229      eval Status=\${${JobType}_Status[${ind}]}
230      eval Nb=\${${JobType}_Nb[${ind}]}
231
232      if [ X${Status} == XOK  ] ; then
233        Color=${ColCpl}
234      else
235        Color=${ColFat}
236      fi
237      printf "  ${Color}%-8s${ColNon} : %3s  |" ${Date} ${Nb}
238    done
239
240    Color=${ColExp}
241
242    # Print monitoring jobs
243    # ---------------------
244    JobType=monitoring
245    if [ $ind -eq 0 ] ; then
246      eval Date=\${${JobType}_Date}
247    else
248      Date=""
249    fi
250    printf "     ${Color}%-4s${ColNon}     |" ${Date}
251
252    # Print atlas jobs
253    # ----------------
254    JobType=atlas
255    eval Date=\${${JobType}_Date[${ind}]}
256    printf "  ${Color}%-12s${ColNon}  |" ${Date}
257
258    printf "\n"
259
260    return
261  fi
262
263  # Print footer
264  # ============
265  if ( ${fg_last} ) ; then
266    printf "|===========================================================================================================|\n"
267    date +"${DateFormat}"
268    return
269  fi
270
271}
272
273#======================================================================#
274
275#D- Task type (computing ,post-processing or checking)
276TaskType=checking
277typeset -i Verbosity=0
278
279CurrentUsr=$( whoami )
280
281if ( [ $# -eq 0 ] ) ; then
282  $0 -h
283  exit
284fi
285
286# Arguments
287# =========
288# Default argument values
289# -----------------------
290TargetUsr=${CurrentUsr}
291HostName=$( hostname | sed -e "s/[0-9].*//" )
292
293fg_color=true
294fg_search=false
295fg_quiet=false
296fg_path=false
297NbHisto=10
298
299# Get arguments from command line
300# -------------------------------
301while [ $# -ne 0 ] ; do
302  case $1 in
303    -h|--help|-help)
304      echo "usage: $0 [-u user] [-q] [-j n] [-s] job_name"
305      echo "       $0 [-u user] [-q] [-j n] -p config.card_path"
306      echo ""
307      echo "options :"
308      echo "  -h : print this help and exit"
309      echo "  -u : owner of the job"
310      echo "  -q : quiet"
311      echo "  -j : print n post-processing jobs (default is 10)"
312      echo "  -s : search for a new job in \$WORKDIR and fill in "
313      echo "       the catalog before printing information"
314      echo "  -p : give the directory (absolute path) containing "
315      echo "       the config.card instead of the job name."
316      exit ;;
317#    -b|-nocolor)
318#      fg_color=false
319#      shift 1 ;;
320    -j|-job-number)
321      NbHisto="$2"
322      shift 2 ;;
323    -p|-config-path)
324      ConfigPath="$2"
325      fg_path=true
326      shift 2 ;;
327    -q|-quiet)
328      fg_quiet=true
329      shift 1 ;;
330    -s|-search)
331      fg_search=true
332      shift 1 ;;
333    -u|-user)
334      TargetUsr="$2"
335      shift 2 ;;
336    -*)
337      $0 -h
338      exit ;;
339    *)
340      break ;;
341  esac
342done
343
344
345if ( ( ! $fg_path ) && [ $# -lt 1 ] ) ; then
346  $0 -h
347  exit
348fi
349
350if ( ( ${fg_path} ) && ( ${fg_search} ) ) ; then
351  echo "You cannot use -s and -p at the same time"
352  exit
353fi
354
355
356# Load libIGCM library
357# ====================
358libIGCM=${libIGCM:=::modipsl::/libIGCM}
359
360. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
361     ( ${DEBUG_debug} ) && IGCM_debug_Check
362. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
363     ( ${DEBUG_debug} ) && IGCM_card_Check
364. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
365     ( ${DEBUG_debug} ) && IGCM_date_Check
366#-------
367. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
368. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
369
370
371JobName=$1
372
373if ( ${fg_quiet} ) ; then
374  NbHisto=1
375fi
376
377echo "Target user = ${TargetUsr}"
378
379# Define the catalog in which the known simulations are stored
380SimuCatalog="${HOME}/.simucatalog.dat"
381if [ ! -s ${SimuCatalog} ] ; then
382  touch ${SimuCatalog}
383fi
384
385# Date format
386DateFormat="%d/%m/%y %R:%S"
387
388# Find SUBMIT_DIR in catalog
389# ==========================
390SearchCatalog
391
392if [ ! X${SUBMIT_DIR} == X ] ; then
393
394  echo "Submit:  >${SUBMIT_DIR}<"
395  cd $SUBMIT_DIR
396
397
398  # Extract usefull information from run.card and config.card
399  # =========================================================
400
401  RunFile="${SUBMIT_DIR}/run.card"
402  ConfFile="${SUBMIT_DIR}/config.card"
403
404  IGCM_config_CommonConfiguration ${SUBMIT_DIR}/config.card
405
406
407  if [ -s ${RunFile} ] ; then
408    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodState
409    PeriodState=${run_Configuration_PeriodState}
410  else
411    PeriodState="Waiting"
412  fi
413
414  if ( [ X${PeriodState} == XRunning ] || [ X${PeriodState} == XOnQueue ] ) ; then
415    #NbRun=$( ccc_mstat -f | grep -c ${JobName} )
416    IGCM_sys_CountJobInQueue ${JobName} NbRun
417
418    if [ ${NbRun} -eq 0 ] ; then
419      PeriodState="Fatal"
420    fi
421  fi
422
423  DateBegin=${config_UserChoices_DateBegin}
424  DateEnd=${config_UserChoices_DateEnd}
425  TagName=${config_UserChoices_TagName}
426  ExperimentName=${config_UserChoices_ExperimentName}
427  SpaceName=${config_UserChoices_SpaceName}
428
429
430  if ( [ ! X${PeriodState} == XWaiting ] && [ ! X${PeriodState} == XCompleted ] ) ; then
431    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateBegin
432    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateEnd
433    IGCM_card_DefineVariableFromOption ${RunFile} Configuration CumulPeriod
434    PeriodDateBegin=${run_Configuration_PeriodDateBegin}
435    PeriodDateEnd=${run_Configuration_PeriodDateEnd}
436    CumulPeriod=${run_Configuration_CumulPeriod}
437  else
438    PeriodDateBegin="."
439    PeriodDateEnd="."
440    CumulPeriod="."
441  fi
442
443  DATA_DIR=${R_SAVE}
444  POST_DIR=${R_BUFR}/Out
445  CWORK_DIR=${R_FIGR}
446  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
447    RebuildJob="rebuild_fromArchive"
448  else
449    RebuildJob="rebuild_fromWorkdir"
450  fi
451
452  if [ ${TargetUsr} != ${CurrentUsr} ] ; then
453    DATA_DIR=$( ChangeUsr ${DATA_DIR})
454    POST_DIR=$( ChangeUsr ${POST_DIR} )
455    CWORK_DIR=$( ChangeUsr ${CWORK_DIR} )
456    REBUILD_DIR=$( ChangeUsr ${REBUILD_DIR} )
457  fi
458
459  echo "Data:    >${DATA_DIR}<"
460  echo "Rebuild: >${REBUILD_DIR}<"
461  echo "Post:    >${POST_DIR}<"
462  echo "Work:    >${CWORK_DIR}<"
463
464  if [ $PeriodState != "Waiting" ] ; then
465
466    # Check pending rebuilds
467    # ======================
468
469    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
470    if [ ${#RebuildList[*]} -gt 0 ] ; then
471      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )
472
473      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
474      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
475    else
476      NbRebuild="."
477
478      FirstRebuild="."
479      LastRebuild="."
480    fi
481  else
482    NbRebuild="."
483
484    FirstRebuild="."
485    LastRebuild="."
486  fi
487
488
489  if [ $PeriodState != "Waiting" ] ; then
490
491    # Check last REBUILD and PACK* jobs
492    # =================================
493    # Define input parameters
494    # -----------------------
495    set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug"
496
497    for JobType in ${JobType_list[*]} ; do
498      typeset    name1="${JobType}_String"
499      typeset    name2="${JobType}_Field"
500      if [ X${JobType} == X${RebuildJob} ] ; then
501        eval ${name1}=IGCM_sys_PutBuffer_Out
502        eval ${name2}=4
503      else
504        eval ${name1}=IGCM_sys_Put_Out
505        eval ${name2}=3
506      fi
507    done
508
509    # Check jobs
510    # ----------
511    NbLines=0
512    for JobType in ${JobType_list[*]} ; do
513      eval String=\${${JobType}_String}
514      eval Field=\${${JobType}_Field}
515
516      set -A FileList $( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} )
517
518      if [ ${#FileList[*]} -gt ${NbLines} ] ; then
519        NbLines=${#FileList[*]}
520      fi
521
522      (( ind = 0 ))
523      for FileName in ${FileList[*]} ; do
524        LastDate=$( basename ${FileName} | gawk -F"." '{ print $(NF-1) }' )
525
526        Error=$( gawk -v String=${String} \
527                     'BEGIN { x=0 } ($1~String) && ($3~"error.") { x=x+1 } END { print x }' \
528                     ${POST_DIR}/${JobType}.${LastDate}.out )
529
530        Match=$( gawk -v String=${String} \
531                     'BEGIN { x=0 } ($1~String) && ($3!~"error.") { x=x+1 } END { print x }' \
532                     ${POST_DIR}/${JobType}.${LastDate}.out )
533        (( Nb = ${Match} - ${Error} ))
534
535        if ( [ ${Error} -eq 0 ] && [ ${Nb} -gt 0 ] ) ; then
536          Status=OK
537        else
538          Status=KO
539        fi
540
541        eval ${JobType}_Date[$ind]=${LastDate}
542        eval ${JobType}_Status[$ind]=${Status}
543        eval ${JobType}_Nb[$ind]=${Nb}
544       
545        (( ind = ind + 1 ))
546      done
547    done
548
549    # Check last MONITORING jobs
550    # ==========================
551    JobType=monitoring
552    if [ -d ${CWORK_DIR}/MONITORING ] ; then
553      LastDate=$( cdo showyear ${CWORK_DIR}/MONITORING/files/ATM_bils_global_ave.nc 2> /dev/null | \
554                      gawk '{ print $NF }' )
555      eval ${JobType}_Date=${LastDate}
556    fi
557
558    # Check last ATLAS jobs
559    # =====================
560    JobType=atlas
561    if [ -d ${CWORK_DIR}/ATLAS ] ; then
562      set -A FileList $( ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto} )
563
564      if [ ${#FileList[*]} -gt ${NbLines} ] ; then
565        NbLines=${#FileList[*]}
566      fi
567
568      (( ind = 0 ))
569      for FileName in ${FileList[*]} ; do
570        eval ${JobType}_Date[$ind]=${FileName}
571        (( ind = ind + 1 ))
572      done
573    fi
574
575
576    # Time of last write on run.card
577    # ==============================
578    LastWrite=$( ls -l --time-style=+"${DateFormat}" ${SUBMIT_DIR}/run.card | gawk '{print $6 " " $7}' )
579
580  else
581
582    NbLines=0
583
584  fi
585   
586
587  # Print results
588  # =============
589  AffichResult -f
590  ind=0
591  while [ $ind -lt $NbLines ] ; do
592    AffichResult -j
593    (( ind = ind + 1 ))
594  done
595  AffichResult -l
596
597fi
598
Note: See TracBrowser for help on using the repository browser.