source: trunk/libIGCM/AA_RunChecker @ 843

Last change on this file since 843 was 843, checked in by labetoulle, 11 years ago

Bugfix in RunChecker? : variable initilization.
  • Property svn:keywords set to Revision Author Date
File size: 17.0 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sonia Labetoulle
5# Contact: sonia.labetoulle__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2012)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#======================================================================#
15# Display a report on how a simulation is faring.
16
17
18function ChangeUsr {
19
20  CurrentGrp=$( groups $CurrentUsr | gawk '{print $3}' )
21  TargetGrp=$( groups $TargetUsr | gawk '{print $3}' )
22
23  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
24                -e "s/${CurrentGrp}/${TargetGrp}/"
25
26}
27
28
29function SearchCatalog {
30
31  typeset num
32  unset SUBMIT_DIR
33
34  fg_new=false
35
36  if [ ! X${JobName} == X ] ; then
37    NbOcc=$( gawk -v JobName=${JobName} \
38             'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
39  else
40    NbOcc=0
41  fi
42
43  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_path} ) ) ; then
44    set -A FileList $( ls ${ConfigPath}/Job_* )
45    if [ X$FileList == X ] ; then
46      NbOcc=0
47    else
48      NbOcc=${#FileList[@]}
49      fg_new=true
50    fi
51  fi
52
53  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_search} ) ) ; then
54    SEARCH_DIR=${WORKDIR}
55    if [ ${TargetUsr} != ${CurrentUsr} ] ; then
56      SEARCH_DIR=$( ChangeUsr ${SEARCH_DIR})
57    fi
58    echo "${JobName} not in Catalog, we'll try to find it in ${SEARCH_DIR}"
59
60    set -A FileList $( find ${SEARCH_DIR}/ \
61                            -path ${SEARCH_DIR}/IGCM_OUT -prune -o \
62                            -name Job_${JobName} -print )
63    if [ X$FileList == X ] ; then
64      NbOcc=0
65    else
66      NbOcc=${#FileList[@]}
67      fg_new=true
68    fi
69  fi
70
71  if [ ${NbOcc} -gt 1 ] ; then
72    echo "More than one job"
73    ind=0
74    while [ ${ind} -lt ${NbOcc} ] ; do
75      printf '%2i) %-30s\n' ${ind} ${FileList[${ind}]}
76      (( ind = ind + 1 ))
77    done
78    echo "Give your choice number or 'q' to quit : "
79    read Choice
80    if [ X${Choice} == Xq ] ; then
81      exit
82    else
83      fg_new=true
84      FileList=${FileList[${Choice}]}
85      NbOcc=1
86    fi
87  fi
88
89  case ${NbOcc} in
90    0)
91      echo "${JobName} not found."
92      echo "You can try : *) '-s' option to automatically search your \$WORKDIR ; "
93      echo "              *) '-p' option to provide the directory (absolute path) "
94      echo "                      containing the config.card ; "
95      echo "              *)  manually editing your ${SimuCatalog}."
96      exit ;;
97    1)
98      if ( ${fg_new} ) ; then
99        JobName=${JobName:=$( basename ${FileList} | gawk -F"_" '{ print $2 }' )}
100        SUBMIT_DIR=$( dirname ${FileList} )
101        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}"
102        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" >> ${SimuCatalog}
103        sort -u ${SimuCatalog} > ${SimuCatalog}.tmp
104        mv ${SimuCatalog}.tmp ${SimuCatalog}
105      elif ( [ ${TargetUsr} == $( gawk -v JobName=${JobName} \
106                                      '$1 ~ JobName {print $2}' \
107                                      ${SimuCatalog} ) ] \
108          && [ ${HostName}  == $( gawk -v JobName=${JobName} \
109                                      '$1 ~ JobName {print $3}' \
110                                      ${SimuCatalog} ) ] ) ; then
111        JobName=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
112        SUBMIT_DIR=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
113      else
114        echo "${JobName} not in Catalog."
115        exit
116      fi
117      ;;
118    *)
119      break ;;
120  esac
121
122  return
123
124}
125
126
127function AffichResult {
128
129  fg_first=false
130  fg_last=false
131  fg_job=false
132
133  while [ $# -ne 0 ] ; do
134    case ${1} in
135      -f|--first)
136        fg_first=true
137        shift 1 ;;
138      -l|--last)
139        fg_last=true
140        shift 1 ;;
141      -j|--job)
142        fg_job=true
143        shift 1 ;;
144      -*)
145        echo "usage: ${0}"
146        echo "       options = -f; -l"
147        exit ;;
148      *)
149        break ;;
150    esac
151  done
152
153
154  # Define colors
155  # =============
156  ColEsc="\033["
157  ColNon="${ColEsc}0m"       # Return to normal
158  ColExp="${ColEsc}1m"       # Blanc - gras
159  ColFat="${ColEsc}1;31m"    # Fatal
160  ColCpl="${ColEsc}1;32m"    # Completed
161  ColAtt="${ColEsc}1;30m"    # Waiting
162  ColDef="${ColEsc}1;34m"    # Default
163  ColRbl="${ColEsc}31m"      # Rebuild
164
165
166
167  # Print header
168  # ============
169  if ( ${fg_first} ) ; then
170    Dum=""
171    (( len = 67 - ${#JobName} ))
172    echo "|===========================================================================================================|"
173    printf "| JobName = ${ColExp}%-${#JobName}s${ColNon}" ${JobName}
174    printf "%-${len}s" ${Dum} 
175    printf "run.card : ${ColExp}%-18s${ColNon}|\n" "${LastWrite}"
176    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
177    echo "|                         |             |                         |             |     Pending Rebuilds      |"
178    echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
179    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
180
181    printf "| %-10s - %-10s | " \
182           $DateBegin $DateEnd 
183
184    case $PeriodState in
185      Fatal)
186        Color=${ColFat}
187        ;;
188      Completed)
189        Color=${ColCpl}
190        ;;
191      Waiting|OnQueue)
192        Color=${ColAtt}
193        ;;
194      *)
195        Color=${ColDef}
196        ;;
197    esac
198    printf "${Color}%-11s${ColNon} | " $PeriodState
199
200    printf "%-10s - %-10s | %11s | " \
201           $PeriodDateBegin $PeriodDateEnd $CumulPeriod
202
203    if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
204      printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
205             $NbRebuild $FirstRebuild $LastRebuild
206    else
207      printf "%3s : %-8s : %-8s |\n" \
208             $NbRebuild $FirstRebuild $LastRebuild
209    fi
210
211    if [ ${NbLines} -gt 0 ] ; then
212      printf "|-----------------------------------------------------------------------------------------------------------|\n"
213      printf "|                                                      Last                                                 |\n"
214      printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
215      printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
216    fi
217
218    return
219  fi
220
221  # Print Post-processing job status
222  # ================================
223  if ( ${fg_job} ) ; then
224    printf "|"
225
226    # Print rebuild and pack jobs
227    # ---------------------------
228    for JobType in ${JobType_list[*]} ; do
229      eval Date=\${${JobType}_Date[${ind}]}
230      eval Status=\${${JobType}_Status[${ind}]}
231      eval Nb=\${${JobType}_Nb[${ind}]}
232
233      if [ X${Status} == XOK  ] ; then
234        Color=${ColCpl}
235      else
236        Color=${ColFat}
237      fi
238      printf "  ${Color}%-8s${ColNon} : %3s  |" ${Date} ${Nb}
239    done
240
241    Color=${ColExp}
242
243    # Print monitoring jobs
244    # ---------------------
245    JobType=monitoring
246    if [ $ind -eq 0 ] ; then
247      eval Date=\${${JobType}_Date}
248    else
249      Date=""
250    fi
251    printf "     ${Color}%-4s${ColNon}     |" ${Date}
252
253    # Print atlas jobs
254    # ----------------
255    JobType=atlas
256    eval Date=\${${JobType}_Date[${ind}]}
257    printf "  ${Color}%-12s${ColNon}  |" ${Date}
258
259    printf "\n"
260
261    return
262  fi
263
264  # Print footer
265  # ============
266  if ( ${fg_last} ) ; then
267    printf "|===========================================================================================================|\n"
268    date +"${DateFormat}"
269    return
270  fi
271
272}
273
274#======================================================================#
275
276#D- Task type (computing, post-processing or checking)
277TaskType=checking
278typeset -i Verbosity=0
279
280CurrentUsr=$( whoami )
281
282if ( [ $# -eq 0 ] ) ; then
283  $0 -h
284  exit
285fi
286
287# Arguments
288# =========
289# Default argument values
290# -----------------------
291TargetUsr=${CurrentUsr}
292HostName=$( hostname | sed -e "s/[0-9].*//" )
293
294fg_color=true
295fg_search=false
296fg_quiet=false
297fg_path=false
298NbHisto=20
299
300# Get arguments from command line
301# -------------------------------
302while [ $# -ne 0 ] ; do
303  case $1 in
304    -h|--help|-help)
305      echo "usage: $0 [-u user] [-q] [-j n] [-s] job_name"
306      echo "       $0 [-u user] [-q] [-j n] -p config.card_path"
307      echo ""
308      echo "options :"
309      echo "  -h : print this help and exit"
310      echo "  -u : owner of the job"
311      echo "  -q : quiet"
312      echo "  -j : print n post-processing jobs (default is 20)"
313      echo "  -s : search for a new job in \$WORKDIR and fill in "
314      echo "       the catalog before printing information"
315      echo "  -p : give the directory (absolute path) containing "
316      echo "       the config.card instead of the job name."
317      exit 0 ;;
318    -j|-job-number)
319      NbHisto="$2"
320      shift 2 ;;
321    -p|-config-path)
322      ConfigPath="$2"
323      fg_path=true
324      shift 2 ;;
325    -q|-quiet)
326      fg_quiet=true
327      shift 1 ;;
328    -s|-search)
329      fg_search=true
330      shift 1 ;;
331    -u|-user)
332      TargetUsr="$2"
333      shift 2 ;;
334    -*)
335      $0 -h
336      exit ;;
337    *)
338      break ;;
339  esac
340done
341
342
343if ( ( ! ${fg_path} ) && [ $# -lt 1 ] ) ; then
344  $0 -h
345  exit
346fi
347
348if ( ( ${fg_path} ) && ( ${fg_search} ) ) ; then
349  echo "You cannot use -s and -p at the same time"
350  exit
351fi
352
353
354# Load libIGCM library
355# ====================
356libIGCM=${libIGCM:=::modipsl::/libIGCM}
357
358DEBUG_debug=false
359DEBUG_sys=false
360
361. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
362. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
363. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
364#-------
365. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
366. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
367#-------
368( ${DEBUG_debug} ) && IGCM_debug_Check
369( ${DEBUG_debug} ) && IGCM_card_Check
370( ${DEBUG_debug} ) && IGCM_date_Check
371
372JobName=$1
373
374if ( ${fg_quiet} ) ; then
375  NbHisto=1
376fi
377
378echo "Target user = ${TargetUsr}"
379
380# Define the catalog in which the known simulations are stored
381SimuCatalog="${HOME}/.simucatalog.dat"
382if [ ! -s ${SimuCatalog} ] ; then
383  touch ${SimuCatalog}
384fi
385
386# Date format
387DateFormat="%d/%m/%y %R:%S"
388
389# Find SUBMIT_DIR in catalog
390# ==========================
391SearchCatalog
392
393if [ ! X${SUBMIT_DIR} == X ] ; then
394
395  echo "Submit:  >${SUBMIT_DIR}<"
396  IGCM_sys_Cd $SUBMIT_DIR
397
398
399  # Extract usefull information from run.card and config.card
400  # =========================================================
401
402  RunFile="${SUBMIT_DIR}/run.card"
403  ConfFile="${SUBMIT_DIR}/config.card"
404
405  IGCM_config_CommonConfiguration ${SUBMIT_DIR}/config.card
406
407
408  if [ -s ${RunFile} ] ; then
409    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodState
410    PeriodState=${run_Configuration_PeriodState}
411  else
412    PeriodState="Waiting"
413  fi
414
415  if ( [ X${PeriodState} == XRunning ] || [ X${PeriodState} == XOnQueue ] ) ; then
416    #NbRun=$( ccc_mstat -f | grep -c ${JobName} )
417    IGCM_sys_CountJobInQueue ${JobName} NbRun
418
419    if [ ${NbRun} -eq 0 ] ; then
420      PeriodState="Fatal"
421    fi
422  fi
423
424  DateBegin=${config_UserChoices_DateBegin}
425  DateEnd=${config_UserChoices_DateEnd}
426  TagName=${config_UserChoices_TagName}
427  ExperimentName=${config_UserChoices_ExperimentName}
428  SpaceName=${config_UserChoices_SpaceName}
429
430
431  if ( [ ! X${PeriodState} == XWaiting ] && [ ! X${PeriodState} == XCompleted ] ) ; then
432    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateBegin
433    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateEnd
434    IGCM_card_DefineVariableFromOption ${RunFile} Configuration CumulPeriod
435    PeriodDateBegin=${run_Configuration_PeriodDateBegin}
436    PeriodDateEnd=${run_Configuration_PeriodDateEnd}
437    CumulPeriod=${run_Configuration_CumulPeriod}
438  else
439    PeriodDateBegin="."
440    PeriodDateEnd="."
441    CumulPeriod="."
442  fi
443
444  DATA_DIR=${R_SAVE}
445  POST_DIR=${R_BUFR}/Out
446  CWORK_DIR=${R_FIGR}
447  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
448    RebuildJob="rebuild_fromArchive"
449  else
450    RebuildJob="rebuild_fromWorkdir"
451  fi
452
453  if [ ${TargetUsr} != ${CurrentUsr} ] ; then
454    DATA_DIR=$( ChangeUsr ${DATA_DIR})
455    POST_DIR=$( ChangeUsr ${POST_DIR} )
456    CWORK_DIR=$( ChangeUsr ${CWORK_DIR} )
457    REBUILD_DIR=$( ChangeUsr ${REBUILD_DIR} )
458  fi
459
460  echo "Data:    >${DATA_DIR}<"
461  echo "Rebuild: >${REBUILD_DIR}<"
462  echo "Post:    >${POST_DIR}<"
463  echo "Work:    >${CWORK_DIR}<"
464
465  # Are packs activated or not ?
466  # ============================
467  if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && \
468       [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
469    Pack=true
470  else
471    Pack=false
472  fi
473
474
475  if [ $PeriodState != "Waiting" ] ; then
476
477    # Check pending rebuilds
478    # ======================
479
480    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
481    if [ ${#RebuildList[*]} -gt 0 ] ; then
482      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )
483
484      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
485      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
486    else
487      NbRebuild="."
488
489      FirstRebuild="."
490      LastRebuild="."
491    fi
492  else
493    NbRebuild="."
494
495    FirstRebuild="."
496    LastRebuild="."
497  fi
498
499
500  if [ $PeriodState != "Waiting" ] ; then
501
502    # Check last REBUILD and PACK* jobs
503    # =================================
504    # Define input parameters
505    # -----------------------
506    set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug"
507
508    for JobType in ${JobType_list[*]} ; do
509      typeset    name1="${JobType}_String"
510      typeset    name2="${JobType}_Field"
511      typeset    name3="${JobType}_Activ"
512      if [ X${JobType} == X${RebuildJob} ] ; then
513        if ( ${Pack} ) ; then
514          eval ${name1}=IGCM_sys_PutBuffer_Out
515        else
516          eval ${name1}=IGCM_sys_Put_Out
517        fi
518        eval ${name2}=4
519        eval ${name3}=true
520      else
521        eval ${name1}=IGCM_sys_Put_Out
522        eval ${name2}=3
523        if ( ${Pack} ) ; then
524          eval ${name3}=true
525        else
526          eval ${name3}=false
527        fi
528      fi
529    done
530
531    # Check jobs
532    # ----------
533    NbLines=0
534    for JobType in ${JobType_list[*]} ; do
535      eval String=\${${JobType}_String}
536      eval Field=\${${JobType}_Field}
537      eval Activ=\${${JobType}_Activ}
538
539      if ( ${Activ} ) ; then
540
541        set -A FileList $( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} )
542
543        if [ ${#FileList[*]} -gt ${NbLines} ] ; then
544          NbLines=${#FileList[*]}
545        fi
546
547        (( ind = 0 ))
548        for FileName in ${FileList[*]} ; do
549          LastDate=""
550          LastDate=$( basename ${FileName} | gawk -F"." '{ print $(NF-1) }' )
551
552          set -- $( gawk -v String=${String} \
553                         'BEGIN { nb_ok = 0 ; nb_ko = 0 } \
554                         ($1 ~ String) { \
555                           if ($3 !~ /error./) { \
556                             nb_ok = nb_ok + 1 \
557                           } else { \
558                             nb_ko = nb_ko + 1 \
559                           } \
560                         } \
561                         END { print nb_ok " " nb_ko }' \
562                         ${POST_DIR}/${JobType}.${LastDate}.out )
563          Match=$1
564          Error=$2
565
566          (( Nb = ${Match} - ${Error} ))
567
568          if ( [ ${Error} -eq 0 ] && [ ${Nb} -gt 0 ] ) ; then
569            Status=OK
570          else
571            Status=KO
572          fi
573
574          eval ${JobType}_Date[$ind]=${LastDate}
575          eval ${JobType}_Status[$ind]=${Status}
576          eval ${JobType}_Nb[$ind]=${Nb}
577
578          (( ind = ind + 1 ))
579        done
580
581      else
582
583          eval ${JobType}_Date[0]=""
584          eval ${JobType}_Status[0]=""
585          eval ${JobType}_Nb[0]=""
586
587      fi
588
589    done
590
591    # Check last MONITORING jobs
592    # ==========================
593    LastDate=""
594    JobType=monitoring
595    IGCM_sys_TestDirArchive ${CWORK_DIR}/MONITORING
596    RC=$?
597    if [ $RC == 0 ] ; then
598      FileTmp=$( IGCM_sys_RshArchive "ls ${CWORK_DIR}/MONITORING/files/*.nc | head -n 1" )
599      IGCM_sys_GetDate_Monitoring ${FileTmp} LastDate
600      eval ${JobType}_Date=${LastDate}
601    fi
602
603    # Check last ATLAS jobs
604    # =====================
605    JobType=atlas
606    IGCM_sys_TestDirArchive ${CWORK_DIR}/ATLAS
607    RC=$?
608    if [ $RC == 0 ] ; then
609      set -A FileList $( IGCM_sys_RshArchive "ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto}" )
610
611      if [ ${#FileList[*]} -gt ${NbLines} ] ; then
612        NbLines=${#FileList[*]}
613      fi
614
615      (( ind = 0 ))
616      for FileName in ${FileList[*]} ; do
617        eval ${JobType}_Date[$ind]=${FileName}
618        (( ind = ind + 1 ))
619      done
620    fi
621
622
623    # Time of last write on run.card
624    # ==============================
625    LastWrite=$( ls -l --time-style=+"${DateFormat}" ${SUBMIT_DIR}/run.card | gawk '{print $6 " " $7}' )
626
627  else
628
629    NbLines=0
630
631  fi
632
633
634  # Print results
635  # =============
636  AffichResult -f
637  ind=0
638  while [ $ind -lt $NbLines ] ; do
639    AffichResult -j
640    (( ind = ind + 1 ))
641  done
642  AffichResult -l
643
644fi
645
Note: See TracBrowser for help on using the repository browser.