source: trunk/libIGCM/AA_RunChecker @ 755

Last change on this file since 755 was 755, checked in by labetoulle, 12 years ago
  • Implementation of th '-p' option to give a config.card path instead of a job name.
  • Bufix in the '-n' search in $WORKDIR.
  • Property svn:executable set to *
  • Property svn:keywords set to Revision Author Date
File size: 14.2 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sonia Labetoulle
5# Contact: sonia.labetoulle__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2012)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#======================================================================#
15# Display a report on how a simulation is faring.
16
17
18function ChangeUsr {
19
20  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
21                -e "s/${CurrentGrp}/${TargetGrp}/"
22
23}
24
25
26function SearchCatalog {
27
28  typeset num
29  unset SUBMIT_DIR
30
31  if ( $fg_path ) ; then
32    set -A FileList
33    FileList=$( ls ${ConfigPath}/Job_* )
34    if [ X$FileList == X ] ; then
35      NbOcc=0
36    else
37      NbOcc=${#FileList[@]}
38      fg_new=true
39    fi
40  else
41    NbOcc=$( awk -v JobName=${JobName} \
42             'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
43  fi
44
45  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_new} ) ) ; then
46    SEARCH_DIR=${WORKDIR}
47    if [ ${TargetUsr} != ${CurrentUsr} ] ; then
48      SEARCH_DIR=$( ChangeUsr ${SEARCH_DIR})
49    fi
50    echo "${JobName} not in Catalog, we'll try to find it in ${SEARCH_DIR}"
51
52    set -A FileList
53    FileList=$( find ${SEARCH_DIR}/ -name "Job_${JobName}" )
54    if [ X$FileList == X ] ; then
55      NbOcc=0
56    else
57      NbOcc=${#FileList[@]}
58    fi
59  fi
60
61  case ${NbOcc} in
62    0)
63      echo "${JobName} not found."
64      echo "You can try : '-n' option to automatically search your \$WORKDIR, "
65      echo "              '-p' option to provide the config.card path, "
66      echo "               manually editing your ${SimuCatalog}"
67      exit ;;
68    1)
69      if ( ${fg_new} ) ; then
70        JobName=${JobName:=$( basename ${FileList} | awk -F"_" '{ print $2 }' )}
71        SUBMIT_DIR=$( dirname ${FileList} )
72        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}"
73        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" >> ${SimuCatalog}
74      elif ( [ ${TargetUsr} == $( awk -v JobName=${JobName} \
75                                      '$1 ~ JobName {print $2}' \
76                                      ${SimuCatalog} ) ] \
77          && [ ${HostName}  == $( awk -v JobName=${JobName} \
78                                      '$1 ~ JobName {print $3}' \
79                                      ${SimuCatalog} ) ] ) ; then
80        JobName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
81        SUBMIT_DIR=$( awk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
82      else
83        echo "${JobName} not in Catalog."
84        exit
85      fi
86      break ;;
87    *)
88      echo "More than one job"
89      break ;;
90  esac
91
92  return
93
94}
95
96
97function AffichResult {
98
99  fg_first=false
100  fg_last=false
101  fg_job=false
102
103  while [ $# -ne 0 ] ; do
104    case ${1} in
105      -f|--first)
106        fg_first=true
107        shift 1 ;;
108      -l|--last)
109        fg_last=true
110        shift 1 ;;
111      -j|--job)
112        fg_job=true
113        shift 1 ;;
114      -*)
115        echo "usage: ${0}"
116        echo "       options = -f; -l"
117        exit ;;
118      *)
119        break ;;
120    esac
121  done
122
123
124  # Define colors
125  # =============
126  ColEsc="\033["
127  ColNon="${ColEsc}0m"       # Return to normal
128  ColExp="${ColEsc}1m"       # Blanc - gras
129  ColFat="${ColEsc}1;31m"    # Fatal
130  ColCpl="${ColEsc}1;32m"    # Completed
131  ColAtt="${ColEsc}1;30m"    # Waiting
132  ColDef="${ColEsc}1;34m"    # Default
133  ColRbl="${ColEsc}31m"      # Rebuild
134
135
136 
137  # Print header
138  # ============
139  if ( ${fg_first} ) ; then
140    Dum=""
141    (( len = 67 - ${#JobName} ))
142    echo "|===========================================================================================================|"
143    printf "| JobName = ${ColExp}%-${#JobName}s${ColNon}" ${JobName}
144    printf "%-${len}s" ${Dum} 
145    printf "run.card : ${ColExp}%-17s${ColNon}|\n" "${LastWrite} "
146    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
147    echo "|                         |             |                         |             |     Pending Rebuilds      |"
148    echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
149    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
150
151    printf "| %-10s - %-10s | " \
152           $DateBegin $DateEnd 
153
154    case $PeriodState in
155      Fatal)
156        Color=${ColFat}
157        break ;;
158      Completed)
159        Color=${ColCpl}
160        break ;;
161      Waiting|OnQueue)
162        Color=${ColAtt}
163        break ;;
164      *)
165        Color=${ColDef}
166        break ;;
167    esac
168    printf "${Color}%-11s${ColNon} | " $PeriodState
169
170    printf "%-10s - %-10s | %11s | " \
171           $PeriodDateBegin $PeriodDateEnd $CumulPeriod
172
173    if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
174      printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
175             $NbRebuild $FirstRebuild $LastRebuild
176    else
177      printf "%3s : %-8s : %-8s |\n" \
178             $NbRebuild $FirstRebuild $LastRebuild
179    fi
180
181    printf "|-----------------------------------------------------------------------------------------------------------|\n"
182    printf "|                                                      Last                                                 |\n"
183    printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
184    printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
185
186    return
187  fi
188
189  # Print Post-processing job status
190  # ================================
191  if ( ${fg_job} ) ; then
192    printf "|"
193
194    # Print rebuild and pack jobs
195    # ---------------------------
196    for JobType in ${JobType_list[*]} ; do
197      eval Date=\${${JobType}_Date[${ind}]}
198      eval Status=\${${JobType}_Status[${ind}]}
199
200      if [ X${Status} == XOK  ] ; then
201        Color=${ColCpl}
202      else
203        Color=${ColFat}
204      fi
205      printf "     ${Color}%-8s${ColNon}     |" ${Date}
206    done
207
208    Color=${ColExp}
209
210    # Print monitoring jobs
211    # ---------------------
212    JobType=monitoring
213    if [ $ind -eq 0 ] ; then
214      eval Date=\${${JobType}_Date}
215    else
216      Date=""
217    fi
218    printf "     ${Color}%-4s${ColNon}     |" ${Date}
219
220    # Print atlas jobs
221    # ----------------
222    JobType=atlas
223    eval Date=\${${JobType}_Date[${ind}]}
224    printf "  ${Color}%-12s${ColNon}  |" ${Date}
225
226    printf "\n"
227
228    return
229  fi
230
231  # Print footer
232  # ============
233  if ( ${fg_last} ) ; then
234    printf "|===========================================================================================================|\n"
235    date +"${DateFormat}"
236    return
237  fi
238
239}
240
241#======================================================================#
242
243#D- Task type (computing or post-processing)
244TaskType=post-processing
245typeset -i Verbosity=0
246
247CurrentUsr=$( whoami )
248CurrentGrp=$( groups $CurrentUsr | awk '{print $3}' )
249
250if ( [ $# -eq 0 ] ) ; then
251  $0 -h
252  exit
253fi
254
255# Arguments
256# =========
257# Default argument values
258# -----------------------
259TargetUsr=${CurrentUsr}
260HostName=$( hostname | sed -e "s/[0-9].*//" )
261
262fg_color=true
263fg_new=false
264fg_quiet=false
265fg_path=false
266NbHisto=10
267
268# Get arguments from command line
269# -------------------------------
270while [ $# -ne 0 ] ; do
271  case $1 in
272    -h|--help|-help)
273      echo "usage: $0 [-u user] [-n] [-q] [-j n] job_name"
274      echo "       $0 [-u user] [-n] [-q] [-j n] -p config.card_path"
275      echo ""
276      echo "options :"
277      echo "  -h : print this help and exit"
278      echo "  -u : owner of the job"
279      echo "  -q : quiet"
280      echo "  -j : print n post-processing jobs (default is 10)"
281      echo "  -n : look for a new job in \$WORKDIR and fill in "
282      echo "       the catalog before printing information"
283      echo "  -p : give the config.card path to a new simulation, "
284      echo "       instead of the job name."
285      exit ;;
286#    -b|-nocolor)
287#      fg_color=false
288#      shift 1 ;;
289    -n|-new)
290      fg_new=true
291      shift 1 ;;
292    -q|-quiet)
293      fg_quiet=true
294      shift 1 ;;
295    -p|-config-path)
296      ConfigPath="$2"
297      fg_path=true
298      shift 2 ;;
299    -j|-job-number)
300      NbHisto="$2"
301      shift 2 ;;
302    -u|-user)
303      TargetUsr="$2"
304      shift 2 ;;
305    -*)
306      $0 -h
307      exit ;;
308    *)
309      break ;;
310  esac
311done
312
313if ( [ ! $fg_path ] && [ $# -lt 1 ] ) ; then
314  $0 -h
315  exit
316fi
317
318
319# Load libIGCM library
320# ====================
321libIGCM=${libIGCM:=::modipsl::/libIGCM}
322
323. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
324     ( ${DEBUG_debug} ) && IGCM_debug_Check
325. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
326     ( ${DEBUG_debug} ) && IGCM_card_Check
327. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
328     ( ${DEBUG_debug} ) && IGCM_date_Check
329#-------
330. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
331. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
332
333
334JobName=$1
335
336if ( ${fg_quiet} ) ; then
337  NbHisto=1
338fi
339
340echo "TargetUsr = ${TargetUsr}"
341echo "HostName = ${HostName}"
342
343TargetGrp=$( groups $TargetUsr | awk '{print $3}' )
344
345# Define the catalog in which the known simulations are stored
346SimuCatalog="$( ccc_home )/.simucatalog.dat"
347if [ ! -s ${SimuCatalog} ] ; then
348  touch ${SimuCatalog}
349fi
350
351# Date format
352DateFormat="%d/%m/%y %R:%S"
353
354# Find SUBMIT_DIR in catalog
355# ==========================
356SearchCatalog
357
358if [ ! X${SUBMIT_DIR} == X ] ; then
359
360  echo "Submit:  >${SUBMIT_DIR}<"
361  cd $SUBMIT_DIR
362
363
364  # Extract usefull information from run.card and config.card
365  # =========================================================
366
367  RunFile="${SUBMIT_DIR}/run.card"
368  ConfFile="${SUBMIT_DIR}/config.card"
369
370  IGCM_config_CommonConfiguration ${SUBMIT_DIR}/config.card
371
372
373  if [ -s ${RunFile} ] ; then
374    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodState
375    PeriodState=${run_Configuration_PeriodState}
376  else
377    PeriodState="Waiting"
378  fi
379
380  if ( [ X${PeriodState} == XRunning ] || [ X${PeriodState} == XOnQueue ] ) ; then
381    NbRun=$( ccc_mstat -f | grep -c ${JobName} )
382
383    if [ ${NbRun} -eq 0 ] ; then
384      PeriodState="Fatal"
385    fi
386  fi
387
388  DateBegin=${config_UserChoices_DateBegin}
389  DateEnd=${config_UserChoices_DateEnd}
390  TagName=${config_UserChoices_TagName}
391  ExperimentName=${config_UserChoices_ExperimentName}
392  SpaceName=${config_UserChoices_SpaceName}
393
394  IGCM_card_DefineVariableFromOption ${RunFile} Configuration CumulPeriod
395  CumulPeriod=${run_Configuration_CumulPeriod}
396
397  if ( [ ! X${PeriodState} == XWaiting ] && [ ! X${PeriodState} == XCompleted ] ) ; then
398    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateBegin
399    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateEnd
400    PeriodDateBegin=${run_Configuration_PeriodDateBegin}
401    PeriodDateEnd=${run_Configuration_PeriodDateEnd}
402  else
403    PeriodDateBegin="."
404    PeriodDateEnd="."
405  fi
406
407  DATA_DIR=${R_SAVE}
408  POST_DIR=${R_BUFR}/Out
409  CWORK_DIR=${R_FIGR}
410  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
411    REBUILD_DIR=${R_SAVE}/TMP
412    RebuildJob="rebuild_fromArchive"
413  else
414    REBUILD_DIR=${BIG_DIR}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
415    RebuildJob="rebuild_fromWorkdir"
416  fi
417
418  if [ ${TargetUsr} != ${CurrentUsr} ] ; then
419    DATA_DIR=$( ChangeUsr ${DATA_DIR})
420    POST_DIR=$( ChangeUsr ${POST_DIR} )
421    CWORK_DIR=$( ChangeUsr ${CWORK_DIR} )
422    REBUILD_DIR=$( ChangeUsr ${REBUILD_DIR} )
423  fi
424
425  echo "Data:    >${DATA_DIR}<"
426  echo "Rebuild: >${REBUILD_DIR}<"
427  echo "Post:    >${POST_DIR}<"
428  echo "Work:    >${CWORK_DIR}<"
429
430  if [ $PeriodState != "Waiting" ] ; then
431
432    # Check pending rebuilds
433    # ======================
434
435    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
436    if [ ${#RebuildList[*]} -gt 0 ] ; then
437      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )
438
439      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
440      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
441    else
442      NbRebuild="."
443
444      FirstRebuild="."
445      LastRebuild="."
446    fi
447  else
448    NbRebuild="."
449
450    FirstRebuild="."
451    LastRebuild="."
452  fi
453
454
455  # Check last REBUILD and PACK* jobs
456  # =================================
457  # Define input parameters
458  # -----------------------
459  set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug"
460
461  for JobType in ${JobType_list[*]} ; do
462    typeset    name1="${JobType}_String"
463    typeset    name2="${JobType}_Field"
464    if [ X${JobType} == X${RebuildJob} ] ; then
465      eval ${name1}=IGCM_sys_PutBuffer_Out
466      eval ${name2}=4
467    else
468      eval ${name1}=IGCM_sys_Put_Out
469      eval ${name2}=3
470    fi
471  done
472
473  # Check jobs
474  # ----------
475  for JobType in ${JobType_list[*]} ; do
476    eval String=\${${JobType}_String}
477    eval Field=\${${JobType}_Field}
478
479    FileList=$( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} )
480
481    (( ind = 0 ))
482    for FileName in ${FileList} ; do
483      LastDate=$( basename ${FileName} | awk -F"." '{ print $(NF-1) }' )
484
485      dum=$( basename $( grep ^${String} ${POST_DIR}/${JobType}.${LastDate}.out | \
486               tail -1 | awk -v Field=${Field} '{ print $( Field ) }' ) | \
487               awk -F"_" '{ print $3 }' )
488
489      if [[ X${dum} == X${LastDate} ]] ; then
490        Status=OK
491      else
492        Status=KO
493      fi
494
495      eval ${JobType}_Date[$ind]=${LastDate}
496      eval ${JobType}_Status[$ind]=${Status}
497     
498      (( ind = ind + 1 ))
499    done
500  done
501
502  # Check last MONITORING jobs
503  # ==========================
504  JobType=monitoring
505  LastDate=$( cdo showyear ${CWORK_DIR}/MONITORING/files/ATM_bils_global_ave.nc 2> /dev/null | \
506                  awk '{ print $NF }' )
507  eval ${JobType}_Date=${LastDate}
508
509  # Check last ATLAS jobs
510  # =====================
511  JobType=atlas
512  FileList=$( ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto} )
513
514  (( ind = 0 ))
515  for FileName in ${FileList} ; do
516    eval ${JobType}_Date[$ind]=${FileName}
517    (( ind = ind + 1 ))
518  done
519
520
521  # Time of last write on run.card
522  # ==============================
523  LastWrite=$( ls -l --time-style=+"${DateFormat}" ${SUBMIT_DIR}/run.card | awk '{print $6 " " $7}' )
524   
525
526  # Print results
527  # =============
528  AffichResult -f
529  ind=0
530  while [ $ind -lt $NbHisto ] ; do
531    AffichResult -j
532    (( ind = ind + 1 ))
533  done
534  AffichResult -l
535
536fi
537
Note: See TracBrowser for help on using the repository browser.