source: trunk/libIGCM/AA_RunChecker @ 753

Last change on this file since 753 was 753, checked in by labetoulle, 12 years ago

Use IGCM_config_CommonConfiguration to define paths (R_OUT, R_SAVE, R_BUFR, ...) in RunChecker?.
See #93.
  • Property svn:executable set to *
  • Property svn:keywords set to Revision Author Date
File size: 11.5 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sonia Labetoulle
5# Contact: sonia.labetoulle__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2012)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#======================================================================#
15
16# Display a report on how a simulation is faring.
17
18function ChangeUsr {
19
20#  echo ">$1<"
21
22  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
23                -e "s/${CurrentGrp}/${TargetGrp}/"
24
25}
26
27
28function SearchCatalog {
29
30  typeset num
31  unset SUBMIT_DIR
32
33  NbOcc=$( awk -v JobName=${JobName} \
34           'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
35
36  if [ ${NbOcc} -eq 0 ] ; then
37    if [ X${fg_new} == X.true. ] ; then
38      echo "${JobName} not in Catalog, we'll try to find it in ${WORK_DIR}"
39     
40      set -A FileList
41      FileList=$( find ${WORK_DIR}/ -name "Job_${JobName}" )
42      echo $FileList
43      (( NbOcc = ${#FileList[@]} ))
44    else
45      echo "${JobName} not in Catalog."
46      echo "Use option -n / --new to look for it and add it to the catalog."
47      exit
48    fi
49  fi
50
51
52  if [ ${NbOcc} -eq 0 ] ; then
53    echo "${JobName} not found."
54    exit
55  else
56    if [ ${NbOcc} -eq 1 ] ; then
57
58      if ( [ ${TargetUsr} == $( awk -v JobName=${JobName} '$1 ~ JobName {print $2}' ${SimuCatalog} ) ] && [ ${HostName} == $( awk -v JobName=${JobName} '$1 ~ JobName {print $3}' ${SimuCatalog} ) ] ) ; then
59        JobName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
60        TargetUsr=$( awk -v JobName=${JobName} '$1 ~ JobName {print $2}' ${SimuCatalog} )
61        HostName=$( awk -v JobName=${JobName} '$1 ~ JobName {print $3}' ${SimuCatalog} )
62        SUBMIT_DIR=$( awk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
63      else
64        echo "${JobName} not in Catalog."
65        exit
66      fi
67    fi
68  fi
69
70}
71
72
73function AffichResult {
74
75  fg_first=".false."
76  fg_last=".false."
77
78  while [ $# -ne 0 ] ; do
79    case ${1} in
80      -f|--first)
81        fg_first=".true."
82        shift 1 ;;
83      -l|--last)
84        fg_last=".true."
85        shift 1 ;;
86      -*)
87        echo "usage: ${0}"
88        echo "       options = -f; -l"
89        exit ;;
90      *)
91        break ;;
92    esac
93  done
94
95
96  # Define colors
97  ColEsc="\033["
98  ColNon="${ColEsc}0m"       # Return to normal
99  ColExp="${ColEsc}1m"       # Blanc - gras
100  ColFat="${ColEsc}1;31m"    # Fatal
101  ColCpl="${ColEsc}1;32m"    # Completed
102  ColAtt="${ColEsc}1;30m"    # Waiting
103  ColDef="${ColEsc}1;34m"    # Default
104  ColRbl="${ColEsc}31m"      # Rebuild
105
106
107 
108  if [ X${fg_first} == X.true. ] ; then
109    Dum=""
110    (( len = 67 - ${#JobName} ))
111    echo "|===========================================================================================================|"
112    printf "| JobName = ${ColExp}%-${#JobName}s${ColNon}" ${JobName}
113    printf "%-${len}s" ${Dum} 
114    printf "run.card : ${ColExp}%-17s${ColNon}|\n" "${LastWrite} "
115    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
116    echo "|                         |             |                         |             |     Pending Rebuilds      |"
117    echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
118    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
119    return
120  fi
121
122  if [ X${fg_last} == X.true. ] ; then
123    printf "|===========================================================================================================|\n"
124    date +"${DateFormat}"
125    return
126  fi
127
128  printf "| %-10s - %-10s | " \
129         $DateBegin $DateEnd 
130
131  case $PeriodState in
132    Fatal)
133      Color=${ColFat}
134      break ;;
135    Completed)
136      Color=${ColCpl}
137      break ;;
138    Waiting|OnQueue)
139      Color=${ColAtt}
140      break ;;
141    *)
142      Color=${ColDef}
143      break ;;
144  esac
145  printf "${Color}%-11s${ColNon} | " $PeriodState
146
147  printf "%-10s - %-10s | %11s | " \
148         $PeriodDateBegin $PeriodDateEnd $CumulPeriod
149
150  if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
151    printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
152           $NbRebuild $FirstRebuild $LastRebuild
153  else
154    printf "%3s : %-8s : %-8s |\n" \
155           $NbRebuild $FirstRebuild $LastRebuild
156  fi
157
158  printf "|-----------------------------------------------------------------------------------------------------------|\n"
159  printf "|                                                      Last                                                 |\n"
160  printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
161  printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
162
163  if [ X${Result[1]} == XOK  ] ; then
164    Color=${ColCpl}
165  else
166    Color=${ColFat}
167  fi
168  printf "|     ${Color}%-8s${ColNon}     |" ${LastDate[1]}
169
170  if [ X${Result[2]} == XOK  ] ; then
171    Color=${ColCpl}
172  else
173    Color=${ColFat}
174  fi
175  printf "     ${Color}%-8s${ColNon}     |" ${LastDate[2]}
176
177  if [ X${Result[3]} == XOK  ] ; then
178    Color=${ColCpl}
179  else
180    Color=${ColFat}
181  fi
182  printf "     ${Color}%-8s${ColNon}     |" ${LastDate[3]}
183
184  if [ X${Result[4]} == XOK  ] ; then
185    Color=${ColCpl}
186  else
187    Color=${ColFat}
188  fi
189  printf "     ${Color}%-8s${ColNon}     |" ${LastDate[4]}
190
191  Color=${ColExp}
192  printf "     ${Color}%-4s${ColNon}     |  ${Color}%-12s${ColNon}  |\n" \
193         ${LastDate[5]} ${LastDate[6]}
194}
195
196#======================================================================#
197
198#D- Task type (computing or post-processing)
199TaskType=post-processing
200typeset -i Verbosity=0
201
202CurrentUsr=$( whoami )
203CurrentGrp=$( groups $CurrentUsr | awk '{print $3}' )
204
205libIGCM=${libIGCM:=::modipsl::/libIGCM}
206
207. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
208     ( ${DEBUG_debug} ) && IGCM_debug_Check
209. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
210     ( ${DEBUG_debug} ) && IGCM_card_Check
211. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
212     ( ${DEBUG_debug} ) && IGCM_date_Check
213#-------
214. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
215. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
216
217if ( [ $# -eq 0 ] ) ; then
218  $0 -h
219  exit
220fi
221
222# Default argument values
223TargetUsr=${CurrentUsr}
224HostName=$( hostname | sed -e "s/[0-9].*//" )
225
226fg_color=".true."
227fg_new=".false."
228fg_quiet=".false."
229(( NbHisto = 1 ))
230
231# Get arguments from command line
232while [ $# -ne 0 ] ; do
233  case $1 in
234    -h|--help|-help)
235      echo "usage: $0 [-u TargetUsr] JobName"
236      exit ;;
237    -b|--nocolor)
238      fg_color=".false."
239      shift 1 ;;
240    -n|--new)
241      fg_new=".true."
242      shift 1 ;;
243    -q|--quiet)
244      fg_quiet=".true."
245      shift 1 ;;
246    -p|--config-path)
247      ConfigPath="$2"
248      shift 2 ;;
249    -u|--user)
250      TargetUsr="$2"
251      shift 2 ;;
252    -*)
253      $0 -h
254      exit ;;
255    *)
256      break ;;
257  esac
258done
259
260JobName=$1
261
262echo "TargetUsr = ${TargetUsr}"
263echo "HostName = ${HostName}"
264
265TargetGrp=$( groups $TargetUsr | awk '{print $3}' )
266
267# Define the catalog in which the known simulations are stored
268SimuCatalog="$( ccc_home )/.simucatalog.dat"
269if [ ! -s ${SimuCatalog} ] ; then
270  touch ${SimuCatalog}
271fi
272
273# Date format
274DateFormat="%d/%m/%y %R:%S"
275
276# Find SUBMIT_DIR from catalog
277
278SearchCatalog
279
280# Go to SUBMIT_DIR
281if [ ! X${SUBMIT_DIR} == X ] ; then
282
283  echo "Submit:  >${SUBMIT_DIR}<"
284  cd $SUBMIT_DIR
285
286
287  # Extract usefull information from run.card and config.card
288
289  RunFile="${SUBMIT_DIR}/run.card"
290  ConfFile="${SUBMIT_DIR}/config.card"
291
292  IGCM_config_CommonConfiguration ${SUBMIT_DIR}/config.card
293
294
295  if [ -s ${RunFile} ] ; then
296    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodState
297    PeriodState=${run_Configuration_PeriodState}
298  else
299    PeriodState="Waiting"
300  fi
301
302  if ( [ X${PeriodState} == XRunning ] || [ X${PeriodState} == XOnQueue ] ) ; then
303    NbRun=$( ccc_mstat -f | grep -c ${JobName} )
304
305    if [ ${NbRun} -eq 0 ] ; then
306      PeriodState="Fatal"
307    fi
308  fi
309
310  DateBegin=${config_UserChoices_DateBegin}
311  DateEnd=${config_UserChoices_DateEnd}
312  TagName=${config_UserChoices_TagName}
313  ExperimentName=${config_UserChoices_ExperimentName}
314  SpaceName=${config_UserChoices_SpaceName}
315
316  IGCM_card_DefineVariableFromOption ${RunFile} Configuration CumulPeriod
317  CumulPeriod=${run_Configuration_CumulPeriod}
318
319  if ( [ ! X${PeriodState} == XWaiting ] && [ ! X${PeriodState} == XCompleted ] ) ; then
320    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateBegin
321    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateEnd
322    PeriodDateBegin=${run_Configuration_PeriodDateBegin}
323    PeriodDateEnd=${run_Configuration_PeriodDateEnd}
324  else
325    PeriodDateBegin="."
326    PeriodDateEnd="."
327  fi
328
329  DATA_DIR=${R_SAVE}
330  POST_DIR=${R_BUFR}/Out
331  CWORK_DIR=${R_FIGR}
332  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
333    REBUILD_DIR=${R_SAVE}/TMP
334  else
335    REBUILD_DIR=${BIG_DIR}/${config_UserChoices_TagName}/${config_UserChoices_JobName}
336  fi
337
338  if [ ${TargetUsr} != ${CurrentUsr} ] ; then
339    DATA_DIR=$( ChangeUsr ${DATA_DIR})
340    POST_DIR=$( ChangeUsr ${POST_DIR} )
341    CWORK_DIR=$( ChangeUsr ${CWORK_DIR} )
342    REBUILD_DIR=$( ChangeUsr ${REBUILD_DIR} )
343  fi
344
345  echo "Data:    >${DATA_DIR}<"
346  echo "Rebuild: >${REBUILD_DIR}<"
347  echo "Post:    >${POST_DIR}<"
348  echo "Work:    >${CWORK_DIR}<"
349
350  if [ $PeriodState != "Waiting" ] ; then
351
352    # Check missing rebuilds
353
354    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
355    if [ ${#RebuildList[*]} -gt 0 ] ; then
356      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )
357
358      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
359      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
360    else
361      NbRebuild="."
362
363      FirstRebuild="."
364      LastRebuild="."
365    fi
366  else
367    NbRebuild="."
368
369    FirstRebuild="."
370    LastRebuild="."
371  fi
372
373  typeset -A LastDate Result
374
375  # Check last REBUILD
376  FileType=rebuild_*
377  (( ind = 1 ))
378
379  LastDate[$ind]=$( ls ${POST_DIR}/${FileType}.*.out | tail -n ${NbHisto} | awk -F"." '{ print $2 }' )
380
381  dum=$( basename $( grep ^IGCM_sys_PutBuffer_Out ${POST_DIR}/${FileType}.${LastDate[$ind]}.out | tail -1 | awk '{ print $4 }' ) | awk -F"_" '{ print $3 }' )
382
383  if [[ ${dum} == ${LastDate[$ind]} ]] ; then
384    Result[$ind]=OK
385  else
386    Result[$ind]=KO
387  fi
388
389  # Check last PACK jobs
390  for FileType in pack_output pack_restart pack_debug ; do
391    (( ind = ind + 1 ))
392
393    LastDate[$ind]=$( ls ${POST_DIR}/${FileType}.*.out | tail -n ${NbHisto} | awk -F"." '{ print $2 }' )
394
395    dum=$( basename $( grep ^IGCM_sys_Put_Out ${POST_DIR}/${FileType}.${LastDate[$ind]}.out | tail -1 | awk '{ print $3 }' ) | awk -F"_" '{ print $3 }' )
396
397    if [[ ${dum} == ${LastDate[$ind]} ]] ; then
398      Result[$ind]=OK
399    else
400      Result[$ind]=KO
401    fi
402  done
403
404
405  # Check last MONITORING jobs
406
407  #set -vx
408
409  FileType=monitoring
410  (( ind = ind + 1 ))
411
412  LastDate[$ind]=$( cdo showyear ${CWORK_DIR}/MONITORING/files/ATM_bils_global_ave.nc 2> /dev/null | awk '{ print $NF }' )
413
414  # Check last ATLAS jobs
415
416  FileType=atlas
417  (( ind = ind + 1 ))
418
419  LastDate[$ind]=$( ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto} )
420
421
422  # Time of last write on run.card
423  LastWrite=$( ls -l --time-style=+"${DateFormat}" ${SUBMIT_DIR}/run.card | awk '{print $6 " " $7}' )
424   
425
426  # Print results
427  AffichResult -f
428  AffichResult
429  AffichResult -l
430
431fi
432
Note: See TracBrowser for help on using the repository browser.