source: trunk/libIGCM/libIGCM_check/libIGCM_check.ksh @ 958

Last change on this file since 958 was 958, checked in by labetoulle, 11 years ago

RunChecker? : add JobID to output (#169).
File size: 13.3 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision:: 939                                      $ Revision of last commit
7# $Author:: sdipsl                                     $ Author of last commit
8# $Date:: 2013-08-23 11:38:24 +0200 (Fri, 23 Aug 2013) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#===================================
15function IGCM_check_CommonDef
16{
17
18  IGCM_debug_PushStack "IGCM_check_CommonDef" $@
19
20  # Define colors
21  # -------------
22  ColEsc="\033["
23  ColNon="${ColEsc}0m"       # Return to normal
24  ColExp="${ColEsc}1m"       # Blanc - gras
25  ColFat="${ColEsc}1;31m"    # Fatal
26  ColCpl="${ColEsc}1;32m"    # Completed
27  ColAtt="${ColEsc}1;30m"    # Waiting
28  ColDef="${ColEsc}1;34m"    # Default
29  ColRbl="${ColEsc}31m"      # Rebuild
30
31  # Define date format
32  # ------------------
33  DateFormat="%d/%m/%y %R:%S"
34
35
36  # What kind of rebuild ?
37  # ----------------------
38  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
39    RebuildJob="rebuild_fromArchive"
40    IGCM_sys_TestDirArchive ${REBUILD_DIR}
41    RebuildExists=$?
42  else
43    RebuildJob="rebuild_fromWorkdir"
44    IGCM_sys_TestDir ${REBUILD_DIR}
45    RebuildExists=$?
46  fi
47
48  # Are packs activated or not ?
49  # ----------------------------
50  if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && \
51       [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
52    Pack=true
53  else
54    Pack=false
55  fi
56
57
58  # Define input parameters
59  # -----------------------
60  set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug"
61
62  for JobType in ${JobType_list[*]} ; do
63    typeset    name1="${JobType}_String"
64    typeset    name2="${JobType}_Field"
65    typeset    name3="${JobType}_Activ"
66    if [ X${JobType} == X${RebuildJob} ] ; then
67      if ( ${Pack} ) ; then
68        eval ${name1}=IGCM_sys_PutBuffer_Out
69      else
70        eval ${name1}=IGCM_sys_Put_Out
71      fi
72      eval ${name2}=4
73      eval ${name3}=true
74    else
75      eval ${name1}=IGCM_sys_Put_Out
76      eval ${name2}=3
77      if ( ${Pack} ) ; then
78        eval ${name3}=true
79      else
80        eval ${name3}=false
81      fi
82    fi
83  done
84
85  IGCM_debug_PopStack "IGCM_check_CommonDef"
86
87}
88
89#===================================
90function IGCM_check_ChangeUsr
91{
92
93  IGCM_debug_PushStack "IGCM_check_ChangeUsr" $@
94
95  CurrentGrp=$( groups $CurrentUsr | gawk '{print $3}' )
96  TargetGrp=$( groups $TargetUsr | gawk '{print $3}' )
97
98  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
99                -e "s/${CurrentGrp}/${TargetGrp}/"
100
101  IGCM_debug_PopStack "IGCM_check_ChangeUsr"
102
103}
104
105
106
107#===================================
108function IGCM_check_SearchCatalog
109{
110
111  IGCM_debug_PushStack "IGCM_check_SearchCatalog" $@
112
113  typeset num
114  unset SUBMIT_DIR
115
116  fg_new=false
117
118  if [ ! X${JobName} == X ] ; then
119    NbOcc=$( gawk -v JobName=${JobName} \
120             'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
121  else
122    NbOcc=0
123  fi
124
125  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_path} ) ) ; then
126    set -A FileList $( ls ${ConfigPath}/Job_* )
127    if [ X$FileList == X ] ; then
128      NbOcc=0
129    else
130      NbOcc=${#FileList[@]}
131      fg_new=true
132    fi
133  fi
134
135  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_search} ) ) ; then
136    SEARCH_DIR=${WORKDIR}
137    if [ ${TargetUsr} != ${CurrentUsr} ] ; then
138      SEARCH_DIR=$( IGCM_check_ChangeUsr ${SEARCH_DIR} )
139    fi
140    echo "${JobName} not in Catalog, we'll try to find it in ${SEARCH_DIR}"
141
142    set -A FileList $( find ${SEARCH_DIR}/ \
143                            -path ${SEARCH_DIR}/IGCM_OUT -prune -o \
144                            -name Job_${JobName} -print )
145    if [ X$FileList == X ] ; then
146      NbOcc=0
147    else
148      NbOcc=${#FileList[@]}
149      fg_new=true
150    fi
151  fi
152
153  if [ ${NbOcc} -gt 1 ] ; then
154    echo "More than one job"
155    ind=0
156    while [ ${ind} -lt ${NbOcc} ] ; do
157      printf '%2i) %-30s\n' ${ind} ${FileList[${ind}]}
158      (( ind = ind + 1 ))
159    done
160    echo "Give your choice number or 'q' to quit : "
161    read Choice
162    if [ X${Choice} == Xq ] ; then
163      return 1
164    else
165      fg_new=true
166      FileList=${FileList[${Choice}]}
167      NbOcc=1
168    fi
169  fi
170
171  case ${NbOcc} in
172    0)
173      if ( ${fg_search} ) ; then
174        echo "${JobName} not found, neither in catalog nor in \$WORKDIR."
175      else
176        echo "${JobName} not found in catalog."
177      fi
178      echo "You can try : *) '-s' option to automatically search your \$WORKDIR ; "
179      echo "              *) '-p' option to provide the directory (absolute path) "
180      echo "                      containing the config.card ; "
181      echo "              *)  manually editing your ${SimuCatalog}."
182      return 1 ;;
183    1)
184      if ( ${fg_new} ) ; then
185        JobName=${JobName:=$( basename ${FileList} | gawk -F"_" '{ print $2 }' )}
186        SUBMIT_DIR=$( dirname ${FileList} )
187        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}"
188        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" >> ${SimuCatalog}
189        sort -u ${SimuCatalog} > ${SimuCatalog}.tmp
190        mv ${SimuCatalog}.tmp ${SimuCatalog}
191      elif ( [ ${TargetUsr} == $( gawk -v JobName=${JobName} \
192                                      '$1 ~ JobName {print $2}' \
193                                      ${SimuCatalog} ) ] \
194          && [ ${HostName}  == $( gawk -v JobName=${JobName} \
195                                      '$1 ~ JobName {print $3}' \
196                                      ${SimuCatalog} ) ] ) ; then
197        JobName=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
198        SUBMIT_DIR=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
199      else
200        echo "${JobName} not in Catalog."
201        return 1
202      fi
203      ;;
204    *)
205      break ;;
206  esac
207
208  IGCM_debug_PopStack "IGCM_check_SearchCatalog"
209
210}
211
212#===================================
213function IGCM_check_PrintHeader
214{
215
216  IGCM_debug_PushStack "IGCM_check_PrintHeader" $@
217
218  echo "|===========================================================================================================|"
219  printf "| JobName = ${ColExp}%-26.26s${ColNon} JobID = ${ColExp}%-36.36s${ColNon}date : ${ColExp}%-18s${ColNon}|\n" \
220         ${JobName} ${JobID} "$( date +"${DateFormat}" )"
221
222  printf "| User    = ${ColExp}%-56.56s${ColNon}last run.card write : ${ColExp}%-18s${ColNon}|\n" \
223         ${TargetUsr} "${LastWrite}"
224
225  printf "|-----------------------------------------------------------------------------------------------------------|\n"
226
227  printf "| Submit  | ${ColExp}%-95.95s${ColNon} |\n" ${SUBMIT_DIR}
228  printf "| Data    | ${ColExp}%-95.95s${ColNon} |\n" ${DATA_DIR}
229  printf "| Rebuild | ${ColExp}%-95.95s${ColNon} |\n" ${REBUILD_DIR}
230  printf "| Post    | ${ColExp}%-95.95s${ColNon} |\n" ${POST_DIR}
231  printf "| Work    | ${ColExp}%-95.95s${ColNon} |\n" ${CWORK_DIR}
232
233  echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
234  echo "|                         |             |                         |             |     Pending Rebuilds      |"
235  echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
236  echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
237
238  printf "| %-10s - %-10s | " \
239         $DateBegin $DateEnd
240
241  case $PeriodState in
242    Fatal)
243      Color=${ColFat}
244      ;;
245    Completed)
246      Color=${ColCpl}
247      ;;
248    Waiting|OnQueue)
249      Color=${ColAtt}
250      ;;
251    *)
252      Color=${ColDef}
253      ;;
254  esac
255  printf "${Color}%-11s${ColNon} | " $PeriodState
256
257  printf "%-10s - %-10s | %11s | " \
258         $PeriodDateBegin $PeriodDateEnd $CumulPeriod
259
260  if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
261    printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
262           $NbRebuild $FirstRebuild $LastRebuild
263  else
264    printf "%3s : %-8s : %-8s |\n" \
265           $NbRebuild $FirstRebuild $LastRebuild
266  fi
267
268  if [ ${NbLines} -gt 0 ] ; then
269    printf "|-----------------------------------------------------------------------------------------------------------|\n"
270    printf "|                                                      Last                                                 |\n"
271    printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
272    printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
273  fi
274
275  IGCM_debug_PopStack "IGCM_check_PrintHeader"
276
277}
278
279#===================================
280function IGCM_check_PrintJob
281{
282
283  IGCM_debug_PushStack "IGCM_check_PrintJob" $@
284
285  printf "|"
286
287  # Print rebuild and pack jobs
288  # ---------------------------
289  for JobType in ${JobType_list[*]} ; do
290    eval Date=\${${JobType}_Date[${ind}]}
291    eval Status=\${${JobType}_Status[${ind}]}
292    eval Nb=\${${JobType}_Nb[${ind}]}
293
294    if [ X${Status} == XOK  ] ; then
295      Color=${ColCpl}
296    else
297      Color=${ColFat}
298    fi
299    printf "  ${Color}%-8s${ColNon} : %3s  |" ${Date} ${Nb}
300  done
301
302  Color=${ColExp}
303
304  # Print monitoring jobs
305  # ---------------------
306  JobType=monitoring
307  if [ $ind -eq 0 ] ; then
308    eval Date=\${${JobType}_Date}
309  else
310    Date=""
311  fi
312  printf "     ${Color}%-4s${ColNon}     |" ${Date}
313
314  # Print atlas jobs
315  # ----------------
316  JobType=atlas
317  eval Date=\${${JobType}_Date[${ind}]}
318  printf "  ${Color}%-12s${ColNon}  |" ${Date}
319
320  printf "\n"
321
322  IGCM_debug_PopStack "IGCM_check_PrintJob"
323
324}
325
326#===================================
327function IGCM_check_PrintFooter
328{
329
330  IGCM_debug_PushStack "IGCM_check_PrintFooter" $@
331
332  printf "|===========================================================================================================|\n"
333  #date +"${DateFormat}"
334
335  IGCM_debug_PopStack "IGCM_check_PrintFooter"
336
337}
338
339#===================================
340function IGCM_check_CheckPendingRebuild
341{
342
343  IGCM_debug_PushStack "IGCM_check_PendingRebuild" $@
344
345  NbRebuild="."
346  FirstRebuild="."
347  LastRebuild="."
348
349  if [ ${RebuildExists} == 0 ] ; then
350    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
351    if [ ${#RebuildList[*]} -gt 0 ] ; then
352      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )
353
354      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
355      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
356    fi
357  fi
358
359  IGCM_debug_PopStack "IGCM_check_PendingRebuild"
360
361}
362
363#===================================
364function IGCM_check_CheckRebPackJobs
365{
366
367  IGCM_debug_PushStack "IGCM_check_CheckRebPackJobs" $@
368
369  NbLines=0
370  for JobType in ${JobType_list[*]} ; do
371    eval String=\${${JobType}_String}
372    eval Field=\${${JobType}_Field}
373    eval Activ=\${${JobType}_Activ}
374
375    if ( ${Activ} ) ; then
376
377      set -A FileList $( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} )
378
379      if [ ${#FileList[*]} -gt ${NbLines} ] ; then
380        NbLines=${#FileList[*]}
381      fi
382
383      (( ind = 0 ))
384      for FileName in ${FileList[*]} ; do
385        LastDate=""
386        LastDate=$( basename ${FileName} | gawk -F"." '{ print $(NF-1) }' )
387
388        set -- $( gawk -v String=${String} \
389                       'BEGIN { nb_ok = 0 ; nb_ko = 0 } \
390                       ($1 ~ String) { \
391                         if ($3 !~ /error./) { \
392                           nb_ok = nb_ok + 1 \
393                         } else { \
394                           nb_ko = nb_ko + 1 \
395                         } \
396                       } \
397                       END { print nb_ok " " nb_ko }' \
398                       ${POST_DIR}/${JobType}.${LastDate}.out )
399        Match=$1
400        Error=$2
401
402        (( Nb = ${Match} - ${Error} ))
403
404        if ( [ ${Error} -eq 0 ] && [ ${Nb} -gt 0 ] ) ; then
405          Status=OK
406        else
407          Status=KO
408        fi
409
410        eval ${JobType}_Date[$ind]=${LastDate}
411        eval ${JobType}_Status[$ind]=${Status}
412        eval ${JobType}_Nb[$ind]=${Nb}
413
414        (( ind = ind + 1 ))
415      done
416
417    else
418
419        eval ${JobType}_Date[0]=""
420        eval ${JobType}_Status[0]=""
421        eval ${JobType}_Nb[0]=""
422
423    fi
424
425  done
426
427  IGCM_debug_PopStack "IGCM_check_CheckRebPackJobs"
428
429}
430
431#===================================
432function IGCM_check_CheckMonitoring
433{
434
435  IGCM_debug_PushStack "IGCM_check_CheckMonitoring" $@
436
437    LastDate=""
438    JobType=monitoring
439    IGCM_sys_TestDirArchive ${CWORK_DIR}/MONITORING
440    RC=$?
441    if [ $RC == 0 ] ; then
442      FileTmp=$( IGCM_sys_RshArchive "ls ${CWORK_DIR}/MONITORING/files/*.nc | head -n 1" )
443      IGCM_sys_GetDate_Monitoring ${FileTmp} LastDate
444      eval ${JobType}_Date=${LastDate}
445    fi
446
447  IGCM_debug_PopStack "IGCM_check_CheckMonitoring"
448
449}
450
451#===================================
452function IGCM_check_CheckAtlas
453{
454
455  IGCM_debug_PushStack "IGCM_check_CheckAtlas" $@
456
457  JobType=atlas
458  IGCM_sys_TestDirArchive ${CWORK_DIR}/ATLAS
459  RC=$?
460  if [ $RC == 0 ] ; then
461    set -A FileList $( IGCM_sys_RshArchive "ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto}" )
462
463    if [ ${#FileList[*]} -gt ${NbLines} ] ; then
464      NbLines=${#FileList[*]}
465    fi
466
467    (( ind = 0 ))
468    for FileName in ${FileList[*]} ; do
469      eval ${JobType}_Date[$ind]=${FileName}
470      (( ind = ind + 1 ))
471    done
472  fi
473
474  IGCM_debug_PopStack "IGCM_check_CheckAtlas"
475
476}
477
478#===================================
Note: See TracBrowser for help on using the repository browser.