source: tags/libIGCM_v2.2/libIGCM_check/libIGCM_check.ksh

Last change on this file was 961, checked in by labetoulle, 11 years ago

Bugfix for RunChecker? : restore -p option (problem with the loop used by -r option, #131).
File size: 13.5 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision:: 939                                      $ Revision of last commit
7# $Author:: sdipsl                                     $ Author of last commit
8# $Date:: 2013-08-23 11:38:24 +0200 (Fri, 23 Aug 2013) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#===================================
15function IGCM_check_CommonDef
16{
17
18  IGCM_debug_PushStack "IGCM_check_CommonDef" $@
19
20  # Define colors
21  # -------------
22  ColEsc="\033["
23  ColNon="${ColEsc}0m"       # Return to normal
24  ColExp="${ColEsc}1m"       # Blanc - gras
25  ColFat="${ColEsc}1;31m"    # Fatal
26  ColCpl="${ColEsc}1;32m"    # Completed
27  ColAtt="${ColEsc}1;30m"    # Waiting
28  ColDef="${ColEsc}1;34m"    # Default
29  ColRbl="${ColEsc}31m"      # Rebuild
30
31  # Define date format
32  # ------------------
33  DateFormat="%d/%m/%y %R:%S"
34
35
36  # What kind of rebuild ?
37  # ----------------------
38  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
39    RebuildJob="rebuild_fromArchive"
40    IGCM_sys_TestDirArchive ${REBUILD_DIR}
41    RebuildExists=$?
42  else
43    RebuildJob="rebuild_fromWorkdir"
44    IGCM_sys_TestDir ${REBUILD_DIR}
45    RebuildExists=$?
46  fi
47
48  # Are packs activated or not ?
49  # ----------------------------
50  if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && \
51       [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
52    Pack=true
53  else
54    Pack=false
55  fi
56
57
58  # Define input parameters
59  # -----------------------
60  set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug"
61
62  for JobType in ${JobType_list[*]} ; do
63    typeset    name1="${JobType}_String"
64    typeset    name2="${JobType}_Field"
65    typeset    name3="${JobType}_Activ"
66    if [ X${JobType} == X${RebuildJob} ] ; then
67      if ( ${Pack} ) ; then
68        eval ${name1}=IGCM_sys_PutBuffer_Out
69      else
70        eval ${name1}=IGCM_sys_Put_Out
71      fi
72      eval ${name2}=4
73      eval ${name3}=true
74    else
75      eval ${name1}=IGCM_sys_Put_Out
76      eval ${name2}=3
77      if ( ${Pack} ) ; then
78        eval ${name3}=true
79      else
80        eval ${name3}=false
81      fi
82    fi
83  done
84
85  IGCM_debug_PopStack "IGCM_check_CommonDef"
86
87}
88
89#===================================
90function IGCM_check_ChangeUsr
91{
92
93  IGCM_debug_PushStack "IGCM_check_ChangeUsr" $@
94
95  CurrentGrp=$( groups $CurrentUsr | gawk '{print $3}' )
96  TargetGrp=$( groups $TargetUsr | gawk '{print $3}' )
97
98  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
99                -e "s/${CurrentGrp}/${TargetGrp}/"
100
101  IGCM_debug_PopStack "IGCM_check_ChangeUsr"
102
103}
104
105
106
107#===================================
108function IGCM_check_SearchCatalog
109{
110
111  IGCM_debug_PushStack "IGCM_check_SearchCatalog" $@
112
113  typeset num
114  unset SUBMIT_DIR
115
116  fg_new=false
117
118  if [ ! X${JobName} == X ] ; then
119    NbOcc=$( gawk -v JobName=${JobName} \
120             'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
121  else
122    NbOcc=0
123  fi
124
125  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_path} ) ) ; then
126
127    if [ ! "$( echo ${ConfigPath} | cut -c 1 )" == "/" ] ; then
128      ConfigPath="${PWD}/${ConfigPath}"
129    fi
130
131    set -A FileList $( ls ${ConfigPath}/Job_* )
132    if [ X$FileList == X ] ; then
133      NbOcc=0
134    else
135      NbOcc=${#FileList[@]}
136      fg_new=true
137    fi
138  fi
139
140  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_search} ) ) ; then
141    SEARCH_DIR=${WORKDIR}
142    if [ ${TargetUsr} != ${CurrentUsr} ] ; then
143      SEARCH_DIR=$( IGCM_check_ChangeUsr ${SEARCH_DIR} )
144    fi
145    echo "${JobName} not in Catalog, we'll try to find it in ${SEARCH_DIR}"
146
147    set -A FileList $( find ${SEARCH_DIR}/ \
148                            -path ${SEARCH_DIR}/IGCM_OUT -prune -o \
149                            -name Job_${JobName} -print )
150    if [ X$FileList == X ] ; then
151      NbOcc=0
152    else
153      NbOcc=${#FileList[@]}
154      fg_new=true
155    fi
156  fi
157
158  if [ ${NbOcc} -gt 1 ] ; then
159    echo "More than one job"
160    ind=0
161    while [ ${ind} -lt ${NbOcc} ] ; do
162      printf '%2i) %-30s\n' ${ind} ${FileList[${ind}]}
163      (( ind = ind + 1 ))
164    done
165    echo "Give your choice number or 'q' to quit : "
166    read Choice
167    if [ X${Choice} == Xq ] ; then
168      return 1
169    else
170      fg_new=true
171      FileList=${FileList[${Choice}]}
172      NbOcc=1
173    fi
174  fi
175
176  case ${NbOcc} in
177    0)
178      if ( ${fg_search} ) ; then
179        echo "${JobName} not found, neither in catalog nor in \$WORKDIR."
180      else
181        echo "${JobName} not found in catalog."
182      fi
183      echo "You can try : *) '-s' option to automatically search your \$WORKDIR ; "
184      echo "              *) '-p' option to provide the directory (absolute path) "
185      echo "                      containing the config.card ; "
186      echo "              *)  manually editing your ${SimuCatalog}."
187      return 1 ;;
188    1)
189      if ( ${fg_new} ) ; then
190#        JobName=${JobName:=$( basename ${FileList} | gawk -F"_" '{ print $2 }' )}
191        JobName=$( basename ${FileList} | gawk -F"_" '{ print $2 }' )
192        SUBMIT_DIR=$( dirname ${FileList} )
193        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}"
194        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" >> ${SimuCatalog}
195        sort -u ${SimuCatalog} > ${SimuCatalog}.tmp
196        mv ${SimuCatalog}.tmp ${SimuCatalog}
197      elif ( [ ${TargetUsr} == $( gawk -v JobName=${JobName} \
198                                      '$1 ~ JobName {print $2}' \
199                                      ${SimuCatalog} ) ] \
200          && [ ${HostName}  == $( gawk -v JobName=${JobName} \
201                                      '$1 ~ JobName {print $3}' \
202                                      ${SimuCatalog} ) ] ) ; then
203        JobName=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
204        SUBMIT_DIR=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
205      else
206        echo "${JobName} not in Catalog."
207        return 1
208      fi
209      ;;
210    *)
211      break ;;
212  esac
213
214  IGCM_debug_PopStack "IGCM_check_SearchCatalog"
215
216}
217
218#===================================
219function IGCM_check_PrintHeader
220{
221
222  IGCM_debug_PushStack "IGCM_check_PrintHeader" $@
223
224  echo "|===========================================================================================================|"
225  printf "| JobName = ${ColExp}%-26.26s${ColNon} JobID = ${ColExp}%-36.36s${ColNon}date : ${ColExp}%-18s${ColNon}|\n" \
226         ${JobName} ${JobID} "$( date +"${DateFormat}" )"
227
228  printf "| User    = ${ColExp}%-56.56s${ColNon}last run.card write : ${ColExp}%-18s${ColNon}|\n" \
229         ${TargetUsr} "${LastWrite}"
230
231  printf "|-----------------------------------------------------------------------------------------------------------|\n"
232
233  printf "| Submit  | ${ColExp}%-95.95s${ColNon} |\n" ${SUBMIT_DIR}
234  printf "| Data    | ${ColExp}%-95.95s${ColNon} |\n" ${DATA_DIR}
235  printf "| Rebuild | ${ColExp}%-95.95s${ColNon} |\n" ${REBUILD_DIR}
236  printf "| Post    | ${ColExp}%-95.95s${ColNon} |\n" ${POST_DIR}
237  printf "| Work    | ${ColExp}%-95.95s${ColNon} |\n" ${CWORK_DIR}
238
239  echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
240  echo "|                         |             |                         |             |     Pending Rebuilds      |"
241  echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
242  echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
243
244  printf "| %-10s - %-10s | " \
245         $DateBegin $DateEnd
246
247  case $PeriodState in
248    Fatal)
249      Color=${ColFat}
250      ;;
251    Completed)
252      Color=${ColCpl}
253      ;;
254    Waiting|OnQueue)
255      Color=${ColAtt}
256      ;;
257    *)
258      Color=${ColDef}
259      ;;
260  esac
261  printf "${Color}%-11s${ColNon} | " $PeriodState
262
263  printf "%-10s - %-10s | %11s | " \
264         $PeriodDateBegin $PeriodDateEnd $CumulPeriod
265
266  if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
267    printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
268           $NbRebuild $FirstRebuild $LastRebuild
269  else
270    printf "%3s : %-8s : %-8s |\n" \
271           $NbRebuild $FirstRebuild $LastRebuild
272  fi
273
274  if [ ${NbLines} -gt 0 ] ; then
275    printf "|-----------------------------------------------------------------------------------------------------------|\n"
276    printf "|                                                      Last                                                 |\n"
277    printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
278    printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
279  fi
280
281  IGCM_debug_PopStack "IGCM_check_PrintHeader"
282
283}
284
285#===================================
286function IGCM_check_PrintJob
287{
288
289  IGCM_debug_PushStack "IGCM_check_PrintJob" $@
290
291  printf "|"
292
293  # Print rebuild and pack jobs
294  # ---------------------------
295  for JobType in ${JobType_list[*]} ; do
296    eval Date=\${${JobType}_Date[${ind}]}
297    eval Status=\${${JobType}_Status[${ind}]}
298    eval Nb=\${${JobType}_Nb[${ind}]}
299
300    if [ X${Status} == XOK  ] ; then
301      Color=${ColCpl}
302    else
303      Color=${ColFat}
304    fi
305    printf "  ${Color}%-8s${ColNon} : %3s  |" ${Date} ${Nb}
306  done
307
308  Color=${ColExp}
309
310  # Print monitoring jobs
311  # ---------------------
312  JobType=monitoring
313  if [ $ind -eq 0 ] ; then
314    eval Date=\${${JobType}_Date}
315  else
316    Date=""
317  fi
318  printf "     ${Color}%-4s${ColNon}     |" ${Date}
319
320  # Print atlas jobs
321  # ----------------
322  JobType=atlas
323  eval Date=\${${JobType}_Date[${ind}]}
324  printf "  ${Color}%-12s${ColNon}  |" ${Date}
325
326  printf "\n"
327
328  IGCM_debug_PopStack "IGCM_check_PrintJob"
329
330}
331
332#===================================
333function IGCM_check_PrintFooter
334{
335
336  IGCM_debug_PushStack "IGCM_check_PrintFooter" $@
337
338  printf "|===========================================================================================================|\n"
339  #date +"${DateFormat}"
340
341  IGCM_debug_PopStack "IGCM_check_PrintFooter"
342
343}
344
345#===================================
346function IGCM_check_CheckPendingRebuild
347{
348
349  IGCM_debug_PushStack "IGCM_check_PendingRebuild" $@
350
351  NbRebuild="."
352  FirstRebuild="."
353  LastRebuild="."
354
355  if [ ${RebuildExists} == 0 ] ; then
356    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
357    if [ ${#RebuildList[*]} -gt 0 ] ; then
358      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )
359
360      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
361      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
362    fi
363  fi
364
365  IGCM_debug_PopStack "IGCM_check_PendingRebuild"
366
367}
368
369#===================================
370function IGCM_check_CheckRebPackJobs
371{
372
373  IGCM_debug_PushStack "IGCM_check_CheckRebPackJobs" $@
374
375  NbLines=0
376  for JobType in ${JobType_list[*]} ; do
377    eval String=\${${JobType}_String}
378    eval Field=\${${JobType}_Field}
379    eval Activ=\${${JobType}_Activ}
380
381    if ( ${Activ} ) ; then
382
383      set -A FileList $( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} )
384
385      if [ ${#FileList[*]} -gt ${NbLines} ] ; then
386        NbLines=${#FileList[*]}
387      fi
388
389      (( ind = 0 ))
390      for FileName in ${FileList[*]} ; do
391        LastDate=""
392        LastDate=$( basename ${FileName} | gawk -F"." '{ print $(NF-1) }' )
393
394        set -- $( gawk -v String=${String} \
395                       'BEGIN { nb_ok = 0 ; nb_ko = 0 } \
396                       ($1 ~ String) { \
397                         if ($3 !~ /error./) { \
398                           nb_ok = nb_ok + 1 \
399                         } else { \
400                           nb_ko = nb_ko + 1 \
401                         } \
402                       } \
403                       END { print nb_ok " " nb_ko }' \
404                       ${POST_DIR}/${JobType}.${LastDate}.out )
405        Match=$1
406        Error=$2
407
408        (( Nb = ${Match} - ${Error} ))
409
410        if ( [ ${Error} -eq 0 ] && [ ${Nb} -gt 0 ] ) ; then
411          Status=OK
412        else
413          Status=KO
414        fi
415
416        eval ${JobType}_Date[$ind]=${LastDate}
417        eval ${JobType}_Status[$ind]=${Status}
418        eval ${JobType}_Nb[$ind]=${Nb}
419
420        (( ind = ind + 1 ))
421      done
422
423    else
424
425        eval ${JobType}_Date[0]=""
426        eval ${JobType}_Status[0]=""
427        eval ${JobType}_Nb[0]=""
428
429    fi
430
431  done
432
433  IGCM_debug_PopStack "IGCM_check_CheckRebPackJobs"
434
435}
436
437#===================================
438function IGCM_check_CheckMonitoring
439{
440
441  IGCM_debug_PushStack "IGCM_check_CheckMonitoring" $@
442
443    LastDate=""
444    JobType=monitoring
445    IGCM_sys_TestDirArchive ${CWORK_DIR}/MONITORING
446    RC=$?
447    if [ $RC == 0 ] ; then
448      FileTmp=$( IGCM_sys_RshArchive "ls ${CWORK_DIR}/MONITORING/files/*.nc | head -n 1" )
449      IGCM_sys_GetDate_Monitoring ${FileTmp} LastDate
450      eval ${JobType}_Date=${LastDate}
451    fi
452
453  IGCM_debug_PopStack "IGCM_check_CheckMonitoring"
454
455}
456
457#===================================
458function IGCM_check_CheckAtlas
459{
460
461  IGCM_debug_PushStack "IGCM_check_CheckAtlas" $@
462
463  JobType=atlas
464  IGCM_sys_TestDirArchive ${CWORK_DIR}/ATLAS
465  RC=$?
466  if [ $RC == 0 ] ; then
467    set -A FileList $( IGCM_sys_RshArchive "ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto}" )
468
469    if [ ${#FileList[*]} -gt ${NbLines} ] ; then
470      NbLines=${#FileList[*]}
471    fi
472
473    (( ind = 0 ))
474    for FileName in ${FileList[*]} ; do
475      eval ${JobType}_Date[$ind]=${FileName}
476      (( ind = ind + 1 ))
477    done
478  fi
479
480  IGCM_debug_PopStack "IGCM_check_CheckAtlas"
481
482}
483
484#===================================
Note: See TracBrowser for help on using the repository browser.