source: trunk/libIGCM/libIGCM_check/libIGCM_tools.ksh @ 952

Last change on this file since 952 was 952, checked in by labetoulle, 11 years ago

RunChecker? functions renamed and moved to libIGCM_check/libIGCM_tools.ksh (See #107).
File size: 13.1 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision:: 939                                      $ Revision of last commit
7# $Author:: sdipsl                                     $ Author of last commit
8# $Date:: 2013-08-23 11:38:24 +0200 (Fri, 23 Aug 2013) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#===================================
15function IGCM_check_CommonDef
16{
17
18  IGCM_debug_PushStack "IGCM_check_CommonDef" $@
19
20  # Define colors
21  # -------------
22  ColEsc="\033["
23  ColNon="${ColEsc}0m"       # Return to normal
24  ColExp="${ColEsc}1m"       # Blanc - gras
25  ColFat="${ColEsc}1;31m"    # Fatal
26  ColCpl="${ColEsc}1;32m"    # Completed
27  ColAtt="${ColEsc}1;30m"    # Waiting
28  ColDef="${ColEsc}1;34m"    # Default
29  ColRbl="${ColEsc}31m"      # Rebuild
30
31  # Define date format
32  # ------------------
33  DateFormat="%d/%m/%y %R:%S"
34
35
36  # What kind of rebuild ?
37  # ----------------------
38  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
39    RebuildJob="rebuild_fromArchive"
40    IGCM_sys_TestDirArchive ${REBUILD_DIR}
41    RebuildExists=$?
42  else
43    RebuildJob="rebuild_fromWorkdir"
44    IGCM_sys_TestDir ${REBUILD_DIR}
45    RebuildExists=$?
46  fi
47
48  # Are packs activated or not ?
49  # ----------------------------
50  if ( [ ! X${config_Post_PackFrequency} = X${NULL_STR} ] && \
51       [ ! X${config_Post_PackFrequency} = XNONE ] ) ; then
52    Pack=true
53  else
54    Pack=false
55  fi
56
57
58  # Define input parameters
59  # -----------------------
60  set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug"
61
62  for JobType in ${JobType_list[*]} ; do
63    typeset    name1="${JobType}_String"
64    typeset    name2="${JobType}_Field"
65    typeset    name3="${JobType}_Activ"
66    if [ X${JobType} == X${RebuildJob} ] ; then
67      if ( ${Pack} ) ; then
68        eval ${name1}=IGCM_sys_PutBuffer_Out
69      else
70        eval ${name1}=IGCM_sys_Put_Out
71      fi
72      eval ${name2}=4
73      eval ${name3}=true
74    else
75      eval ${name1}=IGCM_sys_Put_Out
76      eval ${name2}=3
77      if ( ${Pack} ) ; then
78        eval ${name3}=true
79      else
80        eval ${name3}=false
81      fi
82    fi
83  done
84
85  IGCM_debug_PopStack "IGCM_check_CommonDef"
86
87}
88
89#===================================
90function IGCM_check_ChangeUsr
91{
92
93  IGCM_debug_PushStack "IGCM_check_ChangeUsr" $@
94
95  CurrentGrp=$( groups $CurrentUsr | gawk '{print $3}' )
96  TargetGrp=$( groups $TargetUsr | gawk '{print $3}' )
97
98  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
99                -e "s/${CurrentGrp}/${TargetGrp}/"
100
101  IGCM_debug_PopStack "IGCM_check_ChangeUsr"
102
103}
104
105
106
107#===================================
108function IGCM_check_SearchCatalog
109{
110
111  IGCM_debug_PushStack "IGCM_check_SearchCatalog" $@
112
113  typeset num
114  unset SUBMIT_DIR
115
116  fg_new=false
117
118  if [ ! X${JobName} == X ] ; then
119    NbOcc=$( gawk -v JobName=${JobName} \
120             'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
121  else
122    NbOcc=0
123  fi
124
125  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_path} ) ) ; then
126    set -A FileList $( ls ${ConfigPath}/Job_* )
127    if [ X$FileList == X ] ; then
128      NbOcc=0
129    else
130      NbOcc=${#FileList[@]}
131      fg_new=true
132    fi
133  fi
134
135  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_search} ) ) ; then
136    SEARCH_DIR=${WORKDIR}
137    if [ ${TargetUsr} != ${CurrentUsr} ] ; then
138      echo $SEARCH_DIR
139      SEARCH_DIR=$( IGCM_check_ChangeUsr ${SEARCH_DIR} )
140      echo $SEARCH_DIR
141    fi
142    echo "${JobName} not in Catalog, we'll try to find it in ${SEARCH_DIR}"
143
144    set -A FileList $( find ${SEARCH_DIR}/ \
145                            -path ${SEARCH_DIR}/IGCM_OUT -prune -o \
146                            -name Job_${JobName} -print )
147    if [ X$FileList == X ] ; then
148      NbOcc=0
149    else
150      NbOcc=${#FileList[@]}
151      fg_new=true
152    fi
153  fi
154
155  if [ ${NbOcc} -gt 1 ] ; then
156    echo "More than one job"
157    ind=0
158    while [ ${ind} -lt ${NbOcc} ] ; do
159      printf '%2i) %-30s\n' ${ind} ${FileList[${ind}]}
160      (( ind = ind + 1 ))
161    done
162    echo "Give your choice number or 'q' to quit : "
163    read Choice
164    if [ X${Choice} == Xq ] ; then
165      exit
166    else
167      fg_new=true
168      FileList=${FileList[${Choice}]}
169      NbOcc=1
170    fi
171  fi
172
173  case ${NbOcc} in
174    0)
175      echo "${JobName} not found."
176      echo "You can try : *) '-s' option to automatically search your \$WORKDIR ; "
177      echo "              *) '-p' option to provide the directory (absolute path) "
178      echo "                      containing the config.card ; "
179      echo "              *)  manually editing your ${SimuCatalog}."
180      exit ;;
181    1)
182      if ( ${fg_new} ) ; then
183        JobName=${JobName:=$( basename ${FileList} | gawk -F"_" '{ print $2 }' )}
184        SUBMIT_DIR=$( dirname ${FileList} )
185        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}"
186        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" >> ${SimuCatalog}
187        sort -u ${SimuCatalog} > ${SimuCatalog}.tmp
188        mv ${SimuCatalog}.tmp ${SimuCatalog}
189      elif ( [ ${TargetUsr} == $( gawk -v JobName=${JobName} \
190                                      '$1 ~ JobName {print $2}' \
191                                      ${SimuCatalog} ) ] \
192          && [ ${HostName}  == $( gawk -v JobName=${JobName} \
193                                      '$1 ~ JobName {print $3}' \
194                                      ${SimuCatalog} ) ] ) ; then
195        JobName=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
196        SUBMIT_DIR=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
197      else
198        echo "${JobName} not in Catalog."
199        exit
200      fi
201      ;;
202    *)
203      break ;;
204  esac
205
206  IGCM_debug_PopStack "IGCM_check_SearchCatalog"
207
208}
209
210#===================================
211function IGCM_check_PrintHeader
212{
213
214  IGCM_debug_PushStack "IGCM_check_PrintHeader" $@
215
216  echo "|===========================================================================================================|"
217  printf "| JobName = ${ColExp}%-71.71s${ColNon}date : ${ColExp}%-18s${ColNon}|\n" \
218         ${JobName} "$( date +"${DateFormat}" )"
219
220  printf "| User    = ${ColExp}%-56.56s${ColNon}last run.card write : ${ColExp}%-18s${ColNon}|\n" \
221         ${TargetUsr} "${LastWrite}"
222
223  printf "|-----------------------------------------------------------------------------------------------------------|\n"
224
225  printf "| Submit  | ${ColExp}%-95.95s${ColNon} |\n" ${SUBMIT_DIR}
226  printf "| Data    | ${ColExp}%-95.95s${ColNon} |\n" ${DATA_DIR}
227  printf "| Rebuild | ${ColExp}%-95.95s${ColNon} |\n" ${REBUILD_DIR}
228  printf "| Post    | ${ColExp}%-95.95s${ColNon} |\n" ${POST_DIR}
229  printf "| Work    | ${ColExp}%-95.95s${ColNon} |\n" ${CWORK_DIR}
230
231  echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
232  echo "|                         |             |                         |             |     Pending Rebuilds      |"
233  echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
234  echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
235
236  printf "| %-10s - %-10s | " \
237         $DateBegin $DateEnd
238
239  case $PeriodState in
240    Fatal)
241      Color=${ColFat}
242      ;;
243    Completed)
244      Color=${ColCpl}
245      ;;
246    Waiting|OnQueue)
247      Color=${ColAtt}
248      ;;
249    *)
250      Color=${ColDef}
251      ;;
252  esac
253  printf "${Color}%-11s${ColNon} | " $PeriodState
254
255  printf "%-10s - %-10s | %11s | " \
256         $PeriodDateBegin $PeriodDateEnd $CumulPeriod
257
258  if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
259    printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
260           $NbRebuild $FirstRebuild $LastRebuild
261  else
262    printf "%3s : %-8s : %-8s |\n" \
263           $NbRebuild $FirstRebuild $LastRebuild
264  fi
265
266  if [ ${NbLines} -gt 0 ] ; then
267    printf "|-----------------------------------------------------------------------------------------------------------|\n"
268    printf "|                                                      Last                                                 |\n"
269    printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
270    printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
271  fi
272
273  IGCM_debug_PopStack "IGCM_check_PrintHeader"
274
275}
276
277#===================================
278function IGCM_check_PrintJob
279{
280
281  IGCM_debug_PushStack "IGCM_check_PrintJob" $@
282
283  printf "|"
284
285  # Print rebuild and pack jobs
286  # ---------------------------
287  for JobType in ${JobType_list[*]} ; do
288    eval Date=\${${JobType}_Date[${ind}]}
289    eval Status=\${${JobType}_Status[${ind}]}
290    eval Nb=\${${JobType}_Nb[${ind}]}
291
292    if [ X${Status} == XOK  ] ; then
293      Color=${ColCpl}
294    else
295      Color=${ColFat}
296    fi
297    printf "  ${Color}%-8s${ColNon} : %3s  |" ${Date} ${Nb}
298  done
299
300  Color=${ColExp}
301
302  # Print monitoring jobs
303  # ---------------------
304  JobType=monitoring
305  if [ $ind -eq 0 ] ; then
306    eval Date=\${${JobType}_Date}
307  else
308    Date=""
309  fi
310  printf "     ${Color}%-4s${ColNon}     |" ${Date}
311
312  # Print atlas jobs
313  # ----------------
314  JobType=atlas
315  eval Date=\${${JobType}_Date[${ind}]}
316  printf "  ${Color}%-12s${ColNon}  |" ${Date}
317
318  printf "\n"
319
320  IGCM_debug_PopStack "IGCM_check_PrintJob"
321
322}
323
324#===================================
325function IGCM_check_PrintFooter
326{
327
328  IGCM_debug_PushStack "IGCM_check_PrintFooter" $@
329
330  printf "|===========================================================================================================|\n"
331  #date +"${DateFormat}"
332
333  IGCM_debug_PopStack "IGCM_check_PrintFooter"
334
335}
336
337#===================================
338function IGCM_check_CheckPendingRebuild
339{
340
341  IGCM_debug_PushStack "IGCM_check_PendingRebuild" $@
342
343  NbRebuild="."
344  FirstRebuild="."
345  LastRebuild="."
346
347  if [ ${RebuildExists} == 0 ] ; then
348    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
349    if [ ${#RebuildList[*]} -gt 0 ] ; then
350      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )
351
352      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
353      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
354    fi
355  fi
356
357  IGCM_debug_PopStack "IGCM_check_PendingRebuild"
358
359}
360
361#===================================
362function IGCM_check_CheckRebPackJobs
363{
364
365  IGCM_debug_PushStack "IGCM_check_CheckRebPackJobs" $@
366
367  NbLines=0
368  for JobType in ${JobType_list[*]} ; do
369    eval String=\${${JobType}_String}
370    eval Field=\${${JobType}_Field}
371    eval Activ=\${${JobType}_Activ}
372
373    if ( ${Activ} ) ; then
374
375      set -A FileList $( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} )
376
377      if [ ${#FileList[*]} -gt ${NbLines} ] ; then
378        NbLines=${#FileList[*]}
379      fi
380
381      (( ind = 0 ))
382      for FileName in ${FileList[*]} ; do
383        LastDate=""
384        LastDate=$( basename ${FileName} | gawk -F"." '{ print $(NF-1) }' )
385
386        set -- $( gawk -v String=${String} \
387                       'BEGIN { nb_ok = 0 ; nb_ko = 0 } \
388                       ($1 ~ String) { \
389                         if ($3 !~ /error./) { \
390                           nb_ok = nb_ok + 1 \
391                         } else { \
392                           nb_ko = nb_ko + 1 \
393                         } \
394                       } \
395                       END { print nb_ok " " nb_ko }' \
396                       ${POST_DIR}/${JobType}.${LastDate}.out )
397        Match=$1
398        Error=$2
399
400        (( Nb = ${Match} - ${Error} ))
401
402        if ( [ ${Error} -eq 0 ] && [ ${Nb} -gt 0 ] ) ; then
403          Status=OK
404        else
405          Status=KO
406        fi
407
408        eval ${JobType}_Date[$ind]=${LastDate}
409        eval ${JobType}_Status[$ind]=${Status}
410        eval ${JobType}_Nb[$ind]=${Nb}
411
412        (( ind = ind + 1 ))
413      done
414
415    else
416
417        eval ${JobType}_Date[0]=""
418        eval ${JobType}_Status[0]=""
419        eval ${JobType}_Nb[0]=""
420
421    fi
422
423  done
424
425  IGCM_debug_PopStack "IGCM_check_CheckRebPackJobs"
426
427}
428
429#===================================
430function IGCM_check_CheckMonitoring
431{
432
433  IGCM_debug_PushStack "IGCM_check_CheckMonitoring" $@
434
435    LastDate=""
436    JobType=monitoring
437    IGCM_sys_TestDirArchive ${CWORK_DIR}/MONITORING
438    RC=$?
439    if [ $RC == 0 ] ; then
440      FileTmp=$( IGCM_sys_RshArchive "ls ${CWORK_DIR}/MONITORING/files/*.nc | head -n 1" )
441      IGCM_sys_GetDate_Monitoring ${FileTmp} LastDate
442      eval ${JobType}_Date=${LastDate}
443    fi
444
445  IGCM_debug_PopStack "IGCM_check_CheckMonitoring"
446
447}
448
449#===================================
450function IGCM_check_CheckAtlas
451{
452
453  IGCM_debug_PushStack "IGCM_check_CheckAtlas" $@
454
455  JobType=atlas
456  IGCM_sys_TestDirArchive ${CWORK_DIR}/ATLAS
457  RC=$?
458  if [ $RC == 0 ] ; then
459    set -A FileList $( IGCM_sys_RshArchive "ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto}" )
460
461    if [ ${#FileList[*]} -gt ${NbLines} ] ; then
462      NbLines=${#FileList[*]}
463    fi
464
465    (( ind = 0 ))
466    for FileName in ${FileList[*]} ; do
467      eval ${JobType}_Date[$ind]=${FileName}
468      (( ind = ind + 1 ))
469    done
470  fi
471
472  IGCM_debug_PopStack "IGCM_check_CheckAtlas"
473
474}
475
476#===================================
Note: See TracBrowser for help on using the repository browser.