source: TOOLS/PACK_IPSL/showPackProgress.sh @ 3640

Last change on this file since 3640 was 2035, checked in by aclsce, 11 years ago

Modified to speed up the script.

  • Property svn:executable set to *
File size: 16.7 KB
RevLine 
[1822]1#!/bin/bash
2
[1853]3# On renseigne les variables d'environnement
4. load_ipslPack_env.sh
[1822]5
[1900]6export JOB_DIR=${TMP_MIGR_DATA}
7export EXE_DIR=${LS_SUBCWD:-${PWD}}
8
[1822]9SCRIPT_NAME=$(basename ${0} )
10
[1853]11export IGCM_TMP="${IGCM_DEM}/tmp"  # gpdebug : added
[1822]12
13export config_card=${IGCM_DEM}/config_card.liste
14
15source ${EXE_DIR}/DEM_utilities.sh
16
[1853]17export timeHandlingFile="${USER_OUTPUT_PROGRESS}/timeHandlingFile.txt"
[1862]18export timeLaunchStartFile="${USER_OUTPUT_PROGRESS}/timeLaunchStartFile.txt"
19export timeEndFile="${USER_OUTPUT_PROGRESS}/timeEndFile.txt"
[1822]20
[1849]21
[1822]22function getSimuName
23{   
24   config_card=$1
25   
[1853]26   dirSimu=$( dirname $config_card )
27   simuName=`echo ${dirSimu} | sed "s;${IGCM_DEM}/;;" `
[1822]28   echo $simuName
29
30}
31
32function getTimeTaken
33{
34   statFile=$1
35   if [ ! -e $statFile ]
36   then
37      echo "inconnu : pas de fichier status"
38      return
39   fi
40   
41   timeLine=`sed -n '1p' $statFile `
42   goodFormat=`echo $timeLine | grep -e '^meantime:[[:digit:]]*\.[[:digit:]]\{1,3\}$' | wc -l `
43   if [ "x$goodFormat" == "x0" ]
44   then
45       echo "inconnu : mauvais format du temps"
46       return
47   fi
48   
49   timeTaken=`echo $timeLine | sed 's;^meantime:;;' `
50   echo $timeTaken
51   
52}
53
54function progressStatus
55{
56   statFile=$1
[1911]57   echo "statFile=$1" >> ${JOB_DIR}/showPackProgress.log
[1822]58   if [ ! -e $statFile ]
59   then
60      echo "Non traitee"
61      return
62   fi
[1911]63   echo "blabla" >> ${JOB_DIR}/showPackProgress.log
64   echo "----------------------------------------------" >> ${JOB_DIR}/showPackProgress.log
[1822]65   # echo "progressStatus..." # gpdebug : a virer
66   
67   # ------------------------------------------------------------------------------------------------
68   listLineNumWithKey=`grep -n -E '(COMPLETED|FAILED|DELEGATE)' $statFile | awk -F":" '{print $1}' ` 
69   # echo "listLineNumWithKey=$listLineNumWithKey" # gpdebug : a virer
70   
71  if [ "x$listLineNumWithKey" == "x" ]
72  then
73     echo "Statut illisible"
74     return
75  fi
76
77   # On trouve la derniere ligne du fichier status comportant un mot cle
78   max=0
79   for lineNum in $listLineNumWithKey
80   do
81      if [ $lineNum -gt $max ]
82      then
83          max=$lineNum
84      fi
85   done
86
87   # echo "max=$max" # gpdebug : a virer
88
89   lastLineWithKey=`sed -n "${max}p" $statFile `
90   # echo "lastLineWithKey=$lastLineWithKey" # gpdebug : a virer   
91   # ------------------------------------------------------------------------------------------------
92     
93   # lastLine=`tail -1 $statFile ` # gpdebug : a virer
94   
95   completed=`echo $lastLineWithKey | grep COMPLETED | sed 's; (.*)$;;' `
96   if [ "x${completed}" != "x" ]
97   then
98      echo $completed
99      return
100   fi
101   
102  failed=`echo $lastLineWithKey | grep FAILED | cut -d";" -f1 `
103  if [ "x${failed}" != "x" ]
104  then
105     echo $failed
106     return         
107  fi
108 
109  delegate=`echo $lastLineWithKey | grep DELEGATE `
110  if [ "x${delegate}" != "x" ]
111  then
112     echo "DELEGATE"
113     return
114  fi
115 
116}
117
118
[1849]119function getWaitingTime
120{
121    timeNow_=$1
122    stillWaiting_=1
123   
124    launchTimeString=`cat $timeHandlingFile | grep -e 'launch time:[[:digit:].]*$' `
125    if [ "x${launchTimeString}" == "x" ]
126    then
127        echo "no launch time"
128        return
129    fi
130    launchTime_=`echo $launchTimeString | awk -F":" '{print $2}' `
131   
132   
133    startExecTimeString=`cat $timeHandlingFile | grep -e 'start time:[[:digit:].]*$' `
134    if [ "x${startExecTimeString}" == "x" ]
135    then
136        execTime_=${timeNow_}   
137    else
138        execTime_=`echo $startExecTimeString | awk -F":" '{print $2}' `
139        stillWaiting_=0
140    fi
141   
142    waitingTime_=$( awk 'BEGIN { print '${execTime_}'-'${launchTime_}' }' )
143    waitingTime_=$( awk 'BEGIN { print '${waitingTime_}'/'1000' }' )
144    if [ "x$stillWaiting_" == "x1" ]
145    then
146       waitingTime_="${waitingTime_}, still waiting..."
147    fi
148   
149    echo $waitingTime_
150}
151   
152   
153function getTimeSinceExecutionStart
154{ 
155   
156    timeNow_=$1
157    stillWaiting_=1   
158   
159    execTimeString=`cat $timeHandlingFile | grep -e 'start time:[[:digit:].]*$' `
160    if [ "x${execTimeString}" == "x" ]
161    then
162        echo "no start time"
163        return
164    fi
165    execTime_=`echo $execTimeString | awk -F":" '{print $2}' `
166   
167       
168    endTimeString=`cat $timeHandlingFile | grep -e 'end time:[[:digit:].]*$' `
169    if [ "x${endTimeString}" == "x" ]
170    then
171        endTime_=${timeNow_}
172    else
173        endTime_=`echo $endTimeString | awk -F":" '{print $2}' `
174        stillWaiting_=0
175    fi
176   
177    timeSinceExec_=$( awk 'BEGIN { print '${endTime_}'-'${execTime_}' }' )
178    timeSinceExec_=$( awk 'BEGIN { print '${timeSinceExec_}'/'1000' }' )
179    if [ "x$stillWaiting_" == "x1" ]
180    then
181       timeSinceExec_="${timeSinceExec_}, still executing..."
182    fi
183   
184    echo $timeSinceExec_
185}
186
[1862]187function getTimeSinceLaunchStart
188{
189    timeNow_=$1
190    stillWaiting_=1
191   
192    launchTimeString=`cat $timeLaunchStartFile | grep -e 'launch time:[[:digit:].]*$' `
193    if [ "x${launchTimeString}" == "x" ]
194    then
195        echo "no launch time"
196        return
197    fi
198    launchTime_=`echo $launchTimeString | awk -F":" '{print $2}' `
199   
200   
201    endTimeString=`cat $timeEndFile | grep -e 'end time:[[:digit:].]*$' `
202    if [ "x${endTimeString}" == "x" ]
203    then
204        endTime_=${timeNow_}
205    else
206        endTime_=`echo $endTimeString | awk -F":" '{print $2}' `
207        stillWaiting_=0
208    fi
209   
210    timeSinceLaunch_=$( awk 'BEGIN { print '${endTime_}'-'${launchTime_}' }' )
211    timeSinceLaunch_=$( awk 'BEGIN { print '${timeSinceLaunch_}'/'1000' }' )
212    if [ "x$stillWaiting_" == "x1" ]
213    then
214       timeSinceLaunch_="${timeSinceLaunch_}, still executing..."
215    fi
216   
217    echo $timeSinceLaunch_
[1853]218
[1862]219}
[1853]220
[1862]221
[1853]222function getListProgress
223{
224     set=$1     
225     
226     listName=`echo $(basename $set) | sed 's;\.list$;;' | sed 's;\.nc$;;' `
[1899]227
[1853]228     statusfile=${set%%.list}.status
229     status=$( progressStatus $statusfile ) # gpdebug : a retablir
230     timeTaken="???"
[1899]231
232     if [ -e ${statusfile} ]
233     then
[2035]234         
235         if [ -s $set ] ; then
236             if [[ "$listName" == *"_restart_"* ]] || [[ "$listName" == *"_debug_"* ]]
237             then
238                 purge_list=$( awk '{print $0}' $set | cut -c 5- )
239             else
240                 purge_list=$( awk '{print $0}' $set )
241             fi
242             grep -v -F "$purge_list" ${IGCM_DEM}/Listing_progress.txt > ${IGCM_DEM}/Listing_progress_tmp.txt
243             mv ${IGCM_DEM}/Listing_progress_tmp.txt ${IGCM_DEM}/Listing_progress.txt
[1899]244         fi
245     fi
246
[1868]247     packFailed=`echo $status | grep -E '(DELEGATE|FAILED)' | wc -l `
[1853]248     if [ "x${packFailed}" != "x0" ]
249     then
250        nbOfListsFailed=$(( $nbOfListsFailed + 1 ))
251        simuOK=$(( $simuOK && 0 )) # marque une simu pas OK
252     fi
253 
254     packSuccess=`echo $status | grep "COMPLETED" | wc -l `
255     if [ "x${packSuccess}" != "x0" ]
256     then
257        timeTaken=$( getTimeTaken $statusfile )
258        nbOfListsPacked=$(( $nbOfListsPacked + 1 ))
259     fi
260     
[1868]261     packNottreated=`echo $status | grep -E '(Non traitee|illisible)' | wc -l `
[1864]262     if [ "x${packNottreated}" != "x0" ]
263     then
264         nbOfListsNottreated=$(( $nbOfListsNottreated + 1 ))
[1868]265         simuOK=$(( $simuOK && 0 )) # marque une simu pas OK
[1876]266         simutreated=$(( $simutreated && 0 )) # marque une simu non completement traitee
[1864]267     fi
268
[1911]269     echo "       $listName : $status | time : $timeTaken" >> ${JOB_DIR}/simuPrint.txt
[1853]270     
271     timeInSecondsOK=`echo $timeTaken | grep -e '^[[:digit:].]*$' | wc -l `
272     if [ "x${timeInSecondsOK}" != "x0" ]
273     then
274         totalTime=$(awk 'BEGIN { print '$totalTime'+'$timeTaken' }')
275     fi
276
277
278
279}
280
[1822]281#################################
282####### Main script #############
283#################################
[1853]284
285########  Options du script ########################################################
286focusOnSimu=
287outputFile=
[1899]288outputFile_simus=
[1853]289detailed_mode=1
[1871]290publish_mode=1
[1853]291while [ $# -gt 0 ]
292do
293   # echo "boucle sur les arguments du script ..."
294   # echo "@=$@"
295   case $1 in
296   -s)  echo "option -s selectionnee : afficher une simulation"
297        shift
298        focusOnSimu=$1
299        is_simu_an_option=`echo $focusOnSimu | grep -e '^-' | wc -l `
300        if [ "x${is_simu_an_option}" != "x0" ] || [ "x$focusOnSimu" == "x" ]
301        then
302           echo "L'option -s doit etre suivie d'une simulation !"
303           exit 1
304        fi
305        ;;
306   -o)  echo "option -o selectionnee : fichier d'output"
307        shift
308        outputFile=$1
309        is_output_an_option=`echo $outputFile | grep -e '^-' | wc -l `
310        if [ "x${is_output_an_option}" != "x0" ] || [ "x$outputFile" == "x" ]
311        then
312           echo "L'option -o doit etre suivie d'un fichier d'output !"
313           exit 1
314        fi
315        ;;
316   --not-detailled)  echo "option '--not-detailled' selectionnee"
317                     detailed_mode=0
318                     ;;
[1871]319   -p) echo "option -p selectionnee : publication"
[1899]320       publish_mode=0
[1978]321       outputFile="${MONIT_DIR}/monit_${USER}.txt"
322       outputFile_simus="${MONIT_DIR}/${USER}_SIMUS/monit_${USER}_simus.txt"
323       if [ -d ${MONIT_DIR}/${USER}_SIMUS ]
[1899]324       then
[1978]325           rm -f ${MONIT_DIR}/${USER}_SIMUS/*
[1899]326       else
[1978]327           mkdir -p ${MONIT_DIR}/${USER}_SIMUS
[1899]328       fi
329       echo "Simulation;Status" > ${outputFile_simus}
[1853]330   esac
331   shift
332done
333
334# echo "focusOnSimu=$focusOnSimu"
335# echo "outputFile=$outputFile"
336# echo "detailed_mode=$detailed_mode"
337######################################################################################
338# exit 0 # gpdebug : a virer
339
340
341
[1911]342> ${JOB_DIR}/showPackProgress.log # vider le fichier de log
[1822]343
[1853]344
[1849]345# Recuperation des temps d'attente et ecoule depuis de demarrage des traitements en batch
[1862]346if [ -e $timeLaunchStartFile ]
[1849]347then
348    timeNow=$( getDateMilliSeconds )
[1862]349    # waitingTime=$( getWaitingTime $timeNow ) # supprime pour le moment : pb si +sieurs instances ==> on simplifie
350    # timeSinceExecutionStart=$( getTimeSinceExecutionStart $timeNow ) # supprime pour le moment : pb si +sieurs instances ==> on simplifie
351    timeSinceLaunchStart=$( getTimeSinceLaunchStart $timeNow ) # added
[1849]352else
[1862]353    # waitingTime="no handling time file available." # supprime pour le moment : pb si +sieurs instances ==> on simplifie
354    # timeFromExecutionStart="no handling time file available." # supprime pour le moment : pb si +sieurs instances ==> on simplifie
355    timeSinceLaunchStart="no launch time file available."
[1849]356fi
357
358
[1822]359totalNbOfList=0
360for CONFIG in $( awk '{print $1}' ${IGCM_DEM}/config_card.liste )
361do
362   PATH_SIMU=$( dirname $CONFIG )
363   nbListInConfig=`find $PATH_SIMU -name "*list" | wc -l `
364   totalNbOfList=$(( $totalNbOfList + $nbListInConfig ))
365done
366
367# echo "totalNbOfList=$totalNbOfList"
368# exit 0 # gpdebug : a virer
369
[1871]370TotalNbSimu=0
371nbOfSimuPacked=0
372nbOfSimuFailed=0
373nbOfSimuNottreated=0
[1822]374nbOfListsPacked=0
375nbOfListsFailed=0
[1864]376nbOfListsNottreated=0
[1899]377TotalNbInodes=0
[1848]378TotalSimuTar=0
[1849]379totalTime=0
[1899]380cp ${IGCM_DEM}/Listing.txt ${IGCM_DEM}/Listing_progress.txt
[1871]381TotalNbSimu=`cat ${IGCM_DEM}/config_card.liste |wc -l`
[1853]382
[1871]383if [ "x${outputFile}" == "x" ] && [ "x${publish_mode}" == "x0" ]
384then
385# Publication
386echo "Simulation;Status" > ${outputFile}_${USER}
387fi
388
[1822]389for CONFIG in $( awk '{print $1}' ${IGCM_DEM}/config_card.liste ) ; do
390
391    DEM_state=$( DEM_read_state ${IGCM_DEM}/config_card.liste ${CONFIG} )
[1853]392   
393    # Eviter les simus autre que celle selectionnee par l'option -s
394    showCurrentSimu=`echo $CONFIG | grep "${focusOnSimu}" | wc -l `
395    if [ "x${showCurrentSimu}" == "x0" ]
396    then
397        continue
398    fi
[1822]399
[1853]400    simuName=$( getSimuName $CONFIG )
401   
[1911]402    > ${JOB_DIR}/simuPrint.txt
[1822]403
[1853]404    export PATH_SIMU=$( dirname $CONFIG )
405    # echo "PATH_SIMU = $PATH_SIMU"
406   
407    simuOK=1
[1871]408    simutreated=1
409 
[1853]410    if [ ! -e "${PATH_SIMU}/tar_full_simul.list" ]
411    then
412       
[1822]413         listOfDir="output_ncrcat output_tar restart_tar debug_tar store_cp work_cp"
414         for dir in $listOfDir
415         do
[1911]416             echo "   ${dir} :" >> ${JOB_DIR}/simuPrint.txt
[1822]417             if [ ! -d $PATH_SIMU/${dir} ]
418             then
[1911]419                echo "      ${dir} n'existe pas pour cette simu" >> ${JOB_DIR}/simuPrint.txt
[1871]420#               simuOK=$(( $simuOK && 0 )) # marque une simu pas OK
[1822]421                continue
422             fi
423           
424             listFilesInDir=`find $PATH_SIMU/${dir} -name "*list" | wc -l `
425             if [ "x$listFilesInDir" == "x0" ]
426             then
[1911]427                echo "       -- no list --" >> ${JOB_DIR}/simuPrint.txt
[1822]428                continue           
429             fi
430           
431             setList=$( ls $PATH_SIMU/${dir}/*list )
432
433             for set in $setList
434             do
[1853]435                  getListProgress $set
[1822]436             done
437         done
[1853]438    else         
439         getListProgress ${PATH_SIMU}/tar_full_simul.list
[1848]440         TotalSimuTar=$(( $TotalSimuTar + 1 ))
[1822]441    fi
[1853]442   
443    resSimu="OK"
444    if [ "x${simuOK}" == "x0" ]
445    then
446        resSimu="not OK"
447    fi
[1871]448
449    if [ "x${simutreated}" == "x0" ]
450    then
451        resSimu="not treated"
452    fi
453
454    if [ "x${resSimu}" == "xnot OK" ]
455    then
456       nbOfSimuFailed=$(( $nbOfSimuFailed + 1 )) 
457    elif [ "x${resSimu}" == "xnot treated" ]
458    then
459       nbOfSimuNottreated=$(( $nbOfSimuNottreated + 1 ))
460    else
461        nbOfSimuPacked=$(( $nbOfSimuPacked + 1 ))
462    fi
463
[1853]464    if [ "x${outputFile}" == "x" ]
465    then
466        echo "***************************************************"
467        echo "simulation : $simuName  ===> $resSimu"
468        echo "***************************************************"
469        if [ "x${detailed_mode}" == "x1" ]
470        then
[1911]471            cat ${JOB_DIR}/simuPrint.txt  $outRedirectStatement
[1853]472        fi
473    else
[1871]474        if [ "x${publish_mode}" == "x1" ]
475        then
[1853]476        echo "***************************************************" >> ${outputFile}
477        echo "simulation : $simuName  ===> $resSimu"               >> ${outputFile}
478        echo "***************************************************" >> ${outputFile}
479        if [ "x${detailed_mode}" == "x1" ]
480        then
[1911]481            cat ${JOB_DIR}/simuPrint.txt  $outRedirectStatement               >> ${outputFile}
[1853]482        fi     
[1871]483        else
484# Publication
[1899]485            echo "$simuName;$resSimu" >> ${outputFile_simus}
486            if [ "x${resSimu}" == "xnot OK" ]
487            then
488                simuName_modif=$( echo $simuName | sed s%"/"%"_"%g )
[1978]489                cat ${JOB_DIR}/simuPrint.txt $outRedirectStatemen > ${MONIT_DIR}/${USER}_SIMUS/${simuName_modif}".log"
[1899]490            fi
[1871]491        fi
[1853]492    fi
[1911]493    rm ${JOB_DIR}/simuPrint.txt
[1822]494done
[1899]495NbInodeBefore=`cat ${IGCM_DEM}/Listing.txt | grep "f " | wc -l`
496#NbInodeBefore=`cat ${IGCM_DEM}/Listing.txt  | wc -l`
[1913]497 NbInodeAfterStore=0
[1863]498 if [ -d "${OUTPUT_STORE}" ]
499 then
[1899]500     NbInodeAfterStore=$( find $OUTPUT_STORE -type f -printf "%y %s %p \n" | wc -l )
[1863]501 fi
[1913]502 NbInodeAfterWork=0
[1863]503 if [ -d "${OUTPUT_WORK}" ]
504 then
[1899]505     NbInodeAfterWork=$( find $OUTPUT_WORK -type f -printf "%y %s %p \n" | wc -l )
[1863]506 fi
[1822]507
[1899]508# NbInodeAfter=`cat ${IGCM_DEM}/ListingEnd.txt | grep "f " | wc -l`
509 NbInodeAfter=$(( $NbInodeAfterStore + $NbInodeAfterWork ))
510
511if [ -e ${IGCM_DEM}/Listing_progress.txt ]
512then
513    NbInodesprogressed=`cat ${IGCM_DEM}/Listing_progress.txt | grep "f " | wc -l`
514#    NbInodesprogressed=`cat ${IGCM_DEM}/Listing_progress.txt | wc -l`
[1885]515fi
[1899]516TotalNbInodes=$(( $NbInodeBefore - $NbInodesprogressed ))
[1885]517
[1899]518
[1863]519if [ "x${outputFile}" == "x" ]
520then
521    echo
522    echo "##################"
523    echo "#####  BILAN #####"
524    echo "##################"
[1871]525    echo "nb of Simulations packed with success : ${nbOfSimuPacked} / ${TotalNbSimu}"
526    echo "nb of Simus failed : ${nbOfSimuFailed}"
527    echo "nb of Simus not treated : ${nbOfSimuNottreated}"
[1863]528    echo "nb of Lists packed with success : ${nbOfListsPacked} / ${totalNbOfList}"
529    echo "nb of fails : $nbOfListsFailed"
[1864]530    echo "nb of not treated : ${nbOfListsNottreated}"
[1899]531    echo "nb of inodes packed : $TotalNbInodes"
[1863]532    echo "nb simul full tared : $TotalSimuTar"
533    echo "nb inode before : $NbInodeBefore"
[1899]534    echo "nb inode after WORK : $NbInodeAfterWork"
535    echo "nb inode after STORE : $NbInodeAfterStore"
[1863]536    echo "Total time for elementary operations : $totalTime" 
537    echo "Time since launch start : $timeSinceLaunchStart"
[1822]538
[1863]539else
[1871]540    if [ "x${publish_mode}" == "x1" ]
541        then
[1863]542    echo
543    echo "##################" >> ${outputFile}
544    echo "#####  BILAN #####" >> ${outputFile}
545    echo "##################" >> ${outputFile}
[1871]546    echo "nb of Simulations packed with success : ${nbOfSimuPacked} / ${TotalNbSimu}" >> ${outputFile}
547    echo "nb of Simus failed : ${nbOfSimuFailed}" >> ${outputFile}
548    echo "nb of Simus not treated : ${nbOfSimuNottreated}" >> ${outputFile}
[1863]549    echo "nb of Lists packed with success : ${nbOfListsPacked} / ${totalNbOfList}" >> ${outputFile}
550    echo "nb of fails : $nbOfListsFailed" >> ${outputFile}
[1864]551    echo "nb of not treated : ${nbOfListsNottreated}" >> ${outputFile}
[1899]552    echo "nb of inodes packed : $TotalNbInodes" >> ${outputFile}
[1863]553    echo "nb simul full tared : $TotalSimuTar" >> ${outputFile}
554    echo "nb inode before : $NbInodeBefore" >> ${outputFile}
555    echo "nb inode after : $NbInodeAfter" >> ${outputFile}
556    echo "Total time for elementary operations : $totalTime" >> ${outputFile}
557    echo "Time since launch start : $timeSinceLaunchStart" >> ${outputFile}
[1871]558    else
559# Publication
[1899]560    echo "User;Inode NB;Inode Processed;Inode Treatment Percentage;Inode After;Inode store;Inode percentage STORE-WORK;Simu NB;Simu OK; Simu FAILED;Simu Percentage;List NB;List OK;List FAILED;List Percentage" > ${outputFile}
561        echo "${USER};$NbInodeBefore;$TotalNbInodes;PIE2;$NbInodeAfter;$NbInodeAfterStore;PIE2;${TotalNbSimu};${nbOfSimuPacked};${nbOfSimuFailed};PIE3;${totalNbOfList};${nbOfListsPacked};$nbOfListsFailed;PIE3" >> ${outputFile}
[1871]562    fi
[1863]563fi
Note: See TracBrowser for help on using the repository browser.