source: TOOLS/PACK_IPSL/showPackProgress.sh @ 1849

#!/bin/bash

export JOB_DIR=${LS_SUBCWD:-${PWD}}
export EXE_DIR=${JOB_DIR}

# gpdebug : pour titane
# ---------------------
# export INPUT_DMF_DATA=${SCRATCHDIR}/datasLabo/dataDMNFS_simple1
# export OUTPUT_STORE=${SCRATCHDIR}/PSEUDO_DMNFS_STORE
# export OUTPUT_WORK=${SCRATCHDIR}/PSEUDO_DMNFS_WORK


# gpdebug : pour curie
# --------------------
export INPUT_DMF_DATA=${SCRATCHDIR}/dataDMNFS_FromTitane
export OUTPUT_STORE=${SCRATCHDIR}/dataDMNFS_FromTitane_STORE 
export OUTPUT_WORK=${SCRATCHDIR}/dataDMNFS_FromTitane_WORK

SCRIPT_NAME=$(basename ${0} )

export IGCM_DEM=${SCRATCHDIR}/IGCM_DEM
export IGCM_TMP=${SCRATCHDIR}/IGCM_DEM/tmp

export config_card=${IGCM_DEM}/config_card.liste

source ${EXE_DIR}/DEM_utilities.sh

export timeHandlingFile="timeHandlingFile.txt"


function getSimuName
{   
   config_card=$1
   
   baseName=$( basename $config_card )
   simuName=`echo ${baseName} | sed 's;config_card_;;' `
   echo $simuName

}

function getTimeTaken
{
   statFile=$1
   if [ ! -e $statFile ]
   then
      echo "inconnu : pas de fichier status"
      return
   fi
   
   timeLine=`sed -n '1p' $statFile `
   goodFormat=`echo $timeLine | grep -e '^meantime:[[:digit:]]*\.[[:digit:]]\{1,3\}$' | wc -l `
   if [ "x$goodFormat" == "x0" ]
   then
       echo "inconnu : mauvais format du temps"
       return
   fi
   
   timeTaken=`echo $timeLine | sed 's;^meantime:;;' `
   echo $timeTaken
   
}

function progressStatus
{
   statFile=$1
   echo "statFile=$1" >> showPackProgress.log
   if [ ! -e $statFile ]
   then
      echo "Non traitee"
      return
   fi
   echo "blabla" >> showPackProgress.log
   echo "----------------------------------------------" >> showPackProgress.log
   # echo "progressStatus..." # gpdebug : a virer
   
   # ------------------------------------------------------------------------------------------------
   listLineNumWithKey=`grep -n -E '(COMPLETED|FAILED|DELEGATE)' $statFile | awk -F":" '{print $1}' ` 
   # echo "listLineNumWithKey=$listLineNumWithKey" # gpdebug : a virer
   
  if [ "x$listLineNumWithKey" == "x" ]
  then
     echo "Statut illisible"
     return
  fi

   # On trouve la derniere ligne du fichier status comportant un mot cle 
   max=0
   for lineNum in $listLineNumWithKey
   do
      if [ $lineNum -gt $max ]
      then
          max=$lineNum
      fi
   done

   # echo "max=$max" # gpdebug : a virer

   lastLineWithKey=`sed -n "${max}p" $statFile `
   # echo "lastLineWithKey=$lastLineWithKey" # gpdebug : a virer   
   # ------------------------------------------------------------------------------------------------
     
   # lastLine=`tail -1 $statFile ` # gpdebug : a virer
   
   completed=`echo $lastLineWithKey | grep COMPLETED | sed 's; (.*)$;;' `
   if [ "x${completed}" != "x" ]
   then 
      echo $completed
      return
   fi
   
  failed=`echo $lastLineWithKey | grep FAILED | cut -d";" -f1 `
  if [ "x${failed}" != "x" ]
  then
     echo $failed
     return         
  fi
  
  delegate=`echo $lastLineWithKey | grep DELEGATE `
  if [ "x${delegate}" != "x" ]
  then
     echo "DELEGATE"
     return
  fi
  
}


function getWaitingTime
{
    timeNow_=$1
    stillWaiting_=1
    
    launchTimeString=`cat $timeHandlingFile | grep -e 'launch time:[[:digit:].]*$' `
    if [ "x${launchTimeString}" == "x" ]
    then
        echo "no launch time"
        return
    fi
    launchTime_=`echo $launchTimeString | awk -F":" '{print $2}' `
    
    
    startExecTimeString=`cat $timeHandlingFile | grep -e 'start time:[[:digit:].]*$' `
    if [ "x${startExecTimeString}" == "x" ]
    then
        execTime_=${timeNow_}   
    else
        execTime_=`echo $startExecTimeString | awk -F":" '{print $2}' `
        stillWaiting_=0
    fi
    
    waitingTime_=$( awk 'BEGIN { print '${execTime_}'-'${launchTime_}' }' )
    waitingTime_=$( awk 'BEGIN { print '${waitingTime_}'/'1000' }' )
    if [ "x$stillWaiting_" == "x1" ]
    then
       waitingTime_="${waitingTime_}, still waiting..."
    fi
    
    echo $waitingTime_
}
    
    
function getTimeSinceExecutionStart
{  
    
    timeNow_=$1
    stillWaiting_=1    
    
    execTimeString=`cat $timeHandlingFile | grep -e 'start time:[[:digit:].]*$' `
    if [ "x${execTimeString}" == "x" ]
    then
        echo "no start time"
        return
    fi
    execTime_=`echo $execTimeString | awk -F":" '{print $2}' `
    
        
    endTimeString=`cat $timeHandlingFile | grep -e 'end time:[[:digit:].]*$' `
    if [ "x${endTimeString}" == "x" ]
    then
        endTime_=${timeNow_}
    else
        endTime_=`echo $endTimeString | awk -F":" '{print $2}' `
        stillWaiting_=0
    fi
    
    timeSinceExec_=$( awk 'BEGIN { print '${endTime_}'-'${execTime_}' }' )
    timeSinceExec_=$( awk 'BEGIN { print '${timeSinceExec_}'/'1000' }' )
    if [ "x$stillWaiting_" == "x1" ]
    then
       timeSinceExec_="${timeSinceExec_}, still executing..."
    fi
    
    echo $timeSinceExec_
}

#################################
####### Main script #############
#################################
> showPackProgress.log # vider le fichier de log

# Recuperation des temps d'attente et ecoule depuis de demarrage des traitements en batch
if [ -e $timeHandlingFile ]
then
    timeNow=$( getDateMilliSeconds )
    waitingTime=$( getWaitingTime $timeNow )
    timeSinceExecutionStart=$( getTimeSinceExecutionStart $timeNow )
else
    waitingTime="no handling time file available."
    timeFromExecutionStart="no handling time file available."
fi


totalNbOfList=0
for CONFIG in $( awk '{print $1}' ${IGCM_DEM}/config_card.liste )
do
   PATH_SIMU=$( dirname $CONFIG )
   nbListInConfig=`find $PATH_SIMU -name "*list" | wc -l `
   totalNbOfList=$(( $totalNbOfList + $nbListInConfig ))
done

# echo "totalNbOfList=$totalNbOfList"
# exit 0 # gpdebug : a virer

nbOfListsPacked=0
nbOfListsFailed=0
TotalNbInodes=0
TotalSimuTar=0
totalTime=0

for CONFIG in $( awk '{print $1}' ${IGCM_DEM}/config_card.liste ) ; do 

    DEM_state=$( DEM_read_state ${IGCM_DEM}/config_card.liste ${CONFIG} )

    if [[ "${DEM_state}" != "TarDone" ]] ; then

         simuName=$( getSimuName $CONFIG )
         echo "*******************************************"
         echo "simulation : $simuName"
         echo "*******************************************"
         export PATH_SIMU=$( dirname $CONFIG )
         # echo "PATH_SIMU = $PATH_SIMU"
        
         listOfDir="output_ncrcat output_tar restart_tar debug_tar store_cp work_cp"
         for dir in $listOfDir
         do
             echo "   ${dir} :"
             if [ ! -d $PATH_SIMU/${dir} ]
             then
                echo "      ${dir} n'existe pas pour cette simu"
                continue
             fi
           
             listFilesInDir=`find $PATH_SIMU/${dir} -name "*list" | wc -l `
             if [ "x$listFilesInDir" == "x0" ]
             then
                echo "       -- no list --"
                continue           
             fi
           
             setList=$( ls $PATH_SIMU/${dir}/*list )

             for set in $setList
             do
                  listName=`echo $(basename $set) | sed 's;\.list$;;' | sed 's;\.nc$;;' `
                  ListNbInode=`cat $set |wc -l`
                  TotalNbInodes=$(( $TotalNbInodes + $ListNbInode ))
                  statusfile=${set%%.list}.status
                  status=$( progressStatus $statusfile ) # gpdebug : a retablir
                  timeTaken="???"
                  # progressStatus $statusfile # gpdebug : a virer
                  # exit 0 # gpdebug : a virer
               
                  packFailed=`echo $status | grep -E '(DELEGATE|FAILED|illisible)' | wc -l `
                  if [ "x${packFailed}" != "x0" ]
                  then
                     nbOfListsFailed=$(( $nbOfListsFailed + 1 ))
                  fi
               
                  packSuccess=`echo $status | grep "COMPLETED" | wc -l `
                  if [ "x${packSuccess}" != "x0" ]
                  then
                     timeTaken=$( getTimeTaken $statusfile )
                     nbOfListsPacked=$(( $nbOfListsPacked + 1 ))
                  fi
                  
                  echo "       $listName : $status | time : $timeTaken"
                  
                  timeInSecondsOK=`echo $timeTaken | grep -e '^[[:digit:].]*$' | wc -l `
                  if [ "x${timeInSecondsOK}" != "x0" ]
                  then
                      totalTime=$(awk 'BEGIN { print '$totalTime'+'$timeTaken' }')
                  fi
             done
         done
    else
#si "${DEM_state}" == "TarDone"

         simuName=$( getSimuName $CONFIG )
         echo "*******************************************"
         echo "simulation : $simuName TAR"
         echo "*******************************************"
         TotalSimuTar=$(( $TotalSimuTar + 1 ))
    fi
done

NbInodeBefore=`cat ${IGCM_DEM}/Listing.txt |wc -l`
find $IGCM_DEM -printf "%y %s %p \n" >> ${IGCM_DEM}/ListingEnd.txt
NbInodeAfter=`cat ${IGCM_DEM}/ListingEnd.txt |wc -l`
echo
echo "##################"
echo "#####  BILAN #####"
echo "##################"
echo "nb of Lists packed with success : ${nbOfListsPacked} / ${totalNbOfList}"
echo "nb of fails : $nbOfListsFailed"
echo "nb of inodes packed : $TotalNbInodes"
echo "nb simul full tared : $TotalSimuTar"
echo "nb inode before : $NbInodeBefore"
echo "nb inode after : $NbInodeAfter"
echo "Total time for elementary operations : $totalTime  (sum of times on several proc)" 
echo "waiting time : $waitingTime"
echo "Time since execution start : $timeSinceExecutionStart"