source: trunk/libIGCM/AA_RunChecker @ 795

#!/bin/ksh

#**************************************************************
# Author: Sonia Labetoulle
# Contact: sonia.labetoulle__at__ipsl.jussieu.fr
# $Revision::                                          $ Revision of last commit
# $Author::                                            $ Author of last commit
# $Date::                                              $ Date of last commit
# IPSL (2012)
#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
#
#**************************************************************

#======================================================================#
# Display a report on how a simulation is faring.


function ChangeUsr {

  CurrentGrp=$( groups $CurrentUsr | gawk '{print $3}' )
  TargetGrp=$( groups $TargetUsr | gawk '{print $3}' )

  echo $1 | sed -e "s/${CurrentUsr}/${TargetUsr}/" \
                -e "s/${CurrentGrp}/${TargetGrp}/"

}


function SearchCatalog {

  typeset num
  unset SUBMIT_DIR

  fg_new=false

  if [ ! X${JobName} == X ] ; then
    NbOcc=$( gawk -v JobName=${JobName} \
             'BEGIN {x=0}  $1 ~ JobName {++x} END {print x}' ${SimuCatalog} )
  else
    NbOcc=0
  fi

  if ( [ ${NbOcc} -eq 0 ] && ( $fg_path ) ) ; then
    set -A FileList $( ls ${ConfigPath}/Job_* )
    if [ X$FileList == X ] ; then
      NbOcc=0
    else
      NbOcc=${#FileList[@]}
      fg_new=true
    fi
  fi

  if ( [ ${NbOcc} -eq 0 ] && ( ${fg_search} ) ) ; then
    SEARCH_DIR=${WORKDIR}
    if [ ${TargetUsr} != ${CurrentUsr} ] ; then
      SEARCH_DIR=$( ChangeUsr ${SEARCH_DIR})
    fi
    echo "${JobName} not in Catalog, we'll try to find it in ${SEARCH_DIR}"

    set -A FileList $( find ${SEARCH_DIR}/ \
                            -path ${SEARCH_DIR}/IGCM_OUT -prune -o \
                            -name Job_${JobName} -print )
    if [ X$FileList == X ] ; then
      NbOcc=0
    else
      NbOcc=${#FileList[@]}
      fg_new=true
    fi
  fi

  if [ ${NbOcc} -gt 1 ] ; then
    echo "More than one job"
    ind=0
    while [ ${ind} -lt ${NbOcc} ] ; do
      printf '%2i) %-30s\n' ${ind} ${FileList[${ind}]}
      (( ind = ind + 1 ))
    done
    echo "Give your choice number or 'q' to quit : "
    read Choice
    if [ X${Choice} == Xq ] ; then
      exit
    else
      fg_new=true
      FileList=${FileList[${Choice}]}
      NbOcc=1
    fi
  fi

  case ${NbOcc} in
    0)
      echo "${JobName} not found."
      echo "You can try : *) '-s' option to automatically search your \$WORKDIR, "
      echo "              *) '-p' option to provide the config.card path, "
      echo "              *)  manually editing your ${SimuCatalog}"
      exit ;;
    1)
      if ( ${fg_new} ) ; then
        JobName=${JobName:=$( basename ${FileList} | gawk -F"_" '{ print $2 }' )}
        SUBMIT_DIR=$( dirname ${FileList} )
        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}"
        echo "${JobName} ${TargetUsr} ${HostName} ${SUBMIT_DIR}" >> ${SimuCatalog}
        sort -u ${SimuCatalog} > ${SimuCatalog}.tmp
        mv ${SimuCatalog}.tmp ${SimuCatalog}
      elif ( [ ${TargetUsr} == $( gawk -v JobName=${JobName} \
                                      '$1 ~ JobName {print $2}' \
                                      ${SimuCatalog} ) ] \
          && [ ${HostName}  == $( gawk -v JobName=${JobName} \
                                      '$1 ~ JobName {print $3}' \
                                      ${SimuCatalog} ) ] ) ; then
        JobName=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $1}' ${SimuCatalog} )
        SUBMIT_DIR=$( gawk -v JobName=${JobName} '$1 ~ JobName {print $4}' ${SimuCatalog} )
      else
        echo "${JobName} not in Catalog."
        exit
      fi
      break ;;
    *)
      break ;;
  esac

  return

}


function AffichResult {

  fg_first=false
  fg_last=false
  fg_job=false

  while [ $# -ne 0 ] ; do
    case ${1} in
      -f|--first)
        fg_first=true
        shift 1 ;;
      -l|--last)
        fg_last=true
        shift 1 ;;
      -j|--job)
        fg_job=true
        shift 1 ;;
      -*)
        echo "usage: ${0}"
        echo "       options = -f; -l"
        exit ;;
      *)
        break ;;
    esac
  done


  # Define colors
  # =============
  ColEsc="\033["
  ColNon="${ColEsc}0m"       # Return to normal
  ColExp="${ColEsc}1m"       # Blanc - gras
  ColFat="${ColEsc}1;31m"    # Fatal
  ColCpl="${ColEsc}1;32m"    # Completed
  ColAtt="${ColEsc}1;30m"    # Waiting
  ColDef="${ColEsc}1;34m"    # Default
  ColRbl="${ColEsc}31m"      # Rebuild


  
  #Â Print header
  # ============
  if ( ${fg_first} ) ; then
    Dum=""
    (( len = 67 - ${#JobName} ))
    echo "|===========================================================================================================|"
    printf "| JobName = ${ColExp}%-${#JobName}s${ColNon}" ${JobName}
    printf "%-${len}s" ${Dum} 
    printf "run.card : ${ColExp}%-18s${ColNon}|\n" "${LastWrite}"
    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"
    echo "|                         |             |                         |             |     Pending Rebuilds      |"
    echo "| Date Begin - DateEnd    | PeriodState | Current Period          | CumulPeriod | Nb  : from     : to       |"
    echo "|-------------------------|-------------|-------------------------|-------------|-----:----------:----------|"

    printf "| %-10s - %-10s | " \
           $DateBegin $DateEnd 

    case $PeriodState in
      Fatal)
        Color=${ColFat}
        break ;;
      Completed)
        Color=${ColCpl}
        break ;;
      Waiting|OnQueue)
        Color=${ColAtt}
        break ;;
      *)
        Color=${ColDef}
        break ;;
    esac
    printf "${Color}%-11s${ColNon} | " $PeriodState

    printf "%-10s - %-10s | %11s | " \
           $PeriodDateBegin $PeriodDateEnd $CumulPeriod

    if ( [ X${NbRebuild} != X. ] && [ X${NbRebuild} != X0 ] ) ; then
      printf "${ColRbl}%3s : %-8s : %-8s${ColNon} |\n" \
             $NbRebuild $FirstRebuild $LastRebuild
    else
      printf "%3s : %-8s : %-8s |\n" \
             $NbRebuild $FirstRebuild $LastRebuild
    fi

    if [ ${NbLines} -gt 0 ] ; then
      printf "|-----------------------------------------------------------------------------------------------------------|\n"
      printf "|                                                      Last                                                 |\n"
      printf "|     Rebuild      |   Pack_Output    |   Pack_Restart   |    Pack_Debug    |  Monitoring  |     Atlas      |\n"
      printf "|------------------|------------------|------------------|------------------|--------------|----------------|\n"
    fi

    return
  fi

  # Print Post-processing job status
  # ================================
  if ( ${fg_job} ) ; then
    printf "|"

    #Â Print rebuild and pack jobs
    #Â ---------------------------
    for JobType in ${JobType_list[*]} ; do 
      eval Date=\${${JobType}_Date[${ind}]}
      eval Status=\${${JobType}_Status[${ind}]}
      eval Nb=\${${JobType}_Nb[${ind}]}

      if [ X${Status} == XOK  ] ; then
        Color=${ColCpl}
      else
        Color=${ColFat}
      fi
      printf "  ${Color}%-8s${ColNon} : %3s  |" ${Date} ${Nb}
    done

    Color=${ColExp}

    #Â Print monitoring jobs
    #Â ---------------------
    JobType=monitoring
    if [ $ind -eq 0 ] ; then
      eval Date=\${${JobType}_Date}
    else
      Date=""
    fi
    printf "     ${Color}%-4s${ColNon}     |" ${Date}

    #Â Print atlas jobs
    #Â ----------------
    JobType=atlas
    eval Date=\${${JobType}_Date[${ind}]}
    printf "  ${Color}%-12s${ColNon}  |" ${Date}

    printf "\n"

    return
  fi

  #Â Print footer
  #Â ============
  if ( ${fg_last} ) ; then
    printf "|===========================================================================================================|\n"
    date +"${DateFormat}"
    return
  fi

}

#======================================================================#

#D- Task type (computing ,post-processing or checking)
TaskType=checking
typeset -i Verbosity=0

CurrentUsr=$( whoami )

if ( [ $# -eq 0 ] ) ; then
  $0 -h
  exit
fi

# Arguments
# =========
# Default argument values
#Â -----------------------
TargetUsr=${CurrentUsr}
HostName=$( hostname | sed -e "s/[0-9].*//" )

fg_color=true
fg_search=false
fg_quiet=false
fg_path=false
NbHisto=10

# Get arguments from command line
# -------------------------------
while [ $# -ne 0 ] ; do
  case $1 in
    -h|--help|-help)
      echo "usage: $0 [-u user] [-q] [-j n] [-s] job_name"
      echo "       $0 [-u user] [-q] [-j n] -p config.card_path"
      echo ""
      echo "options :"
      echo "  -h : print this help and exit"
      echo "  -u : owner of the job"
      echo "  -q : quiet"
      echo "  -j : print n post-processing jobs (default is 10)"
      echo "  -s : search for a new job in \$WORKDIR and fill in "
      echo "       the catalog before printing information"
      echo "  -p : give the directory (absolute path) containing "
      echo "       the config.card instead of the job name."
      exit ;;
#    -b|-nocolor)
#      fg_color=false
#      shift 1 ;;
    -j|-job-number)
      NbHisto="$2"
      shift 2 ;;
    -p|-config-path)
      ConfigPath="$2"
      fg_path=true
      shift 2 ;;
    -q|-quiet)
      fg_quiet=true
      shift 1 ;;
    -s|-search)
      fg_search=true
      shift 1 ;;
    -u|-user)
      TargetUsr="$2"
      shift 2 ;;
    -*)
      $0 -h
      exit ;;
    *)
      break ;;
  esac
done


if ( ( ! $fg_path ) && [ $# -lt 1 ] ) ; then
  $0 -h
  exit
fi

if ( ( ${fg_path} ) && ( ${fg_search} ) ) ; then
  echo "You cannot use -s and -p at the same time"
  exit
fi


# Load libIGCM library
# ====================
libIGCM=${libIGCM:=::modipsl::/libIGCM}

. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
     ( ${DEBUG_debug} ) && IGCM_debug_Check
. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
     ( ${DEBUG_debug} ) && IGCM_card_Check
. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
     ( ${DEBUG_debug} ) && IGCM_date_Check
#-------
. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
. ${libIGCM}/libIGCM_config/libIGCM_config.ksh


JobName=$1

if ( ${fg_quiet} ) ; then
  NbHisto=1
fi

echo "Target user = ${TargetUsr}"

# Define the catalog in which the known simulations are stored
SimuCatalog="${HOME}/.simucatalog.dat"
if [ ! -s ${SimuCatalog} ] ; then
  touch ${SimuCatalog}
fi

# Date format
DateFormat="%d/%m/%y %R:%S"

# Find SUBMIT_DIR in catalog
# ==========================
SearchCatalog

if [ ! X${SUBMIT_DIR} == X ] ; then

  echo "Submit:  >${SUBMIT_DIR}<"
  cd $SUBMIT_DIR


  # Extract usefull information from run.card and config.card
  # =========================================================

  RunFile="${SUBMIT_DIR}/run.card"
  ConfFile="${SUBMIT_DIR}/config.card"

  IGCM_config_CommonConfiguration ${SUBMIT_DIR}/config.card


  if [ -s ${RunFile} ] ; then
    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodState
    PeriodState=${run_Configuration_PeriodState}
  else
    PeriodState="Waiting"
  fi

  if ( [ X${PeriodState} == XRunning ] || [ X${PeriodState} == XOnQueue ] ) ; then
    #NbRun=$( ccc_mstat -f | grep -c ${JobName} )
    IGCM_sys_CountJobInQueue ${JobName} NbRun

    if [ ${NbRun} -eq 0 ] ; then
      PeriodState="Fatal"
    fi
  fi

  DateBegin=${config_UserChoices_DateBegin}
  DateEnd=${config_UserChoices_DateEnd}
  TagName=${config_UserChoices_TagName}
  ExperimentName=${config_UserChoices_ExperimentName}
  SpaceName=${config_UserChoices_SpaceName}


  if ( [ ! X${PeriodState} == XWaiting ] && [ ! X${PeriodState} == XCompleted ] ) ; then
    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateBegin
    IGCM_card_DefineVariableFromOption ${RunFile} Configuration PeriodDateEnd
    IGCM_card_DefineVariableFromOption ${RunFile} Configuration CumulPeriod
    PeriodDateBegin=${run_Configuration_PeriodDateBegin}
    PeriodDateEnd=${run_Configuration_PeriodDateEnd}
    CumulPeriod=${run_Configuration_CumulPeriod}
  else
    PeriodDateBegin="."
    PeriodDateEnd="."
    CumulPeriod="."
  fi

  DATA_DIR=${R_SAVE}
  POST_DIR=${R_BUFR}/Out
  CWORK_DIR=${R_FIGR}
  if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
    RebuildJob="rebuild_fromArchive"
  else
    RebuildJob="rebuild_fromWorkdir"
  fi

  if [ ${TargetUsr} != ${CurrentUsr} ] ; then
    DATA_DIR=$( ChangeUsr ${DATA_DIR})
    POST_DIR=$( ChangeUsr ${POST_DIR} )
    CWORK_DIR=$( ChangeUsr ${CWORK_DIR} )
    REBUILD_DIR=$( ChangeUsr ${REBUILD_DIR} )
  fi

  echo "Data:    >${DATA_DIR}<"
  echo "Rebuild: >${REBUILD_DIR}<"
  echo "Post:    >${POST_DIR}<"
  echo "Work:    >${CWORK_DIR}<"

  if [ $PeriodState != "Waiting" ] ; then

    # Check pending rebuilds
    # ======================

    set -A RebuildList $( find ${REBUILD_DIR}/ -name "REBUILD_*" | sort )
    if [ ${#RebuildList[*]} -gt 0 ] ; then
      NbRebuild=$( IGCM_sys_CountFileArchive ${REBUILD_DIR} )

      FirstRebuild=$( basename ${RebuildList[0]} | cut -f2 -d\_ )
      LastRebuild=$( basename ${RebuildList[ (( NbRebuild=${NbRebuild}-1 )) ]} | cut -f2 -d\_ )
    else
      NbRebuild="."

      FirstRebuild="."
      LastRebuild="."
    fi
  else
    NbRebuild="."

    FirstRebuild="."
    LastRebuild="."
  fi


  if [ $PeriodState != "Waiting" ] ; then

    # Check last REBUILD and PACK* jobs
    # =================================
    # Define input parameters
    # -----------------------
    set -A JobType_list "${RebuildJob}" "pack_output" "pack_restart" "pack_debug"

    for JobType in ${JobType_list[*]} ; do 
      typeset    name1="${JobType}_String"
      typeset    name2="${JobType}_Field"
      if [ X${JobType} == X${RebuildJob} ] ; then
        eval ${name1}=IGCM_sys_PutBuffer_Out
        eval ${name2}=4
      else
        eval ${name1}=IGCM_sys_Put_Out
        eval ${name2}=3
      fi
    done

    #Â Check jobs
    # ----------
    NbLines=0
    for JobType in ${JobType_list[*]} ; do 
      eval String=\${${JobType}_String}
      eval Field=\${${JobType}_Field}

      set -A FileList $( ls ${POST_DIR}/${JobType}.*.out | tail -n ${NbHisto} )

      if [ ${#FileList[*]} -gt ${NbLines} ] ; then
        NbLines=${#FileList[*]}
      fi

      (( ind = 0 ))
      for FileName in ${FileList[*]} ; do
        LastDate=$( basename ${FileName} | gawk -F"." '{ print $(NF-1) }' )

        Error=$( gawk -v String=${String} \
                     'BEGIN { x=0 } ($1~String) && ($3~"error.") { x=x+1 } END { print x }' \
                     ${POST_DIR}/${JobType}.${LastDate}.out )

        Match=$( gawk -v String=${String} \
                     'BEGIN { x=0 } ($1~String) && ($3!~"error.") { x=x+1 } END { print x }' \
                     ${POST_DIR}/${JobType}.${LastDate}.out )
        (( Nb = ${Match} - ${Error} ))

        if ( [ ${Error} -eq 0 ] && [ ${Nb} -gt 0 ] ) ; then
          Status=OK
        else
          Status=KO
        fi

        eval ${JobType}_Date[$ind]=${LastDate}
        eval ${JobType}_Status[$ind]=${Status}
        eval ${JobType}_Nb[$ind]=${Nb}
        
        (( ind = ind + 1 ))
      done
    done

    # Check last MONITORING jobs
    # ==========================
    JobType=monitoring
    if [ -d ${CWORK_DIR}/MONITORING ] ; then
      LastDate=$( cdo showyear ${CWORK_DIR}/MONITORING/files/ATM_bils_global_ave.nc 2> /dev/null | \
                      gawk '{ print $NF }' )
      eval ${JobType}_Date=${LastDate}
    fi

    # Check last ATLAS jobs
    # =====================
    JobType=atlas
    if [ -d ${CWORK_DIR}/ATLAS ] ; then
      set -A FileList $( ls ${CWORK_DIR}/ATLAS | tail -n ${NbHisto} )

      if [ ${#FileList[*]} -gt ${NbLines} ] ; then
        NbLines=${#FileList[*]}
      fi

      (( ind = 0 ))
      for FileName in ${FileList[*]} ; do
        eval ${JobType}_Date[$ind]=${FileName}
        (( ind = ind + 1 ))
      done
    fi


    # Time of last write on run.card
    # ==============================
    LastWrite=$( ls -l --time-style=+"${DateFormat}" ${SUBMIT_DIR}/run.card | gawk '{print $6 " " $7}' )

  else

    NbLines=0

  fi
    

  # Print results
  # =============
  AffichResult -f
  ind=0
  while [ $ind -lt $NbLines ] ; do
    AffichResult -j
    (( ind = ind + 1 ))
  done
  AffichResult -l

fi