source: TOOLS/PACK_IPSL/parallelPack.sh @ 2010

#!/bin/bash

function insertDirectives
{
    
    local batchFile="${TMP_MIGR_DATA}/launch_and_measureTime.sh"
    local lineBeginMutableDrves=`grep -n "batch directives : begin" $batchFile | awk -F":" '{print $1}' `
    local lineHeaderMutableDrves=`grep -n "mutable directives" $batchFile | awk -F":" '{print $1}' `
    local lineEndMutableDrves=`grep -n "batch directives : end" $batchFile | awk -F":" '{print $1}' `
    if [ "x${lineBeginMutableDrves}" == "x" ] || [ "x${lineHeaderMutableDrves}" == "x" ] || [ "x${lineEndMutableDrves}" == "x" ]
    then
        echo "Examen du fichier $batchFile :"
        echo "Il manque les balises commentees 'mutable directives' et 'batch directives : end' "
        exit 1
    fi
    
    # echo "lineHeaderMutableDrves=$lineHeaderMutableDrves"
    # echo "lineEndMutableDrves=$lineEndMutableDrves"
    
    if [ $lineHeaderMutableDrves -ge $lineEndMutableDrves ]
    then
       echo "Examen du fichier $batchFile :"
       echo "les lignes 'mutable directives' et 'batch directives : end' n'existent pas ou ne sont"
       echo "pas bien placees l'une par rapport a l'autre ==> STOP."
       exit 1
    fi
    
    # Suppression des anciennes directives
    local firstLineToMove=$(( $lineHeaderMutableDrves + 1 ))
    local lastLineToMove=$(( $lineEndMutableDrves - 1 ))
    if [ $firstLineToMove -lt $lineEndMutableDrves ]
    then
        sed -i "${firstLineToMove},${lastLineToMove}d" $batchFile
    fi
        
    # insertion des directives chargees a partir de "load_batch_directives.sh"
    local lineEndMutableDrves=`grep -n "batch directives : end" $batchFile | awk -F":" '{print $1}' `
    sed -i "${lineEndMutableDrves}i\#MSUB -Qos $QosType" $batchFile
    sed -i "${lineEndMutableDrves}i\#MSUB -q $queueType" $batchFile
    sed -i "${lineEndMutableDrves}i\#MSUB -A $projectName" $batchFile
    sed -i "${lineEndMutableDrves}i\#MSUB -T $computationTime" $batchFile
    sed -i "${lineEndMutableDrves}i\#MSUB -n $nbProcs" $batchFile
    
    # insertion des directives specifiant les sorties des operations de pack
    sed -i "${lineEndMutableDrves}i\#MSUB -e ${DETAILLED_OUTPUT}/pack_ipsl_%I.e" $batchFile
    sed -i "${lineEndMutableDrves}i\#MSUB -o ${DETAILLED_OUTPUT}/pack_ipsl_%I.o" $batchFile

}

function check_machine
{
    local host_name=`hostname `
    local isMachineTitane=`echo $host_name | grep "titane" | wc -l `
    local isMachineCurie=`echo $host_name | grep "curie" | wc -l `
    
    if [ "x${isMachineTitane}" == "x0" ] && [ "x${isMachineCurie}" == "x0" ]
    then
        echo "Ce programme doit etre lance a partir 'curie' ou 'titane'. STOP."
        exit 1
    fi

}

function check_glost_launch
{
    local file_exec='glost_launch'
    if [ ! -e $file_exec ]
    then
        echo "Il n'y a pas l'executable glost_launch."
        echo "Vous devez compiler l'outil via make. "
        exit 1
    fi
}

function checkParameters
{
    isCheckVarOK=`echo $doYouWantCheck | grep -E '^(yes|no)$' | wc -l `
    if [ "x${isCheckVarOK}" == "x0" ]
    then
        echo "La variable 'doYouWantCheck' doit etre yes ou no"
        exit 1
    fi
    
    isRestartVarOK=`echo $forceRestartFromBegin | grep -E '^(yes|no)$' | wc -l `
    if [ "x${isRestartVarOK}" == "x0" ]
    then
        echo "La variable 'forceRestartFromBegin' doit etre yes ou no"
        exit 1
    fi
}


function getNumeroOfPreviousTry
{
   local num_try="0"
   ici=$PWD
   cd $USER_OUTPUT_PROGRESS
   
   tryDirs=`ls | grep -e 'TRY__[[:digit:]]\{1,2\}$' `
   for dir in $tryDirs
   do
      num=`echo $dir | awk -F"__" '{print $NF}' `
      if [ $num -gt $num_try ]
      then
          num_try=$num 
      fi
   done  
   cd $ici
   echo $num_try
   
}

# indique si l'on recommence les traitement depuis le debut ou non 
restartExecFromZero=0

# indique si l'on veut effacer les fichiers d'entrée ou non
deleteFiles=0

# On charge les variables d'environnement 
. load_ipslPack_env.sh

# On charge les directives batch
. load_batch_directives.sh

# On charge les parametres
. loadParameters.sh

checkParameters

if [ "x${forceRestartFromBegin}" == "xyes" ]
then
     restartExecFromZero=1
fi


# on verifie que la machine est 'curie' ou 'titane'
check_machine



# on verifie que l outil glost a bien ete compile
check_glost_launch


########  pour option -f #############
while [ $# -gt 0 ]
do
   echo "boucle sur les arguments du script ..."
   echo "@=$@"
   case $1 in
   -f)  restartExecFromZero=1 
        echo "force restart from the beginning ..."
        ;;
   -d)  deleteFiles=1
        echo "force to delete input files ..."
        ;;
   -h|--help|-help)
        echo "---------------------------------------------------"
        echo "Usage: parallelPack.sh [-f]"
        echo "Description:"
        echo "   Launches the simultaneous treatments (ncrcat, tar, cap) of"
        echo "   file lists contained in the IGCM_DEM directory."
        echo "   In case of machine crash, this script handle restarts from the point"
        echo "   the machine crashed."
        echo
        echo "Options:"
        echo "   -h, --help, -help"
        echo "        Print this manual"
        echo "   -f"
        echo "        Force to restart the treatments from the beginning"
        echo "   -d"
        echo "       force to delete input files ... "
        echo "---------------------------------------------------"
        exit
        ;;
   esac
   shift
done

# copie du fichier launch_and_measureTime.sh sur ${TMP_MIGR_DATA}
cp launch_and_measureTime.sh ${TMP_MIGR_DATA}/.

export restartExecFromZero=${restartExecFromZero}
export deleteFiles=${deleteFiles}
######################################

#########  On insert les directives batch en en-tete du script "launch_and_measureTime.sh"         ##########
#########  lance plus loin. Seules les directives de nom du job et de sorties restent inchangees   ##########

insertDirectives

# exit 0 # gpdebug : a virer

# export inputCmd="${USER_OUTPUT_PROGRESS}/inputCmd.list"
# export nextInputCmd="${USER_OUTPUT_PROGRESS}/nextInputCmd.list"
# export output="${USER_OUTPUT_PROGRESS}/packOutput.log"
# export reportFile="${USER_OUTPUT_PROGRESS}/report.log"

########## Si l'option -f est activee, on detruit tous les fichiers status  ##########
if [ "x$restartExecFromZero" == "x1" ]
then
    echo "Deleting  all status files ..."
    for CONFIG in $( awk '{print $1}' ${IGCM_DEM}/config_card.liste )
    do
       PATH_SIMU=$( dirname $CONFIG )
       ListStatFileInSimu=`find $PATH_SIMU -type f -name "*status" `
       if [ "x$ListStatFileInSimu" != "x" ]
       then
           for file in $ListStatFileInSimu
           do
              if [ -e $file ]
              then
                 rm -f $file
              fi
       
           done      
       fi   
    done
    echo "Deletion done."
fi


# Par ailleurs, on vide les fichiers de cmds, d'output et de rapport si option -f activee
if [ "x${restartExecFromZero}" == "x1" ]
then
    rm -rf ${USER_OUTPUT_PROGRESS}/TRY__*
fi

# exit 0 # a virer

#####################################################################################

export JOB_DIR=${TMP_MIGR_DATA}
export EXE_DIR=${LS_SUBCWD:-${PWD}}
source ${EXE_DIR}/DEM_utilities.sh

############## gpdebug : Pour gestion des erreurs ######################################### 
# Execute la function errorReceive() si reception d'un signal TERM
trap 'DEM_errorReceive' TERM
# liste des processus à détruire en cas d'erreur bloquante
export listPID=$$
# fichier pour la transmission des messages d'erreur d'un fils au prog principal
export errorMsgFile="${TMP_MIGR_DATA}/errorMsg.txt"
############## gpdebug : fin ##############################################################

check_nco_version # verification de la version de nco #######################
check_libccc_user # verif que le module libccc_user est charge ##############
check_cdo         # verif que le module cdo est charge ######################

# exit 0 # a virer

export badFailureFile=${USER_OUTPUT_PROGRESS}/badFailure.txt
> $badFailureFile # nouvel essai, on vide le fichier d'erreur
tryNumFile=${USER_OUTPUT_PROGRESS}/numero_current_try.txt
numPreviousTry=$( getNumeroOfPreviousTry )
# echo "numPreviousTry=$numPreviousTry"
numNewTry=$(( $numPreviousTry + 1 ))


# Pas de 4eme essai permis
# ---------------------------- 
if [ $numNewTry -ge 3 ]
then
    echo "Pas de 3eme essai permis"
    exit 1
fi

# echo "numNewTry=$numNewTry"
mkdir -p ${USER_OUTPUT_PROGRESS}/TRY__${numNewTry}
echo $numNewTry > $tryNumFile

# exit 0 # a virer



export timeLaunchStartFile="${USER_OUTPUT_PROGRESS}/timeLaunchStartFile.txt"
> $timeLaunchStartFile
startTime=$( getDateMilliSeconds )
echo "launch time:$startTime" > $timeLaunchStartFile

export timeEndFile="${USER_OUTPUT_PROGRESS}/timeEndFile.txt"
> $timeEndFile

SCRIPT_NAME=$(basename ${0} )

DEM_log -0 "Demarrage de ${SCRIPT_NAME}"

# ccc_msub -A tgcc0013 launch_and_measureTime.sh
ccc_msub ${TMP_MIGR_DATA}/launch_and_measureTime.sh

if [ $? -ne 0 ]
then
    echo "Errreur : le job batch ${TMP_MIGR_DATA}/launch_and_measureTime.sh n a pas pu etre soumis correctement !"
    echo "Verifier les parametres batch de votre job "
    exit 1
fi

tpsAvantRendreLaMain=15

elapsedTime=0
while [ $elapsedTime -lt $tpsAvantRendreLaMain ]
do
    sleep 5
    elapsedTime=$(( $elapsedTime + 5 ))
    isFailureFileEmpty=`cat $badFailureFile | wc -l `
    if [ "x${isFailureFileEmpty}" != "x0" ]
    then
       sleep 2 # on attend que le fichier soit correctement rempli
       cat $badFailureFile
       exit 0
    fi
done

if [ ${elapsedTime} -ge $tpsAvantRendreLaMain ]
then
    echo "Les $tpsAvantRendreLaMain sec sont ecoulees, les traitements continuent..."
fi


DEM_log -0 "Fin de ${SCRIPT_NAME}"