source: TOOLS/PACK_IPSL/archive_debug.sh @ 3620

#!/bin/bash

#on veut archiver les fichiers Debug. 
#Ils seront tous regroupes dans un repertoire DEBUG commun  
# on les renommes (repertoire DEBUG) de la meme maniere que les fichiers restart
# puis on les archive via tar par periode 
#Creation des listes dans un repertoire debug_tar

config_card=${1}
LISTE_DMNFS=${2}


export JOB_DIR=${JOB_DIR:-${TMP_MIGR_DATA}}
export IGCM_DEM=${IGCM_DEM:-${SCRATCHDIR}/IGCM_DEM}
export IGCM_TMP=${IGCM_TMP:-${IGCM_DEM}/tmp}
mkdir -p ${IGCM_DEM}
mkdir -p ${IGCM_TMP}
clean_tmp=${clean_tmp:-yes}
EXE_DIR=$(dirname ${0})
SCRIPT_NAME=$(basename ${0} )

source ${EXE_DIR}/DEM_utilities.sh

# gpdebug : verif si le present script a ete execute jusqu'au bout #########
MonitoringInfo="${SCRIPT_NAME}-->OK"
simuMonitoring_check "$MonitoringInfo" "$config_card"

DEM_log -0 "Demarrage de ${SCRIPT_NAME}"

#### 0- lecture de config.card 
#### on veut les renseignements suivants : 
#### JobName, PATH_SIMUL, DateBegin et DateEnd
eval $(grep JobName ${config_card}) 
eval $(grep PATH_SIMUL_FULL ${config_card}) 
eval $(grep DateBegin ${config_card}) 
eval $(grep DateEnd ${config_card}) 
eval $(grep IGCM_DEM_SIMU ${config_card})


#### Preparation de l'environnement
# repertoire dans lequel seront renommes les fichiers de debug
STORE_DEBUG=$IGCM_DEM_SIMU/DEBUG
mkdir -p ${STORE_DEBUG}

# repertoire dans lequel nous stockerons les listes de debug a archiver
DEBUG_TAR=$IGCM_DEM_SIMU/debug_tar
mkdir -p ${DEBUG_TAR}
STORE_CP=$IGCM_DEM_SIMU/store_cp
mkdir -p ${STORE_CP}

# lecture de periodpack dans ${IGCM_DEM_SIMU}/period_pack.txt
period_pack=`awk '{print $0} ' ${IGCM_DEM_SIMU}/period_pack.txt`

#### 1- renommer les fichiers ######
DEM_log -1 "Renommer les fichiers de Debug"

cd ${STORE_DEBUG}
if [ "x${STORE_DEBUG}" != "x" ]
then
rm -rf *
fi
# On copie les fichiers de Debug sur le scratchdir en changeant le nom et conservant la date
grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Debug | grep "f "| awk '{print $3}' | sed '/^.*Debug$/d' | sort >> ${IGCM_TMP}/${$}.liste_debug_files.txt

# Test si il y a un repertoire Debug
listDebugNbLines=`cat ${IGCM_TMP}/${$}.liste_debug_files.txt | wc -l `
if [ "x${listDebugNbLines}" == "x0" ]
then
    DEM_log -0 "Pas de repertoire Debug pour cette simulation"
else
# cas particulier : les fichiers Bands sont conserves sans etre archives
    grep ${JobName}_Bands ${IGCM_TMP}/${$}.liste_debug_files.txt >> ${STORE_CP}/cp_files.list
    sed "/${JobName}_Bands/d" ${IGCM_TMP}/${$}.liste_debug_files.txt > ${IGCM_TMP}/${$}.liste_debug_files.txt_tmp
    mv ${IGCM_TMP}/${$}.liste_debug_files.txt_tmp ${IGCM_TMP}/${$}.liste_debug_files.txt

# on s assure que les fichiers restant sont bien des fichiers dans Debug et pas des fichiers dans des sous-repertoires"
    for file in `cat ${IGCM_TMP}/${$}.liste_debug_files.txt ` ; do
        dir=$( echo $file | sed -e "s%.*Debug/%%")
        testdir=$(dirname ${dir})
        if [ $testdir != "." ]; then
            echo $( grep ${file} ${IGCM_TMP}/${$}.liste_debug_files.txt)  >> ${IGCM_TMP}/${$}.liste_no_debug_files.txt
        fi
    done
    if [ -f ${IGCM_TMP}/${$}.liste_no_debug_files.txt ]; then
        join -t % -v 1 ${IGCM_TMP}/${$}.liste_debug_files.txt ${IGCM_TMP}/${$}.liste_no_debug_files.txt > ${IGCM_TMP}/${$}.info_tmp.txt
        mv ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.liste_debug_files.txt
        rm -f ${IGCM_TMP}/${$}.liste_no_debug_files.txt
    fi
    gawk -F/ '{dest_name=$(NF-2)("_") $(NF); system("ln -s " $0 " "dest_name)}' ${IGCM_TMP}/${$}.liste_debug_files.txt
#on fait de meme avec les fichiers out_gcm.e stockes dans le repertoire Out/ qui seront prefixes par Out_
    rm -f ${IGCM_TMP}/${$}.liste_debug_files.txt
    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep -w Out | awk '{print $3}' | sed '/^.*Out$/d' > ${IGCM_TMP}/${$}.liste_debug_files.txt
    gawk -F/ '{dest_name=$(NF-1)("_") $(NF); system("ln -s " $0 " "dest_name)}' ${IGCM_TMP}/${$}.liste_debug_files.txt


    ls -l ${STORE_DEBUG} > ${IGCM_TMP}/${$}.new_liste_debug_files.txt


#creation des listes en fonction des suffixes
   if [ "x${ON_IDRIS}" == "xno" ]
   then
       awk '{if (NR >1) print $9}' ${IGCM_TMP}/${$}.new_liste_debug_files.txt > ${IGCM_TMP}/${$}.new_liste_debug_files.txt_tmp
   else
       awk '{if (NR >1) print $8}' ${IGCM_TMP}/${$}.new_liste_debug_files.txt > ${IGCM_TMP}/${$}.new_liste_debug_files.txt_tmp       
   fi
sort ${IGCM_TMP}/${$}.new_liste_debug_files.txt_tmp > ${IGCM_TMP}/${$}.order_new_liste_debug_files.txt
ref=`awk -F_ '{if (NR==1) print $1}' ${IGCM_TMP}/${$}.order_new_liste_debug_files.txt`



i=1
for new_file in `awk  '{print $0}' ${IGCM_TMP}/${$}.order_new_liste_debug_files.txt`; do 
    new_ref=$( echo ${new_file} | sed -e "s/_.*//") 

    if [ ${new_ref} == ${ref} ]; then
        echo ${new_file} >> ${IGCM_TMP}/${$}.debug_list_${i}.txt
    else
        ref=${new_ref}
        (( i = $i + 1 )) 
        echo ${new_file} >> ${IGCM_TMP}/${$}.debug_list_${i}.txt
    fi
done

j=1
while [[ $j -le $i  ]]; do 


### 2- creer les listes de fichiers par periode ####
    DEM_log -1 "Creation des listes de fichiers par periode"
### 2.1- extraire l'annee et le mois de DateBegin ###
    date_begin=${DateBegin}
    date_end=${DateEnd}
    
    year_begin=$(echo ${date_begin} | cut -c 1-4 )
    month_begin=$(echo ${date_begin} | cut -c 5-6 )
    year_end=$(echo ${date_end} | cut -c 1-4 )
    month_end=$(echo ${date_end} | cut -c 5-6 )


### 2.2- creation des listes ###
    date_begin_pack=${date_begin}
    date_end_simulation=${date_end}

    month_begin_pack=${month_begin}
    year_begin_pack=${year_begin}

    number_pack=1

    while [ ${date_begin_pack} -le ${date_end_simulation} ]
    do 

        DEM_log -1 "On calcule la date de fin du pack "
        month_end_pack=${month_begin_pack}
        year_end_pack=`expr ${year_begin_pack} + ${period_pack}`
        
        date_end_pack=${year_end_pack}${month_end_pack}01   
        for file in `awk '{print $1}' ${IGCM_TMP}/${$}.debug_list_${j}.txt`; do 
            extract_date_file=`echo ${file}  | sed -e "s/.*\${JobName}_//"`
            date_file=`echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g'`
            if [[  "x${date_file}" != "x" && ! -n ${date_file//[0-9]/} ]]; then
                if [  ${date_file} -lt ${date_end_pack} ]
                then 
                    if [ ${date_file} -ge ${date_begin_pack} ]
                    then 
                        echo ${file} >> ${IGCM_TMP}/${$}.liste_pack_debug_${number_pack}.txt
                        grep -v ${file} ${IGCM_TMP}/${$}.debug_list_${j}.txt > ${IGCM_TMP}/${$}.debug_list_${j}.txt.tmp
                        mv ${IGCM_TMP}/${$}.debug_list_${j}.txt.tmp ${IGCM_TMP}/${$}.debug_list_${j}.txt
                        
                    fi
                fi 
            fi
            if [  ${date_file} -gt ${date_end_pack} ]
            then 
                break
            fi
            
        done
        month_begin_pack=${month_end_pack}
        year_begin_pack=${year_end_pack}

        date_begin_pack=${year_begin_pack}${month_begin_pack}01

        number_pack=`expr ${number_pack} + 1`
    done

    ((j = $j + 1 ))
done

compt_pack=1
while [[ $compt_pack -le $number_pack ]]; do 

#on archive les debug - le nom de l'archive contiendra ${yearbegin}${monthbegin}_${yearend}${monthend} 
# end etant la date du dernier fichier contenu et non pas la premiere date suivant 
DEM_log -1 "Recherche de la date de fin reel pour nommer l'archive "
#On classe par ordre alphabetique les fichiers contenus dans liste_pack_debug 
#la date du dernier fichier donnera la date de fin du pack
  if [[ -f ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt ]] ; then
   sort ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt > ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt.tmp

   first_file=$(head -n 1 ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt.tmp )
   extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_//" )
   DateBegin_Debug=$(echo ${extract_date_file}| sed 's%\([0-9]\{8\}\)_.*$%\1%g' )

   last_file=$(tail -n 1 ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt.tmp )
   extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" )
   DateEnd_Debug=$(   echo ${extract_date_file}| sed 's%.*\([0-9]\{8\}\)_.*$%\1%g' )
   mv ${IGCM_TMP}/${$}.liste_pack_debug_${compt_pack}.txt.tmp ${DEBUG_TAR}/${JobName}_debug_${DateBegin_Debug}_${DateEnd_Debug}.list
  fi



DEM_log -1 "On definit le datebegin de la  prochaine periode pack"
#on definit la periode pack suivante

   compt_pack=`expr ${compt_pack} + 1`
done

fi

cd ${EXE_DIR}

# gpdebug : le script n a pas rencontre d'erreur. On l'inscrit dans le fichier de suivi
simuMonitoring_OK "$MonitoringInfo" "$config_card"