source: TOOLS/PACK_IPSL/find_bad_files_in_pack.sh @ 3640

#!/bin/bash
# for a list of login we look if some file like "*JobName*" were packed (instead of "JobName*"
# Create for each simulation a list with all these "bad" files. 
# author : Anne COZIC 
# date : 19/07/2013

STORELOG="/ccc/scratch/cont003/dsm/p86cozic/pack_ipsl_logs"

for LOGIN in `awk  '{print $0}' ${STORELOG}/liste_login` ; do 

echo  "----------------- $LOGIN"
IGCM_TMP=$SCRATCHDIR/TMP_PACK
IGCM_DEM=$SCRATCHDIR/SAVE_PACK/${LOGIN}
mkdir -p ${IGCM_TMP}
rm -rf  ${IGCM_TMP}/*

mkdir -p $SCRATCHDIR/SAVE_PACK
mkdir -p ${IGCM_DEM}
rm -rf  ${IGCM_DEM}/*

cd ${STORELOG}
if [ -s ${LOGIN}.Listing.txt.gz ]; then 
    gunzip ${LOGIN}.Listing.txt.gz
    LISTE_DMNFS=${LOGIN}.Listing.txt

else
    tar xf ${LOGIN}.ipsl_logs.tgz ${LOGIN}/IGCM_DEM/Listing.txt
    mv ${LOGIN}/IGCM_DEM/Listing.txt  ${LOGIN}.Listing.txt
    rm -rf ${LOGIN}/
    LISTE_DMNFS=${LOGIN}.Listing.txt
fi 

if [ -s $LISTE_DMNFS ]; then 

listRestart=`grep "IGCM_OUT/" ${LISTE_DMNFS} | grep Restart/ | awk '{print $3}' | sed 's%/Restart.*%%' `
if [[ ${listRestart} != "" ]] ; then 
    echo "${listRestart}" >> ${IGCM_TMP}/${$}.liste_Restart.txt
fi

while [[ -s ${IGCM_TMP}/${$}.liste_Restart.txt ]] 
do 
DIR=$( awk '{if (NR==1) print $0}' ${IGCM_TMP}/${$}.liste_Restart.txt ) 

#gestion des cas particulier tels que la creation de l'etat initial avec lmdz (JobName/ATM/Output/Restart/ au lieu de JobName/ATM/Restart/) 
    PATH_SIM=$( dirname ${DIR} ) 
    echo ${DIR}
    lastdir=$( echo $PATH_SIM | sed -e "s%.*/%%") 
    if [ $lastdir != "ATM" ]; then
        dirname ${DIR} >> ${IGCM_TMP}/${$}.path_simul.txt
        grep -vw ${PATH_SIM} ${IGCM_TMP}/${$}.liste_Restart.txt > ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ; mv ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ${IGCM_TMP}/${$}.liste_Restart.txt
    else
        dirname ${PATH_SIM} >> ${IGCM_TMP}/${$}.path_simul.txt
        PATH_SIM_TMP=$( dirname ${PATH_SIM} ) 
        grep -vw ${PATH_SIM_TMP} ${IGCM_TMP}/${$}.liste_Restart.txt > ${IGCM_TMP}/${$}.liste_Restart.txt.tmp; mv ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ${IGCM_TMP}/${$}.liste_Restart.txt

    fi 



done
echo "----------------------------------------------fin listerestart"

if [[ -s ${IGCM_TMP}/${$}.path_simul.txt ]] ; then 
# On retire les doublons pour avoir la liste des simulations a tester
awk '{if (x[$0] != "") next ; print $0 ; x[$0]=$0}' <${IGCM_TMP}/${$}.path_simul.txt  >> ${IGCM_TMP}/${$}.liste_simul
awk '{if (x[$0] != "") next ; print $0 ; x[$0]=$0}' <${IGCM_TMP}/${$}.liste_simul  >> ${IGCM_TMP}/${$}.liste_simul.tmp${$}
mv ${IGCM_TMP}/${$}.liste_simul.tmp${$} ${IGCM_DEM}/liste_simul



for PATH_SIMUL_FULL in `awk  '{print $0}' ${IGCM_DEM}/liste_simul` ; do 
    rm -f ${IGCM_TMP}/*
    echo "PATH_SIMUL_FULL = " $PATH_SIMUL_FULL

    # Etablir la liste des fichiers de restart
    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Restart | awk '{print $2 " " $3}' | grep -e '.*/Restart/.*\..*' |grep -v "jnl" | sort  >> ${IGCM_TMP}/${$}.liste_restart_files_config.txt


    # Etablir la liste des fichiers output
    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Output  | awk '{print $2 " "  $3}' | grep -e '.*/Output/.*\.nc$'  | sort >> ${IGCM_TMP}/${$}.liste_output_files_config.txt

    # Lecture du nom du job
    JobName=`awk -F/ '{if (NR==1) print $(NF-3)}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt`

    if [ $JobName = "ATM" ]; then
        JobName=`awk -F/ '{if (NR==1) print $(NF-4)}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt`
    fi 

    echo "JobName = "$JobName

    sed -e "s%.*${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt
    cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" >  ${IGCM_TMP}/${$}.info_tmp_bis.txt
    join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt > ${IGCM_TMP}/${$}.output.tmp
    sort -k 2 ${IGCM_TMP}/${$}.output.tmp > ${IGCM_TMP}/${$}.output_bis.tmp
    mv ${IGCM_TMP}/${$}.output_bis.tmp ${IGCM_TMP}/${$}.liste_output_files_config.txt


    rm -f ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt ${IGCM_TMP}/${$}.output.tmp ${IGCM_TMP}/${$}.output_bis.tmp
## maintenant on va extraire de cette liste les fichiers qui sont du type *${JobName} et non pas ${JobName}
    sed -e "s%.*/${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt
    cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" >  ${IGCM_TMP}/${$}.info_tmp_bis.txt

## la liste que l'on cherche est  ${IGCM_TMP}/${$}.info_tmp_bis.txt
    if [ -s ${IGCM_TMP}/${$}.info_tmp_bis.txt ]; then 
        mv ${IGCM_TMP}/${$}.info_tmp_bis.txt ${IGCM_DEM}/${JobName}.info_bug_pack_output.txt
    fi
done

fi 
fi 
done