source: TOOLS/PACK_IPSL/find_bad_files_in_pack.sh @ 2120

Last change on this file since 2120 was 2120, checked in by acosce, 8 years ago

Add to script to debug some problems create by pack_ipsl :
Somes files with name as *JobName?* (instead of JobName?*) were packed
So these scripts look for all these files and find in which pack they are.

File size: 4.5 KB
Line 
1#!/bin/bash
2# for a list of login we look if some file like "*JobName*" were packed (instead of "JobName*"
3# Create for each simulation a list with all these "bad" files.
4# author : Anne COZIC
5# date : 19/07/2013
6
7STORELOG="/ccc/scratch/cont003/dsm/p86cozic/pack_ipsl_logs"
8
9for LOGIN in `awk  '{print $0}' ${STORELOG}/liste_login` ; do
10
11echo  "----------------- $LOGIN"
12IGCM_TMP=$SCRATCHDIR/TMP_PACK
13IGCM_DEM=$SCRATCHDIR/SAVE_PACK/${LOGIN}
14mkdir -p ${IGCM_TMP}
15rm -rf  ${IGCM_TMP}/*
16
17mkdir -p $SCRATCHDIR/SAVE_PACK
18mkdir -p ${IGCM_DEM}
19rm -rf  ${IGCM_DEM}/*
20
21cd ${STORELOG}
22if [ -s ${LOGIN}.Listing.txt.gz ]; then
23    gunzip ${LOGIN}.Listing.txt.gz
24    LISTE_DMNFS=${LOGIN}.Listing.txt
25
26else
27    tar xf ${LOGIN}.ipsl_logs.tgz ${LOGIN}/IGCM_DEM/Listing.txt
28    mv ${LOGIN}/IGCM_DEM/Listing.txt  ${LOGIN}.Listing.txt
29    rm -rf ${LOGIN}/
30    LISTE_DMNFS=${LOGIN}.Listing.txt
31fi
32
33if [ -s $LISTE_DMNFS ]; then
34
35listRestart=`grep "IGCM_OUT/" ${LISTE_DMNFS} | grep Restart/ | awk '{print $3}' | sed 's%/Restart.*%%' `
36if [[ ${listRestart} != "" ]] ; then
37    echo "${listRestart}" >> ${IGCM_TMP}/${$}.liste_Restart.txt
38fi
39
40while [[ -s ${IGCM_TMP}/${$}.liste_Restart.txt ]]
41do
42DIR=$( awk '{if (NR==1) print $0}' ${IGCM_TMP}/${$}.liste_Restart.txt )
43
44#gestion des cas particulier tels que la creation de l'etat initial avec lmdz (JobName/ATM/Output/Restart/ au lieu de JobName/ATM/Restart/)
45    PATH_SIM=$( dirname ${DIR} )
46    echo ${DIR}
47    lastdir=$( echo $PATH_SIM | sed -e "s%.*/%%")
48    if [ $lastdir != "ATM" ]; then
49        dirname ${DIR} >> ${IGCM_TMP}/${$}.path_simul.txt
50        grep -vw ${PATH_SIM} ${IGCM_TMP}/${$}.liste_Restart.txt > ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ; mv ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ${IGCM_TMP}/${$}.liste_Restart.txt
51    else
52        dirname ${PATH_SIM} >> ${IGCM_TMP}/${$}.path_simul.txt
53        PATH_SIM_TMP=$( dirname ${PATH_SIM} )
54        grep -vw ${PATH_SIM_TMP} ${IGCM_TMP}/${$}.liste_Restart.txt > ${IGCM_TMP}/${$}.liste_Restart.txt.tmp; mv ${IGCM_TMP}/${$}.liste_Restart.txt.tmp ${IGCM_TMP}/${$}.liste_Restart.txt
55
56    fi
57
58
59
60done
61echo "----------------------------------------------fin listerestart"
62
63if [[ -s ${IGCM_TMP}/${$}.path_simul.txt ]] ; then
64# On retire les doublons pour avoir la liste des simulations a tester
65awk '{if (x[$0] != "") next ; print $0 ; x[$0]=$0}' <${IGCM_TMP}/${$}.path_simul.txt  >> ${IGCM_TMP}/${$}.liste_simul
66awk '{if (x[$0] != "") next ; print $0 ; x[$0]=$0}' <${IGCM_TMP}/${$}.liste_simul  >> ${IGCM_TMP}/${$}.liste_simul.tmp${$}
67mv ${IGCM_TMP}/${$}.liste_simul.tmp${$} ${IGCM_DEM}/liste_simul
68
69
70
71for PATH_SIMUL_FULL in `awk  '{print $0}' ${IGCM_DEM}/liste_simul` ; do
72    rm -f ${IGCM_TMP}/*
73    echo "PATH_SIMUL_FULL = " $PATH_SIMUL_FULL
74
75    # Etablir la liste des fichiers de restart
76    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Restart | awk '{print $2 " " $3}' | grep -e '.*/Restart/.*\..*' |grep -v "jnl" | sort  >> ${IGCM_TMP}/${$}.liste_restart_files_config.txt
77
78
79    # Etablir la liste des fichiers output
80    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Output  | awk '{print $2 " "  $3}' | grep -e '.*/Output/.*\.nc$'  | sort >> ${IGCM_TMP}/${$}.liste_output_files_config.txt
81
82    # Lecture du nom du job
83    JobName=`awk -F/ '{if (NR==1) print $(NF-3)}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt`
84
85    if [ $JobName = "ATM" ]; then
86        JobName=`awk -F/ '{if (NR==1) print $(NF-4)}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt`
87    fi
88
89    echo "JobName = "$JobName
90
91    sed -e "s%.*${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt
92    cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" >  ${IGCM_TMP}/${$}.info_tmp_bis.txt
93    join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt > ${IGCM_TMP}/${$}.output.tmp
94    sort -k 2 ${IGCM_TMP}/${$}.output.tmp > ${IGCM_TMP}/${$}.output_bis.tmp
95    mv ${IGCM_TMP}/${$}.output_bis.tmp ${IGCM_TMP}/${$}.liste_output_files_config.txt
96
97
98    rm -f ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt ${IGCM_TMP}/${$}.output.tmp ${IGCM_TMP}/${$}.output_bis.tmp
99## maintenant on va extraire de cette liste les fichiers qui sont du type *${JobName} et non pas ${JobName}
100    sed -e "s%.*/${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt
101    cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" >  ${IGCM_TMP}/${$}.info_tmp_bis.txt
102
103## la liste que l'on cherche est  ${IGCM_TMP}/${$}.info_tmp_bis.txt
104    if [ -s ${IGCM_TMP}/${$}.info_tmp_bis.txt ]; then
105        mv ${IGCM_TMP}/${$}.info_tmp_bis.txt ${IGCM_DEM}/${JobName}.info_bug_pack_output.txt
106    fi
107done
108
109fi
110fi
111done
Note: See TracBrowser for help on using the repository browser.