source: TOOLS/PACK_IPSL/write_liste_pack.sh @ 1906

Last change on this file since 1906 was 1900, checked in by aclsce, 12 years ago

Modified to write log files in TMP_MIGR_DATA directory (instead of current directory)
File size: 10.4 KB
Line 
1#!/bin/bash
2
3
4# Script permettant de creer les listes de fichiers pour ncrcat
5# si un trou est detecte dans la simulation alors cette liste sera
6# pour un tar
7
8# $1= PATH/config_card d'une simulation avec
9# TagName/JobName/ExperimentName/SpaceName/DateBegin/DateEnd/PATH_SIMUL
10
11
12# Pour commencer on travaille avec le fichier information_dmnfs
13# /cscratch/cont003/p86mart/FROM_CCRT_INFORMATIONS_DMNFS/information_dmnfs_2011-09-21
14
15config_card=${1}
16LISTE_DMNFS=${2}
17
18# Declaration des variables
19
20# ----
21export JOB_DIR=${JOB_DIR:-${TMP_MIGR_DATA}}
22export IGCM_DEM=${IGCM_DEM:-${SCRATCHDIR}/IGCM_DEM}
23export IGCM_TMP=${IGCM_TMP:-${IGCM_DEM}/tmp}
24mkdir -p ${IGCM_DEM}
25mkdir -p ${IGCM_TMP}
26clean_tmp=${clean_tmp:-yes}
27EXE_DIR=$(dirname ${0} )
28SCRIPT_NAME=$(basename ${0} )
29source ${EXE_DIR}/DEM_utilities.sh
30
31# gpdebug : verif si le present script a ete execute jusqu'au bout #########
32MonitoringInfo="${SCRIPT_NAME}-->OK"
33simuMonitoring_check "$MonitoringInfo" "$config_card"
34
35# gpdebug : Execute la function errorReceive() si reception d'un signal TERM
36trap 'DEM_errorReceive' TERM ########
37
38# gpdebug : on augmente la liste des proc à détruire en cas d'erreur bloquante du présent proc
39export listPID="$$ ${listPID}" #########
40
41DEM_log -0 "Demmarrage de ${SCRIPT_NAME}"
42cd ${JOB_DIR}
43# ----
44
45DEM_log -0 "config_card : ${config_card}"
46DEM_log -0 "LISTE_DMNFS : ${LISTE_DMNFS}"
47
48eval $(grep JobName         ${config_card}) 
49eval $(grep PATH_SIMUL_FULL ${config_card}) 
50eval $(grep DateBegin       ${config_card}) 
51eval $(grep DateEnd         ${config_card}) 
52eval $(grep IGCM_DEM_SIMU   ${config_card})
53
54SIMUL=${PATH_SIMUL_FULL}/${JobName}
55
56
57DEM_state=$( DEM_read_state ${IGCM_DEM}/config_card.liste ${config_card} )
58# gpdebug : si c'est une premiere exec (pas une reprise)
59if [[ "x${execEveryStep}" == "1" ]] && [[ "${DEM_state}" != "FindSizePackDone" ]] ; then
60    DEM_errorSend "${SCRIPT_NAME}:${LINENO}:Etat incorrect dans ${IGCM_DEM}/config_card.liste"
61fi
62
63
64DEM_write_state ${IGCM_DEM}/config_card.liste ${config_card} WriteListPackRunning
65
66# creation des repertoires with_tar et with_ncrcat
67OUTPUT_NCRCAT=${IGCM_DEM_SIMU}/output_ncrcat
68OUTPUT_TAR=${IGCM_DEM_SIMU}/output_tar
69
70mkdir -p ${OUTPUT_NCRCAT}
71mkdir -p ${OUTPUT_TAR}
72
73# lecture de periodpack dans ${IGCM_DEM_SIMU}/period_pack.txt
74period_pack=$( awk '{print $0} ' ${IGCM_DEM_SIMU}/period_pack.txt )
75
76year_begin=$(  echo ${DateBegin} | cut -c 1-4 )
77month_begin=$( echo ${DateBegin} | cut -c 5-6 )
78year_end=$(    echo ${DateEnd}   | cut -c 1-4 )
79month_end=$(   echo ${DateEnd}   | cut -c 5-6 )
80
81awk '{print $2}' ${IGCM_DEM_SIMU}/liste_output_files_config.txt >  ${IGCM_DEM_SIMU}/info_Output_light.txt
82
83# on cree la liste des fichiers pour chaque type_file
84for type_file in $( awk '{print $0}' ${IGCM_DEM_SIMU}/info_name_file.txt ) ; do 
85   #sed -e "/${type_file}/!d" ${IGCM_DEM_SIMU}/info_Output_light.txt > ${IGCM_TMP}/${$}.info_${type_file}.txt
86    grep ${type_file} ${IGCM_DEM_SIMU}/info_Output_light.txt > ${IGCM_TMP}/${$}.info_${type_file}.txt
87    sort -u ${IGCM_TMP}/${$}.info_${type_file}.txt > ${IGCM_TMP}/${$}.tmp_info_${type_file}.txt.tmp
88    cp ${IGCM_TMP}/${$}.tmp_info_${type_file}.txt.tmp ${IGCM_TMP}/${$}.info_${type_file}.txt
89    [[ "${clean_tmp}" = "yes" ]] && rm -f ${IGCM_TMP}/${$}.tmp_info_${type_file}.txt.tmp
90done
91
92
93# 3- Maintenant pour chaque type de fichier de sortie nous allons creer la liste
94# des fichiers contenus dans chaque pack
95
96# Recherche des dates du premier pack
97
98date_begin_pack=${DateBegin}
99date_end_simulation=${DateEnd}
100
101month_begin_pack=${month_begin}
102year_begin_pack=${year_begin}
103
104number_pack=1
105
106#
107while [[ ${date_begin_pack} -le ${date_end_simulation}  ]]
108do
109
110   month_end_pack=${month_begin_pack}
111   year_end_pack=$(( ${year_begin_pack} +${period_pack} ))
112   
113   date_end_pack=${year_end_pack}${month_end_pack}01   
114
115   DEM_log -2 "Entree boucle 3 : year_begin_pack month_begin_pack year_end_pack month_end_pack : ${year_begin_pack} ${month_begin_pack} ${year_end_pack} ${month_end_pack}"
116   DEM_log -2 "Entree boucle 3 : date_end_pack : ${date_end_pack}"
117   
118   for type_file in `awk '{print $0}' ${IGCM_DEM_SIMU}/info_name_file.txt`
119   do
120       DEM_log -2 "Boucle 3 : ${type_file} ${number_pack}"
121       taille=0
122
123       # On passe en revue chaque type de fichier
124       for file in `awk '{print $0}' ${IGCM_TMP}/${$}.info_${type_file}.txt`
125       do 
126           # Extraire la date de debut et la date de fin de la periode d'output traitee par le fichier   
127           extract_date_file=$( echo ${file}             | sed -e "s%.*${JobName}_%%" )
128           date_debut=$(        echo ${extract_date_file}| sed 's%\([0-9]\{8\}\)_.*$%\1%g' )
129           date_fin=$(          echo ${extract_date_file}| sed 's%.*_\([0-9]\{8\}\)_.*$%\1%g' )
130           
131           if [  ${date_debut} -lt ${date_end_pack} ]
132           then
133               if [ ${date_fin} -ge ${date_begin_pack} ]
134               then
135                   echo ${file} >> ${OUTPUT_NCRCAT}/ncrcat_${type_file}_${date_begin_pack}_${date_end_pack}.list
136                   echo ${date_debut} | cut -c 1-6 >>  ${IGCM_TMP}/${$}.date_${type_file}.txt
137                   grep -v ${file} ${IGCM_TMP}/${$}.info_${type_file}.txt > ${IGCM_TMP}/${$}.info_${type_file}.txt.tmp
138                   cp ${IGCM_TMP}/${$}.info_${type_file}.txt.tmp ${IGCM_TMP}/${$}.info_${type_file}.txt
139                   [[ "${clean_tmp}" = "yes" ]] && rm -f ${IGCM_TMP}/${$}.info_${type_file}.txt.tmp
140               fi
141           else
142               break
143           fi
144           
145       done
146       if [[ -f ${IGCM_TMP}/${$}.date_${type_file}.txt ]] ; then
147           sort -u ${IGCM_TMP}/${$}.date_${type_file}.txt > ${IGCM_TMP}/${$}.date_tmp.txt
148           mv ${IGCM_TMP}/${$}.date_tmp.txt ${IGCM_TMP}/${$}.date_${type_file}.txt
149       else
150           DEM_log -2 "Pas de fichier cree :  ${IGCM_TMP}/${$}.date_${type_file}.txt - Date_end_pack : ${date_end_pack}"
151       fi
152
153#On renomme le fichier ncrcat avec la date reelle de fin et non pas la premiere date suivant la fin
154       sort ${OUTPUT_NCRCAT}/ncrcat_${type_file}_${date_begin_pack}_${date_end_pack}.list > ${IGCM_TMP}/${$}.ncrcat.tmp
155       last_file=$(tail -n 1 ${IGCM_TMP}/${$}.ncrcat.tmp )
156       extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" )
157       date_fin=$(echo ${extract_date_file}| sed 's%.*_\([0-9]\{8\}\)_.*$%\1%g' )
158       mv ${OUTPUT_NCRCAT}/ncrcat_${type_file}_${date_begin_pack}_${date_end_pack}.list ${OUTPUT_NCRCAT}/${JobName}_${date_begin_pack}_${date_fin}_${type_file}.list
159   done
160   
161   
162   month_begin_pack=${month_end_pack}
163   year_begin_pack=${year_end_pack}
164   date_begin_pack=${year_begin_pack}${month_begin_pack}01
165   
166   number_pack=$(( ${number_pack} + 1 ))
167
168   DEM_log -2 "Sortie boucle : year_begin_pack month_begin_pack year_end_pack month_end_pack : ${year_begin_pack} ${month_begin_pack} ${year_end_pack} ${month_end_pack}"
169   DEM_log -2 "Sortie boucle : date_begin_pack : ${date_begin_pack}"
170   
171done
172
173
174## Recherche des trous eventuels dans cette simulation
175
176## Creation de la liste des annees comprisent entre DateBegin et DateEnd >>  ${IGCM_TMP}/${$}.liste_date.txt
177force_sequence (){
178    awk '{
179        if (NR==1) {avant=$1; print $1}
180        else { for (i=avant+1 ; i<=$1 ; i++) printf "%04i\n", i}}'
181}
182
183DEM_log -1 "year_begin - year_end : ${year_begin} - ${year_end}"
184
185echo ${year_begin} > ${IGCM_TMP}/${$}.liste_date.txt
186echo ${year_end}  >> ${IGCM_TMP}/${$}.liste_date.txt
187
188force_sequence < ${IGCM_TMP}/${$}.liste_date.txt > ${IGCM_TMP}/${$}.tmp ; mv ${IGCM_TMP}/${$}.tmp ${IGCM_TMP}/${$}.liste_date.txt
189
190
191## il faut savoir si les fichiers d'output sont annuels ou mensuels ?
192file=$( head -n 2 ${IGCM_DEM_SIMU}/liste_output_files_config.txt )
193
194extract_date_file=$( echo ${file}             | sed -e "s%.*${JobName}_%%" )
195date_debut=$(        echo ${extract_date_file}| sed 's%\([0-9]\{8\}\)_.*$%\1%g')
196date_fin=$(          echo ${extract_date_file}| sed 's%.*_\([0-9]\{8\}\)_.*$%\1%g')
197month_begin=$(       echo ${date_debut}       | cut -c 5-6 )
198month_end=$(         echo ${date_fin}         | cut -c 5-6 )
199
200DEM_log -1 "Recherche de la frequence"
201if [ ${month_end} -eq ${month_begin} ]
202then
203    DEM_log -1 "Mensuel"
204    frequency="mensuel"
205    while read an ; do
206        for month in 1 2 3 4 5 6 7 8 9 10 11 12 ; do
207            printf "%s%02d\n" ${an} ${month}  >> ${IGCM_TMP}/${$}.toutes_dates
208        done
209    done < ${IGCM_TMP}/${$}.liste_date.txt
210    [[ "${clean_tmp}" = "yes" ]] && rm -f ${IGCM_TMP}/${$}.liste_date.txt
211fi
212if [[ ${month_end} -eq "12" && ${month_begin} -eq "01" ]]
213then
214    DEM_log -1 "Annuel"
215    frequency="annuel"
216    while read an ; do
217        i=1
218        printf "%s%02d\n" ${an} ${i}  >> ${IGCM_TMP}/${$}.toutes_dates
219    done < ${IGCM_TMP}/${$}.liste_date.txt   
220fi
221
222
223# Maintenant il faut comparer la liste des dates avec la liste des dates des fichiers
224# et pour des dates manquantes transformer le fichier ncrcat_date correspondant en fichier
225# tar_date
226DEM_log -1 "Recherche des dates manquantes"
227for type_file in `awk '{print $0}' ${IGCM_DEM_SIMU}/info_name_file.txt`
228do
229    DEM_log -2 "${type_file}"
230    rm -f ${IGCM_DEM_SIMU}/${type_file}_manquant.list
231   
232    if [ -f ${IGCM_TMP}/${$}.date_${type_file}.txt ] ; then
233        join -v 1  ${IGCM_TMP}/${$}.toutes_dates ${IGCM_TMP}/${$}.date_${type_file}.txt > ${IGCM_DEM_SIMU}/${type_file}_manquant.list
234       
235        # On change le fichier ncrcat pour chaque date manquante en fichier tar
236        if [[ -f ${IGCM_DEM_SIMU}/${type_file}_manquant.list && -s ${IGCM_DEM_SIMU}/${type_file}_manquant.list ]]; then
237            for date in `awk '{print $0}' ${IGCM_DEM_SIMU}/${type_file}_manquant.list` ; do
238                ls -l ${OUTPUT_NCRCAT}/${JobName}_*_${type_file}.list > ${IGCM_TMP}/${$}.liste_ncrcat
239                for pack in $( awk '{print $9}' ${IGCM_TMP}/${$}.liste_ncrcat ) ; do
240                    extract_date_file=$( echo ${pack}       | sed -e "s%.*${JobName}_%%" )
241                    date_debut=$( echo ${extract_date_file} | sed 's%\([0-9]\{8\}\)_.*$%\1%g'     | cut -c 1-6  )
242                    date_fin=$(   echo ${extract_date_file} | sed 's%.*_\([0-9]\{8\}\)_.*$%\1%g' | cut -c 1-6  )
243                    if [[ ${date} -ge ${date_debut} && ${date} -lt ${date_fin} ]]; then
244                        mv ${pack} ${OUTPUT_TAR}/.
245                        break
246                    else
247                        rm -f ${IGCM_DEM_SIMU}/${type_file}_manquant.list
248                    fi
249                   
250                done
251                [[ "${clean_tmp}" = "yes" ]] && rm -f ${IGCM_TMP}/${$}.liste_ncrcat
252            done
253        elif [[ -f ${IGCM_DEM_SIMU}/${type_file}_manquant.list && ! -s ${IGCM_DEM_SIMU}/${type_file}_manquant.list ]]; then
254            rm -f ${IGCM_DEM_SIMU}/${type_file}_manquant.list
255        fi
256    fi
257done
258
259DEM_write_state ${IGCM_DEM}/config_card.liste ${config_card} WriteListPackDone
260
261# gpdebug : le script n'a pas rencontre d'erreur. On l'inscrit dans le fichier de suivi
262simuMonitoring_OK "$MonitoringInfo" "$config_card"
263
264DEM_log -0 "Fin de ${SCRIPT_NAME}"
265
266exit
Note: See TracBrowser for help on using the repository browser.