source: TOOLS/PACK_IPSL/create_config_card.sh @ 1849

Last change on this file since 1849 was 1849, checked in by acosce, 12 years ago

Add parallel version - global double check - minor bug

File size: 11.1 KB
Line 
1#!/bin/bash
2
3##
4## Job permettant de recreer une partie du config.card d'une simulation
5## Champs vises : JobName / DateBegin / DateEnd / Calendar
6##
7
8# Prend en argument un fichier contenant une liste de simulations
9# cree pour chacune son config.card
10# Ex : ./create_config_card.job liste_simul_fichier_param_test1.txt liste_fichier_dmnfs
11# Ex : ./create_config_card.job liste_simul_param_SL.txt /cscratch/cont003/labetoul/FROM_CCRT_INFORMATIONS_DMNFS/information_dmnfs_2012-03-21
12
13
14LISTE_SIMUL=${1}
15LISTE_DMNFS=${2}
16
17# ----
18export JOB_DIR=${JOB_DIR:-${PWD}}
19export IGCM_DEM=${IGCM_DEM:-${SCRATCHDIR}/IGCM_DEM}
20export IGCM_TMP=${IGCM_TMP:-${IGCM_DEM}/tmp}
21mkdir -p ${IGCM_DEM}
22mkdir -p ${IGCM_TMP}
23clean_tmp=${clean_tmp:-yes}
24EXE_DIR=$(dirname ${0})
25SCRIPT_NAME=$(basename ${0} )
26source ${EXE_DIR}/DEM_utilities.sh
27
28# gpdebug : verif si le present script a ete execute jusqu'au bout #########
29MonitoringInfo="${SCRIPT_NAME}-->OK"
30generalMonitoring "$MonitoringInfo"
31
32DEM_log -0 "Demarrage de ${SCRIPT_NAME}"
33
34# gpdebug : on augmente la liste des proc à détruire en cas d'erreur bloquante du présent proc
35export listPID="$$ ${listPID}" #########
36
37cd ${JOB_DIR}
38# ----
39
40DEM_log -0 "LISTE_SIMUL : ${LISTE_SIMUL}"
41DEM_log -0 "LISTE_DMNFS : ${LISTE_DMNFS}"
42
43[[ -f ${IGCM_DEM}/config_card.liste ]] && rm ${IGCM_DEM}/config_card.liste
44
45#ROOT_DIR=/dmnfs05/cont003/labetoul # soit DMFDIR, soit le tampon. a ameliorer ...
46
47
48cd ${JOB_DIR}
49
50for PATH_SIMUL_FULL in `awk  '{print $0}' ${LISTE_SIMUL}` ; do
51
52    DEM_log -3 PATH_SIMUL_FULL : ${PATH_SIMUL_FULL}
53    #PATH_SIMUL_SHORT=${PATH_SIMUL_FULL##${ROOT_DIR}} # Enleve le debut du path
54#    PATH_SIMUL_SHORT=$(echo ${PATH_SIMUL_FULL} | sed 's%/[^/]*/[^/]*/[^/]*/%%' )
55#    PATH_SIMUL_SHORT=$(echo ${PATH_SIMUL_FULL} | sed 's%/ccc/dmfbuf/import_data.2/ccrt/dmnfs12/cont003/bacasable/%%' )
56    echo "gpdebug : PATH_SIMUL_FULL :" # gpdebug : a virer
57    echo "$PATH_SIMUL_FULL :" # gpdebug : a virer
58    PATH_SIMUL_SHORT=$(echo ${PATH_SIMUL_FULL} | sed "s%${SCRATCHDIR}/%%" ) # gpdebug : added
59    echo "gpdebug : PATH_SIMUL_SHORT :" # gpdebug : a virer
60    echo "$PATH_SIMUL_SHORT :" # gpdebug : a virer 
61    DEM_log -3 PATH_SIMUL_SHORT : ${PATH_SIMUL_SHORT}
62   
63    IGCM_DEM_SIMU=${IGCM_DEM}/${PATH_SIMUL_SHORT}
64    mkdir -p ${IGCM_DEM_SIMU}
65    rm -rf ${IGCM_DEM_SIMU}/*
66
67    #creation du repertoire permettant de stocker
68    #les listes de fichiers qui ne sont pas au format output ou restart
69    #les listes de fichiers d'analyse, les monitos et les atlas
70    OTHER_TAR=${IGCM_DEM_SIMU}/other_tar
71    STORE_CP=${IGCM_DEM_SIMU}/store_cp
72    WORK_CP=${IGCM_DEM_SIMU}/work_cp
73
74    DEM_log -2 "creation des repertoires : $OTHER_TAR  - $STORE_CP - $WORK_CP"
75
76    mkdir -p ${OTHER_TAR}
77    mkdir -p ${STORE_CP}
78    mkdir -p ${WORK_CP}
79
80    # on recherche le nom du job
81    JobName=""
82   
83    # Etablir la liste des fichiers d'Analyse
84    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Analyse  | awk '{print $3}' |  sort >> ${IGCM_TMP}/${$}.cp_analyse_files.list
85    awk -F/ '{if ($(NF)=="Analyse") print $0} ' ${IGCM_TMP}/${$}.cp_analyse_files.list > ${STORE_CP}/cp_files.list
86    rm -f ${IGCM_TMP}/${$}.cp_analyse_files.list
87
88    #On recupere egalement le fichier mesh_mask si il existe
89    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep mesh_mask | awk '{print $3}' >> ${STORE_CP}/cp_files.list
90    sed '/mesh_mask/d' ${LISTE_DMNFS} > ${IGCM_TMP}/${$}.Listing_tmp.txt; mv ${IGCM_TMP}/${$}.Listing_tmp.txt ${LISTE_DMNFS}
91
92    #On copie aussi le contenu du repertoire Exe/
93    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep -w Exe | awk '{print $3}' >> ${IGCM_TMP}/${$}.cp_analyse_files.list
94    awk -F/ '{if ($(NF)=="Exe") print $0} ' ${IGCM_TMP}/${$}.cp_analyse_files.list >> ${STORE_CP}/cp_files.list
95    rm -f ${IGCM_TMP}/${$}.cp_analyse_files.list
96   
97
98    # Etablir la liste des fichiers Monito et Atlas
99    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep  MONITORING | awk '{print $3}' | sort >> ${IGCM_TMP}/${$}.cp_monito_files.list
100    awk -F/ '{if ($(NF)=="MONITORING") print $0} ' ${IGCM_TMP}/${$}.cp_monito_files.list > ${WORK_CP}/cp_files.list
101    rm -f ${IGCM_TMP}/${$}.cp_monito_files.list
102
103    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep  ATLAS | awk '{print $3}' | sort >> ${IGCM_TMP}/${$}.cp_atlas_files.list
104    awk -F/ '{if ($(NF)=="ATLAS") print $0} ' ${IGCM_TMP}/${$}.cp_atlas_files.list >> ${WORK_CP}/cp_files.list
105    rm -f ${IGCM_TMP}/${$}.cp_atlas_files.list
106
107    # Etablir la liste des fichiers de restart
108    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Restart | awk '{print $2 " " $3}' | grep -e '.*/Restart/.*\..*' |grep -v "jnl" | sort  >> ${IGCM_TMP}/${$}.liste_restart_files_config.txt
109
110    # Retirons de la liste les fichiers que l'on n'attends pas, en particulier si il  y a des sous-repertoires au repetoire restart/
111    for file in `awk  '{print $2}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt ` ; do
112        dir=$( echo $file | sed -e "s%.*Restart/%%") 
113        testdir=$(dirname ${dir}) 
114        if [ $testdir != "." ]; then
115            echo $( grep ${file} ${IGCM_TMP}/${$}.liste_restart_files_config.txt)  >> ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt
116        fi
117    done
118    if [ -f ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt ]; then
119        join -t % -v 1 ${IGCM_TMP}/${$}.liste_restart_files_config.txt ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt
120        mv ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.liste_restart_files_config.txt
121    fi
122   
123    # Etablir la liste des fichiers output
124    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Output  | awk '{print $2 " "  $3}' | grep -e '.*/Output/.*\.nc$'  | sort >> ${IGCM_TMP}/${$}.liste_output_files_config.txt
125    # Retirons de la liste les fichiers que l'on n'attends pas, en particulier si il  y a des sous-repertoires aux repetoires Output/*/
126    for file in `awk  '{print $2}'  ${IGCM_TMP}/${$}.liste_output_files_config.txt ` ; do
127        dir=$( echo $file | sed -e "s%.*Output/[^/]*/%%") 
128        testdir=$(dirname ${dir}) 
129        if [ $testdir != "." ]; then
130            echo $( grep ${file} ${IGCM_TMP}/${$}.liste_output_files_config.txt ) >> ${IGCM_TMP}/${$}.liste_no_output_files_config.txt
131        fi
132    done
133    if [ -f ${IGCM_TMP}/${$}.liste_no_output_files_config.txt ]; then
134        join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.liste_no_output_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt
135        mv ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.liste_output_files_config.txt
136    fi
137
138
139
140
141    # Lecture du nom du job
142    JobName=`awk -F/ '{if (NR==1) print $(NF-3)}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt`
143
144    if [ $JobName = "ATM" ]; then
145        JobName=`awk -F/ '{if (NR==1) print $(NF-4)}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt`
146    fi
147    DEM_log -2 JobName ${JobName}
148
149
150# Maintenant que nous avons JobName verifions que tous les fichiers de restart/Output sont bien du type JobName_date_*
151    sed -e "s%.*${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_restart_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt
152    cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" >  ${IGCM_TMP}/${$}.info_tmp_bis.txt
153    join -t % -v 1 ${IGCM_TMP}/${$}.liste_restart_files_config.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt > ${IGCM_TMP}/${$}.restart.tmp
154    sort -k 2 ${IGCM_TMP}/${$}.restart.tmp > ${IGCM_TMP}/${$}.restart_bis.tmp
155    mv ${IGCM_TMP}/${$}.restart_bis.tmp ${IGCM_DEM_SIMU}/liste_restart_files_config.txt
156# Sauvegarde des fichiers qui ne sont pas dans la norme
157    cat ${IGCM_TMP}/${$}.info_tmp_bis.txt >> ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt
158    mv ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt ${OTHER_TAR}/tar_no_restart_files.list
159
160
161    sed -e "s%.*${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt
162    cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" >  ${IGCM_TMP}/${$}.info_tmp_bis.txt
163    join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt > ${IGCM_TMP}/${$}.output.tmp
164    sort -k 2 ${IGCM_TMP}/${$}.output.tmp > ${IGCM_TMP}/${$}.output_bis.tmp
165    mv ${IGCM_TMP}/${$}.output_bis.tmp ${IGCM_DEM_SIMU}/liste_output_files_config.txt
166# Sauvegarde des fichiers qui ne sont pas dans la norme
167    cat ${IGCM_TMP}/${$}.info_tmp_bis.txt >> ${IGCM_TMP}/${$}.liste_no_output_files_config.txt
168    mv ${IGCM_TMP}/${$}.liste_no_output_files_config.txt ${OTHER_TAR}/tar_no_output_files.list
169
170
171# Recherche de DateBegin et DateEnd pour les Restarts
172    first_file=$(        head -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
173    extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_//" )
174    DateBegin_Restart=$( echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
175   
176    last_file=$(         tail -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
177    extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" )
178    DateEnd_Restart=$(   echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
179   
180# Recherche de DateBegin et DateEnd pour les Output
181    first_file=$( head -n 1 ${IGCM_DEM_SIMU}/liste_output_files_config.txt )
182    extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_//" )
183    DateBegin_Output=$(  echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
184   
185    last_file=$(         tail -n 1 ${IGCM_DEM_SIMU}/liste_output_files_config.txt )
186    extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" )
187    DateEnd_Output=$(    echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
188   
189    DateBegin=${DateBegin_Restart}
190    DateEnd=${DateEnd_Restart}
191
192    DEM_log -2 "DateBegin DateEnd : ${DateBegin} ${DateEnd}"
193    DEM_log -2 "DateBegin_Output DateEnd_Output : ${DateBegin_Output} ${DateEnd_Output}"
194    DEM_log -2 "DateBegin_Restart DateEnd_Restart : ${DateBegin_Restart} ${DateEnd_Restart}"
195
196    DateBegin=$( DEM_min ${DateBegin_Output} ${DateBegin} ) 
197    DateEnd=$(   DEM_max ${DateEnd_Output}   ${DateEnd}   )
198   
199    DEM_log -2  "DateBegin DateEnd : ${DateBegin} ${DateEnd}"
200   
201    [[ "${clean_tmp}" = "yes" ]] && rm -f ${IGCM_TMP}/${$}.config_name ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.liste_restart_files_config.txt
202   
203    rm -f ${IGCM_DEM_SIMU}/config_card_${JobName}
204    cat<<EOD>${IGCM_DEM_SIMU}/config_card_${JobName}
205JobName=${JobName}
206DateBegin=${DateBegin}
207DateEnd=${DateEnd}
208PATH_SIMUL_FULL=${PATH_SIMUL_FULL}
209IGCM_DEM_SIMU=${IGCM_DEM_SIMU}
210EOD
211   
212    echo "${IGCM_DEM_SIMU}/config_card_${JobName} ListToBeDone" >> ${IGCM_DEM}/config_card.liste
213
214    DEM_log -1 "${IGCM_DEM_SIMU}/config_card_${JobName} ListToBeDone >> ${IGCM_DEM}/config_card.liste"
215
216
217#si la liste des fichiers d'output est vide on tar la simul
218if [[ !  -s ${IGCM_DEM_SIMU}/liste_output_files_config.txt ]]; then
219DEM_log -2 "liste output vide"
220    ./write_liste_tar.sh ${IGCM_DEM_SIMU}/config_card_${JobName}
221fi
222
223# si les listes no_output et no_restart sont vides on les efface
224if [[ !  -s ${OTHER_TAR}/tar_no_output_files.list ]]; then
225    rm -f ${OTHER_TAR}/tar_no_output_files.list
226fi
227
228if [[ !  -s ${OTHER_TAR}/tar_no_restart_files.list ]]; then
229    rm -f ${OTHER_TAR}/tar_no_restart_files.list
230fi
231
232
233done
234
235# gpdebug : le script n'a pas rencontre d'erreur. On l'inscrit dans le fichier de suivi
236echo $MonitoringInfo >> $generalMonitorFile
237
238DEM_log -0 "Fin de ${SCRIPT_NAME}"
239
Note: See TracBrowser for help on using the repository browser.