source: TOOLS/PACK_IPSL/create_config_card.sh @ 1839

Last change on this file since 1839 was 1839, checked in by acosce, 12 years ago

BugFix? when we try to find variables to exclude of ncrcat.
For example before "Point Omega" was find as a integer variable

File size: 10.5 KB
Line 
1#!/bin/bash
2
3##
4## Job permettant de recreer une partie du config.card d'une simulation
5## Champs vises : JobName / DateBegin / DateEnd / Calendar
6##
7
8# Prend en argument un fichier contenant une liste de simulations
9# cree pour chacune son config.card
10# Ex : ./create_config_card.job liste_simul_fichier_param_test1.txt liste_fichier_dmnfs
11# Ex : ./create_config_card.job liste_simul_param_SL.txt /cscratch/cont003/labetoul/FROM_CCRT_INFORMATIONS_DMNFS/information_dmnfs_2012-03-21
12
13
14LISTE_SIMUL=${1}
15LISTE_DMNFS=${2}
16
17# ----
18export JOB_DIR=${JOB_DIR:-${PWD}}
19export IGCM_DEM=${IGCM_DEM:-${SCRATCHDIR}/IGCM_DEM}
20export IGCM_TMP=${IGCM_TMP:-${IGCM_DEM}/tmp}
21mkdir -p ${IGCM_DEM}
22mkdir -p ${IGCM_TMP}
23clean_tmp=${clean_tmp:-yes}
24EXE_DIR=$(dirname ${0})
25SCRIPT_NAME=$(basename ${0} )
26source ${EXE_DIR}/DEM_utilities.sh
27
28# gpdebug : verif si le present script a ete execute jusqu'au bout #########
29MonitoringInfo="${SCRIPT_NAME}-->OK"
30generalMonitoring "$MonitoringInfo"
31
32DEM_log -0 "Demarrage de ${SCRIPT_NAME}"
33
34# gpdebug : on augmente la liste des proc à détruire en cas d'erreur bloquante du présent proc
35export listPID="$$ ${listPID}" #########
36
37cd ${JOB_DIR}
38# ----
39
40DEM_log -0 "LISTE_SIMUL : ${LISTE_SIMUL}"
41DEM_log -0 "LISTE_DMNFS : ${LISTE_DMNFS}"
42
43[[ -f ${IGCM_DEM}/config_card.liste ]] && rm ${IGCM_DEM}/config_card.liste
44
45#ROOT_DIR=/dmnfs05/cont003/labetoul # soit DMFDIR, soit le tampon. a ameliorer ...
46
47
48cd ${JOB_DIR}
49
50for PATH_SIMUL_FULL in `awk  '{print $0}' ${LISTE_SIMUL}` ; do
51
52    DEM_log -3 PATH_SIMUL_FULL : ${PATH_SIMUL_FULL}
53    #PATH_SIMUL_SHORT=${PATH_SIMUL_FULL##${ROOT_DIR}} # Enleve le debut du path
54    PATH_SIMUL_SHORT=$(echo ${PATH_SIMUL_FULL} | sed 's%/ccc/dmfbuf/import_data.2/ccrt/dmnfs12/cont003/bacasable/%%' )
55    DEM_log -3 PATH_SIMUL_SHORT : ${PATH_SIMUL_SHORT}
56   
57    IGCM_DEM_SIMU=${IGCM_DEM}/${PATH_SIMUL_SHORT}
58    mkdir -p ${IGCM_DEM_SIMU}
59    rm -rf ${IGCM_DEM_SIMU}/*
60
61    #creation du repertoire permettant de stocker
62    #les listes de fichiers qui ne sont pas au format output ou restart
63    #les listes de fichiers d'analyse, les monitos et les atlas
64    OTHER_TAR=${IGCM_DEM_SIMU}/other_tar
65    STORE_CP=${IGCM_DEM_SIMU}/store_cp
66    WORK_CP=${IGCM_DEM_SIMU}/work_cp
67
68    DEM_log -2 "creation des repertoires : $OTHER_TAR  - $STORE_CP - $WORK_CP"
69
70    mkdir -p ${OTHER_TAR}
71    mkdir -p ${STORE_CP}
72    mkdir -p ${WORK_CP}
73
74    # on recherche le nom du job
75    JobName=""
76   
77    # Etablir la liste des fichiers d'Analyse
78    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Analyse  | awk '{print $3}' |  sort >> ${IGCM_TMP}/${$}.cp_analyse_files.list
79    awk -F/ '{if ($(NF)=="Analyse") print $0} ' ${IGCM_TMP}/${$}.cp_analyse_files.list > ${STORE_CP}/cp_files.list
80    rm -f ${IGCM_TMP}/${$}.cp_analyse_files.list
81
82    #On recupere egalement le fichier mesh_mask si il existe
83    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep mesh_mask | awk '{print $3}' >> ${STORE_CP}/cp_files.list
84    sed '/mesh_mask/d' ${LISTE_DMNFS} > ${IGCM_TMP}/${$}.Listing_tmp.txt; mv ${IGCM_TMP}/${$}.Listing_tmp.txt ${LISTE_DMNFS}
85
86    #On copie aussi le contenu du repertoire Exe/
87    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep -w Exe | awk '{print $3}' >> ${IGCM_TMP}/${$}.cp_analyse_files.list
88    awk -F/ '{if ($(NF)=="Exe") print $0} ' ${IGCM_TMP}/${$}.cp_analyse_files.list >> ${STORE_CP}/cp_files.list
89    rm -f ${IGCM_TMP}/${$}.cp_analyse_files.list
90   
91
92    # Etablir la liste des fichiers Monito et Atlas
93    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep  MONITORING | awk '{print $3}' | sort >> ${IGCM_TMP}/${$}.cp_monito_files.list
94    awk -F/ '{if ($(NF)=="MONITORING") print $0} ' ${IGCM_TMP}/${$}.cp_monito_files.list > ${WORK_CP}/cp_files.list
95    rm -f ${IGCM_TMP}/${$}.cp_monito_files.list
96
97    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep  ATLAS | awk '{print $3}' | sort >> ${IGCM_TMP}/${$}.cp_atlas_files.list
98    awk -F/ '{if ($(NF)=="ATLAS") print $0} ' ${IGCM_TMP}/${$}.cp_atlas_files.list >> ${WORK_CP}/cp_files.list
99    rm -f ${IGCM_TMP}/${$}.cp_atlas_files.list
100
101    # Etablir la liste des fichiers de restart
102    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Restart | awk '{print $2 " " $3}' | grep -e '.*/Restart/.*\..*' |grep -v "jnl" | sort  >> ${IGCM_TMP}/${$}.liste_restart_files_config.txt
103
104    # Retirons de la liste les fichiers que l'on n'attends pas, en particulier si il  y a des sous-repertoires au repetoire restart/
105    for file in `awk  '{print $2}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt ` ; do
106        dir=$( echo $file | sed -e "s%.*Restart/%%") 
107        testdir=$(dirname ${dir}) 
108        if [ $testdir != "." ]; then
109            echo $( grep ${file} ${IGCM_TMP}/${$}.liste_restart_files_config.txt)  >> ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt
110        fi
111    done
112    if [ -f ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt ]; then
113        join -t % -v 1 ${IGCM_TMP}/${$}.liste_restart_files_config.txt ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt
114        mv ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.liste_restart_files_config.txt
115    fi
116   
117    # Etablir la liste des fichiers output
118    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Output  | awk '{print $2 " "  $3}' | grep -e '.*/Output/.*\.nc$'  | sort >> ${IGCM_TMP}/${$}.liste_output_files_config.txt
119    # Retirons de la liste les fichiers que l'on n'attends pas, en particulier si il  y a des sous-repertoires aux repetoires Output/*/
120    for file in `awk  '{print $2}'  ${IGCM_TMP}/${$}.liste_output_files_config.txt ` ; do
121        dir=$( echo $file | sed -e "s%.*Output/[^/]*/%%") 
122        testdir=$(dirname ${dir}) 
123        if [ $testdir != "." ]; then
124            echo $( grep ${file} ${IGCM_TMP}/${$}.liste_output_files_config.txt ) >> ${IGCM_TMP}/${$}.liste_no_output_files_config.txt
125        fi
126    done
127    if [ -f ${IGCM_TMP}/${$}.liste_no_output_files_config.txt ]; then
128        join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.liste_no_output_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt
129        mv ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.liste_output_files_config.txt
130    fi
131
132
133
134
135    # Lecture du nom du job
136    JobName=`awk -F/ '{if (NR==1) print $(NF-3)}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt`
137    DEM_log -2 JobName ${JobName}
138
139
140# Maintenant que nous avons JobName verifions que tous les fichiers de restart/Output sont bien du type JobName_date_*
141    sed -e "s%.*${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_restart_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt
142    cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" >  ${IGCM_TMP}/${$}.info_tmp_bis.txt
143    join -t % -v 1 ${IGCM_TMP}/${$}.liste_restart_files_config.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt > ${IGCM_TMP}/${$}.restart.tmp
144    sort -k 2 ${IGCM_TMP}/${$}.restart.tmp > ${IGCM_TMP}/${$}.restart_bis.tmp
145    mv ${IGCM_TMP}/${$}.restart_bis.tmp ${IGCM_DEM_SIMU}/liste_restart_files_config.txt
146# Sauvegarde des fichiers qui ne sont pas dans la norme
147    cat ${IGCM_TMP}/${$}.info_tmp_bis.txt >> ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt
148    mv ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt ${OTHER_TAR}/tar_no_restart_files.list
149
150
151    sed -e "s%.*${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt
152    cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" >  ${IGCM_TMP}/${$}.info_tmp_bis.txt
153    join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt > ${IGCM_TMP}/${$}.output.tmp
154    sort -k 2 ${IGCM_TMP}/${$}.output.tmp > ${IGCM_TMP}/${$}.output_bis.tmp
155    mv ${IGCM_TMP}/${$}.output_bis.tmp ${IGCM_DEM_SIMU}/liste_output_files_config.txt
156# Sauvegarde des fichiers qui ne sont pas dans la norme
157    cat ${IGCM_TMP}/${$}.info_tmp_bis.txt >> ${IGCM_TMP}/${$}.liste_no_output_files_config.txt
158    mv ${IGCM_TMP}/${$}.liste_no_output_files_config.txt ${OTHER_TAR}/tar_no_output_files.list
159
160
161# Recherche de DateBegin et DateEnd pour les Restarts
162    first_file=$(        head -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
163    extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_//" )
164    DateBegin_Restart=$( echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
165   
166    last_file=$(         tail -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
167    extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" )
168    DateEnd_Restart=$(   echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
169   
170# Recherche de DateBegin et DateEnd pour les Output
171    first_file=$( head -n 1 ${IGCM_DEM_SIMU}/liste_output_files_config.txt )
172    extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_//" )
173    DateBegin_Output=$(  echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
174   
175    last_file=$(         tail -n 1 ${IGCM_DEM_SIMU}/liste_output_files_config.txt )
176    extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" )
177    DateEnd_Output=$(    echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
178   
179    DateBegin=${DateBegin_Restart}
180    DateEnd=${DateEnd_Restart}
181
182    DEM_log -2 "DateBegin DateEnd : ${DateBegin} ${DateEnd}"
183    DEM_log -2 "DateBegin_Output DateEnd_Output : ${DateBegin_Output} ${DateEnd_Output}"
184    DEM_log -2 "DateBegin_Restart DateEnd_Restart : ${DateBegin_Restart} ${DateEnd_Restart}"
185
186    DateBegin=$( DEM_min ${DateBegin_Output} ${DateBegin} ) 
187    DateEnd=$(   DEM_max ${DateEnd_Output}   ${DateEnd}   )
188   
189    DEM_log -2  "DateBegin DateEnd : ${DateBegin} ${DateEnd}"
190   
191    [[ "${clean_tmp}" = "yes" ]] && rm -f ${IGCM_TMP}/${$}.config_name ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.liste_restart_files_config.txt
192   
193    rm -f ${IGCM_DEM_SIMU}/config_card_${JobName}
194    cat<<EOD>${IGCM_DEM_SIMU}/config_card_${JobName}
195JobName=${JobName}
196DateBegin=${DateBegin}
197DateEnd=${DateEnd}
198PATH_SIMUL_FULL=${PATH_SIMUL_FULL}
199IGCM_DEM_SIMU=${IGCM_DEM_SIMU}
200EOD
201   
202    echo "${IGCM_DEM_SIMU}/config_card_${JobName} ListToBeDone" >> ${IGCM_DEM}/config_card.liste
203
204    DEM_log -1 "${IGCM_DEM_SIMU}/config_card_${JobName} ListToBeDone >> ${IGCM_DEM}/config_card.liste"
205
206
207#si la liste des fichiers d'output est vide on tar la simul
208if [[ !  -s ${IGCM_DEM_SIMU}/liste_output_files_config.txt ]]; then
209DEM_log -2 "liste output vide"
210    ./write_liste_tar.sh ${IGCM_DEM_SIMU}/config_card_${JobName}
211fi
212
213# si les listes no_output et no_restart sont vides on les efface
214if [[ !  -s ${OTHER_TAR}/tar_no_output_files.list ]]; then
215    rm -f ${OTHER_TAR}/tar_no_output_files.list
216fi
217
218if [[ !  -s ${OTHER_TAR}/tar_no_restart_files.list ]]; then
219    rm -f ${OTHER_TAR}/tar_no_restart_files.list
220fi
221
222
223done
224
225# gpdebug : le script n'a pas rencontre d'erreur. On l'inscrit dans le fichier de suivi
226echo $MonitoringInfo >> $generalMonitorFile
227
228DEM_log -0 "Fin de ${SCRIPT_NAME}"
229
Note: See TracBrowser for help on using the repository browser.