source: TOOLS/create_config_card.sh @ 2134

Last change on this file since 2134 was 2133, checked in by acosce, 11 years ago

New tools allows packing simulation already pack (so pack with a bigger frequency)

File size: 9.0 KB
Line 
1#!/bin/bash
2##
3## Job permettant de recreer une partie du config.card d'une simulation
4## Champs vises : JobName / DateBegin / DateEnd / Calendar
5##
6
7# Prend en argument un fichier contenant une liste de simulations
8# cree pour chacune son config.card
9# Ex : ./create_config_card.job liste_simul_fichier_param_test1.txt liste_fichier_dmnfs
10# Ex : ./create_config_card.job liste_simul_param_SL.txt /cscratch/cont003/labetoul/FROM_CCRT_INFORMATIONS_DMNFS/information_dmnfs_2012-03-21
11
12
13LISTE_SIMUL=${1}
14LISTE_DMNFS=${2}
15
16# ----
17export JOB_DIR=${JOB_DIR:-${TMP_MIGR_DATA}}
18export IGCM_DEM=${IGCM_DEM:-${SCRATCHDIR}/IGCM_DEM}
19export IGCM_TMP=${IGCM_TMP:-${IGCM_DEM}/tmp}
20mkdir -p ${IGCM_DEM}
21mkdir -p ${IGCM_TMP}
22clean_tmp=${clean_tmp:-yes}
23EXE_DIR=$(dirname ${0})
24SCRIPT_NAME=$(basename ${0} )
25source ${EXE_DIR}/DEM_utilities.sh
26
27# gpdebug : verif si le present script a ete execute jusqu'au bout #########
28MonitoringInfo="${SCRIPT_NAME}-->OK"
29generalMonitoring "$MonitoringInfo"
30
31DEM_log -0 "Demarrage de ${SCRIPT_NAME}"
32
33# gpdebug : on augmente la liste des proc à détruire en cas d'erreur bloquante du présent proc
34export listPID="$$ ${listPID}" #########
35
36cd ${JOB_DIR}
37# ----
38
39DEM_log -0 "LISTE_SIMUL : ${LISTE_SIMUL}"
40DEM_log -0 "LISTE_DMNFS : ${LISTE_DMNFS}"
41
42[[ -f ${IGCM_DEM}/config_card.liste ]] && rm ${IGCM_DEM}/config_card.liste
43
44
45for PATH_SIMUL_FULL in `awk  '{print $0}' ${LISTE_SIMUL}` ; do
46
47    DEM_log -3 PATH_SIMUL_FULL : ${PATH_SIMUL_FULL}
48    PATH_SIMUL_SHORT=$(echo ${PATH_SIMUL_FULL} | sed "s%${INPUT_DMF_DATA}/%%" ) # gpdebug : added
49    DEM_log -3 PATH_SIMUL_SHORT : ${PATH_SIMUL_SHORT}
50
51    IGCM_DEM_SIMU=${IGCM_DEM}/${PATH_SIMUL_SHORT}
52    mkdir -p ${IGCM_DEM_SIMU}
53    rm -rf ${IGCM_DEM_SIMU}/*
54
55### note dans cette version on ne gere pas les fichiers Analyse / Monitoring ils sont deja la ou il faut et comme il faut
56    # on recherche le nom du job
57    JobName=""
58   
59
60    # Etablir la liste des fichiers de restart
61    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep RESTART | awk '{print $2 " " $3}' | grep -e '.*/RESTART/.*\..*' |grep -v "jnl" | sort  >> ${IGCM_TMP}/${$}.liste_restart_files_config.txt
62
63    # Retirons de la liste les fichiers que l'on n'attends pas, en particulier si il  y a des sous-repertoires au repetoire RESTART/
64    for file in `awk  '{print $2}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt ` ; do
65        dir=$( echo $file | sed -e "s%.*RESTART/%%") 
66        testdir=$(dirname ${dir}) 
67        if [ $testdir != "." ]; then
68            echo $( grep ${file} ${IGCM_TMP}/${$}.liste_restart_files_config.txt)  >> ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt
69        fi
70    done
71    if [ -f ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt ]; then
72        join -t % -v 1 ${IGCM_TMP}/${$}.liste_restart_files_config.txt ${IGCM_TMP}/${$}.liste_no_restart_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt
73        mv ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.liste_restart_files_config.txt
74    fi
75   
76    # Etablir la liste des fichiers output
77    grep ${PATH_SIMUL_FULL}/ ${LISTE_DMNFS} | grep Output  | awk '{print $2 " "  $3}' | grep -e '.*/Output/.*\.nc$'  | sort >> ${IGCM_TMP}/${$}.liste_output_files_config.txt
78    # Retirons de la liste les fichiers que l'on n'attends pas, en particulier si il  y a des sous-repertoires aux repetoires Output/*/
79    for file in `awk  '{print $2}'  ${IGCM_TMP}/${$}.liste_output_files_config.txt ` ; do
80        dir=$( echo $file | sed -e "s%.*Output/[^/]*/%%") 
81        testdir=$(dirname ${dir}) 
82        if [ $testdir != "." ]; then
83            echo $( grep ${file} ${IGCM_TMP}/${$}.liste_output_files_config.txt ) >> ${IGCM_TMP}/${$}.liste_no_output_files_config.txt
84        fi
85    done
86    if [ -f ${IGCM_TMP}/${$}.liste_no_output_files_config.txt ]; then
87        join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.liste_no_output_files_config.txt > ${IGCM_TMP}/${$}.info_tmp.txt
88        mv ${IGCM_TMP}/${$}.info_tmp.txt ${IGCM_TMP}/${$}.liste_output_files_config.txt
89    fi
90
91    # Lecture du nom du job
92    JobName=`awk -F/ '{if (NR==1) print $(NF-2)}'  ${IGCM_TMP}/${$}.liste_restart_files_config.txt`
93
94    DEM_log -2 JobName ${JobName}
95
96   # traitons les listes de restart pour conserver ceux qui correspondent au format $JobName_restart_date_date.tar et $JobName_date_date_restart.tar
97   # on enleve les fichiers qui ne sont pas en .tar
98   grep -w "tar" ${IGCM_TMP}/${$}.liste_restart_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt; mv ${IGCM_TMP}/${$}.info_tmp.txt  ${IGCM_TMP}/${$}.liste_restart_files_config.txt
99   # dans ces fichiers on enleve ceux qui n'ont pas la chaine "restart" (attention suivant d'ou provient la simulation (pack ou libigcm) la chaine restart n'est pas situee au meme endroite
100   grep "restart"  ${IGCM_TMP}/${$}.liste_restart_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt; mv ${IGCM_TMP}/${$}.info_tmp.txt  ${IGCM_TMP}/${$}.liste_restart_files_config.txt
101   # dans ces fichiers on enleve ceux qui n'ont pas le jobName
102   grep $JobName ${IGCM_TMP}/${$}.liste_restart_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt; mv ${IGCM_TMP}/${$}.info_tmp.txt  ${IGCM_TMP}/${$}.liste_restart_files_config.txt   
103
104   mv ${IGCM_TMP}/${$}.liste_restart_files_config.txt    ${IGCM_DEM_SIMU}/liste_restart_files_config.txt
105
106# Maintenant que nous avons JobName verifions que tous les fichiers de Output sont bien du type JobName_date_*
107    sed -e "s%.*/${JobName}_\([0-9]\{8\}\)_%%" ${IGCM_TMP}/${$}.liste_output_files_config.txt  >  ${IGCM_TMP}/${$}.info_tmp.txt
108    cat ${IGCM_TMP}/${$}.info_tmp.txt |grep "/" >  ${IGCM_TMP}/${$}.info_tmp_bis.txt
109    join -t % -v 1 ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.info_tmp_bis.txt > ${IGCM_TMP}/${$}.output.tmp
110    sort -k 2 ${IGCM_TMP}/${$}.output.tmp > ${IGCM_TMP}/${$}.output_bis.tmp
111    mv ${IGCM_TMP}/${$}.output_bis.tmp ${IGCM_DEM_SIMU}/liste_output_files_config.txt
112
113
114## Recherche de DateBegin et DateEnd pour les Restarts
115    typeRestart=`awk -F_ '{if (NR==1) print $(NF)}'  ${IGCM_DEM_SIMU}/liste_restart_files_config.txt `
116    echo $typeRestart
117    if [[ "$typeRestart" == "restart.tar" ]]; then
118        first_file=$(        head -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
119        extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_//" | sed -e 's/_restart.tar//g' )
120        DateBegin_Restart=$( echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
121
122        last_file=$(         tail -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
123        extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" | sed -e 's/_restart.tar//g' )
124        DateEnd_Restart=$(   echo ${extract_date_file}| sed 's/.*_\([0-9]\{8\}\).*$/\1/g' )
125    else
126        first_file=$(        head -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
127        extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_restart_//" | sed -e 's/\.tar//g' )
128        DateBegin_Restart=$( echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
129
130        last_file=$(         tail -n 1 ${IGCM_DEM_SIMU}/liste_restart_files_config.txt )
131        extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_restart_//" | sed -e 's/\.tar//g' )
132        DateEnd_Restart=$(   echo ${extract_date_file}| sed 's/.*_\([0-9]\{8\}\).*$/\1/g' )
133       
134    fi
135
136
137# Recherche de DateBegin et DateEnd pour les Output
138    first_file=$( head -n 1 ${IGCM_DEM_SIMU}/liste_output_files_config.txt )
139    extract_date_file=$( echo ${first_file}  | sed -e "s/.*${JobName}_//" )
140    DateBegin_Output=$(  echo ${extract_date_file}| sed 's/\([0-9]\{8\}\)_.*$/\1/g' )
141   
142    last_file=$(         tail -n 1 ${IGCM_DEM_SIMU}/liste_output_files_config.txt )
143    extract_date_file=$( echo ${last_file}  | sed -e "s/.*${JobName}_//" )
144    DateEnd_Output=$(    echo ${extract_date_file}| sed 's/.*_\([0-9]\{8\}\)_.*$/\1/g' )
145   
146    DateBegin=${DateBegin_Restart}
147    DateEnd=${DateEnd_Restart}
148
149    DEM_log -2 "DateBegin DateEnd : ${DateBegin} ${DateEnd}"
150    DEM_log -2 "DateBegin_Output DateEnd_Output : ${DateBegin_Output} ${DateEnd_Output}"
151    DEM_log -2 "DateBegin_Restart DateEnd_Restart : ${DateBegin_Restart} ${DateEnd_Restart}"
152
153    DateBegin=$( DEM_min ${DateBegin_Output} ${DateBegin} ) 
154    DateEnd=$(   DEM_max ${DateEnd_Output}   ${DateEnd}   )
155   
156    DEM_log -2  "DateBegin DateEnd : ${DateBegin} ${DateEnd}"
157   
158    [[ "${clean_tmp}" = "yes" ]] && rm -f ${IGCM_TMP}/${$}.config_name ${IGCM_TMP}/${$}.liste_output_files_config.txt ${IGCM_TMP}/${$}.liste_restart_files_config.txt
159   
160    rm -f ${IGCM_DEM_SIMU}/config_card_${JobName}
161    cat<<EOD>${IGCM_DEM_SIMU}/config_card_${JobName}
162JobName=${JobName}
163DateBegin=${DateBegin}
164DateEnd=${DateEnd}
165PATH_SIMUL_FULL=${PATH_SIMUL_FULL}
166IGCM_DEM_SIMU=${IGCM_DEM_SIMU}
167EOD
168   
169    echo "${IGCM_DEM_SIMU}/config_card_${JobName} ListToBeDone" >> ${IGCM_DEM}/config_card.liste
170
171    DEM_log -1 "${IGCM_DEM_SIMU}/config_card_${JobName} ListToBeDone >> ${IGCM_DEM}/config_card.liste"
172
173#si la liste des fichiers d'output est vide on tar la simul
174if [[ !  -s ${IGCM_DEM_SIMU}/liste_output_files_config.txt ]]; then
175DEM_log -2 "liste output vide"
176    cd ${EXE_DIR}
177    ./write_liste_tar.sh ${IGCM_DEM_SIMU}/config_card_${JobName}
178    cd ${JOB_DIR}
179fi
180
181
182done
183
184# gpdebug : le script n'a pas rencontre d'erreur. On l'inscrit dans le fichier de suivi
185echo $MonitoringInfo >> $generalMonitorFile
186
187DEM_log -0 "Fin de ${SCRIPT_NAME}"
188
Note: See TracBrowser for help on using the repository browser.