source: tags/libIGCM_v1_10/AA_move-and-rename @ 1170

Last change on this file since 1170 was 387, checked in by sdipsl, 13 years ago
  • function IGCM_sys_Get_Master on cesium will try 10 times to scp files from master machine. Hopefully the last cesium consolidation
  • function IGCM_config_Check will check chunck2D/3D consistency with TimeSeriesFrequency?.
  • change the way the output from post-processing script was named. We will now have:

-rebuild_fromWorkdir.20051231.out
-create_ts.20051231.2D.out
-create_ts.20051231.3D.out
-atlas_PISCES.19991231.out
-create_ts.20051231.Chunck2D.ATM.POST_1D_histday.out
-create_ts.20051231.Chunck2D.OCE.POST_1D_grid_T.out

  • remove modipsl headers from AA_SE_Checker, AA_TimeSeries_Checker, AA_move-and-rename
    • they check and submit job. They do not use resources and so there is no point to submit them
    • they will be rename shortly SE_Checker.ksh, TimeSeries_Checker.ksh, move-and-rename.ksh
  • cosmetics
  • Property svn:keywords set to Revision Author Date
File size: 15.2 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Martial Mancip
5# Contact: Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14# Change SpaceName/ExperimentName/JobName names and path for a simulation during the run.
15# !! Be careful that the job is suspended before using this script !!
16# You may call this script with those four variables already defined, or modify directly
17# default values under here :
18
19# New Name Space of this experience
20NEW_SpaceName=${NEW_SpaceName:=DEVT}
21
22# New Expericence class of the run
23NEW_ExperimentName=${NEW_ExperimentName:=NEWExperimentName}
24
25# New JobName of the run
26NEW_JobName=${NEW_JobName:=NEWJobName}
27
28SUBMIT_DIR=${SUBMIT_DIR:=my_path_to_my_SUBMIT_DIR}
29
30# verbosity
31Verbosity=3
32
33########################################################################
34
35# Chemin vers MODIPSL
36MODIPSL=${MODIPSL:=::modipsl::}
37
38# Chemin vers libIGCM
39libIGCM=${libIGCM:=${MODIPSL}/libIGCM}
40# Attention : à changer si la machine de post-traitement n'est pas la frontale du serveur de calcul !
41#             voir précence de la variable MirrorlibIGCM dans votre couche systÚme.
42
43
44DEBUG_sys=false
45DEBUG_debug=false
46
47########################################################################
48
49. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh ;
50. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh     #; IGCM_debug_Check
51. ${libIGCM}/libIGCM_card/libIGCM_card.ksh   #; IGCM_card_Check
52. ${libIGCM}/libIGCM_date/libIGCM_date.ksh   #; IGCM_date_Check
53. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
54
55########################################################################
56# define local functions
57
58# Function to replace OLD JobName in file
59
60function REPLACE_sed
61{
62    IGCM_debug_PushStack "REPLACE_sed"
63   
64    FILEIN=${1}
65    FILEOUT=$( basename ${FILEIN} )
66
67    sed -e "s&${R_SAVE}&${NEW_R_SAVE}&g" -e "s/${config_UserChoices_JobName}/${NEW_JobName}/g" \
68        ${1} > ${RUN_DIR_PATH}/${FILEOUT}
69    RET=$?
70
71    touch --reference=${1} ${RUN_DIR_PATH}/${FILEOUT}
72    IGCM_sys_Mv ${RUN_DIR_PATH}/${FILEOUT} ${2}
73    chmod --reference=${1} ${2}/${FILEOUT}
74
75    IGCM_sys_Rm -f ${1}
76
77    IGCM_debug_Print 3 "REPLACE_sed : ${1} ${2}"
78   
79    IGCM_debug_PopStack "REPLACE_sed"
80    return $RET
81}
82
83function MOVE_and_RENAME_files
84{
85    IGCM_debug_PushStack "MOVE_and_RENAME_files"
86
87    typeset VAR_files FILE_OUT
88
89    VAR_files=${1}
90    eval VAR1=\${${VAR_files[0]}}
91
92    set +A FILE_OUT -- $( eval echo \${${VAR_files}[*]} | sed -e "s&${R_SAVE}&${NEW_R_SAVE}&g" -e "s/${config_UserChoices_JobName}/${NEW_JobName}/g" )
93    IGCM_debug_Print 3 "MOVE_and_RENAME_files all : ${VAR1}"
94    IGCM_debug_Print 3 "    to     ${FILE_OUT}." 
95
96    (( ifile = 0 ))
97    for file_o in ${FILE_OUT[@]} ; do
98        eval IGCM_sys_Mv \${${VAR_files}[${ifile}]} ${file_o}
99        (( ifile = ifile + 1 ))
100    done   
101    RET=$?
102
103    IGCM_debug_PopStack "MOVE_and_RENAME_files"
104    return $RET
105}
106
107function MY_DODS_Cp
108{
109    IGCM_debug_PushStack "MY_DODS_Cp"
110
111    typeset old_JobName old_R_DODS
112    IGCM_sys_Cd ${NEW_R_SAVE}
113    old_JobName=${config_UserChoices_JobName}
114    old_R_DODS=${R_DODS}
115    old_R_SAVE=${R_SAVE}
116
117    config_UserChoices_JobName=${NEW_JobName}
118    R_DODS=${NEW_R_DODS}
119    R_SAVE=${NEW_R_SAVE}
120
121    IGCM_sys_Dods_Cp ${1}
122    IGCM_debug_Print 3 "MY_DODS_Cp : ${config_UserChoices_JobName}/${1}"
123    IGCM_sys_Cd ${RUN_DIR_PATH}
124
125    config_UserChoices_JobName=${old_JobName}
126    R_DODS=${old_R_DODS}
127    R_SAVE=${old_R_SAVE}
128
129    IGCM_debug_PopStack "MY_DODS_Cp"
130}
131
132function MY_DODS_Rm
133{
134    IGCM_debug_PushStack "MY_DODS_Rm"
135    DEBUG_debug=false
136
137    IGCM_sys_Cd ${R_SAVE}
138    IGCM_sys_Dods_Rm ${1}
139    IGCM_debug_Print 3 "MY_DODS_Rm : ${config_UserChoices_JobName}/${1}"
140    IGCM_sys_Cd ${RUN_DIR_PATH}
141
142    IGCM_debug_PopStack "MY_DODS_Rm"
143}
144########################################################################
145
146if [ ! -d ${SUBMIT_DIR} ]; then
147    IGCM_debug_Print 1 "No ${SUBMIT_DIR}, we stop here"
148    exit 1
149fi
150
151# We temporary change ${run_Configuration_PeriodState} to
152if [ -f ${SUBMIT_DIR}/run.card ] ; then
153    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration PeriodState
154    if ( [ ${run_Configuration_PeriodState} != "Start" ] && [ ${run_Configuration_PeriodState} != "Running" ] && [ ${run_Configuration_PeriodState} != "OnQueue" ] && [ ${run_Configuration_PeriodState} != "Continue" ] ) ; then
155        old_run_Configuration_PeriodState=${run_Configuration_PeriodState}
156        IGCM_debug_PrintVariables 2 old_run_Configuration_PeriodState
157        IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState "OnQueue"
158    fi
159fi
160
161IGCM_config_Initialize
162
163if [ -f ${SUBMIT_DIR}/run.card ] ; then
164    IGCM_card_DefineVariableFromOption ${SUBMIT_DIR}/run.card Configuration OldPrefix
165    IGCM_debug_PrintVariables 2 run_Configuration_OldPrefix
166    if [ X${old_run_Configuration_PeriodState} != X"Completed" ] ; then
167        DateEnd=$( IGCM_date_ConvertFormatToGregorian $( echo ${run_Configuration_OldPrefix} | awk -F'_' '{print $2}' ) )
168    else
169        DateEnd=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )
170    fi
171else
172    DateEnd=$( IGCM_date_ConvertFormatToGregorian ${config_UserChoices_DateEnd} )
173fi
174
175IGCM_debug_Print 1 ""
176IGCM_debug_Print 1 "DateEnd for MoveJob_Checker : " ${DateEnd}
177IGCM_date_GetYearMonth ${DateEnd}   YearEnd   MonthEnd
178IGCM_debug_Print 1 "YearEnd MonthEnd for TimeSeries_Checker : " ${YearEnd} ${MonthEnd}
179IGCM_debug_Print 1 ""
180
181#set -vx
182
183#====================================================
184#DEFINE NEW_R_SAVE
185if [ X${config_UserChoices_SpaceName} != X ] ; then
186    NEW_SAVE=${config_UserChoices_TagName}/${NEW_SpaceName}
187else
188    NEW_SAVE=${config_UserChoices_TagName}
189fi
190if [ X${config_UserChoices_ExperimentName} != X ] ; then
191    NEW_SAVE=${NEW_SAVE}/${NEW_ExperimentName}
192fi
193
194NEW_R_SAVE=${R_OUT}/${NEW_SAVE}/${NEW_JobName}
195
196IGCM_debug_Print 1 "Move ${R_SAVE} to ${NEW_R_SAVE}"
197if [ -d ${NEW_R_SAVE} ] ; then
198    IGCM_debug_Print 1 "ERROR : ${NEW_R_SAVE} directory already exist !"
199    IGCM_debug_Print 1 "We won't try to  We stop here."
200    exit 1
201fi
202IGCM_sys_MkdirArchive ${NEW_R_SAVE}
203
204#DEFINE NEW_R_DODS
205NEW_R_DODS=${NEW_SAVE}/${NEW_JobName}
206
207# Define NEW_REBUILD_DIR
208if [ -d ${REBUILD_DIR} ] ; then
209    if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
210        NEW_REBUILD_DIR=${NEW_R_SAVE}/TMP
211        IGCM_sys_MkdirArchive ${NEW_REBUILD_DIR}
212    else
213        NEW_REBUILD_DIR=${BIG_DIR}/${config_UserChoices_TagName}/${NEW_JobName}
214        IGCM_sys_MkdirWork ${NEW_REBUILD_DIR}
215    fi
216fi
217
218#====================================================
219# SAVE LOCAL WORKING DIR
220IGCM_sys_Cd ${RUN_DIR_PATH}
221
222#====================================================
223#Exe
224IGCM_debug_Print 1 "Move Exe"
225set +A listfiles -- $( find ${R_SAVE}/Exe -type f )
226
227IGCM_sys_MkdirArchive ${NEW_R_SAVE}/Exe
228MOVE_and_RENAME_files "listfiles"
229
230#====================================================
231#Out
232IGCM_debug_Print 1 "Move Out"
233OutinDir=${R_SAVE}/Out
234OutNEWDir=${NEW_R_SAVE}/Out
235IGCM_sys_MkdirArchive ${OutNEWDir}
236
237# Change JobName and copy run.card
238if [ -f ${OutinDir}/run.card ] ; then
239    REPLACE_sed ${OutinDir}/run.card ${OutNEWDir}
240fi
241
242# Just copy historical run.card
243unset listfiles
244set +A listfiles -- $( find ${OutinDir} -type f -name "run.card\.[0-9]*\.[0-9]*" )
245if [ ${#listfiles[*]} -gt 0 ] ; then
246    IGCM_sys_Cp ${listfiles[*]} ${OutNEWDir}
247    IGCM_sys_Rm -f ${listfiles[*]}
248fi
249
250# Move and Rename OLD Script_Output
251unset listfiles
252set +A listfiles -- $( find ${OutinDir} -type f -name "Script_Output_*\.[0-9]*\.[0-9]*" )
253MOVE_and_RENAME_files "listfiles"
254
255# Move and Rename OLD Executable Output
256unset listfiles
257set +A listfiles -- $( find ${OutinDir} -type f -name "*_out_${config_Executable_Name}" )
258MOVE_and_RENAME_files "listfiles"
259
260#====================================================
261IGCM_debug_Print 1 "For components"
262IGCM_debug_Print 1 "=============="
263
264#====================================================
265#Output
266# loop over components and frequencies
267IGCM_debug_Print 1 "Move Output"
268for comp in ${config_ListOfComponents[*]} ; do
269    IGCM_debug_Print 2 "------------------"
270    IGCM_debug_Print 2 "for component ${comp}"
271
272    unset FREQDIR
273    set +A FREQDIR -- $( find ${R_SAVE}/${comp}/Output -type d -not -name "*Output" )
274
275    IGCM_debug_Print 3 "List of WriteFrequencies : ${FREQDIR[*]}"
276
277    for ThisfreqDir in ${FREQDIR[*]} ; do
278        freq=$( basename ${ThisfreqDir} )
279        IGCM_debug_Print 2 "${freq}."
280
281        NEW_freqDir=${NEW_R_SAVE}/${comp}/Output/${freq}
282        IGCM_sys_MkdirArchive ${NEW_freqDir}
283
284        unset listfiles
285        set +A listfiles -- $( find ${ThisfreqDir} -type f )
286       
287        MOVE_and_RENAME_files "listfiles"
288    done
289
290    unset COMPFILES
291    set +A COMPFILES -- $( find ${R_SAVE}/${comp}/Output -type f )
292   
293    if [ ${#COMPFILES[*]} -gt 0 ] ; then
294        IGCM_debug_Print 2 "simple Output FILES : ${COMPFILES[*]}."
295
296        MOVE_and_RENAME_files "COMPFILES"
297    fi
298done
299
300#====================================================
301#Restart
302# loop over components
303IGCM_debug_Print 1 "Move Restart"
304for comp in ${config_ListOfComponents[*]} ; do
305    IGCM_debug_Print 2 "------------------"
306    IGCM_debug_Print 2 "for component ${comp}"
307
308    ThisRestDir=${R_SAVE}/${comp}/Restart
309    NEW_RestDir=${NEW_R_SAVE}/${comp}/Restart
310    IGCM_sys_MkdirArchive ${NEW_RestDir}
311
312    unset listfiles
313    set +A listfiles -- $( find ${ThisRestDir} -type f )
314   
315    MOVE_and_RENAME_files "listfiles"
316done
317
318#====================================================
319#DEBUG
320# Move and Rename OLD Executable Output
321# loop over components
322IGCM_debug_Print 1 "Move Debug"
323for comp in ${config_ListOfComponents[*]} ; do
324    IGCM_debug_Print 2 "------------------"
325    IGCM_debug_Print 2 "for component ${comp}"
326
327    ThisDebugDir=${R_SAVE}/${comp}/Debug
328    NEW_DebugDir=${NEW_R_SAVE}/${comp}/Debug
329    IGCM_sys_MkdirArchive ${NEW_DebugDir}
330
331    unset listfiles
332    set +A listfiles -- $( find ${ThisDebugDir} -type f )
333   
334    MOVE_and_RENAME_files "listfiles"
335done
336
337#====================================================
338#POST
339# loop over components and frequencies
340IGCM_debug_Print 1 "Move Analyse"
341for comp in ${config_ListOfComponents[*]} ; do
342    IGCM_debug_Print 2 "------------------"
343    IGCM_debug_Print 2 "for component ${comp}"
344   
345    if [ -d ${R_SAVE}/${comp}/Analyse ] ; then
346        unset TSDIR
347        set +A TSDIR -- $( find ${R_SAVE}/${comp}/Analyse -name "TS_*" -type d )
348       
349        for ThisTSDir in ${TSDIR[*]} ; do
350            TS_Dir=$( basename ${ThisTSDir} )
351            IGCM_debug_Print 2 "${TS_Dir}."
352           
353            NEW_TSDir=${NEW_R_SAVE}/${comp}/Analyse/${TS_Dir}
354            IGCM_sys_MkdirArchive ${NEW_TSDir}
355           
356            MY_DODS_Rm ${comp}/Analyse/${TS_Dir}
357           
358            unset listfiles
359            set +A listfiles -- $( find ${ThisTSDir} -type f )
360           
361            MOVE_and_RENAME_files "listfiles"
362           
363            MY_DODS_Cp ${comp}/Analyse/${TS_Dir}
364        done
365
366        ThisSEDir=${R_SAVE}/${comp}/Analyse/SE
367        IGCM_sys_TestDirArchive ${ThisSEDir}
368        if [ $? = 0 ] ; then
369            IGCM_debug_Print 2 "SE."
370            NEW_SEDir=${NEW_R_SAVE}/${comp}/Analyse/SE
371            IGCM_sys_MkdirArchive ${NEW_SEDir}
372           
373            MY_DODS_Rm ${comp}/Analyse/SE
374           
375            unset listfiles
376            set +A listfiles -- $( find ${ThisSEDir} -type f )
377           
378            MOVE_and_RENAME_files "listfiles"
379           
380            MY_DODS_Cp ${comp}/Analyse/SE
381        fi
382
383        MY_DODS_Rm ${comp}/Analyse
384        MY_DODS_Rm ${comp}
385    else
386        IGCM_debug_Print 2 "nothing to do without Analyse."
387    fi
388
389done
390
391#====================================================
392#GRAPHS
393IGCM_debug_Print 1 "Move ATLAS"
394R_ATLAS=ATLAS
395IGCM_sys_TestDirArchive ${R_SAVE}/${R_ATLAS}
396if [ $? = 0 ] ; then
397#    REPLACE_sed ${NEW_R_SAVE}/${R_ATLAS}/*/*/index.html ${NEW_R_SAVE}/${R_ATLAS}
398
399    MY_DODS_Rm ${R_ATLAS}
400
401    IGCM_sys_Mv ${R_SAVE}/${R_ATLAS} ${NEW_R_SAVE}/${R_ATLAS}
402
403    MY_DODS_Cp ${R_ATLAS}
404fi
405
406IGCM_debug_Print 1 "Move MONITORING"
407R_MONITORING=MONITORING
408IGCM_sys_TestDirArchive ${R_SAVE}/${R_MONITORING}
409if [ $? = 0 ] ; then
410    IGCM_sys_MkdirArchive ${NEW_R_SAVE}/${R_MONITORING}
411    REPLACE_sed ${R_SAVE}/${R_MONITORING}/index.html ${NEW_R_SAVE}/${R_MONITORING}
412    REPLACE_sed ${R_SAVE}/${R_MONITORING}/run.card ${NEW_R_SAVE}/${R_MONITORING}
413    REPLACE_sed ${R_SAVE}/${R_MONITORING}/config.card ${NEW_R_SAVE}/${R_MONITORING}
414    IGCM_sys_Cp -rp ${R_SAVE}/${R_MONITORING} ${NEW_R_SAVE}
415
416    MY_DODS_Rm ${R_MONITORING}
417    IGCM_sys_Rm -rf ${R_SAVE}/${R_MONITORING}
418
419    MY_DODS_Cp ${R_MONITORING}
420fi
421
422
423#====================================================
424#REBUILD
425
426if [ -d ${REBUILD_DIR} ] ; then
427
428    IGCM_debug_Print 1 "Move REBUILD"
429    set +A LISTREBUILDS -- $( find ${REBUILD_DIR} -type d  -not -name "*${config_UserChoices_JobName}" -a -not -name "*TMP" )
430
431    for ThisREBDir in ${LISTREBUILDS[*]} ; do
432        rebdir=$( basename ${ThisREBDir} )
433        IGCM_debug_Print 2 "${rebdir}."
434        if [ X${config_Post_RebuildFromArchive} = Xtrue ] ; then
435            IGCM_sys_MkdirArchive ${NEW_REBUILD_DIR}/${rebdir}
436        else
437            IGCM_sys_MkdirWork ${NEW_REBUILD_DIR}/${rebdir}
438        fi
439        REPLACE_sed ${REBUILD_DIR}/${rebdir}/rebuild.ksh ${NEW_REBUILD_DIR}/${rebdir}
440        IGCM_sys_Mv -f ${REBUILD_DIR}/${rebdir}/* ${NEW_REBUILD_DIR}/${rebdir}/
441    done
442fi
443
444#====================================================
445#List OLD R_SAVE and Ask for erase it !
446
447IGCM_debug_Print 1 "ls of SAVE DIR :"
448ls --color -lR ${R_SAVE}
449
450echo "\033[1;31mPlease verify that ALL OLD DIRECTORY LISTED IS COMPLETLY EMPTY !"
451echo "\033[1;32mIf It is NOT EMPTY, you can TRY AGAIN this script."
452echo "\033[1;31mDo you REALLY want to erase all this directory ?"
453echo -n " Your answer (y/n) : \033[m"
454if [ X${reponse} = X ] ; then
455    read reponse
456fi
457case ${reponse} in
458           
459    oui|OUI|o|y|yes|YES)
460        echo "OK. It will be erased."
461        IGCM_sys_Rm -Rf ${R_SAVE}
462        echo "done."
463        ;;
464    non|NON|n|no|NO)
465        echo "Nothing to do !"
466        ;;
467esac
468
469
470#############################################################################################################
471# Edit config.card, run.card and Job_${NewJobName}
472
473IGCM_debug_Print 1 "edit config.card"
474if [ X${config_UserChoices_SpaceName} != X ] ; then
475    IGCM_card_WriteOption ${SUBMIT_DIR}/config.card UserChoices SpaceName ${NEW_SpaceName}
476fi
477if [ X${config_UserChoices_ExperimentName} != X ] ; then
478    IGCM_card_WriteOption ${SUBMIT_DIR}/config.card UserChoices ExperimentName ${NEW_ExperimentName}
479fi
480IGCM_card_WriteOption ${SUBMIT_DIR}/config.card UserChoices JobName ${NEW_JobName}
481
482if [ -f ${SUBMIT_DIR}/run.card ] ; then
483    IGCM_debug_Print 1 "edit run.card"
484    sed -e "s&${R_SAVE}&${NEW_R_SAVE}&g" -e "s/${config_UserChoices_JobName}/${NEW_JobName}/g" \
485        ${SUBMIT_DIR}/run.card > ${RUN_DIR_PATH}/run.card
486    IGCM_sys_Mv -f ${RUN_DIR_PATH}/run.card ${SUBMIT_DIR}/run.card
487
488    if [ X${old_run_Configuration_PeriodState} != X ] ; then
489        IGCM_card_WriteOption ${SUBMIT_DIR}/run.card Configuration PeriodState ${old_run_Configuration_PeriodState}
490    fi
491fi
492
493if [ -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ] ; then
494    IGCM_debug_Print 1 "edit Job_${NEW_JobName}"
495    sed -e "s&${R_SAVE}&${NEW_R_SAVE}&g" -e "s/${config_UserChoices_JobName}/${NEW_JobName}/g" \
496        ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} > ${SUBMIT_DIR}/Job_${NEW_JobName}
497    chmod --reference=${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ${SUBMIT_DIR}/Job_${NEW_JobName}
498    ls -la ${SUBMIT_DIR}/Job_${NEW_JobName}
499fi
Note: See TracBrowser for help on using the repository browser.