source: TOOLS/PACK_IPSL/enlarge_my_files.sh @ 1766

Last change on this file since 1766 was 1766, checked in by aclsce, 12 years ago

Modified to take into account new name of listfiles.

  • Property svn:executable set to *
File size: 17.7 KB
Line 
1#!/bin/bash
2#------------------------------------------------------
3switchtotar=0
4maxtries=3
5#---------------------------------------------
6while [ $# -ne 0 ]
7do
8        case $1 in
9       --switchtotar)
10                switchtotar=1
11                shift 1 ;;
12       --maxtries)
13                maxtries=$2
14                shift 2 ;;
15       -h|--help|-help)
16                echo "----------------------------------------------------------------------------"
17                echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
18                echo
19                echo "Description:"
20                echo "   Pass a ncrcat, tar or cp command on files from prepared sets of files."
21                echo "   For ncrcat command 2 shots will be done at each try".
22                echo 
23                echo "   A status file and a log file will be created in output_ncrcat directory."
24                echo "   Copy a set to output_tar directory for later processing after N tries."
25                echo
26                echo "Arguments:"
27                echo "   dirin is the input directory where files to concatenate will be read"
28                echo "   dirout is the output directory where concatenated files will be written"
29                echo
30                echo "   The following directory must exist where the command enlarge_my_files is run:"
31                echo "       enlarge_my_files"
32                echo "                       |-- with_ncrcat"
33                echo "                       |   |-- set_000001.list"
34                echo "                       |   |-- set_000002.list"
35                echo "                       |   |-- set_000003.list"
36                echo "                       |   |-- set_000004.list"
37                echo "                       |   \`-- set_000005.list"
38                echo "                       \`-- with_tar"
39                echo
40                echo "Options:"
41                echo "   -h, --help, -help"
42                echo "       Print this manual."
43                echo "   --switchtotar"
44                echo "       Abort ncrcat operation and switch to tar command"
45                echo "   --maxtries N"
46                echo "       Maximum number of tries before switching to tar command" 
47                echo "       Default is 3"
48                echo
49                echo "----------------------------------------------------------------------------"
50                exit ;;
51        -*)
52                `dirname $0`/enlarge_my_files.sh -h
53                exit ;;
54        *)
55                break ;;
56        esac
57done
58
59if [ $# -lt 2 ] ; then
60        `dirname $0`/enlarge_my_files.sh -h
61        exit
62fi
63
64if [[ ! -d $PATH_SIMU/output_ncrcat || ! -d $PATH_SIMU/output_tar || ! -d $PATH_SIMU/restart_tar || ! -d $PATH_SIMU/store_cp || ! -d $PATH_SIMU/work_cp || ! -d $PATH_SIMU/debug_tar || ! -d $PATH_SIMU/other_tar ]] ; then
65        `dirname $0`/enlarge_my_files.sh -h
66        exit
67fi
68
69#------------------------------------------------------
70dirin=$1
71dirout=$2
72dirout_work=$3
73
74#------------------------------------------------------
75function find_varstoexclude {
76nbfile=0
77for file in `cat $1` ; do
78    ncdump -h ${file} | gawk '{if (match($0, /(byte|char|short|int|float|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
79    let nbfile=nbfile+1
80done
81
82varstoexclude=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d','`
83varstoexcludefiltered=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' | paste -s -d','`
84nbvars=`echo $varstoexcludefiltered | gawk -F',' '{print NF}'`
85
86rm -f tmp_$$.txt
87
88return 0
89}
90
91#------------------------------------------------------
92for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do
93
94        #-----------------------------------
95        echo "#-------------------"
96        echo "Set: $set"
97        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
98        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
99        statusfile=${set%%.list}.status
100        logfile=${set%%.list}_${datestr}.log
101
102        #-----------------------------------
103        # Create status file if not exists
104        if [ ! -e $statusfile ] ; then
105                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
106        fi
107
108        #-----------------------------------
109        # Switch to tar command
110        if [ $switchtotar -eq 1 ] ; then
111                echo "====> DELEGATE to tar command with option --switchtotar"
112                echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
113                mv $set $PATH_SIMU/with_tar
114                continue
115        fi
116
117        #-----------------------------------
118        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
119        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
120                printf "====> "
121                tail -1 $statusfile
122                continue
123        fi
124
125        #-----------------------------------
126        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
127        nbtries=`wc -l $statusfile | cut -d' ' -f1`             
128        echo "Nb of tries: $nbtries"
129
130        #-----------------------------------
131        # Set output file name from list name
132
133        fileout1=$( basename $set )
134        fileout=${fileout1%.list}
135
136        filefirst=`head -n 1 $set`
137        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
138
139        outputfile=$diroutputfile/${fileout}
140        mkdir -p $diroutputfile
141        echo "Output file to write: $outputfile"
142
143        #-----------------------------------
144        # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
145        extratar=0
146        find_varstoexclude $set
147        if [ $nbvars -eq 0 ] ; then
148                varstoexclude="t_inst_.*"
149        else
150                # There are others variables than 't_inst_.*' so an extra tar file will be created
151                varstoexclude="t_inst_.*,$varstoexcludefiltered"
152                extratar=1
153                # Copy to with_tar directory
154                cp $set $PATH_SIMU/with_tar
155        fi
156
157        #-----------------------------------
158        # Try 2 times before fire "FAILED"
159        ncrcatcmd=ncrcat
160#       ncrcatoptions="--md5_digest"
161        ncrcatoptions=""
162
163        cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1
164        if [ $? -eq 0 ] ; then
165                datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
166                echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
167                echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
168                echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
169        else
170                # Add --md5_digest
171                cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1
172                if [ $? -eq 0 ] ; then
173                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
174                        echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
175                        echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
176                        echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
177                else
178                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
179                        echo "====> FAILED at try #$nbtries" 
180                        echo "FAILED ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
181                        if [ $nbtries -ge $maxtries ] ; then
182                                echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
183                                echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
184                                cp $set $PATH_SIMU/with_tar
185                        fi
186                fi
187        fi
188       
189done   
190
191#------------------------------------------------------
192
193for set in $( ls $PATH_SIMU/output_tar/*list ) ; do
194
195        #-----------------------------------
196        echo "#-------------------"
197        echo "Set: $set"
198        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
199        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
200        statusfile=${set%%.list}.status
201        logfile=${set%%.list}_${datestr}.log
202
203        #-----------------------------------
204        # Create status file if not exists
205        if [ ! -e $statusfile ] ; then
206                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
207        fi
208
209        #-----------------------------------
210        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
211        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
212                printf "====> "
213                tail -1 $statusfile
214                continue
215        fi
216
217        #-----------------------------------
218        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
219        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
220        # echo "Nb of tries: $nbtries"
221
222        #-----------------------------------
223        # Set output file name from list name
224
225        fileout1=$( basename $set )
226        fileout=${fileout1%.list}.tar
227       
228        filefirst=`head -n 1 $set`
229        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` 
230
231        outputfile=$diroutputfile/${fileout}
232        mkdir -p $diroutputfile
233        echo "Output file to write: $outputfile"
234
235        #-----------------------------------
236        # Try 1 time before fire "FAILED"
237        # Waiting for CCRT command
238        tarcmd=tar
239        taroptions="--format=posix -W -cf"
240       
241        cd $dirinputfile
242
243        set_local=$set".local"
244        for file in $( cat $set) ; do
245            basename $file >> $set".local" 
246        done
247           
248#       #-----------------------------------
249
250        $tarcmd $taroptions $diroutputfile/$outputfile --dereference --files-from $set.local > $logfile 2>&1
251        if [ $? -eq 0 ] ; then
252            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
253            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
254            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
255            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
256        else
257            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
258            echo "====> FAILED at try #$nbtries" 
259            echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
260
261        fi
262        rm -f $set".local" 
263        #-----------------------------------
264       
265done   
266
267
268#------------------------------------------------------
269
270for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do
271
272        #-----------------------------------
273        echo "#-------------------"
274        echo "Set: $set"
275        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
276        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
277        statusfile=${set%%.list}.status
278        logfile=${set%%.list}_${datestr}.log
279
280        #-----------------------------------
281        # Create status file if not exists
282        if [ ! -e $statusfile ] ; then
283                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
284        fi
285
286        #-----------------------------------
287        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
288        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
289                printf "====> "
290                tail -1 $statusfile
291                continue
292        fi
293
294        #-----------------------------------
295        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
296        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
297        # echo "Nb of tries: $nbtries"
298
299        #-----------------------------------
300        # Set output file name from list name
301        dirin1=$PATH_SIMU/RESTART
302        dirin2=${dirin1##${IGCM_DEM}}
303        dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
304        diroutputfile=${OUTPUT_STORE}${dirin3}
305        fileout1=$( basename $set )
306        fileout=${fileout1%.list}.tar
307        outputfile=$diroutputfile/${fileout}
308        mkdir -p $diroutputfile
309        echo "Output file to write: $outputfile"
310
311        #-----------------------------------
312        # Try 1 time before fire "FAILED"
313        # Waiting for CCRT command
314        tarcmd=tar
315        taroptions="--format=posix -W -cf"
316       
317        cd $dirin1
318#       ls
319       
320        $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1
321        if [ $? -eq 0 ] ; then
322            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
323            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
324            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
325            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
326        else
327            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
328            echo "====> FAILED at try #$nbtries" 
329            echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
330
331        fi
332       
333        #-----------------------------------
334       
335done   
336
337#------------------------------------------------------
338for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do
339
340        #-----------------------------------
341        echo "#-------------------"
342        echo "Set: $set"
343        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
344        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
345        statusfile=${set%%.list}.status
346        logfile=${set%%.list}_${datestr}.log
347
348        #-----------------------------------
349        # Create status file if not exists
350        if [ ! -e $statusfile ] ; then
351                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
352        fi
353
354        #-----------------------------------
355        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
356        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
357                printf "====> "
358                tail -1 $statusfile
359                continue
360        fi
361
362        #-----------------------------------
363        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
364        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
365        # echo "Nb of tries: $nbtries"
366
367        #-----------------------------------
368        # Set output file name from list name
369        dirin1=$PATH_SIMU/DEBUG
370        dirin2=${dirin1##${IGCM_DEM}}
371        dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
372        diroutputfile=${OUTPUT_STORE}${dirin3}
373        fileout1=$( basename $set )
374        fileout=${fileout1%.list}.tar
375        outputfile=$diroutputfile/${fileout}
376        mkdir -p $diroutputfile
377        echo "Output file to write: $outputfile"
378
379        #-----------------------------------
380        # Try 1 time before fire "FAILED"
381        # Waiting for CCRT command
382        tarcmd=tar
383        taroptions="--format=posix -W -cf"
384       
385        cd $dirin1
386#       ls
387       
388        $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1
389        if [ $? -eq 0 ] ; then
390            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
391            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
392            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
393            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
394        else
395            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
396            echo "====> FAILED at try #$nbtries" 
397            echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
398
399        fi
400       
401        #-----------------------------------
402       
403done   
404
405#------------------------------------------------------
406for set in $( ls $PATH_SIMU/store_cp/*list ) ; do
407
408        #-----------------------------------
409        echo "#-------------------"
410        echo "Set: $set"
411        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
412        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
413        statusfile=${set%%.list}.status
414        logfile=${set%%.list}_${datestr}.log
415
416        #-----------------------------------
417        # Create status file if not exists
418        if [ ! -e $statusfile ] ; then
419                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
420        fi
421
422        #-----------------------------------
423        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
424        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
425                printf "====> "
426                tail -1 $statusfile
427                continue
428        fi
429
430        #-----------------------------------
431        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
432        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
433        # echo "Nb of tries: $nbtries"
434
435        #-----------------------------------
436        # Set output file name from first and last files from the current set
437        for file in $( cat $set) ; do
438         
439            diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout%"`
440            mkdir -p $diroutputfile
441            echo "Output file to write: $diroutputfile/$( basename $file )"
442           
443        #-----------------------------------
444        # Try 1 time before fire "FAILED"
445        # Waiting for CCRT command
446            cpcmd="cp "
447            cpoptions="-rf"
448            $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1
449        done
450
451        if [ $? -eq 0 ] ; then
452            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
453            echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
454            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
455            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
456        else
457            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
458            echo "====> FAILED at try #$nbtries" 
459            echo "FAILED ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
460
461        fi
462       
463        #-----------------------------------
464       
465done   
466
467#------------------------------------------------------
468for set in $( ls $PATH_SIMU/work_cp/*list ) ; do
469
470        #-----------------------------------
471        echo "#-------------------"
472        echo "Set: $set"
473        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
474        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
475        statusfile=${set%%.list}.status
476        logfile=${set%%.list}_${datestr}.log
477
478        #-----------------------------------
479        # Create status file if not exists
480        if [ ! -e $statusfile ] ; then
481                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
482        fi
483
484        #-----------------------------------
485        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
486        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
487                printf "====> "
488                tail -1 $statusfile
489                continue
490        fi
491
492        #-----------------------------------
493        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
494        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
495        # echo "Nb of tries: $nbtries"
496
497        #-----------------------------------
498        # Set output file name from first and last files from the current set
499        for file in $( cat $set) ; do
500         
501            diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout_work%"`
502            mkdir -p $diroutputfile
503            echo "Output file to write: $diroutputfile/$( basename $file )"
504           
505        #-----------------------------------
506        # Try 1 time before fire "FAILED"
507        # Waiting for CCRT command
508            cpcmd="cp "
509            cpoptions="-rf"
510            $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1
511        done
512
513        if [ $? -eq 0 ] ; then
514            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
515            echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
516            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
517            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
518        else
519            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
520            echo "====> FAILED at try #$nbtries" 
521            echo "FAILED ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
522
523        fi
524       
525        #-----------------------------------
526       
527done   
Note: See TracBrowser for help on using the repository browser.