source: TOOLS/PACK_IPSL/enlarge_my_files.sh @ 1768

Last change on this file since 1768 was 1768, checked in by aclsce, 12 years ago

Fixed some bugs.

  • Property svn:executable set to *
File size: 17.8 KB
Line 
1#!/bin/bash
2#------------------------------------------------------
3switchtotar=0
4maxtries=3
5#---------------------------------------------
6while [ $# -ne 0 ]
7do
8        case $1 in
9       --switchtotar)
10                switchtotar=1
11                shift 1 ;;
12       --maxtries)
13                maxtries=$2
14                shift 2 ;;
15       -h|--help|-help)
16                echo "----------------------------------------------------------------------------"
17                echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
18                echo
19                echo "Description:"
20                echo "   Pass a ncrcat, tar or cp command on files from prepared sets of files."
21                echo "   For ncrcat command 2 shots will be done at each try".
22                echo 
23                echo "   A status file and a log file will be created in output_ncrcat directory."
24                echo "   Copy a set to output_tar directory for later processing after N tries."
25                echo
26                echo "Arguments:"
27                echo "   dirin is the input directory where files to concatenate will be read"
28                echo "   dirout is the output directory where concatenated files will be written"
29                echo
30                echo "   The following directory must exist where the command enlarge_my_files is run:"
31                echo "       enlarge_my_files"
32                echo "                       |-- output_ncrcat"
33                echo "                       |   |-- set_000001.list"
34                echo "                       |   |-- set_000002.list"
35                echo "                       |   |-- set_000003.list"
36                echo "                       |   |-- set_000004.list"
37                echo "                       |   \`-- set_000005.list"
38                echo "                       \`-- output_tar"
39                echo
40                echo "Options:"
41                echo "   -h, --help, -help"
42                echo "       Print this manual."
43                echo "   --switchtotar"
44                echo "       Abort ncrcat operation and switch to tar command"
45                echo "   --maxtries N"
46                echo "       Maximum number of tries before switching to tar command" 
47                echo "       Default is 3"
48                echo
49                echo "----------------------------------------------------------------------------"
50                exit ;;
51        -*)
52                `dirname $0`/enlarge_my_files.sh -h
53                exit ;;
54        *)
55                break ;;
56        esac
57done
58
59if [ $# -lt 2 ] ; then
60        `dirname $0`/enlarge_my_files.sh -h
61        exit
62fi
63
64if [[ ! -d $PATH_SIMU/output_ncrcat || ! -d $PATH_SIMU/output_tar || ! -d $PATH_SIMU/restart_tar || ! -d $PATH_SIMU/store_cp || ! -d $PATH_SIMU/work_cp || ! -d $PATH_SIMU/debug_tar || ! -d $PATH_SIMU/other_tar ]] ; then
65        `dirname $0`/enlarge_my_files.sh -h
66        exit
67fi
68
69#------------------------------------------------------
70dirin=$1
71dirout=$2
72dirout_work=$3
73
74#------------------------------------------------------
75function find_varstoexclude {
76nbfile=0
77for file in `cat $1` ; do
78    ncdump -h ${file} | gawk '{if (match($0, /(byte|char|short|int|float|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
79    let nbfile=nbfile+1
80done
81
82varstoexclude=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d','`
83varstoexcludefiltered=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' | paste -s -d','`
84nbvars=`echo $varstoexcludefiltered | gawk -F',' '{print NF}'`
85
86rm -f tmp_$$.txt
87
88return 0
89}
90
91#------------------------------------------------------
92for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do
93
94        #-----------------------------------
95        echo "#-------------------"
96        echo "Set: $set"
97        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
98        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
99        statusfile=${set%%.list}.status
100        logfile=${set%%.list}_${datestr}.log
101
102        #-----------------------------------
103        # Create status file if not exists
104        if [ ! -e $statusfile ] ; then
105                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
106        fi
107
108        #-----------------------------------
109        # Switch to tar command
110        if [ $switchtotar -eq 1 ] ; then
111                echo "====> DELEGATE to tar command with option --switchtotar"
112                echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
113                mv $set $PATH_SIMU/output_tar
114                continue
115        fi
116
117        #-----------------------------------
118        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
119        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
120                printf "====> "
121                tail -1 $statusfile
122                continue
123        fi
124
125        #-----------------------------------
126        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
127        nbtries=`wc -l $statusfile | cut -d' ' -f1`             
128        echo "Nb of tries: $nbtries"
129
130        #-----------------------------------
131        # Set output file name from list name
132
133        fileout1=$( basename $set )
134        fileout=${fileout1%.list}
135
136        filefirst=`head -n 1 $set`
137        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
138
139        outputfile=$diroutputfile/${fileout}
140        mkdir -p $diroutputfile
141        echo "Output file to write: $outputfile"
142
143        #-----------------------------------
144        # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
145        extratar=0
146        find_varstoexclude $set
147        if [ $nbvars -eq 0 ] ; then
148                varstoexclude="t_inst_.*"
149        else
150                # There are others variables than 't_inst_.*' so an extra tar file will be created
151                varstoexclude="t_inst_.*,$varstoexcludefiltered"
152                extratar=1
153                # Copy to output_tar directory
154                cp $set $PATH_SIMU/output_tar
155        fi
156
157        #-----------------------------------
158        # Try 2 times before fire "FAILED"
159        ncrcatcmd=ncrcat
160        ncrcatoptions="--md5_digest"
161#       ncrcatoptions=""
162
163        cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1
164        if [ $? -eq 0 ] ; then
165                datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
166                echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
167                echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
168                echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
169        else
170                # Add --md5_digest
171                cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1
172                if [ $? -eq 0 ] ; then
173                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
174                        echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
175                        echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
176                        echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
177                else
178                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
179                        echo "====> FAILED at try #$nbtries" 
180                        echo "FAILED ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
181                        if [ $nbtries -ge $maxtries ] ; then
182                                echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
183                                echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
184                                cp $set $PATH_SIMU/output_tar
185                        fi
186                fi
187        fi
188       
189done   
190
191#------------------------------------------------------
192
193for set in $( ls $PATH_SIMU/output_tar/*list ) ; do
194
195        #-----------------------------------
196        echo "#-------------------"
197        echo "Set: $set"
198        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
199        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
200        statusfile=${set%%.list}.status
201        logfile=${set%%.list}_${datestr}.log
202
203        #-----------------------------------
204        # Create status file if not exists
205        if [ ! -e $statusfile ] ; then
206                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
207        fi
208
209        #-----------------------------------
210        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
211        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
212                printf "====> "
213                tail -1 $statusfile
214                continue
215        fi
216
217        #-----------------------------------
218        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
219        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
220        # echo "Nb of tries: $nbtries"
221
222        #-----------------------------------
223        # Set output file name from list name
224
225        fileout1=$( basename $set )
226        fileout=${fileout1%.list}.tar
227       
228        filefirst=`head -n 1 $set`
229        dirinputfile=`dirname $filefirst`
230        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` 
231
232        outputfile=$diroutputfile/${fileout}
233        mkdir -p $diroutputfile
234        echo "Output file to write: $outputfile"
235
236        #-----------------------------------
237        # Try 1 time before fire "FAILED"
238        # Waiting for CCRT command
239        tarcmd=tar
240        taroptions="--format=posix -W -cf"
241       
242        cd $dirinputfile
243
244        set_local=$set".local"
245        for file in $( cat $set) ; do
246            basename $file >> $set".local" 
247        done
248           
249#       #-----------------------------------
250
251        $tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1
252        if [ $? -eq 0 ] ; then
253            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
254            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
255            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
256            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
257        else
258            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
259            echo "====> FAILED at try #$nbtries" 
260            echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
261
262        fi
263        rm -f $set".local" 
264        #-----------------------------------
265       
266done   
267
268
269#------------------------------------------------------
270
271for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do
272
273        #-----------------------------------
274        echo "#-------------------"
275        echo "Set: $set"
276        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
277        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
278        statusfile=${set%%.list}.status
279        logfile=${set%%.list}_${datestr}.log
280
281        #-----------------------------------
282        # Create status file if not exists
283        if [ ! -e $statusfile ] ; then
284                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
285        fi
286
287        #-----------------------------------
288        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
289        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
290                printf "====> "
291                tail -1 $statusfile
292                continue
293        fi
294
295        #-----------------------------------
296        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
297        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
298        # echo "Nb of tries: $nbtries"
299
300        #-----------------------------------
301        # Set output file name from list name
302        dirin1=$PATH_SIMU/RESTART
303        dirin2=${dirin1##${IGCM_DEM}}
304        dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
305        diroutputfile=${OUTPUT_STORE}${dirin3}
306        fileout1=$( basename $set )
307        fileout=${fileout1%.list}.tar
308        outputfile=$diroutputfile/${fileout}
309        mkdir -p $diroutputfile
310        echo "Output file to write: $outputfile"
311
312        #-----------------------------------
313        # Try 1 time before fire "FAILED"
314        # Waiting for CCRT command
315        tarcmd=tar
316        taroptions="--format=posix -W -cf"
317       
318        cd $dirin1
319#       ls
320       
321        $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1
322        if [ $? -eq 0 ] ; then
323            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
324            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
325            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
326            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
327        else
328            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
329            echo "====> FAILED at try #$nbtries" 
330            echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
331
332        fi
333       
334        #-----------------------------------
335       
336done   
337
338#------------------------------------------------------
339for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do
340
341        #-----------------------------------
342        echo "#-------------------"
343        echo "Set: $set"
344        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
345        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
346        statusfile=${set%%.list}.status
347        logfile=${set%%.list}_${datestr}.log
348
349        #-----------------------------------
350        # Create status file if not exists
351        if [ ! -e $statusfile ] ; then
352                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
353        fi
354
355        #-----------------------------------
356        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
357        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
358                printf "====> "
359                tail -1 $statusfile
360                continue
361        fi
362
363        #-----------------------------------
364        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
365        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
366        # echo "Nb of tries: $nbtries"
367
368        #-----------------------------------
369        # Set output file name from list name
370        dirin1=$PATH_SIMU/DEBUG
371        dirin2=${dirin1##${IGCM_DEM}}
372        dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
373        diroutputfile=${OUTPUT_STORE}${dirin3}
374        fileout1=$( basename $set )
375        fileout=${fileout1%.list}.tar
376        outputfile=$diroutputfile/${fileout}
377        mkdir -p $diroutputfile
378        echo "Output file to write: $outputfile"
379
380        #-----------------------------------
381        # Try 1 time before fire "FAILED"
382        # Waiting for CCRT command
383        tarcmd=tar
384        taroptions="--format=posix -W -cf"
385       
386        cd $dirin1
387#       ls
388       
389        $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1
390        if [ $? -eq 0 ] ; then
391            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
392            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
393            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
394            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
395        else
396            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
397            echo "====> FAILED at try #$nbtries" 
398            echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
399
400        fi
401       
402        #-----------------------------------
403       
404done   
405
406#------------------------------------------------------
407for set in $( ls $PATH_SIMU/store_cp/*list ) ; do
408
409        #-----------------------------------
410        echo "#-------------------"
411        echo "Set: $set"
412        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
413        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
414        statusfile=${set%%.list}.status
415        logfile=${set%%.list}_${datestr}.log
416
417        #-----------------------------------
418        # Create status file if not exists
419        if [ ! -e $statusfile ] ; then
420                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
421        fi
422
423        #-----------------------------------
424        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
425        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
426                printf "====> "
427                tail -1 $statusfile
428                continue
429        fi
430
431        #-----------------------------------
432        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
433        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
434        # echo "Nb of tries: $nbtries"
435
436        #-----------------------------------
437        # Set output file name from first and last files from the current set
438        for file in $( cat $set) ; do
439         
440            diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout%"`
441            mkdir -p $diroutputfile
442            echo "Output file to write: $diroutputfile/$( basename $file )"
443           
444        #-----------------------------------
445        # Try 1 time before fire "FAILED"
446        # Waiting for CCRT command
447            cpcmd="cp "
448            cpoptions="-rf"
449            $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1
450        done
451
452        if [ $? -eq 0 ] ; then
453            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
454            echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
455            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
456            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
457        else
458            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
459            echo "====> FAILED at try #$nbtries" 
460            echo "FAILED ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
461
462        fi
463       
464        #-----------------------------------
465       
466done   
467
468#------------------------------------------------------
469for set in $( ls $PATH_SIMU/work_cp/*list ) ; do
470
471        #-----------------------------------
472        echo "#-------------------"
473        echo "Set: $set"
474        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
475        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
476        statusfile=${set%%.list}.status
477        logfile=${set%%.list}_${datestr}.log
478
479        #-----------------------------------
480        # Create status file if not exists
481        if [ ! -e $statusfile ] ; then
482                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
483        fi
484
485        #-----------------------------------
486        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
487        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
488                printf "====> "
489                tail -1 $statusfile
490                continue
491        fi
492
493        #-----------------------------------
494        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
495        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
496        # echo "Nb of tries: $nbtries"
497
498        #-----------------------------------
499        # Set output file name from first and last files from the current set
500        for file in $( cat $set) ; do
501         
502            diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout_work%"`
503            mkdir -p $diroutputfile
504            echo "Output file to write: $diroutputfile/$( basename $file )"
505           
506        #-----------------------------------
507        # Try 1 time before fire "FAILED"
508        # Waiting for CCRT command
509            cpcmd="cp "
510            cpoptions="-rf"
511            $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1
512        done
513
514        if [ $? -eq 0 ] ; then
515            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
516            echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
517            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
518            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
519        else
520            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
521            echo "====> FAILED at try #$nbtries" 
522            echo "FAILED ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
523
524        fi
525       
526        #-----------------------------------
527       
528done   
Note: See TracBrowser for help on using the repository browser.