source: TOOLS/PACK_IPSL/enlarge_my_files.sh @ 1839

Last change on this file since 1839 was 1839, checked in by acosce, 12 years ago

BugFix? when we try to find variables to exclude of ncrcat.
For example before "Point Omega" was find as a integer variable

  • Property svn:executable set to *
File size: 21.0 KB
Line 
1#!/bin/bash
2#------------------------------------------------------
3switchtotar=0
4maxtries=3
5#---------------------------------------------
6while [ $# -ne 0 ]
7do
8        case $1 in
9       --switchtotar)
10                switchtotar=1
11                shift 1 ;;
12       --maxtries)
13                maxtries=$2
14                shift 2 ;;
15       -h|--help|-help)
16                echo "----------------------------------------------------------------------------"
17                echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
18                echo
19                echo "Description:"
20                echo "   Pass a ncrcat, tar or cp command on files from prepared sets of files."
21                echo "   For ncrcat command 2 shots will be done at each try".
22                echo 
23                echo "   A status file and a log file will be created in output_ncrcat directory."
24                echo "   Copy a set to output_tar directory for later processing after N tries."
25                echo
26                echo "Arguments:"
27                echo "   dirin is the input directory where files to concatenate will be read"
28                echo "   dirout is the output directory where concatenated files will be written"
29                echo
30                echo "   The following directory must exist where the command enlarge_my_files is run:"
31                echo "       enlarge_my_files"
32                echo "                       |-- output_ncrcat"
33                echo "                       |   |-- set_000001.list"
34                echo "                       |   |-- set_000002.list"
35                echo "                       |   |-- set_000003.list"
36                echo "                       |   |-- set_000004.list"
37                echo "                       |   \`-- set_000005.list"
38                echo "                       \`-- output_tar"
39                echo
40                echo "Options:"
41                echo "   -h, --help, -help"
42                echo "       Print this manual."
43                echo "   --switchtotar"
44                echo "       Abort ncrcat operation and switch to tar command"
45                echo "   --maxtries N"
46                echo "       Maximum number of tries before switching to tar command" 
47                echo "       Default is 3"
48                echo
49                echo "----------------------------------------------------------------------------"
50                exit ;;
51        -*)
52                `dirname $0`/enlarge_my_files.sh -h
53                exit ;;
54        *)
55                break ;;
56        esac
57done
58
59if [ $# -lt 2 ] ; then
60        `dirname $0`/enlarge_my_files.sh -h
61        exit
62fi
63
64if [[ ! -d $PATH_SIMU/output_ncrcat || ! -d $PATH_SIMU/output_tar || ! -d $PATH_SIMU/restart_tar || ! -d $PATH_SIMU/store_cp || ! -d $PATH_SIMU/work_cp || ! -d $PATH_SIMU/debug_tar || ! -d $PATH_SIMU/other_tar ]] ; then
65        `dirname $0`/enlarge_my_files.sh -h
66        exit
67fi
68
69#------------------------------------------------------
70dirin=$1
71dirout=$2
72dirout_work=$3
73
74#------------------------------------------------------
75function find_varstoexclude {
76nbfile=0
77for file in `cat $1` ; do
78 #   ncdump -h ${file} | gawk '{if (match($0, /(byte|char|short|int|float|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
79    ncdump -h ${file} | grep -Ew "byte|char|short|int|float|double"  | awk -F\( '{split($1,a," ") ; split($2,b,")") ; if (a[2] != b[1]) print a[2]}'  >> tmp_$$.txt
80    let nbfile=nbfile+1
81done
82
83varstoexclude=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d','`
84varstoexcludefiltered=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' | paste -s -d','`
85nbvars=`echo $varstoexcludefiltered | gawk -F',' '{print NF}'`
86
87rm -f tmp_$$.txt
88
89return 0
90}
91
92# gpdebug : pour test showPackProgress.sh *************************
93export RANDOM=$$
94
95function gives_0_or_1
96{
97   bit=-1
98   let "bit = RANDOM % 2"
99   echo $bit
100}
101
102resultCmd=
103# gpdebug : ****************************** fin ********************
104
105
106#********** gpdebug : mesure d'intervalles de temps ***************
107function getDateMilliSeconds
108{
109   test=`date +%s%N`
110   testnum=${#test}
111   millisec=${test:0:$testnum-6}
112   echo "$millisec"
113}
114
115function getTimeDiffSeconds
116{
117   startTimeMilliSec=$1
118   endTimeMilliSec=$( getDateMilliSeconds )
119   
120   diffMilliSec=$(( $endTimeMilliSec - $startTimeMilliSec ))
121   
122   diffSeconds=$(awk 'BEGIN { print '$diffMilliSec'/'1000' }')
123   echo "$diffSeconds" 
124
125}
126# gpdebug : ****************************** fin ********************
127#------------------------------------------------------
128for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do
129
130        #-----------------------------------
131        echo "#-------------------"
132        echo "Set: $set"
133        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
134        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
135        statusfile=${set%%.list}.status
136        logfile=${set%%.list}_${datestr}.log
137
138        #-----------------------------------
139        # Create status file if not exists
140        if [ ! -e $statusfile ] ; then
141                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
142        fi
143
144        #-----------------------------------
145        # Switch to tar command
146        if [ $switchtotar -eq 1 ] ; then
147                echo "====> DELEGATE to tar command with option --switchtotar"
148                echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
149                mv $set $PATH_SIMU/output_tar
150                continue
151        fi
152
153        #-----------------------------------
154        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
155        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
156                printf "====> "
157                tail -1 $statusfile
158                continue
159        fi
160
161        #-----------------------------------
162        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
163        nbtries=`wc -l $statusfile | cut -d' ' -f1`             
164        echo "Nb of tries: $nbtries"
165
166        #-----------------------------------
167        # Set output file name from list name
168
169        fileout1=$( basename $set )
170        fileout=${fileout1%.list}
171
172        filefirst=`head -n 1 $set`
173        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
174
175        outputfile=$diroutputfile/${fileout}
176        mkdir -p $diroutputfile
177        echo "Output file to write: $outputfile"
178
179        #-----------------------------------
180        # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
181        extratar=0
182        find_varstoexclude $set
183        if [ $nbvars -eq 0 ] ; then
184                varstoexclude="t_inst_.*"
185        else
186                # There are others variables than 't_inst_.*' so an extra tar file will be created
187                varstoexclude="t_inst_.*,$varstoexcludefiltered"
188                extratar=1
189                # Copy to output_tar directory
190                echo "====> COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*"
191                echo "COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" >> $statusfile
192                cp $set $PATH_SIMU/output_tar
193        fi
194
195        #-----------------------------------
196        # Try 2 times before fire "FAILED"
197        ncrcatcmd=ncrcat
198        ncrcatoptions="--md5_digest"
199#       ncrcatoptions=""
200
201        startTime=$( getDateMilliSeconds ) # gpdebug : time
202        cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
203        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
204       
205        # resultCmd=$( gives_0_or_1 )
206        # resultCmd=1
207        # if [ $resultCmd -eq 0 ] ; then
208        if [ $? -eq 0 ] ; then
209                sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
210                datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
211                echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
212                echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
213                echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
214        else
215                # Add --md5_digest
216                startTime=$( getDateMilliSeconds ) # gpdebug : time
217                cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
218                meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
219                # resultCmd=$( gives_0_or_1 )
220                # resultCmd=1
221                # if [ $resultCmd -eq 0 ] ; then
222                if [ $? -eq 0 ] ; then
223                        sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
224                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
225                        echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
226                        echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
227                        echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
228                else
229                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
230                        echo "====> FAILED at try #$nbtries" 
231                        echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
232                        if [ $nbtries -ge $maxtries ] ; then
233                                echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
234                                echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
235                                cp $set $PATH_SIMU/output_tar # gpdebug : a retablir
236                        fi
237                fi
238        fi
239       
240done   
241
242#------------------------------------------------------
243
244for set in $( ls $PATH_SIMU/output_tar/*list ) ; do
245
246        #-----------------------------------
247        echo "#-------------------"
248        echo "Set: $set"
249        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
250        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
251        statusfile=${set%%.list}.status
252        logfile=${set%%.list}_${datestr}.log
253
254        #-----------------------------------
255        # Create status file if not exists
256        if [ ! -e $statusfile ] ; then
257                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
258        fi
259
260        #-----------------------------------
261        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
262        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
263                printf "====> "
264                tail -1 $statusfile
265                continue
266        fi
267
268        #-----------------------------------
269        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
270        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
271        # echo "Nb of tries: $nbtries"
272
273        #-----------------------------------
274        # Set output file name from list name
275
276        fileout1=$( basename $set )
277        fileout=${fileout1%.list}.tar
278       
279        filefirst=`head -n 1 $set`
280        dirinputfile=`dirname $filefirst`
281        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` 
282
283        outputfile=$diroutputfile/${fileout}
284        mkdir -p $diroutputfile
285        echo "Output file to write: $outputfile"
286
287        #-----------------------------------
288        # Try 1 time before fire "FAILED"
289        # Waiting for CCRT command
290        tarcmd=tar
291        taroptions="--format=posix -W -cf"
292        #taroptions="--format=posix -cf"
293        cd $dirinputfile
294
295        set_local=$set".local"
296        for file in $( cat $set) ; do
297            basename $file >> $set".local" 
298        done
299           
300#       #-----------------------------------
301        startTime=$( getDateMilliSeconds ) # gpdebug : time
302        $tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1 # gpdebug : a retablir
303        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
304       
305        # resultCmd=$( gives_0_or_1 )
306        # if [ $resultCmd -eq 0 ] ; then
307        if [ $? -eq 0 ] ; then
308            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
309            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
310            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
311            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
312            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
313        else
314            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
315            echo "====> FAILED at try #$nbtries" 
316            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
317
318        fi
319        rm -f $set".local" 
320        #-----------------------------------
321       
322done   
323
324
325#------------------------------------------------------
326
327for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do
328
329        #-----------------------------------
330        echo "#-------------------"
331        echo "Set: $set"
332        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
333        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
334        statusfile=${set%%.list}.status
335        logfile=${set%%.list}_${datestr}.log
336
337        #-----------------------------------
338        # Create status file if not exists
339        if [ ! -e $statusfile ] ; then
340                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
341        fi
342
343        #-----------------------------------
344        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
345        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
346                printf "====> "
347                tail -1 $statusfile
348                continue
349        fi
350
351        #-----------------------------------
352        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
353        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
354        # echo "Nb of tries: $nbtries"
355
356        #-----------------------------------
357        # Set output file name from list name
358        dirin1=$PATH_SIMU/RESTART
359        dirin2=${dirin1##${IGCM_DEM}}
360        dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
361        diroutputfile=${OUTPUT_STORE}${dirin3}
362        fileout1=$( basename $set )
363        fileout=${fileout1%.list}.tar
364        outputfile=$diroutputfile/${fileout}
365        mkdir -p $diroutputfile
366        echo "Output file to write: $outputfile"
367
368        #-----------------------------------
369        # Try 1 time before fire "FAILED"
370        # Waiting for CCRT command
371        tarcmd=tar
372        taroptions="--format=posix -W -cf"
373#       taroptions="--format=posix -cf"
374        cd $dirin1
375#       ls
376       
377        startTime=$( getDateMilliSeconds ) # gpdebug : time
378        $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
379        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
380       
381        # resultCmd=$( gives_0_or_1 )
382        # if [ $resultCmd -eq 0 ] ; then
383        if [ $? -eq 0 ] ; then
384            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
385            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
386            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
387            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
388            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
389        else
390            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
391            echo "====> FAILED at try #$nbtries" 
392            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
393
394        fi
395       
396        #-----------------------------------
397       
398done   
399
400#------------------------------------------------------
401for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do
402
403        #-----------------------------------
404        echo "#-------------------"
405        echo "Set: $set"
406        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
407        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
408        statusfile=${set%%.list}.status
409        logfile=${set%%.list}_${datestr}.log
410
411        #-----------------------------------
412        # Create status file if not exists
413        if [ ! -e $statusfile ] ; then
414                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
415        fi
416
417        #-----------------------------------
418        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
419        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
420                printf "====> "
421                tail -1 $statusfile
422                continue
423        fi
424
425        #-----------------------------------
426        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
427        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
428        # echo "Nb of tries: $nbtries"
429
430        #-----------------------------------
431        # Set output file name from list name
432        dirin1=$PATH_SIMU/DEBUG
433        dirin2=${dirin1##${IGCM_DEM}}
434        dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
435        diroutputfile=${OUTPUT_STORE}${dirin3}
436        fileout1=$( basename $set )
437        fileout=${fileout1%.list}.tar
438        outputfile=$diroutputfile/${fileout}
439        mkdir -p $diroutputfile
440        echo "Output file to write: $outputfile"
441
442        #-----------------------------------
443        # Try 1 time before fire "FAILED"
444        # Waiting for CCRT command
445        tarcmd=tar
446        taroptions="--format=posix -W -cf"
447#       taroptions="--format=posix -cf"
448        cd $dirin1
449#       ls
450
451        startTime=$( getDateMilliSeconds ) # gpdebug : time
452        $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
453        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
454       
455        # resultCmd=$( gives_0_or_1 )
456        # if [ $resultCmd -eq 0 ] ; then
457        if [ $? -eq 0 ] ; then
458            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
459            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
460            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
461            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
462            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
463        else
464            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
465            echo "====> FAILED at try #$nbtries" 
466            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
467
468        fi
469       
470        #-----------------------------------
471       
472done   
473
474#------------------------------------------------------
475for set in $( ls $PATH_SIMU/store_cp/*list ) ; do
476
477        #-----------------------------------
478        echo "#-------------------"
479        echo "Set: $set"
480        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
481        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
482        statusfile=${set%%.list}.status
483        logfile=${set%%.list}_${datestr}.log
484
485        #-----------------------------------
486        # Create status file if not exists
487        if [ ! -e $statusfile ] ; then
488                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
489        fi
490
491        #-----------------------------------
492        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
493        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
494                printf "====> "
495                tail -1 $statusfile
496                continue
497        fi
498
499        #-----------------------------------
500        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
501        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
502        # echo "Nb of tries: $nbtries"
503
504        #-----------------------------------
505        # Set output file name from first and last files from the current set
506        for file in $( cat $set) ; do
507         
508            diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout%"`
509            mkdir -p $diroutputfile
510            echo "Output file to write: $diroutputfile/$( basename $file )"
511           
512        #-----------------------------------
513        # Try 1 time before fire "FAILED"
514        # Waiting for CCRT command
515            cpcmd="cp "
516            cpoptions="-rf"
517            startTime=$( getDateMilliSeconds ) # gpdebug : time
518            $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir
519            meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
520        done
521       
522        # resultCmd=$( gives_0_or_1 )
523        # if [ $resultCmd -eq 0 ] ; then
524        if [ $? -eq 0 ] ; then
525            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
526            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
527            echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
528            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
529            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
530        else
531            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
532            echo "====> FAILED at try #$nbtries" 
533            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
534
535        fi
536       
537        #-----------------------------------
538       
539done   
540
541#------------------------------------------------------
542for set in $( ls $PATH_SIMU/work_cp/*list ) ; do
543
544        #-----------------------------------
545        echo "#-------------------"
546        echo "Set: $set"
547        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
548        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
549        statusfile=${set%%.list}.status
550        logfile=${set%%.list}_${datestr}.log
551
552        #-----------------------------------
553        # Create status file if not exists
554        if [ ! -e $statusfile ] ; then
555                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
556        fi
557
558        #-----------------------------------
559        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
560        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
561                printf "====> "
562                tail -1 $statusfile
563                continue
564        fi
565
566        #-----------------------------------
567        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
568        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
569        # echo "Nb of tries: $nbtries"
570
571        #-----------------------------------
572        # Set output file name from first and last files from the current set
573        for file in $( cat $set) ; do
574         
575            diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout_work%"`
576            mkdir -p $diroutputfile
577            echo "Output file to write: $diroutputfile/$( basename $file )"
578           
579        #-----------------------------------
580        # Try 1 time before fire "FAILED"
581        # Waiting for CCRT command
582            cpcmd="cp "
583            cpoptions="-rf"
584            startTime=$( getDateMilliSeconds ) # gpdebug : time
585            $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1  # gpdebug : a retablir
586            meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : timeS
587        done
588       
589        # resultCmd=$( gives_0_or_1 )
590        # if [ $resultCmd -eq 0 ] ; then
591        if [ $? -eq 0 ] ; then
592            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
593            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
594            echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
595            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
596            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
597        else
598            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
599            echo "====> FAILED at try #$nbtries" 
600            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
601
602        fi
603       
604        #-----------------------------------
605       
606done   
Note: See TracBrowser for help on using the repository browser.