source: TOOLS/PACK_IPSL/enlarge_my_files.sh @ 1840

Last change on this file since 1840 was 1840, checked in by acosce, 12 years ago

Last commit was a mistake

  • Property svn:executable set to *
File size: 20.7 KB
Line 
1#!/bin/bash
2#------------------------------------------------------
3switchtotar=0
4maxtries=3
5#---------------------------------------------
6while [ $# -ne 0 ]
7do
8        case $1 in
9       --switchtotar)
10                switchtotar=1
11                shift 1 ;;
12       --maxtries)
13                maxtries=$2
14                shift 2 ;;
15       -h|--help|-help)
16                echo "----------------------------------------------------------------------------"
17                echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
18                echo
19                echo "Description:"
20                echo "   Pass a ncrcat, tar or cp command on files from prepared sets of files."
21                echo "   For ncrcat command 2 shots will be done at each try".
22                echo 
23                echo "   A status file and a log file will be created in output_ncrcat directory."
24                echo "   Copy a set to output_tar directory for later processing after N tries."
25                echo
26                echo "Arguments:"
27                echo "   dirin is the input directory where files to concatenate will be read"
28                echo "   dirout is the output directory where concatenated files will be written"
29                echo
30                echo "   The following directory must exist where the command enlarge_my_files is run:"
31                echo "       enlarge_my_files"
32                echo "                       |-- output_ncrcat"
33                echo "                       |   |-- set_000001.list"
34                echo "                       |   |-- set_000002.list"
35                echo "                       |   |-- set_000003.list"
36                echo "                       |   |-- set_000004.list"
37                echo "                       |   \`-- set_000005.list"
38                echo "                       \`-- output_tar"
39                echo
40                echo "Options:"
41                echo "   -h, --help, -help"
42                echo "       Print this manual."
43                echo "   --switchtotar"
44                echo "       Abort ncrcat operation and switch to tar command"
45                echo "   --maxtries N"
46                echo "       Maximum number of tries before switching to tar command" 
47                echo "       Default is 3"
48                echo
49                echo "----------------------------------------------------------------------------"
50                exit ;;
51        -*)
52                `dirname $0`/enlarge_my_files.sh -h
53                exit ;;
54        *)
55                break ;;
56        esac
57done
58
59if [ $# -lt 2 ] ; then
60        `dirname $0`/enlarge_my_files.sh -h
61        exit
62fi
63
64if [[ ! -d $PATH_SIMU/output_ncrcat || ! -d $PATH_SIMU/output_tar || ! -d $PATH_SIMU/restart_tar || ! -d $PATH_SIMU/store_cp || ! -d $PATH_SIMU/work_cp || ! -d $PATH_SIMU/debug_tar || ! -d $PATH_SIMU/other_tar ]] ; then
65        `dirname $0`/enlarge_my_files.sh -h
66        exit
67fi
68
69#------------------------------------------------------
70dirin=$1
71dirout=$2
72dirout_work=$3
73
74#------------------------------------------------------
75function find_varstoexclude {
76nbfile=0
77for file in `cat $1` ; do
78    ncdump -h ${file} | gawk '{if (match($0, /(byte|char|short|int|float|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
79    let nbfile=nbfile+1
80done
81
82varstoexclude=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d','`
83varstoexcludefiltered=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' | paste -s -d','`
84nbvars=`echo $varstoexcludefiltered | gawk -F',' '{print NF}'`
85
86rm -f tmp_$$.txt
87
88return 0
89}
90
91# gpdebug : pour test showPackProgress.sh *************************
92export RANDOM=$$
93
94function gives_0_or_1
95{
96   bit=-1
97   let "bit = RANDOM % 2"
98   echo $bit
99}
100
101resultCmd=
102# gpdebug : ****************************** fin ********************
103
104
105#********** gpdebug : mesure d'intervalles de temps ***************
106function getDateMilliSeconds
107{
108   test=`date +%s%N`
109   testnum=${#test}
110   millisec=${test:0:$testnum-6}
111   echo "$millisec"
112}
113
114function getTimeDiffSeconds
115{
116   startTimeMilliSec=$1
117   endTimeMilliSec=$( getDateMilliSeconds )
118   
119   diffMilliSec=$(( $endTimeMilliSec - $startTimeMilliSec ))
120   
121   diffSeconds=$(awk 'BEGIN { print '$diffMilliSec'/'1000' }')
122   echo "$diffSeconds" 
123
124}
125# gpdebug : ****************************** fin ********************
126#------------------------------------------------------
127for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do
128
129        #-----------------------------------
130        echo "#-------------------"
131        echo "Set: $set"
132        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
133        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
134        statusfile=${set%%.list}.status
135        logfile=${set%%.list}_${datestr}.log
136
137        #-----------------------------------
138        # Create status file if not exists
139        if [ ! -e $statusfile ] ; then
140                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
141        fi
142
143        #-----------------------------------
144        # Switch to tar command
145        if [ $switchtotar -eq 1 ] ; then
146                echo "====> DELEGATE to tar command with option --switchtotar"
147                echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
148                mv $set $PATH_SIMU/output_tar
149                continue
150        fi
151
152        #-----------------------------------
153        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
154        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
155                printf "====> "
156                tail -1 $statusfile
157                continue
158        fi
159
160        #-----------------------------------
161        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
162        nbtries=`wc -l $statusfile | cut -d' ' -f1`             
163        echo "Nb of tries: $nbtries"
164
165        #-----------------------------------
166        # Set output file name from list name
167
168        fileout1=$( basename $set )
169        fileout=${fileout1%.list}
170
171        filefirst=`head -n 1 $set`
172        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
173
174        outputfile=$diroutputfile/${fileout}
175        mkdir -p $diroutputfile
176        echo "Output file to write: $outputfile"
177
178        #-----------------------------------
179        # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
180        extratar=0
181        find_varstoexclude $set
182        if [ $nbvars -eq 0 ] ; then
183                varstoexclude="t_inst_.*"
184        else
185                # There are others variables than 't_inst_.*' so an extra tar file will be created
186                varstoexclude="t_inst_.*,$varstoexcludefiltered"
187                extratar=1
188                # Copy to output_tar directory
189                cp $set $PATH_SIMU/output_tar
190        fi
191
192        #-----------------------------------
193        # Try 2 times before fire "FAILED"
194        ncrcatcmd=ncrcat
195        ncrcatoptions="--md5_digest"
196#       ncrcatoptions=""
197
198        startTime=$( getDateMilliSeconds ) # gpdebug : time
199        cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
200        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
201       
202        # resultCmd=$( gives_0_or_1 )
203        # resultCmd=1
204        # if [ $resultCmd -eq 0 ] ; then
205        if [ $? -eq 0 ] ; then
206                sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
207                datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
208                echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
209                echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
210                echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
211        else
212                # Add --md5_digest
213                startTime=$( getDateMilliSeconds ) # gpdebug : time
214                cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
215                meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
216                # resultCmd=$( gives_0_or_1 )
217                # resultCmd=1
218                # if [ $resultCmd -eq 0 ] ; then
219                if [ $? -eq 0 ] ; then
220                        sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
221                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
222                        echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
223                        echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
224                        echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
225                else
226                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
227                        echo "====> FAILED at try #$nbtries" 
228                        echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
229                        if [ $nbtries -ge $maxtries ] ; then
230                                echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
231                                echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
232                                cp $set $PATH_SIMU/output_tar # gpdebug : a retablir
233                        fi
234                fi
235        fi
236       
237done   
238
239#------------------------------------------------------
240
241for set in $( ls $PATH_SIMU/output_tar/*list ) ; do
242
243        #-----------------------------------
244        echo "#-------------------"
245        echo "Set: $set"
246        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
247        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
248        statusfile=${set%%.list}.status
249        logfile=${set%%.list}_${datestr}.log
250
251        #-----------------------------------
252        # Create status file if not exists
253        if [ ! -e $statusfile ] ; then
254                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
255        fi
256
257        #-----------------------------------
258        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
259        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
260                printf "====> "
261                tail -1 $statusfile
262                continue
263        fi
264
265        #-----------------------------------
266        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
267        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
268        # echo "Nb of tries: $nbtries"
269
270        #-----------------------------------
271        # Set output file name from list name
272
273        fileout1=$( basename $set )
274        fileout=${fileout1%.list}.tar
275       
276        filefirst=`head -n 1 $set`
277        dirinputfile=`dirname $filefirst`
278        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` 
279
280        outputfile=$diroutputfile/${fileout}
281        mkdir -p $diroutputfile
282        echo "Output file to write: $outputfile"
283
284        #-----------------------------------
285        # Try 1 time before fire "FAILED"
286        # Waiting for CCRT command
287        tarcmd=tar
288        taroptions="--format=posix -W -cf"
289        #taroptions="--format=posix -cf"
290        cd $dirinputfile
291
292        set_local=$set".local"
293        for file in $( cat $set) ; do
294            basename $file >> $set".local" 
295        done
296           
297#       #-----------------------------------
298        startTime=$( getDateMilliSeconds ) # gpdebug : time
299        $tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1 # gpdebug : a retablir
300        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
301       
302        # resultCmd=$( gives_0_or_1 )
303        # if [ $resultCmd -eq 0 ] ; then
304        if [ $? -eq 0 ] ; then
305            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
306            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
307            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
308            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
309            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
310        else
311            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
312            echo "====> FAILED at try #$nbtries" 
313            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
314
315        fi
316        rm -f $set".local" 
317        #-----------------------------------
318       
319done   
320
321
322#------------------------------------------------------
323
324for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do
325
326        #-----------------------------------
327        echo "#-------------------"
328        echo "Set: $set"
329        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
330        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
331        statusfile=${set%%.list}.status
332        logfile=${set%%.list}_${datestr}.log
333
334        #-----------------------------------
335        # Create status file if not exists
336        if [ ! -e $statusfile ] ; then
337                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
338        fi
339
340        #-----------------------------------
341        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
342        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
343                printf "====> "
344                tail -1 $statusfile
345                continue
346        fi
347
348        #-----------------------------------
349        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
350        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
351        # echo "Nb of tries: $nbtries"
352
353        #-----------------------------------
354        # Set output file name from list name
355        dirin1=$PATH_SIMU/RESTART
356        dirin2=${dirin1##${IGCM_DEM}}
357        dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
358        diroutputfile=${OUTPUT_STORE}${dirin3}
359        fileout1=$( basename $set )
360        fileout=${fileout1%.list}.tar
361        outputfile=$diroutputfile/${fileout}
362        mkdir -p $diroutputfile
363        echo "Output file to write: $outputfile"
364
365        #-----------------------------------
366        # Try 1 time before fire "FAILED"
367        # Waiting for CCRT command
368        tarcmd=tar
369        taroptions="--format=posix -W -cf"
370#       taroptions="--format=posix -cf"
371        cd $dirin1
372#       ls
373       
374        startTime=$( getDateMilliSeconds ) # gpdebug : time
375        $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
376        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
377       
378        # resultCmd=$( gives_0_or_1 )
379        # if [ $resultCmd -eq 0 ] ; then
380        if [ $? -eq 0 ] ; then
381            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
382            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
383            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
384            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
385            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
386        else
387            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
388            echo "====> FAILED at try #$nbtries" 
389            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
390
391        fi
392       
393        #-----------------------------------
394       
395done   
396
397#------------------------------------------------------
398for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do
399
400        #-----------------------------------
401        echo "#-------------------"
402        echo "Set: $set"
403        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
404        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
405        statusfile=${set%%.list}.status
406        logfile=${set%%.list}_${datestr}.log
407
408        #-----------------------------------
409        # Create status file if not exists
410        if [ ! -e $statusfile ] ; then
411                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
412        fi
413
414        #-----------------------------------
415        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
416        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
417                printf "====> "
418                tail -1 $statusfile
419                continue
420        fi
421
422        #-----------------------------------
423        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
424        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
425        # echo "Nb of tries: $nbtries"
426
427        #-----------------------------------
428        # Set output file name from list name
429        dirin1=$PATH_SIMU/DEBUG
430        dirin2=${dirin1##${IGCM_DEM}}
431        dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
432        diroutputfile=${OUTPUT_STORE}${dirin3}
433        fileout1=$( basename $set )
434        fileout=${fileout1%.list}.tar
435        outputfile=$diroutputfile/${fileout}
436        mkdir -p $diroutputfile
437        echo "Output file to write: $outputfile"
438
439        #-----------------------------------
440        # Try 1 time before fire "FAILED"
441        # Waiting for CCRT command
442        tarcmd=tar
443        taroptions="--format=posix -W -cf"
444#       taroptions="--format=posix -cf"
445        cd $dirin1
446#       ls
447
448        startTime=$( getDateMilliSeconds ) # gpdebug : time
449        $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
450        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
451       
452        # resultCmd=$( gives_0_or_1 )
453        # if [ $resultCmd -eq 0 ] ; then
454        if [ $? -eq 0 ] ; then
455            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
456            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
457            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
458            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
459            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
460        else
461            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
462            echo "====> FAILED at try #$nbtries" 
463            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
464
465        fi
466       
467        #-----------------------------------
468       
469done   
470
471#------------------------------------------------------
472for set in $( ls $PATH_SIMU/store_cp/*list ) ; do
473
474        #-----------------------------------
475        echo "#-------------------"
476        echo "Set: $set"
477        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
478        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
479        statusfile=${set%%.list}.status
480        logfile=${set%%.list}_${datestr}.log
481
482        #-----------------------------------
483        # Create status file if not exists
484        if [ ! -e $statusfile ] ; then
485                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
486        fi
487
488        #-----------------------------------
489        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
490        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
491                printf "====> "
492                tail -1 $statusfile
493                continue
494        fi
495
496        #-----------------------------------
497        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
498        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
499        # echo "Nb of tries: $nbtries"
500
501        #-----------------------------------
502        # Set output file name from first and last files from the current set
503        for file in $( cat $set) ; do
504         
505            diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout%"`
506            mkdir -p $diroutputfile
507            echo "Output file to write: $diroutputfile/$( basename $file )"
508           
509        #-----------------------------------
510        # Try 1 time before fire "FAILED"
511        # Waiting for CCRT command
512            cpcmd="cp "
513            cpoptions="-rf"
514            startTime=$( getDateMilliSeconds ) # gpdebug : time
515            $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir
516            meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
517        done
518       
519        # resultCmd=$( gives_0_or_1 )
520        # if [ $resultCmd -eq 0 ] ; then
521        if [ $? -eq 0 ] ; then
522            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
523            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
524            echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
525            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
526            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
527        else
528            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
529            echo "====> FAILED at try #$nbtries" 
530            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
531
532        fi
533       
534        #-----------------------------------
535       
536done   
537
538#------------------------------------------------------
539for set in $( ls $PATH_SIMU/work_cp/*list ) ; do
540
541        #-----------------------------------
542        echo "#-------------------"
543        echo "Set: $set"
544        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
545        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
546        statusfile=${set%%.list}.status
547        logfile=${set%%.list}_${datestr}.log
548
549        #-----------------------------------
550        # Create status file if not exists
551        if [ ! -e $statusfile ] ; then
552                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
553        fi
554
555        #-----------------------------------
556        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
557        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
558                printf "====> "
559                tail -1 $statusfile
560                continue
561        fi
562
563        #-----------------------------------
564        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
565        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
566        # echo "Nb of tries: $nbtries"
567
568        #-----------------------------------
569        # Set output file name from first and last files from the current set
570        for file in $( cat $set) ; do
571         
572            diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout_work%"`
573            mkdir -p $diroutputfile
574            echo "Output file to write: $diroutputfile/$( basename $file )"
575           
576        #-----------------------------------
577        # Try 1 time before fire "FAILED"
578        # Waiting for CCRT command
579            cpcmd="cp "
580            cpoptions="-rf"
581            startTime=$( getDateMilliSeconds ) # gpdebug : time
582            $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1  # gpdebug : a retablir
583            meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : timeS
584        done
585       
586        # resultCmd=$( gives_0_or_1 )
587        # if [ $resultCmd -eq 0 ] ; then
588        if [ $? -eq 0 ] ; then
589            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
590            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
591            echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
592            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
593            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
594        else
595            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
596            echo "====> FAILED at try #$nbtries" 
597            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
598
599        fi
600       
601        #-----------------------------------
602       
603done   
Note: See TracBrowser for help on using the repository browser.