source: TOOLS/PACK_IPSL/enlarge_my_files.sh @ 1853

Last change on this file since 1853 was 1853, checked in by gpincka, 10 years ago

principalement : enchainement du launcher parallele + gestion reprise du code parallele

  • Property svn:executable set to *
File size: 23.6 KB
Line 
1#!/bin/bash
2#------------------------------------------------------
3switchtotar=0
4maxtries=3
5#---------------------------------------------
6while [ $# -ne 0 ]
7do
8        case $1 in
9       --switchtotar)
10                switchtotar=1
11                shift 1 ;;
12       --maxtries)
13                maxtries=$2
14                shift 2 ;;
15       -h|--help|-help)
16                echo "----------------------------------------------------------------------------"
17                echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
18                echo
19                echo "Description:"
20                echo "   Pass a ncrcat, tar or cp command on files from prepared sets of files."
21                echo "   For ncrcat command 2 shots will be done at each try".
22                echo 
23                echo "   A status file and a log file will be created in output_ncrcat directory."
24                echo "   Copy a set to output_tar directory for later processing after N tries."
25                echo
26                echo "Arguments:"
27                echo "   dirin is the input directory where files to concatenate will be read"
28                echo "   dirout is the output directory where concatenated files will be written"
29                echo
30                echo "   The following directory must exist where the command enlarge_my_files is run:"
31                echo "       enlarge_my_files"
32                echo "                       |-- output_ncrcat"
33                echo "                       |   |-- set_000001.list"
34                echo "                       |   |-- set_000002.list"
35                echo "                       |   |-- set_000003.list"
36                echo "                       |   |-- set_000004.list"
37                echo "                       |   \`-- set_000005.list"
38                echo "                       \`-- output_tar"
39                echo
40                echo "Options:"
41                echo "   -h, --help, -help"
42                echo "       Print this manual."
43                echo "   --switchtotar"
44                echo "       Abort ncrcat operation and switch to tar command"
45                echo "   --maxtries N"
46                echo "       Maximum number of tries before switching to tar command" 
47                echo "       Default is 3"
48                echo
49                echo "----------------------------------------------------------------------------"
50                exit ;;
51        -*)
52                `dirname $0`/enlarge_my_files.sh -h
53                exit ;;
54        *)
55                break ;;
56        esac
57done
58
59if [ $# -lt 2 ] ; then
60        `dirname $0`/enlarge_my_files.sh -h
61        exit
62fi
63
64if ( \
65     [ ! -d $PATH_SIMU/output_ncrcat ] || \
66     [ ! -d $PATH_SIMU/output_tar ]    || \
67     [ ! -d $PATH_SIMU/restart_tar ]   || \
68     [ ! -d $PATH_SIMU/store_cp ]      || \
69     [ ! -d $PATH_SIMU/work_cp ]       || \
70     [ ! -d $PATH_SIMU/debug_tar ]     || \
71     [ ! -d $PATH_SIMU/other_tar ] \
72   ) \
73   && \
74   [ ! -e $PATH_SIMU/tar_full_simul.list ]
75then
76        `dirname $0`/enlarge_my_files.sh -h
77        exit
78fi
79
80#------------------------------------------------------
81dirin=$1
82dirout=$2
83dirout_work=$3
84
85#------------------------------------------------------
86function find_varstoexclude {
87nbfile=0
88for file in `cat $1` ; do
89 #   ncdump -h ${file} | gawk '{if (match($0, /(byte|char|short|int|float|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
90    ncdump -h ${file} | grep -Ew "byte|char|short|int|float|double"  | awk -F\( '{split($1,a," ") ; split($2,b,")") ; if (a[2] != b[1]) print a[2]}'  >> tmp_$$.txt
91    let nbfile=nbfile+1
92done
93
94varstoexclude=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d','`
95varstoexcludefiltered=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' | paste -s -d','`
96nbvars=`echo $varstoexcludefiltered | gawk -F',' '{print NF}'`
97
98rm -f tmp_$$.txt
99
100return 0
101}
102
103# gpdebug : pour test showPackProgress.sh *************************
104export RANDOM=$$
105
106function gives_0_or_1
107{
108   bit=-1
109   let "bit = RANDOM % 2"
110   echo $bit
111}
112
113resultCmd=
114# gpdebug : ****************************** fin ********************
115
116source ${EXE_DIR}/DEM_utilities.sh # gpdebug : pour trouver les fonctions de mesure de temps
117
118#------------------------------------------------------
119for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do
120
121        #-----------------------------------
122        echo "#-------------------"
123        echo "Set: $set"
124        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
125        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
126        statusfile=${set%%.list}.status
127        logfile=${set%%.list}_${datestr}.log
128
129        #-----------------------------------
130        # Create status file if not exists
131        if [ ! -e $statusfile ] ; then
132                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
133        fi
134
135        #-----------------------------------
136        # Switch to tar command
137        if [ $switchtotar -eq 1 ] ; then
138                echo "====> DELEGATE to tar command with option --switchtotar"
139                echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
140                mv $set $PATH_SIMU/output_tar
141                continue
142        fi
143
144        #-----------------------------------
145        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
146        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
147                printf "====> "
148                tail -1 $statusfile
149                continue
150        fi
151
152        #-----------------------------------
153        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
154        nbtries=`wc -l $statusfile | cut -d' ' -f1`             
155        echo "Nb of tries: $nbtries"
156
157        #-----------------------------------
158        # Set output file name from list name
159
160        fileout1=$( basename $set )
161        fileout=${fileout1%.list}
162
163        filefirst=`head -n 1 $set`
164        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
165
166        outputfile=$diroutputfile/${fileout}
167        mkdir -p $diroutputfile
168        echo "Output file to write: $outputfile"
169
170        #-----------------------------------
171        # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
172        extratar=0
173        find_varstoexclude $set
174        if [ $nbvars -eq 0 ] ; then
175                varstoexclude="t_inst_.*"
176        else
177                # There are others variables than 't_inst_.*' so an extra tar file will be created
178                varstoexclude="t_inst_.*,$varstoexcludefiltered"
179                extratar=1
180                # Copy to output_tar directory
181                echo "====> COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*"
182                echo "COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" >> $statusfile
183                cp $set $PATH_SIMU/output_tar
184        fi
185
186        #-----------------------------------
187        # Try 2 times before fire "FAILED"
188        ncrcatcmd=ncrcat
189        ncrcatoptions="--md5_digest"
190#       ncrcatoptions=""
191
192        startTime=$( getDateMilliSeconds ) # gpdebug : time
193        resCmd=0
194        cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
195        resCmd=$?
196        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
197       
198        # resultCmd=$( gives_0_or_1 )
199        # resultCmd=1
200        # if [ $resultCmd -eq 0 ] ; then
201        if [ $resCmd -eq 0 ] ; then
202                sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
203                datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
204                echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
205                echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
206                echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
207        else
208                # Add --md5_digest
209                rm -f $outputfile*ncrcat.tmp
210                startTime=$( getDateMilliSeconds ) # gpdebug : time
211                resCmd=0
212                cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
213                resCmd=$?
214                meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
215                # resultCmd=$( gives_0_or_1 )
216                # resultCmd=1
217                # if [ $resultCmd -eq 0 ] ; then
218                if [ $resCmd -eq 0 ] ; then
219                        sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
220                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
221                        echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
222                        echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
223                        echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
224                else
225                        rm -f $outputfile*ncrcat.tmp
226                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
227                        echo "====> FAILED at try #$nbtries" 
228                        echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
229                        if [ $nbtries -ge $maxtries ] ; then
230                                echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
231                                echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
232                                cp $set $PATH_SIMU/output_tar # gpdebug : a retablir
233                        fi
234                fi
235        fi
236       
237done   
238
239#------------------------------------------------------
240
241for set in $( ls $PATH_SIMU/output_tar/*list ) ; do
242
243        #-----------------------------------
244        echo "#-------------------"
245        echo "Set: $set"
246        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
247        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
248        statusfile=${set%%.list}.status
249        logfile=${set%%.list}_${datestr}.log
250
251        #-----------------------------------
252        # Create status file if not exists
253        if [ ! -e $statusfile ] ; then
254                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
255        fi
256
257        #-----------------------------------
258        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
259        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
260                printf "====> "
261                tail -1 $statusfile
262                continue
263        fi
264
265        #-----------------------------------
266        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
267        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
268        # echo "Nb of tries: $nbtries"
269
270        #-----------------------------------
271        # Set output file name from list name
272
273        fileout1=$( basename $set )
274        fileout=${fileout1%.list}.tar
275       
276        filefirst=`head -n 1 $set`
277        dirinputfile=`dirname $filefirst`
278        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` 
279
280        outputfile=$diroutputfile/${fileout}
281        mkdir -p $diroutputfile
282        echo "Output file to write: $outputfile"
283
284        #-----------------------------------
285        # Try 1 time before fire "FAILED"
286        # Waiting for CCRT command
287        tarcmd=tar
288        taroptions="--format=posix -W -cf"
289        #taroptions="--format=posix -cf"
290        cd $dirinputfile
291
292        set_local=$set".local"
293        for file in $( cat $set) ; do
294            basename $file >> $set".local" 
295        done
296           
297#       #-----------------------------------
298        startTime=$( getDateMilliSeconds ) # gpdebug : time
299        resCmd=0
300        $tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1 # gpdebug : a retablir
301        resCmd=$?
302        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
303       
304        # resultCmd=$( gives_0_or_1 )
305        # if [ $resultCmd -eq 0 ] ; then
306        if [ $resCmd -eq 0 ] ; then
307            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
308            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
309            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
310            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
311            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
312        else
313            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
314            echo "====> FAILED at try #$nbtries" 
315            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
316
317        fi
318        rm -f $set".local" 
319        #-----------------------------------
320       
321done   
322
323
324#------------------------------------------------------
325
326for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do
327        #-----------------------------------
328        echo "#-------------------"
329        echo "Set: $set"
330        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
331        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
332        statusfile=${set%%.list}.status
333        logfile=${set%%.list}_${datestr}.log
334
335        #-----------------------------------
336        # Create status file if not exists
337        if [ ! -e $statusfile ] ; then
338                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
339        fi
340
341        #-----------------------------------
342        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
343        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
344                printf "====> "
345                tail -1 $statusfile
346                continue
347        fi
348
349        #-----------------------------------
350        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
351        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
352        # echo "Nb of tries: $nbtries"
353
354        #-----------------------------------
355        # Set output file name from list name
356        dirin1=$PATH_SIMU/RESTART
357        dirin2=${dirin1##${IGCM_DEM}}
358        dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
359        diroutputfile=${OUTPUT_STORE}${dirin3}
360        fileout1=$( basename $set )
361        fileout=${fileout1%.list}.tar
362        outputfile=$diroutputfile/${fileout}
363        mkdir -p $diroutputfile
364        echo "Output file to write: $outputfile"
365
366        #-----------------------------------
367        # Try 1 time before fire "FAILED"
368        # Waiting for CCRT command
369        tarcmd=tar
370        taroptions="--format=posix -W -cf"
371#       taroptions="--format=posix -cf"
372        cd $dirin1
373#       ls
374       
375        startTime=$( getDateMilliSeconds ) # gpdebug : time
376        resCmd=0
377        $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
378        resCmd=$?
379        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
380       
381        # resultCmd=$( gives_0_or_1 )
382        # if [ $resultCmd -eq 0 ] ; then
383        if [ $resCmd -eq 0 ] ; then
384            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
385            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
386            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
387            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
388            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
389        else
390            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
391            echo "====> FAILED at try #$nbtries" 
392            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
393
394        fi
395       
396        #-----------------------------------
397       
398done   
399
400#------------------------------------------------------
401for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do
402
403        #-----------------------------------
404        echo "#-------------------"
405        echo "Set: $set"
406        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
407        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
408        statusfile=${set%%.list}.status
409        logfile=${set%%.list}_${datestr}.log
410
411        #-----------------------------------
412        # Create status file if not exists
413        if [ ! -e $statusfile ] ; then
414                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
415        fi
416
417        #-----------------------------------
418        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
419        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
420                printf "====> "
421                tail -1 $statusfile
422                continue
423        fi
424
425        #-----------------------------------
426        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
427        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
428        # echo "Nb of tries: $nbtries"
429
430        #-----------------------------------
431        # Set output file name from list name
432        dirin1=$PATH_SIMU/DEBUG
433        dirin2=${dirin1##${IGCM_DEM}}
434        dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
435        diroutputfile=${OUTPUT_STORE}${dirin3}
436        fileout1=$( basename $set )
437        fileout=${fileout1%.list}.tar
438        outputfile=$diroutputfile/${fileout}
439        mkdir -p $diroutputfile
440        echo "Output file to write: $outputfile"
441
442        #-----------------------------------
443        # Try 1 time before fire "FAILED"
444        # Waiting for CCRT command
445        tarcmd=tar
446        taroptions="--format=posix -W -cf"
447#       taroptions="--format=posix -cf"
448        cd $dirin1
449#       ls
450
451        startTime=$( getDateMilliSeconds ) # gpdebug : time
452        resCmd=0
453        $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
454        resCmd=$?
455        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
456       
457        # resultCmd=$( gives_0_or_1 )
458        # if [ $resultCmd -eq 0 ] ; then
459        if [ $resCmd -eq 0 ] ; then
460            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
461            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
462            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
463            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
464            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
465        else
466            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
467            echo "====> FAILED at try #$nbtries" 
468            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
469
470        fi
471       
472        #-----------------------------------
473       
474done   
475
476#------------------------------------------------------
477for set in $( ls $PATH_SIMU/store_cp/*list ) ; do
478
479        #-----------------------------------
480        echo "#-------------------"
481        echo "Set: $set"
482        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
483        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
484        statusfile=${set%%.list}.status
485        logfile=${set%%.list}_${datestr}.log
486
487        #-----------------------------------
488        # Create status file if not exists
489        if [ ! -e $statusfile ] ; then
490                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
491        fi
492
493        #-----------------------------------
494        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
495        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
496                printf "====> "
497                tail -1 $statusfile
498                continue
499        fi
500
501        #-----------------------------------
502        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
503        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
504        # echo "Nb of tries: $nbtries"
505
506        #-----------------------------------
507        # Set output file name from first and last files from the current set
508        for file in $( cat $set) ; do
509         
510            diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout%"`
511            mkdir -p $diroutputfile
512            echo "Output file to write: $diroutputfile/$( basename $file )"
513           
514        #-----------------------------------
515        # Try 1 time before fire "FAILED"
516        # Waiting for CCRT command
517            cpcmd="cp "
518            cpoptions="-rf"
519            startTime=$( getDateMilliSeconds ) # gpdebug : time
520            resCmd=0
521            $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir
522            resCmd=$?
523            meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
524        done
525       
526        # resultCmd=$( gives_0_or_1 )
527        # if [ $resultCmd -eq 0 ] ; then
528        if [ $resCmd -eq 0 ] ; then
529            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
530            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
531            echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
532            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
533            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
534        else
535            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
536            echo "====> FAILED at try #$nbtries" 
537            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
538
539        fi
540       
541        #-----------------------------------
542       
543done   
544
545#------------------------------------------------------
546for set in $( ls $PATH_SIMU/work_cp/*list ) ; do
547
548        #-----------------------------------
549        echo "#-------------------"
550        echo "Set: $set"
551        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
552        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
553        statusfile=${set%%.list}.status
554        logfile=${set%%.list}_${datestr}.log
555
556        #-----------------------------------
557        # Create status file if not exists
558        if [ ! -e $statusfile ] ; then
559                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
560        fi
561
562        #-----------------------------------
563        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
564        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
565                printf "====> "
566                tail -1 $statusfile
567                continue
568        fi
569
570        #-----------------------------------
571        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
572        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
573        # echo "Nb of tries: $nbtries"
574
575        #-----------------------------------
576        # Set output file name from first and last files from the current set
577        for file in $( cat $set) ; do
578         
579            diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout_work%"`
580            mkdir -p $diroutputfile
581            echo "Output file to write: $diroutputfile/$( basename $file )"
582           
583        #-----------------------------------
584        # Try 1 time before fire "FAILED"
585        # Waiting for CCRT command
586            cpcmd="cp "
587            cpoptions="-rf"
588            startTime=$( getDateMilliSeconds ) # gpdebug : time
589            resCmd=0
590            $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1  # gpdebug : a retablir
591            resCmd=$?
592            meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : timeS
593        done
594       
595        # resultCmd=$( gives_0_or_1 )
596        # if [ $resultCmd -eq 0 ] ; then
597        if [ $resCmd -eq 0 ] ; then
598            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
599            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
600            echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
601            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
602            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
603        else
604            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
605            echo "====> FAILED at try #$nbtries" 
606            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
607
608        fi
609       
610        #-----------------------------------
611       
612done
613
614# if [ "$baseNameOfListFile" == "tar_full_simul.list" ]
615for set in $( ls $PATH_SIMU/tar_full_simul.list )
616do
617        # set="${PATH_SIMU}/tar_full_simul.list" # gpdebug : foireux ?
618        #-----------------------------------
619        echo "#-------------------"
620        echo "Set: $set"
621        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
622        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
623        statusfile=${set%%.list}.status
624        logfile=${set%%.list}_${datestr}.log
625
626        #-----------------------------------
627        # Create status file if not exists
628        if [ ! -e $statusfile ] ; then
629                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
630        fi
631
632        #-----------------------------------
633        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
634        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
635                printf "====> "
636                tail -1 $statusfile
637                exit 0
638        fi
639
640        #-----------------------------------
641        # Set output file name from list name
642        # echo "PATH_SIMU=$PATH_SIMU"
643        dirOfOutputFile=`echo $PATH_SIMU | sed "s;${IGCM_DEM};${OUTPUT_STORE};" `
644        fileout=${dirnameOfListFile}.tar
645        # echo "dirnameOfListFile=$dirnameOfListFile"
646        outputfile=$dirOfOutputFile/${fileout}
647        mkdir -p $dirOfOutputFile
648        echo "Output file to write: $outputfile  tar_full_simul"
649       
650        # exit 0 # a virer
651
652        #-----------------------------------
653        # Try 1 time before fire "FAILED"
654        # Waiting for CCRT command
655        tarcmd=tar
656        taroptions="--format=posix -W -cf"
657#       taroptions="--format=posix -cf"
658
659        filefirst=`head -n 1 $set`
660        dirinputfile=`dirname $filefirst`
661        cd $dirinputfile
662
663        set_local=${set%%.list}.local
664        > $set_local
665        for file in $( cat $set) ; do
666            basename $file >> $set_local 
667        done
668#       ls
669       
670        startTime=$( getDateMilliSeconds ) # gpdebug : time
671        resCmd=0
672        $tarcmd $taroptions $outputfile --dereference --files-from $set_local > $logfile 2>&1 # gpdebug : a retablir
673        resCmd=$?
674        meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
675       
676        # resultCmd=$( gives_0_or_1 )
677        # if [ $resultCmd -eq 0 ] ; then
678        if [ $resCmd -eq 0 ] ; then
679            sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
680            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
681            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
682            echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
683            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
684        else
685            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
686            echo "====> FAILED at try #$nbtries" 
687            echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
688            exit 1
689        fi
690       
691        #-----------------------------------
692done
Note: See TracBrowser for help on using the repository browser.