source: TOOLS/PACK_IPSL/enlarge_my_files.sh @ 1727

Last change on this file since 1727 was 1727, checked in by aclsce, 13 years ago

Cleaning.

  • Property svn:executable set to *
File size: 9.7 KB
Line 
1#!/bin/bash
2
3#------------------------------------------------------
4switchtotar=0
5maxtries=3
6
7#---------------------------------------------
8while [ $# -ne 0 ]
9do
10        case $1 in
11       --switchtotar)
12                switchtotar=1
13                shift 1 ;;
14       --maxtries)
15                maxtries=$2
16                shift 2 ;;
17       -h|--help|-help)
18                echo "----------------------------------------------------------------------------"
19                echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
20                echo
21                echo "Description:"
22                echo "   Pass a ncrcat command on files from prepared sets of files."
23                echo "   2 shots will be done at each try".
24                echo 
25                echo "   A status file and a log file will be created in enlarge_my_files/with_ncrcat directory."
26                echo "   Copy a set to enlarge_my_files/with_tar directory for later processing after N tries."
27                echo
28                echo "Arguments:"
29                echo "   dirin is the input directory where files to concatenate will be read"
30                echo "   dirout is the output directory where concatenated files will be written"
31                echo
32                echo "   The following directory must exist where the command enlarge_my_files is run:"
33                echo "       enlarge_my_files"
34                echo "                       |-- with_ncrcat"
35                echo "                       |   |-- set_000001.txt"
36                echo "                       |   |-- set_000002.txt"
37                echo "                       |   |-- set_000003.txt"
38                echo "                       |   |-- set_000004.txt"
39                echo "                       |   \`-- set_000005.txt"
40                echo "                       \`-- with_tar"
41                echo
42                echo "Options:"
43                echo "   -h, --help, -help"
44                echo "       Print this manual."
45                echo "   --switchtotar"
46                echo "       Abort ncrcat operation and switch to tar command"
47                echo "   --maxtries N"
48                echo "       Maximum number of tries before switching to tar command" 
49                echo "       Default is 3"
50                echo
51                echo "----------------------------------------------------------------------------"
52                exit ;;
53        -*)
54                `dirname $0`/enlarge_my_files.sh -h
55                exit ;;
56        *)
57                break ;;
58        esac
59done
60
61if [ $# -lt 2 ] ; then
62        `dirname $0`/enlarge_my_files.sh -h
63        exit
64fi
65
66if [[ ! -d $PATH_SIMU/with_ncrcat || ! -d $PATH_SIMU/with_tar ]] ; then
67        `dirname $0`/enlarge_my_files.sh -h
68        exit
69fi
70
71#------------------------------------------------------
72dirin=$1
73dirout=$2
74
75#------------------------------------------------------
76function find_varstoexclude {
77nbfile=0
78for file in `cat $1` ; do
79    ncdump -h ${file} | gawk '{if (match($0, /(byte|char|short|int|float|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
80    let nbfile=nbfile+1
81done
82
83varstoexclude=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d','`
84varstoexcludefiltered=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' | paste -s -d','`
85nbvars=`echo $varstoexcludefiltered | gawk -F',' '{print NF}'`
86
87rm -f tmp_$$.txt
88
89return 0
90}
91
92#------------------------------------------------------
93for set in $PATH_SIMU/with_ncrcat/*list ; do
94
95        #-----------------------------------
96        echo "#-------------------"
97        echo "Set: $set"
98        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
99        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
100        statusfile=${set%%.list}.status
101        logfile=${set%%.list}_${datestr}.log
102
103        #-----------------------------------
104        # Create status file if not exists
105        if [ ! -e $statusfile ] ; then
106                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
107        fi
108
109        #-----------------------------------
110        # Switch to tar command
111        if [ $switchtotar -eq 1 ] ; then
112                echo "====> DELEGATE to tar command with option --switchtotar"
113                echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
114                mv $set $PATH_SIMU/with_tar
115                continue
116        fi
117
118        #-----------------------------------
119        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
120        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
121                printf "====> "
122                tail -1 $statusfile
123                continue
124        fi
125
126        #-----------------------------------
127        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
128        nbtries=`wc -l $statusfile | cut -d' ' -f1`             
129        echo "Nb of tries: $nbtries"
130
131        #-----------------------------------
132        # Set output file name from first and last files from the current set
133        filefirst=`head -n 1 $set`
134        fileoutpart1=`basename $filefirst | cut -d'_' -f-2`
135        filelast=`tail -n 1 $set`
136        fileoutpart2=`basename $filelast | cut -d'_' -f3-`
137
138        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
139#       outputfile=$diroutputfile/ncrcat_${fileoutpart1}_${fileoutpart2}
140        outputfile=$diroutputfile/${fileoutpart1}_${fileoutpart2}
141        mkdir -p $diroutputfile
142        echo "Output file to write: $outputfile"
143
144        #-----------------------------------
145        # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
146        extratar=0
147        find_varstoexclude $set
148        if [ $nbvars -eq 0 ] ; then
149                varstoexclude="t_inst_.*"
150        else
151                # There are others variables than 't_inst_.*' so an extra tar file will be created
152                varstoexclude="t_inst_.*,$varstoexcludefiltered"
153                extratar=1
154                # Copy to with_tar directory
155                cp $set $PATH_SIMU/with_tar
156        fi
157
158        #-----------------------------------
159        # Try 2 times before fire "FAILED"
160        ncrcatcmd=ncrcat
161        #ncrcatoptions="--md5_digest"
162        ncrcatoptions=""
163
164        cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1
165        if [ $? -eq 0 ] ; then
166                datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
167                echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
168                echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
169                echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
170        else
171                # Add --md5_digest
172                cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1
173                if [ $? -eq 0 ] ; then
174                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
175                        echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
176                        echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
177                        echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
178                else
179                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
180                        echo "====> FAILED at try #$nbtries" 
181                        echo "FAILED ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
182                        if [ $nbtries -ge $maxtries ] ; then
183                                echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
184                                echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
185                                cp $set $PATH_SIMU/with_tar
186                        fi
187                fi
188        fi
189       
190        #-----------------------------------
191       
192done   
193
194#------------------------------------------------------
195
196#------------------------------------------------------
197for set in $PATH_SIMU/with_tar/*list ; do
198
199        #-----------------------------------
200        echo "#-------------------"
201        echo "Set: $set"
202        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
203        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
204        statusfile=${set%%.list}.status
205        logfile=${set%%.list}_${datestr}.log
206
207        #-----------------------------------
208        # Create status file if not exists
209        if [ ! -e $statusfile ] ; then
210                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
211        fi
212
213        #-----------------------------------
214        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
215        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
216                printf "====> "
217                tail -1 $statusfile
218                continue
219        fi
220
221        #-----------------------------------
222        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
223        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
224        # echo "Nb of tries: $nbtries"
225
226        #-----------------------------------
227        # Set output file name from list name
228        filefirst=`head -n 1 $set`
229        dirin1=$( dirname $filefirst )
230        fileout1=$( basename $set )
231        fileout=${fileout1%.list}
232        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
233#       outputfile=$diroutputfile/ncrcat_${fileoutpart1}_${fileoutpart2}
234        outputfile=$diroutputfile/${fileout}.tar
235        echo $outputfile
236        mkdir -p $diroutputfile
237        echo "Output file to write: $outputfile"
238
239#       #-----------------------------------
240#       # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
241#       extratar=0
242#       find_varstoexclude $set
243#       if [ $nbvars -eq 0 ] ; then
244#               varstoexclude="t_inst_.*"
245#       else
246#               # There are others variables than 't_inst_.*' so an extra tar file will be created
247#               varstoexclude="t_inst_.*,$varstoexcludefiltered"
248#               extratar=1
249#               # Copy to with_tar directory
250#               cp $set $PATH_SIMU/with_tar
251#       fi
252
253        #-----------------------------------
254        # Try 1 time before fire "FAILED"
255        # Waiting for CCRT command
256        tarcmd=echo
257        taroptions="-cvf"
258       
259#       cd $dirin1
260#       ls
261       
262        $tarcmd "$taroptions $outputfile $(cat $set)" > $logfile 2>&1
263        if [ $? -eq 0 ] ; then
264            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
265            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile ; $extratar" >> $statusfile
266            echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
267            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
268        else
269            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
270            echo "====> FAILED at try #$nbtries" 
271            echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile ; $extratar" >> $statusfile
272
273        fi
274       
275        #-----------------------------------
276       
277done   
278
279
280
Note: See TracBrowser for help on using the repository browser.