source: TOOLS/PACK_IPSL/enlarge_my_files.sh @ 1723

Last change on this file since 1723 was 1717, checked in by aclsce, 12 years ago
  • Added scripts "launch_ipsl_enlarge.sh" and "enlarge_my_files.sh".
  • Added creation and use of directories "with_ncrcat" and "with_tar" to store list of files to treat.
  • Added mv of "liste_simul_*" file into IGCM_DEM directory.
  • Property svn:executable set to *
File size: 9.9 KB
Line 
1#!/bin/bash
2
3#======================================================
4# Author: Patrick.Brockmann@cea.fr
5#
6#======================================================
7
8#------------------------------------------------------
9switchtotar=0
10maxtries=3
11
12#---------------------------------------------
13while [ $# -ne 0 ]
14do
15        case $1 in
16       --switchtotar)
17                switchtotar=1
18                shift 1 ;;
19       --maxtries)
20                maxtries=$2
21                shift 2 ;;
22       -h|--help|-help)
23                echo "----------------------------------------------------------------------------"
24                echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
25                echo
26                echo "Description:"
27                echo "   Pass a ncrcat command on files from prepared sets of files."
28                echo "   2 shots will be done at each try".
29                echo 
30                echo "   A status file and a log file will be created in enlarge_my_files/with_ncrcat directory."
31                echo "   Copy a set to enlarge_my_files/with_tar directory for later processing after N tries."
32                echo
33                echo "Arguments:"
34                echo "   dirin is the input directory where files to concatenate will be read"
35                echo "   dirout is the output directory where concatenated files will be written"
36                echo
37                echo "   The following directory must exist where the command enlarge_my_files is run:"
38                echo "       enlarge_my_files"
39                echo "                       |-- with_ncrcat"
40                echo "                       |   |-- set_000001.txt"
41                echo "                       |   |-- set_000002.txt"
42                echo "                       |   |-- set_000003.txt"
43                echo "                       |   |-- set_000004.txt"
44                echo "                       |   \`-- set_000005.txt"
45                echo "                       \`-- with_tar"
46                echo
47                echo "Options:"
48                echo "   -h, --help, -help"
49                echo "       Print this manual."
50                echo "   --switchtotar"
51                echo "       Abort ncrcat operation and switch to tar command"
52                echo "   --maxtries N"
53                echo "       Maximum number of tries before switching to tar command" 
54                echo "       Default is 3"
55                echo
56                echo "----------------------------------------------------------------------------"
57                exit ;;
58        -*)
59                `dirname $0`/enlarge_my_files.sh -h
60                exit ;;
61        *)
62                break ;;
63        esac
64done
65
66if [ $# -lt 2 ] ; then
67        `dirname $0`/enlarge_my_files.sh -h
68        exit
69fi
70
71if [[ ! -d $PATH_SIMU/with_ncrcat || ! -d $PATH_SIMU/with_tar ]] ; then
72        `dirname $0`/enlarge_my_files.sh -h
73        exit
74fi
75
76#------------------------------------------------------
77dirin=$1
78dirout=$2
79
80#------------------------------------------------------
81function find_varstoexclude {
82nbfile=0
83for file in `cat $1` ; do
84    ncdump -h ${file} | gawk '{if (match($0, /(byte|char|short|int|float|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
85    let nbfile=nbfile+1
86done
87
88varstoexclude=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' | paste -s -d','`
89varstoexcludefiltered=`cat tmp_$$.txt | sort | uniq -c | awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' | paste -s -d','`
90nbvars=`echo $varstoexcludefiltered | gawk -F',' '{print NF}'`
91
92rm -f tmp_$$.txt
93
94return 0
95}
96
97#------------------------------------------------------
98for set in $PATH_SIMU/with_ncrcat/*list ; do
99
100        #-----------------------------------
101        echo "#-------------------"
102        echo "Set: $set"
103        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
104        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
105        statusfile=${set%%.list}.status
106        logfile=${set%%.list}_${datestr}.log
107
108        #-----------------------------------
109        # Create status file if not exists
110        if [ ! -e $statusfile ] ; then
111                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
112        fi
113
114        #-----------------------------------
115        # Switch to tar command
116        if [ $switchtotar -eq 1 ] ; then
117                echo "====> DELEGATE to tar command with option --switchtotar"
118                echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
119                mv $set $PATH_SIMU/with_tar
120                continue
121        fi
122
123        #-----------------------------------
124        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
125        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
126                printf "====> "
127                tail -1 $statusfile
128                continue
129        fi
130
131        #-----------------------------------
132        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
133        nbtries=`wc -l $statusfile | cut -d' ' -f1`             
134        echo "Nb of tries: $nbtries"
135
136        #-----------------------------------
137        # Set output file name from first and last files from the current set
138        filefirst=`head -n 1 $set`
139        fileoutpart1=`basename $filefirst | cut -d'_' -f-2`
140        filelast=`tail -n 1 $set`
141        fileoutpart2=`basename $filelast | cut -d'_' -f3-`
142
143        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
144#       outputfile=$diroutputfile/ncrcat_${fileoutpart1}_${fileoutpart2}
145        outputfile=$diroutputfile/${fileoutpart1}_${fileoutpart2}
146        mkdir -p $diroutputfile
147        echo "Output file to write: $outputfile"
148
149        #-----------------------------------
150        # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
151        extratar=0
152        find_varstoexclude $set
153        if [ $nbvars -eq 0 ] ; then
154                varstoexclude="t_inst_.*"
155        else
156                # There are others variables than 't_inst_.*' so an extra tar file will be created
157                varstoexclude="t_inst_.*,$varstoexcludefiltered"
158                extratar=1
159                # Copy to with_tar directory
160                cp $set $PATH_SIMU/with_tar
161        fi
162
163        #-----------------------------------
164        # Try 2 times before fire "FAILED"
165        ncrcatcmd=ncrcat
166        #ncrcatoptions="--md5_digest"
167        ncrcatoptions=""
168
169        cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1
170        if [ $? -eq 0 ] ; then
171                datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
172                echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
173                echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
174                echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
175        else
176                # Add --md5_digest
177                cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1
178                if [ $? -eq 0 ] ; then
179                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
180                        echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
181                        echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
182                        echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
183                else
184                        datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
185                        echo "====> FAILED at try #$nbtries" 
186                        echo "FAILED ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
187                        if [ $nbtries -ge $maxtries ] ; then
188                                echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
189                                echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
190                                cp $set $PATH_SIMU/with_tar
191                        fi
192                fi
193        fi
194       
195        #-----------------------------------
196       
197done   
198
199#------------------------------------------------------
200
201#------------------------------------------------------
202for set in $PATH_SIMU/with_tar/*list ; do
203
204        #-----------------------------------
205        echo "#-------------------"
206        echo "Set: $set"
207        datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
208        datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
209        statusfile=${set%%.list}.status
210        logfile=${set%%.list}_${datestr}.log
211
212        #-----------------------------------
213        # Create status file if not exists
214        if [ ! -e $statusfile ] ; then
215                echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
216        fi
217
218        #-----------------------------------
219        # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
220        if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then
221                printf "====> "
222                tail -1 $statusfile
223                continue
224        fi
225
226        #-----------------------------------
227        # nbtries = number of lines from status file (first comment line gives 1 for the first try)
228        # nbtries=`wc -l $statusfile | cut -d' ' -f1`           
229        # echo "Nb of tries: $nbtries"
230
231        #-----------------------------------
232        # Set output file name from list name
233        filefirst=`head -n 1 $set`
234        dirin1=$( dirname $filefirst )
235        fileout1=$( basename $set )
236        fileout=${fileout1%.list}
237        diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"`
238#       outputfile=$diroutputfile/ncrcat_${fileoutpart1}_${fileoutpart2}
239        outputfile=$diroutputfile/${fileout}.tar
240        echo $outputfile
241        mkdir -p $diroutputfile
242        echo "Output file to write: $outputfile"
243
244#       #-----------------------------------
245#       # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
246#       extratar=0
247#       find_varstoexclude $set
248#       if [ $nbvars -eq 0 ] ; then
249#               varstoexclude="t_inst_.*"
250#       else
251#               # There are others variables than 't_inst_.*' so an extra tar file will be created
252#               varstoexclude="t_inst_.*,$varstoexcludefiltered"
253#               extratar=1
254#               # Copy to with_tar directory
255#               cp $set $PATH_SIMU/with_tar
256#       fi
257
258        #-----------------------------------
259        # Try 1 time before fire "FAILED"
260        # Waiting for CCRT command
261        tarcmd=echo
262        taroptions="-cvf"
263       
264#       cd $dirin1
265#       ls
266       
267        $tarcmd "$taroptions $outputfile $(cat $set)" > $logfile 2>&1
268        if [ $? -eq 0 ] ; then
269            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
270            echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile ; $extratar" >> $statusfile
271            echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
272            echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
273        else
274            datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
275            echo "====> FAILED at try #$nbtries" 
276            echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile ; $extratar" >> $statusfile
277
278        fi
279       
280        #-----------------------------------
281       
282done   
283
284
285
Note: See TracBrowser for help on using the repository browser.