Context Navigation

source: TOOLS/PACK_IPSL/enlarge_my_files.sh @ 1875

Last change on this file since 1875 was 1875, checked in by gpincka, 12 years ago
portage sur machine ulam de l'idris
Property svn:executable set to ``*
File size: 25.2 KB

Line
1	#!/bin/bash
2	#------------------------------------------------------
3	switchtotar=0
4	maxtries=3
5	#---------------------------------------------
6	while [ $# -ne 0 ]
7	do
8	case $1 in
9	--switchtotar)
10	switchtotar=1
11	shift 1 ;;
12	--maxtries)
13	maxtries=$2
14	shift 2 ;;
15	-h\|--help\|-help)
16	echo "----------------------------------------------------------------------------"
17	echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
18	echo
19	echo "Description:"
20	echo " Pass a ncrcat, tar or cp command on files from prepared sets of files."
21	echo " For ncrcat command 2 shots will be done at each try".
22	echo
23	echo " A status file and a log file will be created in output_ncrcat directory."
24	echo " Copy a set to output_tar directory for later processing after N tries."
25	echo
26	echo "Arguments:"
27	echo " dirin is the input directory where files to concatenate will be read"
28	echo " dirout is the output directory where concatenated files will be written"
29	echo
30	echo " The following directory must exist where the command enlarge_my_files is run:"
31	echo " enlarge_my_files"
32	echo " \|-- output_ncrcat"
33	echo " \| \|-- set_000001.list"
34	echo " \| \|-- set_000002.list"
35	echo " \| \|-- set_000003.list"
36	echo " \| \|-- set_000004.list"
37	echo " \| \`-- set_000005.list"
38	echo " \`-- output_tar"
39	echo
40	echo "Options:"
41	echo " -h, --help, -help"
42	echo " Print this manual."
43	echo " --switchtotar"
44	echo " Abort ncrcat operation and switch to tar command"
45	echo " --maxtries N"
46	echo " Maximum number of tries before switching to tar command"
47	echo " Default is 3"
48	echo
49	echo "----------------------------------------------------------------------------"
50	exit ;;
51	-*)
52	`dirname $0`/enlarge_my_files.sh -h
53	exit ;;
54	*)
55	break ;;
56	esac
57	done
58
59	if [ $# -lt 2 ] ; then
60	`dirname $0`/enlarge_my_files.sh -h
61	exit
62	fi
63
64	if ( \
65	[ ! -d $PATH_SIMU/output_ncrcat ] \|\| \
66	[ ! -d $PATH_SIMU/output_tar ] \|\| \
67	[ ! -d $PATH_SIMU/restart_tar ] \|\| \
68	[ ! -d $PATH_SIMU/store_cp ] \|\| \
69	[ ! -d $PATH_SIMU/work_cp ] \|\| \
70	[ ! -d $PATH_SIMU/debug_tar ] \|\| \
71	[ ! -d $PATH_SIMU/other_tar ] \
72	) \
73	&& \
74	[ ! -e $PATH_SIMU/tar_full_simul.list ]
75	then
76	`dirname $0`/enlarge_my_files.sh -h
77	exit
78	fi
79
80	#------------------------------------------------------
81	dirin=$1
82	dirout=$2
83	dirout_work=$3
84
85	#------------------------------------------------------
86	function find_varstoexclude {
87	nbfile=0
88	for file in `cat $1` ; do
89	# ncdump -h ${file} \| gawk '{if (match($0, /(byte\|char\|short\|int\|float\|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
90	ncdump -h ${file} \| grep -Ew "byte\|char\|short\|int\|float\|double" \| awk -F\( '{split($1,a," ") ; split($2,b,")") ; if (a[2] != b[1]) print a[2]}' >> tmp_$$.txt
91	let nbfile=nbfile+1
92	done
93
94	varstoexclude=`cat tmp_$$.txt \| sort \| uniq -c \| awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' \| paste -s -d','`
95	varstoexcludefiltered=`cat tmp_$$.txt \| sort \| uniq -c \| awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' \| paste -s -d','`
96	nbvars=`echo $varstoexcludefiltered \| gawk -F',' '{print NF}'`
97
98	rm -f tmp_$$.txt
99
100	return 0
101	}
102
103	# gpdebug : pour test showPackProgress.sh *************************
104	export RANDOM=$$
105
106	function gives_0_or_1
107	{
108	bit=-1
109	let "bit = RANDOM % 2"
110	echo $bit
111	}
112
113	resultCmd=
114	# gpdebug : **************************** fin ******************
115
116	source ${EXE_DIR}/DEM_utilities.sh # gpdebug : pour trouver les fonctions de mesure de temps
117
118	if ( \
119	[ -d $PATH_SIMU/output_ncrcat ] \|\| \
120	[ -d $PATH_SIMU/output_tar ] \|\| \
121	[ -d $PATH_SIMU/restart_tar ] \|\| \
122	[ -d $PATH_SIMU/store_cp ] \|\| \
123	[ -d $PATH_SIMU/work_cp ] \|\| \
124	[ -d $PATH_SIMU/debug_tar ] \|\| \
125	[ -d $PATH_SIMU/other_tar ] \
126	)
127	then
128
129	#------------------------------------------------------
130	for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do
131
132	#-----------------------------------
133	echo "#-------------------"
134	echo "Set: $set"
135	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
136	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
137	statusfile=${set%%.list}.status
138	logfile=${set%%.list}_${datestr}.log
139
140	#-----------------------------------
141	# Create status file if not exists
142	if [ ! -e $statusfile ] ; then
143	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
144	fi
145
146	#-----------------------------------
147	# Switch to tar command
148	if [ $switchtotar -eq 1 ] ; then
149	echo "====> DELEGATE to tar command with option --switchtotar"
150	echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
151	mv $set $PATH_SIMU/output_tar
152	continue
153	fi
154
155	#-----------------------------------
156	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
157	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
158	printf "====> "
159	tail -1 $statusfile
160	continue
161	fi
162
163	#-----------------------------------
164	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
165	nbtries=`wc -l $statusfile \| cut -d' ' -f1`
166	echo "Nb of tries: $nbtries"
167
168	#-----------------------------------
169	# Set output file name from list name
170
171	fileout1=$( basename $set )
172	fileout=${fileout1%.list}
173
174	filefirst=`head -n 1 $set`
175	diroutputfile=`dirname $filefirst \| sed -e "s%$dirin%$dirout%"`
176
177	outputfile=$diroutputfile/${fileout}
178	mkdir -p $diroutputfile
179	echo "Output file to write: $outputfile"
180
181	#-----------------------------------
182	# Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
183	extratar=0
184	find_varstoexclude $set
185	if [ $nbvars -eq 0 ] ; then
186	varstoexclude="t_inst_.*"
187	else
188	# There are others variables than 't_inst_.*' so an extra tar file will be created
189	varstoexclude="t_inst_.*,$varstoexcludefiltered"
190	extratar=1
191	# Copy to output_tar directory
192	echo "====> COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*"
193	echo "COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" >> $statusfile
194	cp $set $PATH_SIMU/output_tar
195	fi
196
197	#-----------------------------------
198	# Try 2 times before fire "FAILED"
199	ncrcatcmd=ncrcat
200	ncrcatoptions=""
201	if [ "x${ON_IDRIS}" == "xno" ]
202	then
203	ncrcatoptions="--md5_digest"
204	fi
205	# ncrcatoptions=""
206
207	startTime=$( getDateMilliSeconds ) # gpdebug : time
208	resCmd=0
209	cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
210	resCmd=$?
211	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
212
213	# resultCmd=$( gives_0_or_1 )
214	# resultCmd=1
215	# if [ $resultCmd -eq 0 ] ; then
216	if [ $resCmd -eq 0 ] ; then
217	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
218	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
219	echo "DONE ; $datestr1 ; $datestr2 ; cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
220	echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
221	echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
222	else
223	# Add --md5_digest
224	rm -f $outputfile*ncrcat.tmp
225	startTime=$( getDateMilliSeconds ) # gpdebug : time
226	resCmd=0
227	cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
228	resCmd=$?
229	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
230	# resultCmd=$( gives_0_or_1 )
231	# resultCmd=1
232	# if [ $resultCmd -eq 0 ] ; then
233	if [ $resCmd -eq 0 ] ; then
234	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
235	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
236	echo "DONE ; $datestr1 ; $datestr2 ; cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
237	echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
238	echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
239	else
240	rm -f $outputfile*ncrcat.tmp
241	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
242	echo "====> FAILED at try #$nbtries"
243	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
244	if [ $nbtries -ge $maxtries ] ; then
245	echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
246	echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
247	cp $set $PATH_SIMU/output_tar # gpdebug : a retablir
248	fi
249	fi
250	fi
251
252	done
253
254	#------------------------------------------------------
255
256	for set in $( ls $PATH_SIMU/output_tar/*list ) ; do
257
258	#-----------------------------------
259	echo "#-------------------"
260	echo "Set: $set"
261	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
262	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
263	statusfile=${set%%.list}.status
264	logfile=${set%%.list}_${datestr}.log
265
266	#-----------------------------------
267	# Create status file if not exists
268	if [ ! -e $statusfile ] ; then
269	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
270	fi
271
272	#-----------------------------------
273	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
274	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
275	printf "====> "
276	tail -1 $statusfile
277	continue
278	fi
279
280	#-----------------------------------
281	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
282	# nbtries=`wc -l $statusfile \| cut -d' ' -f1`
283	# echo "Nb of tries: $nbtries"
284
285	#-----------------------------------
286	# Set output file name from list name
287
288	fileout1=$( basename $set )
289	fileout=${fileout1%.list}.tar
290
291	filefirst=`head -n 1 $set`
292	dirinputfile=`dirname $filefirst`
293	diroutputfile=`dirname $filefirst \| sed -e "s%$dirin%$dirout%"`
294
295	outputfile=$diroutputfile/${fileout}
296	mkdir -p $diroutputfile
297	echo "Output file to write: $outputfile"
298
299	#-----------------------------------
300	# Try 1 time before fire "FAILED"
301	# Waiting for CCRT command
302	tarcmd=tar
303	taroptions="--format=posix -W -cf"
304	#taroptions="--format=posix -cf"
305	cd $dirinputfile
306
307	set_local=$set".local"
308	for file in $( cat $set) ; do
309	basename $file >> $set".local"
310	done
311
312	# #-----------------------------------
313	startTime=$( getDateMilliSeconds ) # gpdebug : time
314	resCmd=0
315	$tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1 # gpdebug : a retablir
316	resCmd=$?
317	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
318
319	# resultCmd=$( gives_0_or_1 )
320	# if [ $resultCmd -eq 0 ] ; then
321	if [ $resCmd -eq 0 ] ; then
322	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
323	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
324	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
325	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
326	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
327	else
328	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
329	echo "====> FAILED at try #$nbtries"
330	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
331
332	fi
333	rm -f $set".local"
334	#-----------------------------------
335
336	done
337
338
339	#------------------------------------------------------
340
341	for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do
342	#-----------------------------------
343	echo "#-------------------"
344	echo "Set: $set"
345	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
346	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
347	statusfile=${set%%.list}.status
348	logfile=${set%%.list}_${datestr}.log
349
350	#-----------------------------------
351	# Create status file if not exists
352	if [ ! -e $statusfile ] ; then
353	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
354	fi
355
356	#-----------------------------------
357	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
358	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
359	printf "====> "
360	tail -1 $statusfile
361	continue
362	fi
363
364	#-----------------------------------
365	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
366	# nbtries=`wc -l $statusfile \| cut -d' ' -f1`
367	# echo "Nb of tries: $nbtries"
368
369	#-----------------------------------
370	# Set output file name from list name
371	dirin1=$PATH_SIMU/RESTART
372	dirin2=${dirin1##${IGCM_DEM}}
373	dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
374	diroutputfile=${OUTPUT_STORE}${dirin3}
375	fileout1=$( basename $set )
376	fileout=${fileout1%.list}.tar
377	outputfile=$diroutputfile/${fileout}
378	mkdir -p $diroutputfile
379	echo "Output file to write: $outputfile"
380
381	#-----------------------------------
382	# Try 1 time before fire "FAILED"
383	# Waiting for CCRT command
384	tarcmd=tar
385	taroptions="--format=posix -W -cf"
386	# taroptions="--format=posix -cf"
387	cd $dirin1
388	# ls
389
390	startTime=$( getDateMilliSeconds ) # gpdebug : time
391	resCmd=0
392	$tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
393	resCmd=$?
394	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
395
396	# resultCmd=$( gives_0_or_1 )
397	# if [ $resultCmd -eq 0 ] ; then
398	if [ $resCmd -eq 0 ] ; then
399	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
400	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
401	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
402	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
403	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
404	else
405	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
406	echo "====> FAILED at try #$nbtries"
407	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
408
409	fi
410
411	#-----------------------------------
412
413	done
414
415	#------------------------------------------------------
416	for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do
417
418	#-----------------------------------
419	echo "#-------------------"
420	echo "Set: $set"
421	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
422	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
423	statusfile=${set%%.list}.status
424	logfile=${set%%.list}_${datestr}.log
425
426	#-----------------------------------
427	# Create status file if not exists
428	if [ ! -e $statusfile ] ; then
429	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
430	fi
431
432	#-----------------------------------
433	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
434	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
435	printf "====> "
436	tail -1 $statusfile
437	continue
438	fi
439
440	#-----------------------------------
441	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
442	# nbtries=`wc -l $statusfile \| cut -d' ' -f1`
443	# echo "Nb of tries: $nbtries"
444
445	#-----------------------------------
446	# Set output file name from list name
447	dirin1=$PATH_SIMU/DEBUG
448	dirin2=${dirin1##${IGCM_DEM}}
449	dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
450	diroutputfile=${OUTPUT_STORE}${dirin3}
451	fileout1=$( basename $set )
452	fileout=${fileout1%.list}.tar
453	outputfile=$diroutputfile/${fileout}
454	mkdir -p $diroutputfile
455	echo "Output file to write: $outputfile"
456
457	#-----------------------------------
458	# Try 1 time before fire "FAILED"
459	# Waiting for CCRT command
460	tarcmd=tar
461	taroptions="--format=posix -W -cf"
462	# taroptions="--format=posix -cf"
463	cd $dirin1
464	# ls
465
466	startTime=$( getDateMilliSeconds ) # gpdebug : time
467	resCmd=0
468	$tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
469	resCmd=$?
470	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
471
472	# resultCmd=$( gives_0_or_1 )
473	# if [ $resultCmd -eq 0 ] ; then
474	if [ $resCmd -eq 0 ] ; then
475	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
476	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
477	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
478	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
479	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
480	else
481	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
482	echo "====> FAILED at try #$nbtries"
483	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
484
485	fi
486
487	#-----------------------------------
488
489	done
490
491	#------------------------------------------------------
492	for set in $( ls $PATH_SIMU/store_cp/*list ) ; do
493
494	#-----------------------------------
495	echo "#-------------------"
496	echo "Set: $set"
497	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
498	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
499	statusfile=${set%%.list}.status
500	logfile=${set%%.list}_${datestr}.log
501
502	#-----------------------------------
503	# Create status file if not exists
504	if [ ! -e $statusfile ] ; then
505	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
506	fi
507
508	#-----------------------------------
509	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
510	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
511	printf "====> "
512	tail -1 $statusfile
513	continue
514	fi
515
516	#-----------------------------------
517	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
518	# nbtries=`wc -l $statusfile \| cut -d' ' -f1`
519	# echo "Nb of tries: $nbtries"
520
521	#-----------------------------------
522	# Set output file name from first and last files from the current set
523	for file in $( cat $set) ; do
524
525	diroutputfile=`dirname $file \| sed -e "s%$dirin%$dirout%"`
526	mkdir -p $diroutputfile
527	echo "Output file to write: $diroutputfile/$( basename $file )"
528
529	#-----------------------------------
530	# Try 1 time before fire "FAILED"
531	# Waiting for CCRT command
532	cpcmd="cp "
533	cpoptions="-rf"
534	startTime=$( getDateMilliSeconds ) # gpdebug : time
535	resCmd=0
536	$cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir
537	resCmd=$?
538	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
539	done
540
541	# resultCmd=$( gives_0_or_1 )
542	# if [ $resultCmd -eq 0 ] ; then
543	if [ $resCmd -eq 0 ] ; then
544	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
545	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
546	echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
547	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
548	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
549	else
550	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
551	echo "====> FAILED at try #$nbtries"
552	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
553
554	fi
555
556	#-----------------------------------
557
558	done
559
560	#------------------------------------------------------
561	for set in $( ls $PATH_SIMU/work_cp/*list ) ; do
562
563	#-----------------------------------
564	echo "#-------------------"
565	echo "Set: $set"
566	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
567	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
568	statusfile=${set%%.list}.status
569	logfile=${set%%.list}_${datestr}.log
570
571	#-----------------------------------
572	# Create status file if not exists
573	if [ ! -e $statusfile ] ; then
574	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
575	fi
576
577	#-----------------------------------
578	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
579	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
580	printf "====> "
581	tail -1 $statusfile
582	continue
583	fi
584
585	#-----------------------------------
586	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
587	# nbtries=`wc -l $statusfile \| cut -d' ' -f1`
588	# echo "Nb of tries: $nbtries"
589
590	#-----------------------------------
591	# Set output file name from first and last files from the current set
592	for file in $( cat $set) ; do
593
594	diroutputfile=`dirname $file \| sed -e "s%$dirin%$dirout_work%"`
595	mkdir -p $diroutputfile
596	echo "Output file to write: $diroutputfile/$( basename $file )"
597
598	#-----------------------------------
599	# Try 1 time before fire "FAILED"
600	# Waiting for CCRT command
601	cpcmd="cp "
602	cpoptions="-rf"
603	startTime=$( getDateMilliSeconds ) # gpdebug : time
604	resCmd=0
605	$cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir
606	resCmd=$?
607	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : timeS
608	done
609
610	# resultCmd=$( gives_0_or_1 )
611	# if [ $resultCmd -eq 0 ] ; then
612	if [ $resCmd -eq 0 ] ; then
613	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
614	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
615	echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
616	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
617	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
618	else
619	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
620	echo "====> FAILED at try #$nbtries"
621	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
622
623	fi
624
625	#-----------------------------------
626
627	done
628	fi
629	# if [ "$baseNameOfListFile" == "tar_full_simul.list" ]
630	for set in $( ls ${PATH_SIMU}/*.list \| grep "tar_full_simul" )
631	do
632	# set="${PATH_SIMU}/tar_full_simul.list" # gpdebug : foireux ?
633	#-----------------------------------
634	echo "#-------------------"
635	echo "Set: $set"
636	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
637	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
638	statusfile=${set%%.list}.status
639	logfile=${set%%.list}_${datestr}.log
640
641	#-----------------------------------
642	# Create status file if not exists
643	if [ ! -e $statusfile ] ; then
644	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
645	fi
646
647	#-----------------------------------
648	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
649	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
650	printf "====> "
651	tail -1 $statusfile
652	exit 0
653	fi
654
655	#-----------------------------------
656	# Set output file name from list name
657	# echo "PATH_SIMU=$PATH_SIMU"
658	dirOfOutputFile=`echo $PATH_SIMU \| sed "s;${IGCM_DEM};${OUTPUT_STORE};" `
659	fileout=${dirnameOfListFile}.tar
660	# echo "dirnameOfListFile=$dirnameOfListFile"
661	outputfile=$dirOfOutputFile/${fileout}
662	mkdir -p $dirOfOutputFile
663	echo "Output file to write: $outputfile tar_full_simul"
664
665	# exit 0 # a virer
666
667	#-----------------------------------
668	# Try 1 time before fire "FAILED"
669	# Waiting for CCRT command
670	tarcmd=tar
671	taroptions="--format=posix -W -cf"
672	# taroptions="--format=posix -cf"
673
674	filefirst=`head -n 1 $set`
675	dirinputfile=`dirname $filefirst`
676	cd $dirinputfile
677
678	set_local=${set%%.list}.local
679	> $set_local
680	for file in $( cat $set) ; do
681	basename $file >> $set_local
682	done
683	# ls
684
685	startTime=$( getDateMilliSeconds ) # gpdebug : time
686	resCmd=0
687	$tarcmd $taroptions $outputfile --dereference --files-from $set_local > $logfile 2>&1 # gpdebug : a retablir
688	resCmd=$?
689	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
690
691	# resultCmd=$( gives_0_or_1 )
692	# if [ $resultCmd -eq 0 ] ; then
693	if [ $resCmd -eq 0 ] ; then
694	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
695	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
696	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
697	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
698	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
699	else
700	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
701	echo "====> FAILED at try #$nbtries"
702	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
703	exit 1
704	fi
705
706	#-----------------------------------
707	done

Note: See TracBrowser for help on using the repository browser.

Download in other formats: