Context Navigation

enlarge_my_files.sh @ 3863

Last change on this file since 3863 was 1876, checked in by gpincka, 12 years ago
correction pour simu type 'tar full'
Property svn:executable set to ``*
File size: 25.6 KB

Rev	Line
[1717]	1	#!/bin/bash
	2	#------------------------------------------------------
	3	switchtotar=0
	4	maxtries=3
	5	#---------------------------------------------
	6	while [ $# -ne 0 ]
	7	do
	8	case $1 in
	9	--switchtotar)
	10	switchtotar=1
	11	shift 1 ;;
	12	--maxtries)
	13	maxtries=$2
	14	shift 2 ;;
	15	-h\|--help\|-help)
	16	echo "----------------------------------------------------------------------------"
	17	echo "Usage: enlarge_my_files.sh [--switchtotar] [--maxtries N] dirin dirout"
	18	echo
	19	echo "Description:"
[1752]	20	echo " Pass a ncrcat, tar or cp command on files from prepared sets of files."
	21	echo " For ncrcat command 2 shots will be done at each try".
[1717]	22	echo
[1752]	23	echo " A status file and a log file will be created in output_ncrcat directory."
	24	echo " Copy a set to output_tar directory for later processing after N tries."
[1717]	25	echo
	26	echo "Arguments:"
	27	echo " dirin is the input directory where files to concatenate will be read"
	28	echo " dirout is the output directory where concatenated files will be written"
	29	echo
	30	echo " The following directory must exist where the command enlarge_my_files is run:"
	31	echo " enlarge_my_files"
[1768]	32	echo " \|-- output_ncrcat"
[1752]	33	echo " \| \|-- set_000001.list"
	34	echo " \| \|-- set_000002.list"
	35	echo " \| \|-- set_000003.list"
	36	echo " \| \|-- set_000004.list"
	37	echo " \| \`-- set_000005.list"
[1768]	38	echo " \`-- output_tar"
[1717]	39	echo
	40	echo "Options:"
	41	echo " -h, --help, -help"
	42	echo " Print this manual."
	43	echo " --switchtotar"
	44	echo " Abort ncrcat operation and switch to tar command"
	45	echo " --maxtries N"
	46	echo " Maximum number of tries before switching to tar command"
	47	echo " Default is 3"
	48	echo
	49	echo "----------------------------------------------------------------------------"
	50	exit ;;
	51	-*)
	52	`dirname $0`/enlarge_my_files.sh -h
	53	exit ;;
	54	*)
	55	break ;;
	56	esac
	57	done
	58
	59	if [ $# -lt 2 ] ; then
	60	`dirname $0`/enlarge_my_files.sh -h
	61	exit
	62	fi
	63
[1853]	64	if ( \
	65	[ ! -d $PATH_SIMU/output_ncrcat ] \|\| \
	66	[ ! -d $PATH_SIMU/output_tar ] \|\| \
	67	[ ! -d $PATH_SIMU/restart_tar ] \|\| \
	68	[ ! -d $PATH_SIMU/store_cp ] \|\| \
	69	[ ! -d $PATH_SIMU/work_cp ] \|\| \
	70	[ ! -d $PATH_SIMU/debug_tar ] \|\| \
	71	[ ! -d $PATH_SIMU/other_tar ] \
	72	) \
	73	&& \
	74	[ ! -e $PATH_SIMU/tar_full_simul.list ]
	75	then
[1717]	76	`dirname $0`/enlarge_my_files.sh -h
	77	exit
	78	fi
	79
	80	#------------------------------------------------------
	81	dirin=$1
	82	dirout=$2
[1752]	83	dirout_work=$3
[1717]	84
	85	#------------------------------------------------------
	86	function find_varstoexclude {
	87	nbfile=0
	88	for file in `cat $1` ; do
[1841]	89	# ncdump -h ${file} \| gawk '{if (match($0, /(byte\|char\|short\|int\|float\|double) (.*)\(/, arr)) print arr[2] }' >> tmp_$$.txt
	90	ncdump -h ${file} \| grep -Ew "byte\|char\|short\|int\|float\|double" \| awk -F\( '{split($1,a," ") ; split($2,b,")") ; if (a[2] != b[1]) print a[2]}' >> tmp_$$.txt
[1717]	91	let nbfile=nbfile+1
	92	done
	93
	94	varstoexclude=`cat tmp_$$.txt \| sort \| uniq -c \| awk -v nbfile=$nbfile '{if ($1 != nbfile) {print $2}}' \| paste -s -d','`
	95	varstoexcludefiltered=`cat tmp_$$.txt \| sort \| uniq -c \| awk -v nbfile=$nbfile '{if (($1 != nbfile) && !(match($2, /t_inst_(.*)/))) {print $2}}' \| paste -s -d','`
	96	nbvars=`echo $varstoexcludefiltered \| gawk -F',' '{print NF}'`
	97
	98	rm -f tmp_$$.txt
	99
	100	return 0
	101	}
	102
[1822]	103	# gpdebug : pour test showPackProgress.sh *************************
	104	export RANDOM=$$
	105
	106	function gives_0_or_1
	107	{
	108	bit=-1
	109	let "bit = RANDOM % 2"
	110	echo $bit
	111	}
	112
	113	resultCmd=
	114	# gpdebug : **************************** fin ******************
	115
[1849]	116	source ${EXE_DIR}/DEM_utilities.sh # gpdebug : pour trouver les fonctions de mesure de temps
[1822]	117
[1855]	118	if ( \
	119	[ -d $PATH_SIMU/output_ncrcat ] \|\| \
	120	[ -d $PATH_SIMU/output_tar ] \|\| \
	121	[ -d $PATH_SIMU/restart_tar ] \|\| \
	122	[ -d $PATH_SIMU/store_cp ] \|\| \
	123	[ -d $PATH_SIMU/work_cp ] \|\| \
	124	[ -d $PATH_SIMU/debug_tar ] \|\| \
	125	[ -d $PATH_SIMU/other_tar ] \
	126	)
	127	then
[1717]	128
[1855]	129	#------------------------------------------------------
	130	for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do
[1717]	131
[1855]	132	#-----------------------------------
	133	echo "#-------------------"
	134	echo "Set: $set"
	135	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	136	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	137	statusfile=${set%%.list}.status
	138	logfile=${set%%.list}_${datestr}.log
[1717]	139
[1855]	140	#-----------------------------------
	141	# Create status file if not exists
	142	if [ ! -e $statusfile ] ; then
	143	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	144	fi
[1717]	145
[1855]	146	#-----------------------------------
	147	# Switch to tar command
	148	if [ $switchtotar -eq 1 ] ; then
	149	echo "====> DELEGATE to tar command with option --switchtotar"
	150	echo "DELEGATE to tar command with option --switchtotar" >> $statusfile
	151	mv $set $PATH_SIMU/output_tar
	152	continue
	153	fi
[1717]	154
[1855]	155	#-----------------------------------
	156	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	157	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
	158	printf "====> "
	159	tail -1 $statusfile
	160	continue
	161	fi
[1717]	162
[1855]	163	#-----------------------------------
	164	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
	165	nbtries=`wc -l $statusfile \| cut -d' ' -f1`
	166	echo "Nb of tries: $nbtries"
[1766]	167
[1855]	168	#-----------------------------------
	169	# Set output file name from list name
[1766]	170
[1855]	171	fileout1=$( basename $set )
	172	fileout=${fileout1%.list}
[1717]	173
[1855]	174	filefirst=`head -n 1 $set`
	175	diroutputfile=`dirname $filefirst \| sed -e "s%$dirin%$dirout%"`
[1717]	176
[1855]	177	outputfile=$diroutputfile/${fileout}
	178	mkdir -p $diroutputfile
	179	echo "Output file to write: $outputfile"
[1717]	180
[1855]	181	#-----------------------------------
	182	# Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')
	183	extratar=0
	184	find_varstoexclude $set
	185	if [ $nbvars -eq 0 ] ; then
	186	varstoexclude="t_inst_.*"
	187	else
	188	# There are others variables than 't_inst_.*' so an extra tar file will be created
	189	varstoexclude="t_inst_.*,$varstoexcludefiltered"
	190	extratar=1
	191	# Copy to output_tar directory
	192	echo "====> COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*"
	193	echo "COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" >> $statusfile
	194	cp $set $PATH_SIMU/output_tar
	195	fi
[1717]	196
[1855]	197	#-----------------------------------
	198	# Try 2 times before fire "FAILED"
	199	ncrcatcmd=ncrcat
[1875]	200	ncrcatoptions=""
	201	if [ "x${ON_IDRIS}" == "xno" ]
	202	then
	203	ncrcatoptions="--md5_digest"
	204	fi
[1855]	205	# ncrcatoptions=""
[1717]	206
[1855]	207	startTime=$( getDateMilliSeconds ) # gpdebug : time
	208	resCmd=0
	209	cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
	210	resCmd=$?
	211	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
[1717]	212
[1855]	213	# resultCmd=$( gives_0_or_1 )
	214	# resultCmd=1
	215	# if [ $resultCmd -eq 0 ] ; then
	216	if [ $resCmd -eq 0 ] ; then
	217	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
	218	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	219	echo "DONE ; $datestr1 ; $datestr2 ; cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
	220	echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"
	221	echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile
	222	else
	223	# Add --md5_digest
	224	rm -f $outputfile*ncrcat.tmp
	225	startTime=$( getDateMilliSeconds ) # gpdebug : time
	226	resCmd=0
	227	cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir
	228	resCmd=$?
	229	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
	230	# resultCmd=$( gives_0_or_1 )
	231	# resultCmd=1
	232	# if [ $resultCmd -eq 0 ] ; then
	233	if [ $resCmd -eq 0 ] ; then
	234	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
	235	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	236	echo "DONE ; $datestr1 ; $datestr2 ; cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
	237	echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)"
	238	echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile
	239	else
	240	rm -f $outputfile*ncrcat.tmp
	241	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	242	echo "====> FAILED at try #$nbtries"
	243	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; cat $set \| $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile
	244	if [ $nbtries -ge $maxtries ] ; then
	245	echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)"
	246	echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile
	247	cp $set $PATH_SIMU/output_tar # gpdebug : a retablir
	248	fi
	249	fi
	250	fi
[1717]	251
[1855]	252	done
[1717]	253
[1855]	254	#------------------------------------------------------
[1717]	255
[1855]	256	for set in $( ls $PATH_SIMU/output_tar/*list ) ; do
[1717]	257
[1855]	258	#-----------------------------------
	259	echo "#-------------------"
	260	echo "Set: $set"
	261	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	262	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	263	statusfile=${set%%.list}.status
	264	logfile=${set%%.list}_${datestr}.log
[1717]	265
[1855]	266	#-----------------------------------
	267	# Create status file if not exists
	268	if [ ! -e $statusfile ] ; then
	269	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	270	fi
[1766]	271
[1855]	272	#-----------------------------------
	273	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	274	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
	275	printf "====> "
	276	tail -1 $statusfile
	277	continue
	278	fi
[1766]	279
[1855]	280	#-----------------------------------
	281	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
[1876]	282	nbtries=`wc -l $statusfile \| cut -d' ' -f1`
[1855]	283	# echo "Nb of tries: $nbtries"
[1717]	284
[1855]	285	#-----------------------------------
	286	# Set output file name from list name
[1752]	287
[1855]	288	fileout1=$( basename $set )
	289	fileout=${fileout1%.list}.tar
[1752]	290
[1855]	291	filefirst=`head -n 1 $set`
	292	dirinputfile=`dirname $filefirst`
	293	diroutputfile=`dirname $filefirst \| sed -e "s%$dirin%$dirout%"`
[1752]	294
[1855]	295	outputfile=$diroutputfile/${fileout}
	296	mkdir -p $diroutputfile
	297	echo "Output file to write: $outputfile"
[1752]	298
[1855]	299	#-----------------------------------
	300	# Try 1 time before fire "FAILED"
	301	# Waiting for CCRT command
	302	tarcmd=tar
	303	taroptions="--format=posix -W -cf"
	304	#taroptions="--format=posix -cf"
	305	cd $dirinputfile
[1752]	306
[1855]	307	set_local=$set".local"
	308	for file in $( cat $set) ; do
	309	basename $file >> $set".local"
	310	done
[1752]	311
[1855]	312	# #-----------------------------------
	313	startTime=$( getDateMilliSeconds ) # gpdebug : time
	314	resCmd=0
	315	$tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1 # gpdebug : a retablir
	316	resCmd=$?
	317	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
[1752]	318
[1855]	319	# resultCmd=$( gives_0_or_1 )
	320	# if [ $resultCmd -eq 0 ] ; then
	321	if [ $resCmd -eq 0 ] ; then
	322	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
	323	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	324	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
	325	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
	326	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	327	else
	328	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	329	echo "====> FAILED at try #$nbtries"
	330	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
[1752]	331
[1855]	332	fi
	333	rm -f $set".local"
	334	#-----------------------------------
[1752]	335
[1855]	336	done
[1752]	337
[1717]	338
[1855]	339	#------------------------------------------------------
[1717]	340
[1855]	341	for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do
	342	#-----------------------------------
	343	echo "#-------------------"
	344	echo "Set: $set"
	345	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	346	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	347	statusfile=${set%%.list}.status
	348	logfile=${set%%.list}_${datestr}.log
[1717]	349
[1855]	350	#-----------------------------------
	351	# Create status file if not exists
	352	if [ ! -e $statusfile ] ; then
	353	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	354	fi
[1717]	355
[1855]	356	#-----------------------------------
	357	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	358	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
	359	printf "====> "
	360	tail -1 $statusfile
	361	continue
	362	fi
[1752]	363
[1855]	364	#-----------------------------------
	365	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
[1876]	366	nbtries=`wc -l $statusfile \| cut -d' ' -f1`
[1855]	367	# echo "Nb of tries: $nbtries"
[1752]	368
[1855]	369	#-----------------------------------
	370	# Set output file name from list name
	371	dirin1=$PATH_SIMU/RESTART
	372	dirin2=${dirin1##${IGCM_DEM}}
	373	dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
	374	diroutputfile=${OUTPUT_STORE}${dirin3}
	375	fileout1=$( basename $set )
	376	fileout=${fileout1%.list}.tar
	377	outputfile=$diroutputfile/${fileout}
	378	mkdir -p $diroutputfile
	379	echo "Output file to write: $outputfile"
[1752]	380
[1855]	381	#-----------------------------------
	382	# Try 1 time before fire "FAILED"
	383	# Waiting for CCRT command
	384	tarcmd=tar
	385	taroptions="--format=posix -W -cf"
	386	# taroptions="--format=posix -cf"
	387	cd $dirin1
	388	# ls
[1752]	389
[1855]	390	startTime=$( getDateMilliSeconds ) # gpdebug : time
	391	resCmd=0
	392	$tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
	393	resCmd=$?
	394	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
[1822]	395
[1855]	396	# resultCmd=$( gives_0_or_1 )
	397	# if [ $resultCmd -eq 0 ] ; then
	398	if [ $resCmd -eq 0 ] ; then
	399	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
	400	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	401	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
	402	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
	403	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	404	else
	405	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	406	echo "====> FAILED at try #$nbtries"
	407	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
[1752]	408
[1855]	409	fi
[1752]	410
[1855]	411	#-----------------------------------
[1752]	412
[1855]	413	done
[1752]	414
[1855]	415	#------------------------------------------------------
	416	for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do
[1752]	417
[1855]	418	#-----------------------------------
	419	echo "#-------------------"
	420	echo "Set: $set"
	421	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	422	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	423	statusfile=${set%%.list}.status
	424	logfile=${set%%.list}_${datestr}.log
[1752]	425
[1855]	426	#-----------------------------------
	427	# Create status file if not exists
	428	if [ ! -e $statusfile ] ; then
	429	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	430	fi
[1752]	431
[1855]	432	#-----------------------------------
	433	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	434	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
	435	printf "====> "
	436	tail -1 $statusfile
	437	continue
	438	fi
	439
	440	#-----------------------------------
	441	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
[1876]	442	nbtries=`wc -l $statusfile \| cut -d' ' -f1`
[1855]	443	# echo "Nb of tries: $nbtries"
	444
	445	#-----------------------------------
	446	# Set output file name from list name
	447	dirin1=$PATH_SIMU/DEBUG
	448	dirin2=${dirin1##${IGCM_DEM}}
	449	dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )}
	450	diroutputfile=${OUTPUT_STORE}${dirin3}
	451	fileout1=$( basename $set )
	452	fileout=${fileout1%.list}.tar
	453	outputfile=$diroutputfile/${fileout}
[1752]	454	mkdir -p $diroutputfile
[1855]	455	echo "Output file to write: $outputfile"
	456
	457	#-----------------------------------
	458	# Try 1 time before fire "FAILED"
	459	# Waiting for CCRT command
	460	tarcmd=tar
	461	taroptions="--format=posix -W -cf"
	462	# taroptions="--format=posix -cf"
	463	cd $dirin1
	464	# ls
	465
[1822]	466	startTime=$( getDateMilliSeconds ) # gpdebug : time
[1843]	467	resCmd=0
[1855]	468	$tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir
[1843]	469	resCmd=$?
[1822]	470	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
[1752]	471
[1855]	472	# resultCmd=$( gives_0_or_1 )
	473	# if [ $resultCmd -eq 0 ] ; then
	474	if [ $resCmd -eq 0 ] ; then
	475	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
	476	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	477	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
	478	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
	479	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	480	else
	481	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	482	echo "====> FAILED at try #$nbtries"
	483	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
[1752]	484
[1855]	485	fi
[1752]	486
[1855]	487	#-----------------------------------
[1752]	488
[1855]	489	done
[1752]	490
[1855]	491	#------------------------------------------------------
	492	for set in $( ls $PATH_SIMU/store_cp/*list ) ; do
[1752]	493
[1855]	494	#-----------------------------------
	495	echo "#-------------------"
	496	echo "Set: $set"
	497	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	498	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	499	statusfile=${set%%.list}.status
	500	logfile=${set%%.list}_${datestr}.log
[1752]	501
[1855]	502	#-----------------------------------
	503	# Create status file if not exists
	504	if [ ! -e $statusfile ] ; then
	505	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	506	fi
[1752]	507
[1855]	508	#-----------------------------------
	509	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	510	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
	511	printf "====> "
	512	tail -1 $statusfile
	513	continue
	514	fi
[1853]	515
[1855]	516	#-----------------------------------
	517	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
[1876]	518	nbtries=`wc -l $statusfile \| cut -d' ' -f1`
[1855]	519	# echo "Nb of tries: $nbtries"
	520
	521	#-----------------------------------
	522	# Set output file name from first and last files from the current set
	523	for file in $( cat $set) ; do
	524
	525	diroutputfile=`dirname $file \| sed -e "s%$dirin%$dirout%"`
	526	mkdir -p $diroutputfile
	527	echo "Output file to write: $diroutputfile/$( basename $file )"
	528
	529	#-----------------------------------
	530	# Try 1 time before fire "FAILED"
	531	# Waiting for CCRT command
	532	cpcmd="cp "
	533	cpoptions="-rf"
	534	startTime=$( getDateMilliSeconds ) # gpdebug : time
	535	resCmd=0
	536	$cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir
	537	resCmd=$?
	538	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
	539	done
	540
	541	# resultCmd=$( gives_0_or_1 )
	542	# if [ $resultCmd -eq 0 ] ; then
	543	if [ $resCmd -eq 0 ] ; then
	544	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
	545	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	546	echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
	547	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
	548	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	549	else
	550	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	551	echo "====> FAILED at try #$nbtries"
	552	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
	553
	554	fi
	555
	556	#-----------------------------------
	557
	558	done
	559
	560	#------------------------------------------------------
	561	for set in $( ls $PATH_SIMU/work_cp/*list ) ; do
	562
	563	#-----------------------------------
	564	echo "#-------------------"
	565	echo "Set: $set"
	566	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	567	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	568	statusfile=${set%%.list}.status
	569	logfile=${set%%.list}_${datestr}.log
	570
	571	#-----------------------------------
	572	# Create status file if not exists
	573	if [ ! -e $statusfile ] ; then
	574	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	575	fi
	576
	577	#-----------------------------------
	578	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	579	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
	580	printf "====> "
	581	tail -1 $statusfile
	582	continue
	583	fi
	584
	585	#-----------------------------------
	586	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
[1876]	587	nbtries=`wc -l $statusfile \| cut -d' ' -f1`
[1855]	588	# echo "Nb of tries: $nbtries"
	589
	590	#-----------------------------------
	591	# Set output file name from first and last files from the current set
	592	for file in $( cat $set) ; do
	593
	594	diroutputfile=`dirname $file \| sed -e "s%$dirin%$dirout_work%"`
	595	mkdir -p $diroutputfile
	596	echo "Output file to write: $diroutputfile/$( basename $file )"
	597
	598	#-----------------------------------
	599	# Try 1 time before fire "FAILED"
	600	# Waiting for CCRT command
	601	cpcmd="cp "
	602	cpoptions="-rf"
	603	startTime=$( getDateMilliSeconds ) # gpdebug : time
	604	resCmd=0
	605	$cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir
	606	resCmd=$?
	607	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : timeS
	608	done
	609
	610	# resultCmd=$( gives_0_or_1 )
	611	# if [ $resultCmd -eq 0 ] ; then
	612	if [ $resCmd -eq 0 ] ; then
	613	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
	614	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	615	echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
	616	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
	617	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	618	else
	619	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	620	echo "====> FAILED at try #$nbtries"
	621	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile
	622
	623	fi
	624
	625	#-----------------------------------
	626
	627	done
	628	fi
[1876]	629
[1855]	630	for set in $( ls ${PATH_SIMU}/*.list \| grep "tar_full_simul" )
[1853]	631	do
	632	# set="${PATH_SIMU}/tar_full_simul.list" # gpdebug : foireux ?
	633	#-----------------------------------
	634	echo "#-------------------"
	635	echo "Set: $set"
	636	datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"`
	637	datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	638	statusfile=${set%%.list}.status
	639	logfile=${set%%.list}_${datestr}.log
	640
	641	#-----------------------------------
	642	# Create status file if not exists
	643	if [ ! -e $statusfile ] ; then
	644	echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile
	645	fi
	646
	647	#-----------------------------------
	648	# Skip the set if last line of the status file is "COMPLETED" or "DELEGATE"
	649	if tail -1 $statusfile \| grep -q -E '(COMPLETED\|DELEGATE)' ; then
	650	printf "====> "
	651	tail -1 $statusfile
[1876]	652	continue
[1853]	653	fi
[1876]	654
[1853]	655	#-----------------------------------
[1876]	656	# nbtries = number of lines from status file (first comment line gives 1 for the first try)
	657	nbtries=`wc -l $statusfile \| cut -d' ' -f1`
	658	# echo "Nb of tries: $nbtries"
	659
	660	#-----------------------------------
[1853]	661	# Set output file name from list name
	662	# echo "PATH_SIMU=$PATH_SIMU"
	663	dirOfOutputFile=`echo $PATH_SIMU \| sed "s;${IGCM_DEM};${OUTPUT_STORE};" `
[1876]	664	# echo "dirOfOutputFile=$dirOfOutputFile"
	665	dirOfOutputFile_Up=$( dirname $dirOfOutputFile )
	666	PATH_SIMU_NAME=$( basename $dirOfOutputFile )
	667	fileout=${PATH_SIMU_NAME}.tar
[1853]	668	# echo "dirnameOfListFile=$dirnameOfListFile"
[1876]	669	outputfile=$dirOfOutputFile_Up/${fileout}
	670	# echo "outputfile=$outputfile"
	671	# exit 1 # gpdebug : a virer
	672	mkdir -p $dirOfOutputFile_Up
[1853]	673	echo "Output file to write: $outputfile tar_full_simul"
	674
	675	# exit 0 # a virer
	676
	677	#-----------------------------------
	678	# Try 1 time before fire "FAILED"
	679	# Waiting for CCRT command
	680	tarcmd=tar
	681	taroptions="--format=posix -W -cf"
	682	# taroptions="--format=posix -cf"
	683
	684	filefirst=`head -n 1 $set`
	685	dirinputfile=`dirname $filefirst`
	686	cd $dirinputfile
	687
	688	set_local=${set%%.list}.local
	689	> $set_local
	690	for file in $( cat $set) ; do
	691	basename $file >> $set_local
	692	done
	693	# ls
	694
	695	startTime=$( getDateMilliSeconds ) # gpdebug : time
	696	resCmd=0
	697	$tarcmd $taroptions $outputfile --dereference --files-from $set_local > $logfile 2>&1 # gpdebug : a retablir
	698	resCmd=$?
	699	meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time
	700
	701	# resultCmd=$( gives_0_or_1 )
	702	# if [ $resultCmd -eq 0 ] ; then
	703	if [ $resCmd -eq 0 ] ; then
	704	sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time
	705	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	706	echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile
	707	echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)"
	708	echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile
	709	else
	710	datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"`
	711	echo "====> FAILED at try #$nbtries"
	712	echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile
	713	exit 1
	714	fi
	715
	716	#-----------------------------------
	717	done

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: TOOLS/PACK_IPSL/enlarge_my_files.sh @ 3863

Download in other formats: