Changeset 1855
- Timestamp:
- 09/03/12 15:46:07 (12 years ago)
- File:
-
- 1 edited
-
TOOLS/PACK_IPSL/enlarge_my_files.sh (modified) (1 diff)
Legend:
- Unmodified
- Added
- Removed
-
TOOLS/PACK_IPSL/enlarge_my_files.sh
r1853 r1855 116 116 source ${EXE_DIR}/DEM_utilities.sh # gpdebug : pour trouver les fonctions de mesure de temps 117 117 118 #------------------------------------------------------ 119 for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do 120 121 #----------------------------------- 122 echo "#-------------------" 123 echo "Set: $set" 124 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 125 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 126 statusfile=${set%%.list}.status 127 logfile=${set%%.list}_${datestr}.log 128 129 #----------------------------------- 130 # Create status file if not exists 131 if [ ! -e $statusfile ] ; then 132 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 133 fi 134 135 #----------------------------------- 136 # Switch to tar command 137 if [ $switchtotar -eq 1 ] ; then 138 echo "====> DELEGATE to tar command with option --switchtotar" 139 echo "DELEGATE to tar command with option --switchtotar" >> $statusfile 140 mv $set $PATH_SIMU/output_tar 141 continue 142 fi 143 144 #----------------------------------- 145 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 146 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 147 printf "====> " 148 tail -1 $statusfile 149 continue 150 fi 151 152 #----------------------------------- 153 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 154 nbtries=`wc -l $statusfile | cut -d' ' -f1` 155 echo "Nb of tries: $nbtries" 156 157 #----------------------------------- 158 # Set output file name from list name 159 160 fileout1=$( basename $set ) 161 fileout=${fileout1%.list} 162 163 filefirst=`head -n 1 $set` 164 diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` 165 166 outputfile=$diroutputfile/${fileout} 167 mkdir -p $diroutputfile 168 echo "Output file to write: $outputfile" 169 170 #----------------------------------- 171 # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*') 172 extratar=0 173 find_varstoexclude $set 174 if [ $nbvars -eq 0 ] ; then 175 varstoexclude="t_inst_.*" 176 else 177 # There are others variables than 't_inst_.*' so an extra tar file will be created 178 varstoexclude="t_inst_.*,$varstoexcludefiltered" 179 extratar=1 180 # Copy to output_tar directory 181 echo "====> COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" 182 echo "COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" >> $statusfile 183 cp $set $PATH_SIMU/output_tar 184 fi 185 186 #----------------------------------- 187 # Try 2 times before fire "FAILED" 188 ncrcatcmd=ncrcat 189 ncrcatoptions="--md5_digest" 190 # ncrcatoptions="" 191 192 startTime=$( getDateMilliSeconds ) # gpdebug : time 193 resCmd=0 194 cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir 195 resCmd=$? 196 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 197 198 # resultCmd=$( gives_0_or_1 ) 199 # resultCmd=1 200 # if [ $resultCmd -eq 0 ] ; then 201 if [ $resCmd -eq 0 ] ; then 202 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 203 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 204 echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile 205 echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)" 206 echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile 207 else 208 # Add --md5_digest 209 rm -f $outputfile*ncrcat.tmp 118 if ( \ 119 [ -d $PATH_SIMU/output_ncrcat ] || \ 120 [ -d $PATH_SIMU/output_tar ] || \ 121 [ -d $PATH_SIMU/restart_tar ] || \ 122 [ -d $PATH_SIMU/store_cp ] || \ 123 [ -d $PATH_SIMU/work_cp ] || \ 124 [ -d $PATH_SIMU/debug_tar ] || \ 125 [ -d $PATH_SIMU/other_tar ] \ 126 ) 127 then 128 129 #------------------------------------------------------ 130 for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do 131 132 #----------------------------------- 133 echo "#-------------------" 134 echo "Set: $set" 135 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 136 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 137 statusfile=${set%%.list}.status 138 logfile=${set%%.list}_${datestr}.log 139 140 #----------------------------------- 141 # Create status file if not exists 142 if [ ! -e $statusfile ] ; then 143 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 144 fi 145 146 #----------------------------------- 147 # Switch to tar command 148 if [ $switchtotar -eq 1 ] ; then 149 echo "====> DELEGATE to tar command with option --switchtotar" 150 echo "DELEGATE to tar command with option --switchtotar" >> $statusfile 151 mv $set $PATH_SIMU/output_tar 152 continue 153 fi 154 155 #----------------------------------- 156 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 157 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 158 printf "====> " 159 tail -1 $statusfile 160 continue 161 fi 162 163 #----------------------------------- 164 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 165 nbtries=`wc -l $statusfile | cut -d' ' -f1` 166 echo "Nb of tries: $nbtries" 167 168 #----------------------------------- 169 # Set output file name from list name 170 171 fileout1=$( basename $set ) 172 fileout=${fileout1%.list} 173 174 filefirst=`head -n 1 $set` 175 diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` 176 177 outputfile=$diroutputfile/${fileout} 178 mkdir -p $diroutputfile 179 echo "Output file to write: $outputfile" 180 181 #----------------------------------- 182 # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*') 183 extratar=0 184 find_varstoexclude $set 185 if [ $nbvars -eq 0 ] ; then 186 varstoexclude="t_inst_.*" 187 else 188 # There are others variables than 't_inst_.*' so an extra tar file will be created 189 varstoexclude="t_inst_.*,$varstoexcludefiltered" 190 extratar=1 191 # Copy to output_tar directory 192 echo "====> COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" 193 echo "COPY to tar command because find other variables ($varstoexcludefiltered) than t_inst_*" >> $statusfile 194 cp $set $PATH_SIMU/output_tar 195 fi 196 197 #----------------------------------- 198 # Try 2 times before fire "FAILED" 199 ncrcatcmd=ncrcat 200 ncrcatoptions="--md5_digest" 201 # ncrcatoptions="" 202 203 startTime=$( getDateMilliSeconds ) # gpdebug : time 204 resCmd=0 205 cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir 206 resCmd=$? 207 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 208 209 # resultCmd=$( gives_0_or_1 ) 210 # resultCmd=1 211 # if [ $resultCmd -eq 0 ] ; then 212 if [ $resCmd -eq 0 ] ; then 213 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 214 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 215 echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile 216 echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)" 217 echo "COMPLETED at try #$nbtries (at 1st shot on 2)" >> $statusfile 218 else 219 # Add --md5_digest 220 rm -f $outputfile*ncrcat.tmp 221 startTime=$( getDateMilliSeconds ) # gpdebug : time 222 resCmd=0 223 cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir 224 resCmd=$? 225 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 226 # resultCmd=$( gives_0_or_1 ) 227 # resultCmd=1 228 # if [ $resultCmd -eq 0 ] ; then 229 if [ $resCmd -eq 0 ] ; then 230 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 231 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 232 echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile 233 echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)" 234 echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile 235 else 236 rm -f $outputfile*ncrcat.tmp 237 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 238 echo "====> FAILED at try #$nbtries" 239 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile 240 if [ $nbtries -ge $maxtries ] ; then 241 echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" 242 echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile 243 cp $set $PATH_SIMU/output_tar # gpdebug : a retablir 244 fi 245 fi 246 fi 247 248 done 249 250 #------------------------------------------------------ 251 252 for set in $( ls $PATH_SIMU/output_tar/*list ) ; do 253 254 #----------------------------------- 255 echo "#-------------------" 256 echo "Set: $set" 257 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 258 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 259 statusfile=${set%%.list}.status 260 logfile=${set%%.list}_${datestr}.log 261 262 #----------------------------------- 263 # Create status file if not exists 264 if [ ! -e $statusfile ] ; then 265 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 266 fi 267 268 #----------------------------------- 269 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 270 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 271 printf "====> " 272 tail -1 $statusfile 273 continue 274 fi 275 276 #----------------------------------- 277 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 278 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 279 # echo "Nb of tries: $nbtries" 280 281 #----------------------------------- 282 # Set output file name from list name 283 284 fileout1=$( basename $set ) 285 fileout=${fileout1%.list}.tar 286 287 filefirst=`head -n 1 $set` 288 dirinputfile=`dirname $filefirst` 289 diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` 290 291 outputfile=$diroutputfile/${fileout} 292 mkdir -p $diroutputfile 293 echo "Output file to write: $outputfile" 294 295 #----------------------------------- 296 # Try 1 time before fire "FAILED" 297 # Waiting for CCRT command 298 tarcmd=tar 299 taroptions="--format=posix -W -cf" 300 #taroptions="--format=posix -cf" 301 cd $dirinputfile 302 303 set_local=$set".local" 304 for file in $( cat $set) ; do 305 basename $file >> $set".local" 306 done 307 308 # #----------------------------------- 309 startTime=$( getDateMilliSeconds ) # gpdebug : time 310 resCmd=0 311 $tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1 # gpdebug : a retablir 312 resCmd=$? 313 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 314 315 # resultCmd=$( gives_0_or_1 ) 316 # if [ $resultCmd -eq 0 ] ; then 317 if [ $resCmd -eq 0 ] ; then 318 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 319 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 320 echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile 321 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 322 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 323 else 324 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 325 echo "====> FAILED at try #$nbtries" 326 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile 327 328 fi 329 rm -f $set".local" 330 #----------------------------------- 331 332 done 333 334 335 #------------------------------------------------------ 336 337 for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do 338 #----------------------------------- 339 echo "#-------------------" 340 echo "Set: $set" 341 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 342 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 343 statusfile=${set%%.list}.status 344 logfile=${set%%.list}_${datestr}.log 345 346 #----------------------------------- 347 # Create status file if not exists 348 if [ ! -e $statusfile ] ; then 349 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 350 fi 351 352 #----------------------------------- 353 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 354 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 355 printf "====> " 356 tail -1 $statusfile 357 continue 358 fi 359 360 #----------------------------------- 361 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 362 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 363 # echo "Nb of tries: $nbtries" 364 365 #----------------------------------- 366 # Set output file name from list name 367 dirin1=$PATH_SIMU/RESTART 368 dirin2=${dirin1##${IGCM_DEM}} 369 dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )} 370 diroutputfile=${OUTPUT_STORE}${dirin3} 371 fileout1=$( basename $set ) 372 fileout=${fileout1%.list}.tar 373 outputfile=$diroutputfile/${fileout} 374 mkdir -p $diroutputfile 375 echo "Output file to write: $outputfile" 376 377 #----------------------------------- 378 # Try 1 time before fire "FAILED" 379 # Waiting for CCRT command 380 tarcmd=tar 381 taroptions="--format=posix -W -cf" 382 # taroptions="--format=posix -cf" 383 cd $dirin1 384 # ls 385 386 startTime=$( getDateMilliSeconds ) # gpdebug : time 387 resCmd=0 388 $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir 389 resCmd=$? 390 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 391 392 # resultCmd=$( gives_0_or_1 ) 393 # if [ $resultCmd -eq 0 ] ; then 394 if [ $resCmd -eq 0 ] ; then 395 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 396 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 397 echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile 398 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 399 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 400 else 401 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 402 echo "====> FAILED at try #$nbtries" 403 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile 404 405 fi 406 407 #----------------------------------- 408 409 done 410 411 #------------------------------------------------------ 412 for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do 413 414 #----------------------------------- 415 echo "#-------------------" 416 echo "Set: $set" 417 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 418 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 419 statusfile=${set%%.list}.status 420 logfile=${set%%.list}_${datestr}.log 421 422 #----------------------------------- 423 # Create status file if not exists 424 if [ ! -e $statusfile ] ; then 425 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 426 fi 427 428 #----------------------------------- 429 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 430 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 431 printf "====> " 432 tail -1 $statusfile 433 continue 434 fi 435 436 #----------------------------------- 437 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 438 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 439 # echo "Nb of tries: $nbtries" 440 441 #----------------------------------- 442 # Set output file name from list name 443 dirin1=$PATH_SIMU/DEBUG 444 dirin2=${dirin1##${IGCM_DEM}} 445 dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )} 446 diroutputfile=${OUTPUT_STORE}${dirin3} 447 fileout1=$( basename $set ) 448 fileout=${fileout1%.list}.tar 449 outputfile=$diroutputfile/${fileout} 450 mkdir -p $diroutputfile 451 echo "Output file to write: $outputfile" 452 453 #----------------------------------- 454 # Try 1 time before fire "FAILED" 455 # Waiting for CCRT command 456 tarcmd=tar 457 taroptions="--format=posix -W -cf" 458 # taroptions="--format=posix -cf" 459 cd $dirin1 460 # ls 461 462 startTime=$( getDateMilliSeconds ) # gpdebug : time 463 resCmd=0 464 $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir 465 resCmd=$? 466 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 467 468 # resultCmd=$( gives_0_or_1 ) 469 # if [ $resultCmd -eq 0 ] ; then 470 if [ $resCmd -eq 0 ] ; then 471 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 472 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 473 echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile 474 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 475 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 476 else 477 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 478 echo "====> FAILED at try #$nbtries" 479 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile 480 481 fi 482 483 #----------------------------------- 484 485 done 486 487 #------------------------------------------------------ 488 for set in $( ls $PATH_SIMU/store_cp/*list ) ; do 489 490 #----------------------------------- 491 echo "#-------------------" 492 echo "Set: $set" 493 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 494 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 495 statusfile=${set%%.list}.status 496 logfile=${set%%.list}_${datestr}.log 497 498 #----------------------------------- 499 # Create status file if not exists 500 if [ ! -e $statusfile ] ; then 501 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 502 fi 503 504 #----------------------------------- 505 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 506 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 507 printf "====> " 508 tail -1 $statusfile 509 continue 510 fi 511 512 #----------------------------------- 513 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 514 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 515 # echo "Nb of tries: $nbtries" 516 517 #----------------------------------- 518 # Set output file name from first and last files from the current set 519 for file in $( cat $set) ; do 520 521 diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout%"` 522 mkdir -p $diroutputfile 523 echo "Output file to write: $diroutputfile/$( basename $file )" 524 525 #----------------------------------- 526 # Try 1 time before fire "FAILED" 527 # Waiting for CCRT command 528 cpcmd="cp " 529 cpoptions="-rf" 210 530 startTime=$( getDateMilliSeconds ) # gpdebug : time 211 531 resCmd=0 212 cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 # gpdebug : a retablir532 $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir 213 533 resCmd=$? 214 534 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 215 # resultCmd=$( gives_0_or_1 ) 216 # resultCmd=1 217 # if [ $resultCmd -eq 0 ] ; then 218 if [ $resCmd -eq 0 ] ; then 219 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 220 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 221 echo "DONE ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile 222 echo "====> COMPLETED at try #$nbtries (at 2nd shot on 2)" 223 echo "COMPLETED at try #$nbtries (at 2nd shot on 2)" >> $statusfile 224 else 225 rm -f $outputfile*ncrcat.tmp 226 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 227 echo "====> FAILED at try #$nbtries" 228 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; cat $set | $ncrcatcmd $ncrcatoptions -O -x -v '$varstoexclude' -o $outputfile ; $logfile ; $extratar" >> $statusfile 229 if [ $nbtries -ge $maxtries ] ; then 230 echo "====> DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" 231 echo "DELEGATE to tar command after $nbtries tries (maxtries=$maxtries)" >> $statusfile 232 cp $set $PATH_SIMU/output_tar # gpdebug : a retablir 233 fi 234 fi 235 fi 236 237 done 238 239 #------------------------------------------------------ 240 241 for set in $( ls $PATH_SIMU/output_tar/*list ) ; do 242 243 #----------------------------------- 244 echo "#-------------------" 245 echo "Set: $set" 246 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 247 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 248 statusfile=${set%%.list}.status 249 logfile=${set%%.list}_${datestr}.log 250 251 #----------------------------------- 252 # Create status file if not exists 253 if [ ! -e $statusfile ] ; then 254 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 255 fi 256 257 #----------------------------------- 258 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 259 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 260 printf "====> " 261 tail -1 $statusfile 262 continue 263 fi 264 265 #----------------------------------- 266 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 267 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 268 # echo "Nb of tries: $nbtries" 269 270 #----------------------------------- 271 # Set output file name from list name 272 273 fileout1=$( basename $set ) 274 fileout=${fileout1%.list}.tar 275 276 filefirst=`head -n 1 $set` 277 dirinputfile=`dirname $filefirst` 278 diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` 279 280 outputfile=$diroutputfile/${fileout} 281 mkdir -p $diroutputfile 282 echo "Output file to write: $outputfile" 283 284 #----------------------------------- 285 # Try 1 time before fire "FAILED" 286 # Waiting for CCRT command 287 tarcmd=tar 288 taroptions="--format=posix -W -cf" 289 #taroptions="--format=posix -cf" 290 cd $dirinputfile 291 292 set_local=$set".local" 293 for file in $( cat $set) ; do 294 basename $file >> $set".local" 295 done 296 297 # #----------------------------------- 298 startTime=$( getDateMilliSeconds ) # gpdebug : time 299 resCmd=0 300 $tarcmd $taroptions $outputfile --dereference --files-from $set.local > $logfile 2>&1 # gpdebug : a retablir 301 resCmd=$? 302 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 303 304 # resultCmd=$( gives_0_or_1 ) 305 # if [ $resultCmd -eq 0 ] ; then 306 if [ $resCmd -eq 0 ] ; then 307 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 308 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 309 echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile 310 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 311 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 312 else 313 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 314 echo "====> FAILED at try #$nbtries" 315 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile 316 317 fi 318 rm -f $set".local" 319 #----------------------------------- 320 321 done 322 323 324 #------------------------------------------------------ 325 326 for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do 327 #----------------------------------- 328 echo "#-------------------" 329 echo "Set: $set" 330 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 331 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 332 statusfile=${set%%.list}.status 333 logfile=${set%%.list}_${datestr}.log 334 335 #----------------------------------- 336 # Create status file if not exists 337 if [ ! -e $statusfile ] ; then 338 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 339 fi 340 341 #----------------------------------- 342 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 343 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 344 printf "====> " 345 tail -1 $statusfile 346 continue 347 fi 348 349 #----------------------------------- 350 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 351 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 352 # echo "Nb of tries: $nbtries" 353 354 #----------------------------------- 355 # Set output file name from list name 356 dirin1=$PATH_SIMU/RESTART 357 dirin2=${dirin1##${IGCM_DEM}} 358 dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )} 359 diroutputfile=${OUTPUT_STORE}${dirin3} 360 fileout1=$( basename $set ) 361 fileout=${fileout1%.list}.tar 362 outputfile=$diroutputfile/${fileout} 363 mkdir -p $diroutputfile 364 echo "Output file to write: $outputfile" 365 366 #----------------------------------- 367 # Try 1 time before fire "FAILED" 368 # Waiting for CCRT command 369 tarcmd=tar 370 taroptions="--format=posix -W -cf" 371 # taroptions="--format=posix -cf" 372 cd $dirin1 373 # ls 374 375 startTime=$( getDateMilliSeconds ) # gpdebug : time 376 resCmd=0 377 $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir 378 resCmd=$? 379 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 380 381 # resultCmd=$( gives_0_or_1 ) 382 # if [ $resultCmd -eq 0 ] ; then 383 if [ $resCmd -eq 0 ] ; then 384 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 385 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 386 echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile 387 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 388 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 389 else 390 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 391 echo "====> FAILED at try #$nbtries" 392 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile 393 394 fi 395 396 #----------------------------------- 397 398 done 399 400 #------------------------------------------------------ 401 for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do 402 403 #----------------------------------- 404 echo "#-------------------" 405 echo "Set: $set" 406 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 407 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 408 statusfile=${set%%.list}.status 409 logfile=${set%%.list}_${datestr}.log 410 411 #----------------------------------- 412 # Create status file if not exists 413 if [ ! -e $statusfile ] ; then 414 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 415 fi 416 417 #----------------------------------- 418 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 419 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 420 printf "====> " 421 tail -1 $statusfile 422 continue 423 fi 424 425 #----------------------------------- 426 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 427 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 428 # echo "Nb of tries: $nbtries" 429 430 #----------------------------------- 431 # Set output file name from list name 432 dirin1=$PATH_SIMU/DEBUG 433 dirin2=${dirin1##${IGCM_DEM}} 434 dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )} 435 diroutputfile=${OUTPUT_STORE}${dirin3} 436 fileout1=$( basename $set ) 437 fileout=${fileout1%.list}.tar 438 outputfile=$diroutputfile/${fileout} 439 mkdir -p $diroutputfile 440 echo "Output file to write: $outputfile" 441 442 #----------------------------------- 443 # Try 1 time before fire "FAILED" 444 # Waiting for CCRT command 445 tarcmd=tar 446 taroptions="--format=posix -W -cf" 447 # taroptions="--format=posix -cf" 448 cd $dirin1 449 # ls 450 451 startTime=$( getDateMilliSeconds ) # gpdebug : time 452 resCmd=0 453 $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 # gpdebug : a retablir 454 resCmd=$? 455 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 456 457 # resultCmd=$( gives_0_or_1 ) 458 # if [ $resultCmd -eq 0 ] ; then 459 if [ $resCmd -eq 0 ] ; then 460 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 461 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 462 echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile 463 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 464 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 465 else 466 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 467 echo "====> FAILED at try #$nbtries" 468 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile 469 470 fi 471 472 #----------------------------------- 473 474 done 475 476 #------------------------------------------------------ 477 for set in $( ls $PATH_SIMU/store_cp/*list ) ; do 478 479 #----------------------------------- 480 echo "#-------------------" 481 echo "Set: $set" 482 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 483 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 484 statusfile=${set%%.list}.status 485 logfile=${set%%.list}_${datestr}.log 486 487 #----------------------------------- 488 # Create status file if not exists 489 if [ ! -e $statusfile ] ; then 490 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 491 fi 492 493 #----------------------------------- 494 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 495 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 496 printf "====> " 497 tail -1 $statusfile 498 continue 499 fi 500 501 #----------------------------------- 502 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 503 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 504 # echo "Nb of tries: $nbtries" 505 506 #----------------------------------- 507 # Set output file name from first and last files from the current set 508 for file in $( cat $set) ; do 509 510 diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout%"` 511 mkdir -p $diroutputfile 512 echo "Output file to write: $diroutputfile/$( basename $file )" 513 514 #----------------------------------- 515 # Try 1 time before fire "FAILED" 516 # Waiting for CCRT command 517 cpcmd="cp " 518 cpoptions="-rf" 519 startTime=$( getDateMilliSeconds ) # gpdebug : time 520 resCmd=0 521 $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir 522 resCmd=$? 523 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : time 524 done 525 526 # resultCmd=$( gives_0_or_1 ) 527 # if [ $resultCmd -eq 0 ] ; then 528 if [ $resCmd -eq 0 ] ; then 529 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 530 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 531 echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 532 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 533 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 534 else 535 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 536 echo "====> FAILED at try #$nbtries" 537 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 538 539 fi 540 541 #----------------------------------- 542 543 done 544 545 #------------------------------------------------------ 546 for set in $( ls $PATH_SIMU/work_cp/*list ) ; do 547 548 #----------------------------------- 549 echo "#-------------------" 550 echo "Set: $set" 551 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 552 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 553 statusfile=${set%%.list}.status 554 logfile=${set%%.list}_${datestr}.log 555 556 #----------------------------------- 557 # Create status file if not exists 558 if [ ! -e $statusfile ] ; then 559 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 560 fi 561 562 #----------------------------------- 563 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 564 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 565 printf "====> " 566 tail -1 $statusfile 567 continue 568 fi 569 570 #----------------------------------- 571 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 572 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 573 # echo "Nb of tries: $nbtries" 574 575 #----------------------------------- 576 # Set output file name from first and last files from the current set 577 for file in $( cat $set) ; do 578 579 diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout_work%"` 580 mkdir -p $diroutputfile 581 echo "Output file to write: $diroutputfile/$( basename $file )" 582 583 #----------------------------------- 584 # Try 1 time before fire "FAILED" 585 # Waiting for CCRT command 586 cpcmd="cp " 587 cpoptions="-rf" 588 startTime=$( getDateMilliSeconds ) # gpdebug : time 589 resCmd=0 590 $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir 591 resCmd=$? 592 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : timeS 593 done 594 595 # resultCmd=$( gives_0_or_1 ) 596 # if [ $resultCmd -eq 0 ] ; then 597 if [ $resCmd -eq 0 ] ; then 598 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 599 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 600 echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 601 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 602 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 603 else 604 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 605 echo "====> FAILED at try #$nbtries" 606 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 607 608 fi 609 610 #----------------------------------- 611 612 done 613 535 done 536 537 # resultCmd=$( gives_0_or_1 ) 538 # if [ $resultCmd -eq 0 ] ; then 539 if [ $resCmd -eq 0 ] ; then 540 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 541 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 542 echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 543 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 544 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 545 else 546 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 547 echo "====> FAILED at try #$nbtries" 548 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 549 550 fi 551 552 #----------------------------------- 553 554 done 555 556 #------------------------------------------------------ 557 for set in $( ls $PATH_SIMU/work_cp/*list ) ; do 558 559 #----------------------------------- 560 echo "#-------------------" 561 echo "Set: $set" 562 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 563 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 564 statusfile=${set%%.list}.status 565 logfile=${set%%.list}_${datestr}.log 566 567 #----------------------------------- 568 # Create status file if not exists 569 if [ ! -e $statusfile ] ; then 570 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 571 fi 572 573 #----------------------------------- 574 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 575 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 576 printf "====> " 577 tail -1 $statusfile 578 continue 579 fi 580 581 #----------------------------------- 582 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 583 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 584 # echo "Nb of tries: $nbtries" 585 586 #----------------------------------- 587 # Set output file name from first and last files from the current set 588 for file in $( cat $set) ; do 589 590 diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout_work%"` 591 mkdir -p $diroutputfile 592 echo "Output file to write: $diroutputfile/$( basename $file )" 593 594 #----------------------------------- 595 # Try 1 time before fire "FAILED" 596 # Waiting for CCRT command 597 cpcmd="cp " 598 cpoptions="-rf" 599 startTime=$( getDateMilliSeconds ) # gpdebug : time 600 resCmd=0 601 $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 # gpdebug : a retablir 602 resCmd=$? 603 meantime=$( getTimeDiffSeconds $startTime ) # gpdebug : timeS 604 done 605 606 # resultCmd=$( gives_0_or_1 ) 607 # if [ $resultCmd -eq 0 ] ; then 608 if [ $resCmd -eq 0 ] ; then 609 sed -i "1i\meantime:${meantime}" $statusfile # gpdebug : time 610 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 611 echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 612 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 613 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 614 else 615 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 616 echo "====> FAILED at try #$nbtries" 617 echo "FAILED at try #$nbtries ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 618 619 fi 620 621 #----------------------------------- 622 623 done 624 fi 614 625 # if [ "$baseNameOfListFile" == "tar_full_simul.list" ] 615 for set in $( ls $ PATH_SIMU/tar_full_simul.list)626 for set in $( ls ${PATH_SIMU}/*.list | grep "tar_full_simul" ) 616 627 do 617 628 # set="${PATH_SIMU}/tar_full_simul.list" # gpdebug : foireux ?
Note: See TracChangeset
for help on using the changeset viewer.
