Changeset 1752
- Timestamp:
- 06/06/12 18:01:10 (12 years ago)
- Location:
- TOOLS/PACK_IPSL
- Files:
-
- 2 edited
-
enlarge_my_files.sh (modified) (10 diffs)
-
launch_ipsl_enlarge.sh (modified) (2 diffs)
Legend:
- Unmodified
- Added
- Removed
-
TOOLS/PACK_IPSL/enlarge_my_files.sh
r1727 r1752 1 1 #!/bin/bash 2 3 2 #------------------------------------------------------ 4 3 switchtotar=0 5 4 maxtries=3 6 7 5 #--------------------------------------------- 8 6 while [ $# -ne 0 ] … … 20 18 echo 21 19 echo "Description:" 22 echo " Pass a ncrcat command on files from prepared sets of files."23 echo " 2 shots will be done at each try".20 echo " Pass a ncrcat, tar or cp command on files from prepared sets of files." 21 echo " For ncrcat command 2 shots will be done at each try". 24 22 echo 25 echo " A status file and a log file will be created in enlarge_my_files/with_ncrcat directory."26 echo " Copy a set to enlarge_my_files/with_tar directory for later processing after N tries."23 echo " A status file and a log file will be created in output_ncrcat directory." 24 echo " Copy a set to output_tar directory for later processing after N tries." 27 25 echo 28 26 echo "Arguments:" … … 33 31 echo " enlarge_my_files" 34 32 echo " |-- with_ncrcat" 35 echo " | |-- set_000001. txt"36 echo " | |-- set_000002. txt"37 echo " | |-- set_000003. txt"38 echo " | |-- set_000004. txt"39 echo " | \`-- set_000005. txt"33 echo " | |-- set_000001.list" 34 echo " | |-- set_000002.list" 35 echo " | |-- set_000003.list" 36 echo " | |-- set_000004.list" 37 echo " | \`-- set_000005.list" 40 38 echo " \`-- with_tar" 41 39 echo … … 64 62 fi 65 63 66 if [[ ! -d $PATH_SIMU/ with_ncrcat || ! -d $PATH_SIMU/with_tar ]] ; then64 if [[ ! -d $PATH_SIMU/output_ncrcat || ! -d $PATH_SIMU/output_tar ]] ; then 67 65 `dirname $0`/enlarge_my_files.sh -h 68 66 exit … … 72 70 dirin=$1 73 71 dirout=$2 72 dirout_work=$3 74 73 75 74 #------------------------------------------------------ … … 91 90 92 91 #------------------------------------------------------ 93 for set in $ PATH_SIMU/with_ncrcat/*list; do92 for set in $( ls $PATH_SIMU/output_ncrcat/*list ) ; do 94 93 95 94 #----------------------------------- … … 159 158 # Try 2 times before fire "FAILED" 160 159 ncrcatcmd=ncrcat 161 #ncrcatoptions="--md5_digest"162 ncrcatoptions=""160 ncrcatoptions="--md5_digest" 161 # ncrcatoptions="" 163 162 164 163 cat $set | $ncrcatcmd $ncrcatoptions -O -x -v $varstoexclude -o $outputfile > $logfile 2>&1 … … 188 187 fi 189 188 190 #-----------------------------------191 192 189 done 193 190 194 191 #------------------------------------------------------ 195 192 196 #------------------------------------------------------ 197 for set in $PATH_SIMU/with_tar/*list ; do 193 for set in $( ls $PATH_SIMU/output_tar/*list ) ; do 198 194 199 195 #----------------------------------- … … 227 223 # Set output file name from list name 228 224 filefirst=`head -n 1 $set` 229 dirin1=$( dirname $filefirst ) 230 fileout1=$( basename $set ) 231 fileout=${fileout1%.list} 225 dirinputfile=`dirname $filefirst` 232 226 diroutputfile=`dirname $filefirst | sed -e "s%$dirin%$dirout%"` 233 # outputfile=$diroutputfile/ncrcat_${fileoutpart1}_${fileoutpart2} 234 outputfile=$diroutputfile/${fileout}.tar 235 echo $outputfile 227 mkdir -p $diroutputfile 228 # echo "Output file to write: $outputfile" 229 fileout1=$( basename $set ) 230 outputfile=${fileout1%.list}.tar 231 echo "Output file to write: $outputfile" 232 233 #----------------------------------- 234 # Try 1 time before fire "FAILED" 235 # Waiting for CCRT command 236 tarcmd=tar 237 taroptions="--format=posix -W -cf" 238 239 cd $dirinputfile 240 241 set_local=$set".local" 242 for file in $( cat $set) ; do 243 basename $file >> $set".local" 244 done 245 246 # #----------------------------------- 247 248 $tarcmd $taroptions $diroutputfile/$outputfile --dereference --files-from $set.local > $logfile 2>&1 249 if [ $? -eq 0 ] ; then 250 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 251 echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile 252 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 253 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 254 else 255 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 256 echo "====> FAILED at try #$nbtries" 257 echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile 258 259 fi 260 rm -f $set".local" 261 #----------------------------------- 262 263 done 264 265 266 #------------------------------------------------------ 267 268 for set in $( ls $PATH_SIMU/restart_tar/*list ) ; do 269 270 #----------------------------------- 271 echo "#-------------------" 272 echo "Set: $set" 273 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 274 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 275 statusfile=${set%%.list}.status 276 logfile=${set%%.list}_${datestr}.log 277 278 #----------------------------------- 279 # Create status file if not exists 280 if [ ! -e $statusfile ] ; then 281 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 282 fi 283 284 #----------------------------------- 285 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 286 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 287 printf "====> " 288 tail -1 $statusfile 289 continue 290 fi 291 292 #----------------------------------- 293 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 294 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 295 # echo "Nb of tries: $nbtries" 296 297 #----------------------------------- 298 # Set output file name from list name 299 dirin1=$PATH_SIMU/RESTART 300 dirin2=${dirin1##${IGCM_DEM}} 301 dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )} 302 diroutputfile=${OUTPUT_STORE}${dirin3} 303 fileout1=$( basename $set ) 304 fileout=${fileout1%.list}.tar 305 outputfile=$diroutputfile/${fileout} 236 306 mkdir -p $diroutputfile 237 307 echo "Output file to write: $outputfile" 238 308 239 # #-----------------------------------240 # # Find variables to exclude to have homogeneous files (always exclude 't_inst_.*')241 # extratar=0242 # find_varstoexclude $set243 # if [ $nbvars -eq 0 ] ; then244 # varstoexclude="t_inst_.*"245 # else246 # # There are others variables than 't_inst_.*' so an extra tar file will be created247 # varstoexclude="t_inst_.*,$varstoexcludefiltered"248 # extratar=1249 # # Copy to with_tar directory250 # cp $set $PATH_SIMU/with_tar251 # fi252 253 309 #----------------------------------- 254 310 # Try 1 time before fire "FAILED" 255 311 # Waiting for CCRT command 256 tarcmd= echo257 taroptions="- cvf"258 259 #cd $dirin1312 tarcmd=tar 313 taroptions="--format=posix -W -cf" 314 315 cd $dirin1 260 316 # ls 261 317 262 $tarcmd "$taroptions $outputfile $(cat $set)"> $logfile 2>&1318 $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 263 319 if [ $? -eq 0 ] ; then 264 320 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 265 echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile ; $extratar" >> $statusfile266 echo "====> COMPLETED at try #$nbtries (at 1st shot on 2)"321 echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile 322 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 267 323 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 268 324 else 269 325 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 270 326 echo "====> FAILED at try #$nbtries" 271 echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile ; $extratar" >> $statusfile327 echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile 272 328 273 329 fi … … 277 333 done 278 334 279 280 335 #------------------------------------------------------ 336 for set in $( ls $PATH_SIMU/debug_tar/*list ) ; do 337 338 #----------------------------------- 339 echo "#-------------------" 340 echo "Set: $set" 341 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 342 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 343 statusfile=${set%%.list}.status 344 logfile=${set%%.list}_${datestr}.log 345 346 #----------------------------------- 347 # Create status file if not exists 348 if [ ! -e $statusfile ] ; then 349 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 350 fi 351 352 #----------------------------------- 353 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 354 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 355 printf "====> " 356 tail -1 $statusfile 357 continue 358 fi 359 360 #----------------------------------- 361 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 362 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 363 # echo "Nb of tries: $nbtries" 364 365 #----------------------------------- 366 # Set output file name from list name 367 dirin1=$PATH_SIMU/DEBUG 368 dirin2=${dirin1##${IGCM_DEM}} 369 dirin3=${dirin2##/$( basename ${INPUT_DMF_DATA} )} 370 diroutputfile=${OUTPUT_STORE}${dirin3} 371 fileout1=$( basename $set ) 372 fileout=${fileout1%.list}.tar 373 outputfile=$diroutputfile/${fileout} 374 mkdir -p $diroutputfile 375 echo "Output file to write: $outputfile" 376 377 #----------------------------------- 378 # Try 1 time before fire "FAILED" 379 # Waiting for CCRT command 380 tarcmd=tar 381 taroptions="--format=posix -W -cf" 382 383 cd $dirin1 384 # ls 385 386 $tarcmd $taroptions $outputfile --dereference --files-from $set > $logfile 2>&1 387 if [ $? -eq 0 ] ; then 388 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 389 echo "DONE ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set); $logfile" >> $statusfile 390 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 391 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 392 else 393 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 394 echo "====> FAILED at try #$nbtries" 395 echo "FAILED ; $datestr1 ; $datestr2 ; $tarcmd $taroptions $outputfile $(cat $set) ; $logfile" >> $statusfile 396 397 fi 398 399 #----------------------------------- 400 401 done 402 403 #------------------------------------------------------ 404 for set in $( ls $PATH_SIMU/store_cp/*list ) ; do 405 406 #----------------------------------- 407 echo "#-------------------" 408 echo "Set: $set" 409 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 410 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 411 statusfile=${set%%.list}.status 412 logfile=${set%%.list}_${datestr}.log 413 414 #----------------------------------- 415 # Create status file if not exists 416 if [ ! -e $statusfile ] ; then 417 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 418 fi 419 420 #----------------------------------- 421 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 422 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 423 printf "====> " 424 tail -1 $statusfile 425 continue 426 fi 427 428 #----------------------------------- 429 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 430 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 431 # echo "Nb of tries: $nbtries" 432 433 #----------------------------------- 434 # Set output file name from first and last files from the current set 435 for file in $( cat $set) ; do 436 437 diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout%"` 438 mkdir -p $diroutputfile 439 echo "Output file to write: $diroutputfile/$( basename $file )" 440 441 #----------------------------------- 442 # Try 1 time before fire "FAILED" 443 # Waiting for CCRT command 444 cpcmd="cp " 445 cpoptions="-rf" 446 $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 447 done 448 449 if [ $? -eq 0 ] ; then 450 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 451 echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 452 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 453 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 454 else 455 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 456 echo "====> FAILED at try #$nbtries" 457 echo "FAILED ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 458 459 fi 460 461 #----------------------------------- 462 463 done 464 465 #------------------------------------------------------ 466 for set in $( ls $PATH_SIMU/work_cp/*list ) ; do 467 468 #----------------------------------- 469 echo "#-------------------" 470 echo "Set: $set" 471 datestr=`LC_ALL=C date +"%Y%m%dT%H%M%S"` 472 datestr1=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 473 statusfile=${set%%.list}.status 474 logfile=${set%%.list}_${datestr}.log 475 476 #----------------------------------- 477 # Create status file if not exists 478 if [ ! -e $statusfile ] ; then 479 echo "# Status ; DateStart ; DateEnd ; Command ; TarFileExists" > $statusfile 480 fi 481 482 #----------------------------------- 483 # Skip the set if last line of the status file is "COMPLETED" or "DELEGATE" 484 if tail -1 $statusfile | grep -q -E '(COMPLETED|DELEGATE)' ; then 485 printf "====> " 486 tail -1 $statusfile 487 continue 488 fi 489 490 #----------------------------------- 491 # nbtries = number of lines from status file (first comment line gives 1 for the first try) 492 # nbtries=`wc -l $statusfile | cut -d' ' -f1` 493 # echo "Nb of tries: $nbtries" 494 495 #----------------------------------- 496 # Set output file name from first and last files from the current set 497 for file in $( cat $set) ; do 498 499 diroutputfile=`dirname $file | sed -e "s%$dirin%$dirout_work%"` 500 mkdir -p $diroutputfile 501 echo "Output file to write: $diroutputfile/$( basename $file )" 502 503 #----------------------------------- 504 # Try 1 time before fire "FAILED" 505 # Waiting for CCRT command 506 cpcmd="cp " 507 cpoptions="-rf" 508 $cpcmd $cpoptions $file $diroutputfile > $logfile 2>&1 509 done 510 511 if [ $? -eq 0 ] ; then 512 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 513 echo "DONE ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 514 echo "====> COMPLETED at try #$nbtries (at 1st shot on 1)" 515 echo "COMPLETED at try #$nbtries (at 1st shot on 1)" >> $statusfile 516 else 517 datestr2=`LC_ALL=C date +"%Y-%m-%dT%H:%M:%S"` 518 echo "====> FAILED at try #$nbtries" 519 echo "FAILED ; $datestr1 ; $datestr2 ; $cpcmd $cpoptions $file $diroutputfile ; $logfile" >> $statusfile 520 521 fi 522 523 #----------------------------------- 524 525 done -
TOOLS/PACK_IPSL/launch_ipsl_enlarge.sh
r1727 r1752 15 15 export EXE_DIR=${JOB_DIR} 16 16 17 INPUT_DMF_DATA=${SCRATCHDIR}/PSEUDO_DMNFS 18 OUTPUT_STORE=${SCRATCHDIR}/PSEUDO_DMNFS_STORE 17 export INPUT_DMF_DATA=${SCRATCHDIR}/PSEUDO_DMNFS 18 export OUTPUT_STORE=${SCRATCHDIR}/PSEUDO_DMNFS_STORE 19 export OUTPUT_WORK=${SCRATCHDIR}/PSEUDO_DMNFS_WORK 19 20 20 21 SCRIPT_NAME=$(basename ${0} ) … … 38 39 DEM_write_state ${IGCM_DEM}/config_card.liste ${CONFIG} PackRunning 39 40 export PATH_SIMU=$( dirname $CONFIG ) 40 ${EXE_DIR}/enlarge_my_files.sh ${INPUT_DMF_DATA} ${OUTPUT_STORE} 41 ${EXE_DIR}/enlarge_my_files.sh ${INPUT_DMF_DATA} ${OUTPUT_STORE} ${OUTPUT_WORK} 41 42 42 43 DEM_write_state ${IGCM_DEM}/config_card.liste ${CONFIG} PackDone
Note: See TracChangeset
for help on using the changeset viewer.
