Changeset 1863
- Timestamp:
- 09/05/12 16:40:02 (12 years ago)
- Location:
- TOOLS/PACK_IPSL
- Files:
-
- 7 edited
Legend:
- Unmodified
- Added
- Removed
-
TOOLS/PACK_IPSL/launch_and_measureTime.sh
r1862 r1863 25 25 then 26 26 local cmdToPrint=`echo $cmdReport | awk '{print $NF}' ` 27 echo "./process_list.sh $cmdToPrint" >> ${tasksListFile} 27 case "x$resCmd" in 28 x5) 29 newCmdToPrint=`echo $cmdToPrint | sed 's;output_ncrcat;output_tar;' ` 30 echo "./process_list.sh $newCmdToPrint" >> ${tasksListFile} 31 ;; 32 x10) 33 echo "./process_list.sh $cmdToPrint" >> ${tasksListFile} 34 newCmdToPrint=`echo $cmdToPrint | sed 's;output_ncrcat;output_tar;' ` 35 echo "./process_list.sh $newCmdToPrint" >> ${tasksListFile} 36 ;; 37 *) 38 echo "./process_list.sh $cmdToPrint" >> ${tasksListFile} 39 ;; 40 esac 28 41 fi 29 42 done … … 162 175 #MSUB -r pack_ipsl # Nom du job 163 176 ### mutable directives ### 164 #MSUB -o /ccc/dmfbuf/import_data.2/ccrt/dmnfs12/cont003/bacasable/GUILLAUME/PSEUDO_DMNFS_PROGRESS/zIGCM_OUT/ LMDZ4OR_v2/detailed_pack_output/pack_ipsl_%I.o165 #MSUB -e /ccc/dmfbuf/import_data.2/ccrt/dmnfs12/cont003/bacasable/GUILLAUME/PSEUDO_DMNFS_PROGRESS/zIGCM_OUT/ LMDZ4OR_v2/detailed_pack_output/pack_ipsl_%I.e166 #MSUB -n 4177 #MSUB -o /ccc/dmfbuf/import_data.2/ccrt/dmnfs12/cont003/bacasable/GUILLAUME/PSEUDO_DMNFS_PROGRESS/zIGCM_OUT/detailed_pack_output/pack_ipsl_%I.o 178 #MSUB -e /ccc/dmfbuf/import_data.2/ccrt/dmnfs12/cont003/bacasable/GUILLAUME/PSEUDO_DMNFS_PROGRESS/zIGCM_OUT/detailed_pack_output/pack_ipsl_%I.e 179 #MSUB -n 7 167 180 #MSUB -T 900 168 181 #MSUB -A tgcc0013 … … 233 246 do 234 247 echo "./process_list.sh $file" >> ${inputCmd} 235 done 236 248 done 237 249 done 250 238 251 else 239 252 # if try > 1 && inst == 1 ==> construction liste cmd avec fichiers try - 1, derniere instance … … 299 312 300 313 # a virer ######################################### 301 #if [ ${numNewInstance} -ge 4]302 #then303 #echo >> $badFailureFile304 # echo "4eme instance. STOP." >> $badFailureFile305 #exit 1306 #fi314 if [ ${numNewInstance} -ge 10 ] 315 then 316 echo >> $badFailureFile 317 echo "10eme instance. STOP." >> $badFailureFile 318 exit 1 319 fi 307 320 ################################################### 308 321 # startTime=$( getDateMilliSeconds ) # suppr … … 341 354 # ---------------------- 342 355 everythingOK=`cat $nextInputCmd | wc -l ` 343 if [ "x {everythingOK}" == "x0"]356 if [ "x${everythingOK}" == "x0"] 344 357 then 345 358 echo "Tout s'est fini correctement" >> $badFailureFile -
TOOLS/PACK_IPSL/launch_ipsl_pack.sh
r1859 r1863 68 68 . load_ipslPack_env.sh 69 69 70 if [ "x${IGCM_DEM}" == "x" ] 71 then 72 echo "variable IGCM_DEM non renseignee. STOP." 73 exit 1 74 fi 75 # rm -f ${IGCM_DEM}/Listing.txt # gpdebug : en cas de reprise, on ne peut pas laisser ca en l'etat --> suppr 76 # gpdebug : Donc on rajoute le if qui suit 77 if [ "x${execEveryStep}" == "x1" ] 78 then 79 echo "Destruction du repertoire :" 80 echo "${IGCM_DEM}" 81 rm -rf ${IGCM_DEM} 82 fi 83 70 84 export IGCM_TMP="${IGCM_DEM}/tmp" # gpdebug : added 71 85 # INPUT_DMF_DATA, positionnee par le load_ipslPack_env.sh, sert pour pour PATH_SIMUL_SHORT dans create_config_card.sh … … 102 116 # 0- on cree le listing correspondant au fichier param 103 117 # on efface les tests precedents 104 105 # rm -f ${IGCM_DEM}/Listing.txt # gpdebug : en cas de reprise, on ne peut pas laisser ca en l'etat --> suppr106 # gpdebug : Donc on rajoute le if qui suit107 if [ "x${execEveryStep}" == "x1" ]108 then109 rm -f ${IGCM_DEM}/Listing.txt110 fi111 118 112 119 for DIR in $( awk '{print $0}' ${LISTE_SIMUL} ) ; do -
TOOLS/PACK_IPSL/load_batch_directives.sh
r1853 r1863 2 2 3 3 # Directives Batch pour le code parallele 4 export nbProcs=" 4"4 export nbProcs="7" 5 5 export computationTime="900" 6 6 export timeLimitBeforeEnd="890" -
TOOLS/PACK_IPSL/load_ipslPack_env.sh
r1861 r1863 23 23 #export OUTPUT_PROGRESS="/scratch/cont003/pierreg/PSEUDO_DMNFS_PROGRESS" 24 24 25 26 27 # Determination du chemin pour le rep ${PSEUDO_DMNFS_PROGRESS}/USER (seulement pour la partie de pack) 28 # ------------------------------------------------------------------ 29 # Le rep ${OUTPUT_PROGRESS} contiendra des sous rep USER 30 # ${OUTPUT_PROGRESS}/USER contiendra, pour l'utilisateur USER : 31 # * la liste des taches a executer en parallele par le prog c/mpi 32 # * la nouvelle liste des taches qui n'ont pas ete reussies dans un premier temps 33 # * les sorties des taches executees en parallele (traitement des liste de fichiers) 34 # L'idee est de regrouper ces elements dans un rep ${OUTPUT_PROGRESS}/USER 35 # Pour determiner le rep USER, on parcourt ${IGCM_DEM}/config_card.liste. 36 # La chaine de caract commune a toutes les lignes de ce fichier est le rep USER, 37 # que l'on cree ensuite dans ${OUTPUT_PROGRESS} 38 39 function find_intersection 40 { 41 # Trouve la sous chaine commune de 2 chaines de caract 42 local string1=$1 43 local string2=$2 44 45 local lengthString1=${#string1} 46 local lengthString2=${#string2} 47 48 local longestString= 49 local smallestString= 50 if [ $lengthString1 -ge $lengthString2 ] 51 then 52 longestString=$string1 53 smallestString=$string2 54 else 55 longestString=$string2 56 smallestString=$string1 57 fi 58 59 while [ "x`echo $longestString | grep $smallestString `" == "x" ] 60 do 61 smallestString=$( dirname $smallestString ) 62 done 63 echo "$smallestString" 64 } 65 66 25 67 scriptCallingThisOne="$scriptCallingLoad" 26 68 # echo "scriptCallingThisOne=$scriptCallingThisOne" … … 29 71 then 30 72 31 # Determination du chemin pour le rep ${PSEUDO_DMNFS_PROGRESS}/USER (seulement pour la partie de pack) 32 # ------------------------------------------------------------------ 33 # Le rep ${OUTPUT_PROGRESS} contiendra des sous rep USER 34 # ${OUTPUT_PROGRESS}/USER contiendra, pour l'utilisateur USER : 35 # * la liste des taches a executer en parallele par le prog c/mpi 36 # * la nouvelle liste des taches qui n'ont pas ete reussies dans un premier temps 37 # * les sorties des taches executees en parallele (traitement des liste de fichiers) 38 # L'idee est de regrouper ces elements dans un rep ${OUTPUT_PROGRESS}/USER 39 # Pour determiner le rep USER, on parcourt ${IGCM_DEM}/config_card.liste. 40 # La chaine de caract commune a toutes les lignes de ce fichier est le rep USER, 41 # que l'on cree ensuite dans ${OUTPUT_PROGRESS} 42 43 function find_intersection 44 { 45 # Trouve la sous chaine commune de 2 chaines de caract 46 local string1=$1 47 local string2=$2 48 49 local lengthString1=${#string1} 50 local lengthString2=${#string2} 51 52 local longestString= 53 local smallestString= 54 if [ $lengthString1 -ge $lengthString2 ] 55 then 56 longestString=$string1 57 smallestString=$string2 58 else 59 longestString=$string2 60 smallestString=$string1 61 fi 62 63 while [ "x`echo $longestString | grep $smallestString `" == "x" ] 64 do 65 smallestString=$( dirname $smallestString ) 66 done 67 echo "$smallestString" 68 } 69 73 echo "specific loading for pack script." 74 70 75 COMMON_STRING=`head -n 1 ${IGCM_DEM}/config_card.liste | awk '{print $1}' | xargs dirname ` 71 76 for CONFIG in $( awk '{print $1}' ${IGCM_DEM}/config_card.liste ) ; do -
TOOLS/PACK_IPSL/parallelPack.sh
r1862 r1863 241 241 done 242 242 243 244 245 243 if [ ${elapsedTime} -ge 500 ] 244 then 245 echo "Les 500 sec sont ecoulees, les traitements continuent..." 246 fi 247 248 249 250 -
TOOLS/PACK_IPSL/process_list.sh
r1853 r1863 107 107 echo "DELEGATE to tar command with option --switchtotar" >> $statusfile 108 108 mv $set $PATH_SIMU/output_tar 109 exit 0109 exit 5 110 110 fi 111 111 # echo "output_ncrcat : blabla 2" … … 202 202 cp $set $PATH_SIMU/output_tar # gpdebug : a retablir 203 203 fi 204 exit 1 204 205 # exit 10 : il faut tarer cette liste dans "output_tar" et tenter de la concatener a nouveau 206 # exit 5 : seulement la tarer dans "output_tar" 207 if [ "x${extratar}" == "x1" ] 208 then 209 exit 10 210 else 211 exit 5 212 fi 205 213 fi 206 214 fi -
TOOLS/PACK_IPSL/showPackProgress.sh
r1862 r1863 423 423 # find $IGCM_DEM -printf "%y %s %p \n" >> ${IGCM_DEM}/ListingEnd.txt 424 424 rm -f ${IGCM_DEM}/ListingEnd.txt 425 find $OUTPUT_STORE -printf "%y %s %p \n" >> ${IGCM_DEM}/ListingEnd.txt 426 find $OUTPUT_WORK -printf "%y %s %p \n" >> ${IGCM_DEM}/ListingEnd.txt 425 if [ -d "${OUTPUT_STORE}" ] 426 then 427 find $OUTPUT_STORE -printf "%y %s %p \n" >> ${IGCM_DEM}/ListingEnd.txt 428 fi 429 if [ -d "${OUTPUT_WORK}" ] 430 then 431 find $OUTPUT_WORK -printf "%y %s %p \n" >> ${IGCM_DEM}/ListingEnd.txt 432 fi 427 433 NbInodeAfter=`cat ${IGCM_DEM}/ListingEnd.txt |wc -l` 428 echo 429 echo "##################" 430 echo "##### BILAN #####" 431 echo "##################" 432 echo "nb of Lists packed with success : ${nbOfListsPacked} / ${totalNbOfList}" 433 echo "nb of fails : $nbOfListsFailed" 434 echo "nb of inodes packed : $TotalNbInodes" 435 echo "nb simul full tared : $TotalSimuTar" 436 echo "nb inode before : $NbInodeBefore" 437 echo "nb inode after : $NbInodeAfter" 438 echo "Total time for elementary operations : $totalTime" 439 # echo "waiting time : $waitingTime" # supprime pour le moment : pb si +sieurs instances ==> on simplifie 440 # echo "Time since execution start : $timeSinceExecutionStart" # supprime pour le moment : pb si +sieurs instances ==> on simplifie 441 echo "Time since launch start : $timeSinceLaunchStart" 442 443 444 434 435 if [ "x${outputFile}" == "x" ] 436 then 437 echo 438 echo "##################" 439 echo "##### BILAN #####" 440 echo "##################" 441 echo "nb of Lists packed with success : ${nbOfListsPacked} / ${totalNbOfList}" 442 echo "nb of fails : $nbOfListsFailed" 443 echo "nb of inodes packed : $TotalNbInodes" 444 echo "nb simul full tared : $TotalSimuTar" 445 echo "nb inode before : $NbInodeBefore" 446 echo "nb inode after : $NbInodeAfter" 447 echo "Total time for elementary operations : $totalTime" 448 echo "Time since launch start : $timeSinceLaunchStart" 449 450 else 451 echo 452 echo "##################" >> ${outputFile} 453 echo "##### BILAN #####" >> ${outputFile} 454 echo "##################" >> ${outputFile} 455 echo "nb of Lists packed with success : ${nbOfListsPacked} / ${totalNbOfList}" >> ${outputFile} 456 echo "nb of fails : $nbOfListsFailed" >> ${outputFile} 457 echo "nb of inodes packed : $TotalNbInodes" >> ${outputFile} 458 echo "nb simul full tared : $TotalSimuTar" >> ${outputFile} 459 echo "nb inode before : $NbInodeBefore" >> ${outputFile} 460 echo "nb inode after : $NbInodeAfter" >> ${outputFile} 461 echo "Total time for elementary operations : $totalTime" >> ${outputFile} 462 echo "Time since launch start : $timeSinceLaunchStart" >> ${outputFile} 463 464 fi
Note: See TracChangeset
for help on using the changeset viewer.