| 1 | #!/bin/bash |
|---|
| 2 | |
|---|
| 3 | # --------------------------------------------------------------------- |
|---|
| 4 | # |
|---|
| 5 | # process's mapping |
|---|
| 6 | # *********************** |
|---|
| 7 | # |
|---|
| 8 | # PURPOSE : |
|---|
| 9 | # --------- |
|---|
| 10 | # The execution time si also influenced by the internode |
|---|
| 11 | # communications and by the memory contention between parallel MPI tasks. |
|---|
| 12 | # This program aims at creating a suitable map of the MPI processes |
|---|
| 13 | # to the computing NODE according to different mapping strategies. |
|---|
| 14 | # It acts only on the local scheduler configuration and does not modify |
|---|
| 15 | # the parallel algorithm neither the domain decomposition. |
|---|
| 16 | # The following strategies have been implemented: |
|---|
| 17 | |
|---|
| 18 | # - com: the mapping will reduce the internode communications. The |
|---|
| 19 | # communication pattern defines a graph where each vertex |
|---|
| 20 | # represents an MPI task and an edge is a communication between |
|---|
| 21 | # two tasks. The mapping is obtained by partitioning the graph |
|---|
| 22 | # in a number of subgraph equal to the number of computing nodes; |
|---|
| 23 | # the partitioning minimizes the total number of cutting edges. |
|---|
| 24 | |
|---|
| 25 | # - mem: the mapping will reduce the memory contention between processes |
|---|
| 26 | # by evenly distributing the total amount of allocated memory |
|---|
| 27 | # among the computing nodes. The amount of memory allocated in a |
|---|
| 28 | # computing node is equal to the memory allocated in the other |
|---|
| 29 | # nodes. |
|---|
| 30 | |
|---|
| 31 | # - away: a process will not be mapped to the same node of any of |
|---|
| 32 | # its neighbours |
|---|
| 33 | |
|---|
| 34 | # INPUT : |
|---|
| 35 | # ------------- |
|---|
| 36 | |
|---|
| 37 | # This script requires the following external package: |
|---|
| 38 | # - scotch library: http://www.labri.fr/perso/pelegrin/scotch/ (for the com mapping) |
|---|
| 39 | # - the file "background.png" (that can be extracted from the bathimetry) to visualize the mapping |
|---|
| 40 | |
|---|
| 41 | # npart: number of partitions i.e. number of nodes |
|---|
| 42 | |
|---|
| 43 | # processes layout: the file reports how the MPI processes are disposed on the |
|---|
| 44 | # parallel subdomains. This file is an output of the |
|---|
| 45 | # "cmcc_mppopt_showproc" script (matrix.in) |
|---|
| 46 | |
|---|
| 47 | # processes coordinates: the file reports the cartesian coordinates of the |
|---|
| 48 | # processes subdomain. It is used for the visualization. |
|---|
| 49 | # The file is an output of the |
|---|
| 50 | # "cmcc_mppopt_showproc" script ( typical name: <confname>-<decomposition>_<num_sub_domain> example: nemo025-019x045_672 ) |
|---|
| 51 | |
|---|
| 52 | # number of ocean points: this file reports for each subdomain the number of ocean points |
|---|
| 53 | # and an estimation of the amount of memory required for |
|---|
| 54 | # each subdomain. The file is an output of the |
|---|
| 55 | # "cmcc_mpp_optimizer" script with extension *.layout |
|---|
| 56 | |
|---|
| 57 | |
|---|
| 58 | # MODIFICATIONS: |
|---|
| 59 | # -------------- |
|---|
| 60 | # v1.0: Nov 2014 (I.Epicoco, F.Macchia - CMCC) |
|---|
| 61 | # ---------------------------------------------------------------------- |
|---|
| 62 | |
|---|
| 63 | |
|---|
| 64 | #Warning !!! |
|---|
| 65 | # check $procxnode variable that must be set to the number of cores per node. |
|---|
| 66 | # It also represents the cardinality of each partition |
|---|
| 67 | |
|---|
| 68 | procxnode=16 |
|---|
| 69 | |
|---|
| 70 | |
|---|
| 71 | echo "Usage: partition.sh <proc layout file> <num partitions> <coordinates file> <num ocean point file> <type_of_mapping: {mem, comm, away}>" |
|---|
| 72 | |
|---|
| 73 | matrix=$1 |
|---|
| 74 | #matrix="matrix.in" |
|---|
| 75 | |
|---|
| 76 | #check for the processes layout file |
|---|
| 77 | if [ ! -f $matrix ]; then |
|---|
| 78 | echo "file with process layout not found" |
|---|
| 79 | exit 1 |
|---|
| 80 | fi |
|---|
| 81 | |
|---|
| 82 | maptype=$5 |
|---|
| 83 | if [ -z $maptype ]; then |
|---|
| 84 | maptype="comm" |
|---|
| 85 | fi |
|---|
| 86 | npart=$2 |
|---|
| 87 | #npart=21 |
|---|
| 88 | |
|---|
| 89 | gridfile=$3 |
|---|
| 90 | #gridfile="./nemo116-019x045_672" |
|---|
| 91 | |
|---|
| 92 | #check the process coordinate file |
|---|
| 93 | if [ ! -f $gridfile ]; then |
|---|
| 94 | echo "processes coordinates file not found" |
|---|
| 95 | exit 1 |
|---|
| 96 | fi |
|---|
| 97 | |
|---|
| 98 | layoutfile=$4 |
|---|
| 99 | #layoutfile="./0672_CMCC.layout" |
|---|
| 100 | |
|---|
| 101 | #check the .layout |
|---|
| 102 | if [ ! -f $layoutfile ]; then |
|---|
| 103 | echo "num ocean points file not found" |
|---|
| 104 | exit 1 |
|---|
| 105 | fi |
|---|
| 106 | |
|---|
| 107 | #output file for the for the mapping |
|---|
| 108 | parmetisfile=$maptype.mapping.$npart |
|---|
| 109 | |
|---|
| 110 | #output file with the TASK GEOMETRY definition |
|---|
| 111 | geometry_var=$maptype.geometry.$npart |
|---|
| 112 | |
|---|
| 113 | #output file for BG/Q settings |
|---|
| 114 | bg_geometry=$maptype.bggeometry.$npart |
|---|
| 115 | |
|---|
| 116 | #Extract the number of rows and columns of the domain decomposition |
|---|
| 117 | nline=$(cat $matrix | wc -l) |
|---|
| 118 | echo "rows= $nline" |
|---|
| 119 | ncol=$(($(cat $matrix | wc -w)/$nline)) |
|---|
| 120 | echo "columns= $ncol" |
|---|
| 121 | |
|---|
| 122 | dim=$((procxnode * npart)) |
|---|
| 123 | decomp="$ncol x $nline" #Be aware of the white spaces |
|---|
| 124 | #decomp="8 x 37" #be aware of the white spaces |
|---|
| 125 | |
|---|
| 126 | ############################################################# |
|---|
| 127 | # mapping based on cardinal order |
|---|
| 128 | ############################################################# |
|---|
| 129 | |
|---|
| 130 | if [ $maptype = "card" ]; then |
|---|
| 131 | |
|---|
| 132 | rm $parmetisfile >& /dev/null |
|---|
| 133 | |
|---|
| 134 | for i in `seq 0 $((npart - 1))`; do |
|---|
| 135 | for j in `seq 1 $procxnode`; do |
|---|
| 136 | echo $i >> $parmetisfile |
|---|
| 137 | done |
|---|
| 138 | done |
|---|
| 139 | fi |
|---|
| 140 | |
|---|
| 141 | ############################################################# |
|---|
| 142 | # mapping based on calc2 strategy |
|---|
| 143 | ############################################################# |
|---|
| 144 | |
|---|
| 145 | if [ $maptype = "calc2" ]; then |
|---|
| 146 | |
|---|
| 147 | grep -A $((dim+1)) "$decomp" $layoutfile | grep -v Total | awk '{print $2 " - " $1}' | sort -n -r > oceanpt.def |
|---|
| 148 | rm mapping.tmp >& /dev/null |
|---|
| 149 | for i in `seq 0 $((dim-1))`; do |
|---|
| 150 | line=`sed -n "$((i+1)),$((i+1)) p" oceanpt.def` |
|---|
| 151 | echo "$((i % npart)) - $line" >> mapping.tmp |
|---|
| 152 | done |
|---|
| 153 | |
|---|
| 154 | cat mapping.tmp | awk '{print $5 " " $1}' | sort -n | cut -d " " -f 2 > $parmetisfile |
|---|
| 155 | |
|---|
| 156 | fi |
|---|
| 157 | |
|---|
| 158 | ############################################################# |
|---|
| 159 | # mapping based on memory |
|---|
| 160 | ############################################################# |
|---|
| 161 | if [ $maptype = "calc" -o $maptype = "mem" ]; then |
|---|
| 162 | |
|---|
| 163 | rm oceanpt.* >& /dev/null |
|---|
| 164 | |
|---|
| 165 | #reading the ocean points memory usage |
|---|
| 166 | if [ $maptype = "calc" ]; then |
|---|
| 167 | grep -A $((dim+1)) "$decomp" $layoutfile | grep -v Total | awk '{print $2 " - " $1}' | sort -n -r > oceanpt.def |
|---|
| 168 | else |
|---|
| 169 | grep -A $((dim+1)) "$decomp" $layoutfile | grep -v Total | awk '{print $3 " - " $1}' | sort -n -r > oceanpt.def |
|---|
| 170 | fi |
|---|
| 171 | |
|---|
| 172 | i=0 |
|---|
| 173 | for node in `cat oceanpt.def | cut -d "-" -f 1 | sed -e "s/\.//" -e "s/^0*//"`; do |
|---|
| 174 | ocnpnt[i]=$node |
|---|
| 175 | ((i++)) |
|---|
| 176 | done; |
|---|
| 177 | |
|---|
| 178 | i=0 |
|---|
| 179 | for node in `cat oceanpt.def | cut -d "-" -f 2`; do |
|---|
| 180 | rank[i]=$node |
|---|
| 181 | ((i++)) |
|---|
| 182 | done; |
|---|
| 183 | |
|---|
| 184 | rm weight* mapp_* >& /dev/null |
|---|
| 185 | |
|---|
| 186 | |
|---|
| 187 | for i in `seq 0 $(($npart-1))`; do |
|---|
| 188 | echo "0 - $i" >> weight |
|---|
| 189 | done |
|---|
| 190 | |
|---|
| 191 | for i in `seq 0 $(($dim - 1))`; do |
|---|
| 192 | minrow="`sort -n weight| head -1`" |
|---|
| 193 | minocn=`echo $minrow | cut -d "-" -f 1 | sed -e "s/ //g"` |
|---|
| 194 | minnode=`echo $minrow | cut -d "-" -f 2 | sed -e "s/ //g"` |
|---|
| 195 | |
|---|
| 196 | printf "\r Processing node $((i+1)) of $dim \t" |
|---|
| 197 | |
|---|
| 198 | echo "${rank[$i]}" >> mapp_$minnode |
|---|
| 199 | newvalue=$(($minocn + ${ocnpnt[$i]})) |
|---|
| 200 | if [ `wc -l mapp_$minnode | cut -d " " -f 1` -eq $procxnode ]; then |
|---|
| 201 | grep -v "${minrow}$" weight > weight.new |
|---|
| 202 | echo "$newvalue - $minnode" >> weight.final |
|---|
| 203 | else |
|---|
| 204 | sed -e "s/${minrow}$/$newvalue - $minnode/" weight > weight.new |
|---|
| 205 | fi |
|---|
| 206 | mv weight.new weight |
|---|
| 207 | done |
|---|
| 208 | echo " " |
|---|
| 209 | |
|---|
| 210 | for i in `ls mapp_*`; do |
|---|
| 211 | idx=`echo $i|cut -d "_" -f 2` |
|---|
| 212 | sed -e "s/^\(.*\)/\1 $idx/" $i >> metis.part.memory.temp |
|---|
| 213 | done |
|---|
| 214 | |
|---|
| 215 | rm $parmetisfile >& /dev/null |
|---|
| 216 | |
|---|
| 217 | sort -n metis.part.memory.temp | cut -d " " -f 2 >> $parmetisfile |
|---|
| 218 | rm metis.part.memory.temp |
|---|
| 219 | |
|---|
| 220 | rm oceanpt.* weight* mapp_* >& /dev/null |
|---|
| 221 | |
|---|
| 222 | fi |
|---|
| 223 | |
|---|
| 224 | ############################################################# |
|---|
| 225 | # Creation of the graph file in parmetis format |
|---|
| 226 | ############################################################# |
|---|
| 227 | |
|---|
| 228 | |
|---|
| 229 | el=0 |
|---|
| 230 | #for i in `tail -r $matrix`; do |
|---|
| 231 | for i in `tac $matrix`; do |
|---|
| 232 | m[el]=$(($i+1)) |
|---|
| 233 | ((el++)) |
|---|
| 234 | done; |
|---|
| 235 | |
|---|
| 236 | |
|---|
| 237 | if [ $el -ne $(($nline*$ncol)) ] ; then |
|---|
| 238 | echo "ERROR" |
|---|
| 239 | exit 1 |
|---|
| 240 | fi |
|---|
| 241 | |
|---|
| 242 | for((i=0;i<${#m[@]};i++)) ; do |
|---|
| 243 | if [ ${m[$i]} -ne 0 ] ; then |
|---|
| 244 | |
|---|
| 245 | up=${m[(($i+$ncol))]} ; if [[ $up -ne 0 ]] ; then echo -n "$up " >> $outdir"parmetis.part"; fi |
|---|
| 246 | |
|---|
| 247 | if [ $((i - ncol )) -ge 0 ]; then |
|---|
| 248 | down=${m[(($i-$ncol))]} ; if [[ $down -ne 0 ]] ; then echo -n "$down " >> $outdir"parmetis.part"; fi |
|---|
| 249 | fi |
|---|
| 250 | |
|---|
| 251 | #left boundary |
|---|
| 252 | if [[ $(($i%$ncol)) -eq 0 ]] ; then |
|---|
| 253 | left=${m[(($i+$ncol-1))]} |
|---|
| 254 | else |
|---|
| 255 | left=${m[(($i-1))]} |
|---|
| 256 | fi |
|---|
| 257 | if [[ $left -ne 0 ]] ; then echo -n "$left " >> $outdir"parmetis.part"; fi |
|---|
| 258 | |
|---|
| 259 | #right boundary |
|---|
| 260 | if [[ $(($(($i+1))%$ncol)) -eq 0 ]] ; then |
|---|
| 261 | right=${m[(($i-$ncol+1))]} |
|---|
| 262 | else |
|---|
| 263 | right=${m[(($i+1))]} ; |
|---|
| 264 | fi |
|---|
| 265 | if [[ $right -ne 0 ]] ; then echo -n "$right " >> $outdir"parmetis.part"; fi |
|---|
| 266 | |
|---|
| 267 | #north pole domains |
|---|
| 268 | if [[ $i -ge $(($nline*$ncol-$ncol)) ]] ; then |
|---|
| 269 | near1=${m[(($ncol*((2*$nline-1))-$i-2))]} ; if [[ ( $near1 != 0 ) && ( $near1 != $up ) && ( $near1 != $down ) && ( $near1 != $left ) && ( $near1 != $right ) && ( $near1 != ${m[$i]} ) ]] ; then echo -n "$near1 " >> $outdir"parmetis.part"; fi |
|---|
| 270 | near2=${m[(($ncol*((2*$nline-1))-$i-1))]} ; if [[ $near2 -ne 0 && ( $near2 != $up ) && ( $near2 != $down ) && ( $near2 != $left ) && ( $near2 != $right ) && ( $near2 != $near1 ) && ( $near2 != ${m[$i]} ) ]] ; then echo -n "$near2 " >> $outdir"parmetis.part"; fi |
|---|
| 271 | near3=${m[(($ncol*((2*$nline-1))-$i))]} ; if [[ $near3 -ne 0 && ( $near3 != $up ) && ( $near3 != $down ) && ( $near3 != $left ) && ( $near3 != $right ) && ( $near3 != $near1 ) && ( $near3 != $near2) && ( $near3 != ${m[$i]} ) ]] ; then echo -n "$near3 " >> $outdir"parmetis.part"; fi |
|---|
| 272 | fi |
|---|
| 273 | |
|---|
| 274 | echo "" >> $outdir"parmetis.part" |
|---|
| 275 | fi |
|---|
| 276 | done; |
|---|
| 277 | |
|---|
| 278 | node=$(cat $outdir"parmetis.part" | wc -l) |
|---|
| 279 | edge=$(cat $outdir"parmetis.part" | wc -w) |
|---|
| 280 | edge=$(($edge/2)) |
|---|
| 281 | |
|---|
| 282 | echo "$node $edge"> $outdir"parmetis.in" |
|---|
| 283 | while read line; do |
|---|
| 284 | echo "${line}" >> $outdir"parmetis.in" |
|---|
| 285 | done < $outdir"parmetis.part" |
|---|
| 286 | |
|---|
| 287 | rm $outdir"parmetis.part" |
|---|
| 288 | |
|---|
| 289 | ############################################################# |
|---|
| 290 | # mapping based on communication |
|---|
| 291 | ############################################################# |
|---|
| 292 | |
|---|
| 293 | if [ $maptype = "comm" ]; then |
|---|
| 294 | #execution of scotch (gpart num_part [input graph file] [output map file] -b0) |
|---|
| 295 | |
|---|
| 296 | #translate the parmetis format file into the scotch format file |
|---|
| 297 | gcv parmetis.in scotch.in -ic -os |
|---|
| 298 | |
|---|
| 299 | #partitioning with scotch |
|---|
| 300 | gpart $npart scotch.in scotch.map -b0 |
|---|
| 301 | |
|---|
| 302 | #translate the output file format |
|---|
| 303 | sed -n "s/\t/ /p" scotch.map | cut -d " " -f 2 > $parmetisfile |
|---|
| 304 | |
|---|
| 305 | rm scotch.map scotch.in |
|---|
| 306 | |
|---|
| 307 | #lancia metis (gpmetis [options] graphfile nparts) |
|---|
| 308 | |
|---|
| 309 | #gpmetis -ptype=kway -objtype=vol parmetis.in $npart |
|---|
| 310 | #gpmetis -contig -ufactor=1 -ncuts=50 -niter=200 -ptype=kway -objtype=cut parmetis.in $npart |
|---|
| 311 | |
|---|
| 312 | fi # end maptype==comm |
|---|
| 313 | |
|---|
| 314 | |
|---|
| 315 | ############################################################# |
|---|
| 316 | # Mapping based on neighbour-away algorithm |
|---|
| 317 | ############################################################# |
|---|
| 318 | |
|---|
| 319 | if [ $maptype = "away" ]; then |
|---|
| 320 | |
|---|
| 321 | # the script partitions the input nodes of the graph into "npart" sets. One node is requested to belong |
|---|
| 322 | # to a partition different from each of its neighbours |
|---|
| 323 | |
|---|
| 324 | #input file describing the dependency graph. Format used is ParMetis adiacency list file |
|---|
| 325 | graphfile="parmetis.in" |
|---|
| 326 | |
|---|
| 327 | # create npart temporary files |
|---|
| 328 | for i in `seq 0 $((npart - 1))`; do |
|---|
| 329 | p_filename=`printf "p_%04d" $i` |
|---|
| 330 | rm $p_filename >& /dev/null |
|---|
| 331 | touch $p_filename |
|---|
| 332 | done |
|---|
| 333 | num_nodes=`sed -n "1,1 s/[0-9]*$//p" $graphfile` |
|---|
| 334 | for i in `seq 1 $num_nodes`; do |
|---|
| 335 | # processing node $i |
|---|
| 336 | printf "\r Processing node $i of $num_nodes \t" |
|---|
| 337 | # sort the current partitions |
|---|
| 338 | ordered_file="p_ordered" |
|---|
| 339 | wc -l p_*[0-9] | sed -n "1,$npart s/^ *//p" | sort -n > $ordered_file |
|---|
| 340 | |
|---|
| 341 | # insert the current node $i into a partition where none of its neighbours is present |
|---|
| 342 | neighbour_list=`sed -n "$((i+1)),$((i+1)) p" $graphfile` |
|---|
| 343 | placed=0 |
|---|
| 344 | for j in `seq 1 $npart`; do |
|---|
| 345 | p_filename=`sed -n "$j,$j p" $ordered_file | cut -d " " -f 2` |
|---|
| 346 | insert=1 |
|---|
| 347 | for k in $neighbour_list; do |
|---|
| 348 | flag=`grep "^$k$" $p_filename | wc -l` |
|---|
| 349 | if [ $flag != 0 ]; then |
|---|
| 350 | insert=0 |
|---|
| 351 | break |
|---|
| 352 | fi |
|---|
| 353 | done |
|---|
| 354 | if [ $insert == 0 ]; then |
|---|
| 355 | continue |
|---|
| 356 | fi |
|---|
| 357 | echo $i >> $p_filename |
|---|
| 358 | placed=1 |
|---|
| 359 | break |
|---|
| 360 | done |
|---|
| 361 | if [ $placed == 0 ]; then |
|---|
| 362 | echo "Error node $i can not be placed in any of the partitions" |
|---|
| 363 | exit 1 |
|---|
| 364 | fi |
|---|
| 365 | done |
|---|
| 366 | printf "\n" |
|---|
| 367 | |
|---|
| 368 | rm $ordered_file |
|---|
| 369 | |
|---|
| 370 | # merging of the entire temporary partions files |
|---|
| 371 | rm $parmetisfile >& /dev/null |
|---|
| 372 | for i in `seq 1 $num_nodes`; do |
|---|
| 373 | part=`grep "^$i$" p_*[0-9] | cut -d "_" -f 2 | cut -d ":" -f 1 | sed "s/^0*//"` |
|---|
| 374 | if [ x$part == "x" ]; then |
|---|
| 375 | part=0 |
|---|
| 376 | fi |
|---|
| 377 | printf "%d\n" $part >> $parmetisfile |
|---|
| 378 | done |
|---|
| 379 | |
|---|
| 380 | rm p_*[0-9] |
|---|
| 381 | |
|---|
| 382 | fi # end maptype = "away" |
|---|
| 383 | |
|---|
| 384 | ############################################################# |
|---|
| 385 | # Check for correctness |
|---|
| 386 | ############################################################# |
|---|
| 387 | |
|---|
| 388 | #check if each partition has the same number of elements |
|---|
| 389 | for i in `eval echo {0..$(($npart-1))}`; do |
|---|
| 390 | n=`grep "^$i$" $parmetisfile | wc -l` |
|---|
| 391 | if [ $n -ne $procxnode ] ; then |
|---|
| 392 | echo "ERROR! Node $i has $n proc!" |
|---|
| 393 | fi |
|---|
| 394 | done |
|---|
| 395 | |
|---|
| 396 | ############################################################# |
|---|
| 397 | # Diagnostic |
|---|
| 398 | ############################################################# |
|---|
| 399 | |
|---|
| 400 | # Memory needed for each partitions |
|---|
| 401 | echo "*******************" |
|---|
| 402 | echo "Memory Distribution" |
|---|
| 403 | echo "*******************" |
|---|
| 404 | grep -A $((dim+1)) "$decomp" $layoutfile | grep -v Total | awk '{print $3}' > oceanpt.tmp |
|---|
| 405 | |
|---|
| 406 | for i in `seq 1 $dim`; do |
|---|
| 407 | p=`sed -n "$i,$i p" $parmetisfile` |
|---|
| 408 | weight=`sed -n "$i,$i p" oceanpt.tmp` |
|---|
| 409 | w[p]=`echo 0${w[$p]} + $weight | bc` |
|---|
| 410 | done |
|---|
| 411 | |
|---|
| 412 | rm oceanpt.tmp |
|---|
| 413 | |
|---|
| 414 | max=${w[0]} |
|---|
| 415 | min=${w[0]} |
|---|
| 416 | tot=${w[0]} |
|---|
| 417 | min_p=0 |
|---|
| 418 | max_p=0 |
|---|
| 419 | echo "0 - ${w[0]} GB" |
|---|
| 420 | for i in `seq 1 $((npart-1))`; do |
|---|
| 421 | echo "$i - ${w[$i]} GB" |
|---|
| 422 | t=`echo "${w[$i]} > $max" | bc` |
|---|
| 423 | if [ $t == 1 ]; then max=${w[$i]}; max_p=$i; fi |
|---|
| 424 | t=`echo "${w[$i]} < $min" | bc` |
|---|
| 425 | if [ $t == 1 ]; then min=${w[$i]}; min_p=$i; fi |
|---|
| 426 | tot=`echo "$tot + ${w[$i]}" | bc` |
|---|
| 427 | done |
|---|
| 428 | mean=`echo "scale = 3; $tot/$npart"|bc ` |
|---|
| 429 | echo "Max memory $max GB on partition $max_p" |
|---|
| 430 | echo "Min memory $min GB on partition $min_p" |
|---|
| 431 | echo "Mean memory $mean GB" |
|---|
| 432 | echo "Memory unbalance (Max/Mean)" `echo "scale = 2; $max/$mean" |bc` "%" |
|---|
| 433 | |
|---|
| 434 | # Number of interconnection (intra- and inter- node) between processes |
|---|
| 435 | echo "*********************************" |
|---|
| 436 | echo "Interconnection between processes" |
|---|
| 437 | echo "*********************************" |
|---|
| 438 | |
|---|
| 439 | com=0 |
|---|
| 440 | comtot=0 |
|---|
| 441 | |
|---|
| 442 | i=1 |
|---|
| 443 | while read p; do |
|---|
| 444 | part[i]=$p |
|---|
| 445 | ((i++)) |
|---|
| 446 | done < $parmetisfile |
|---|
| 447 | |
|---|
| 448 | l=0 |
|---|
| 449 | while read line; do |
|---|
| 450 | if [[ l -ne 0 ]] ; then |
|---|
| 451 | for i in `echo $line`; do |
|---|
| 452 | if [[ ${part[$i]} -ne ${part[$l]} ]] ; then |
|---|
| 453 | ((com++)) |
|---|
| 454 | fi |
|---|
| 455 | ((comtot++)) |
|---|
| 456 | done |
|---|
| 457 | fi |
|---|
| 458 | ((l++)) |
|---|
| 459 | done < $outdir"parmetis.in" |
|---|
| 460 | |
|---|
| 461 | echo "Tot comunicazioni: $comtot" |
|---|
| 462 | echo "Tot comunicazioni internodo: $com" |
|---|
| 463 | |
|---|
| 464 | |
|---|
| 465 | ############################################################# |
|---|
| 466 | #writing the TASK_GEOMETRY |
|---|
| 467 | ############################################################# |
|---|
| 468 | |
|---|
| 469 | number_of_groups=`cat $parmetisfile | sort -u | wc -l` |
|---|
| 470 | |
|---|
| 471 | echo -n 'export LSB_PJL_TASK_GEOMETRY="{' > $geometry_var |
|---|
| 472 | #for j in `seq 0 $((number_of_groups - 1))`; do |
|---|
| 473 | for j in `eval echo {0..$((number_of_groups - 1))}`; do |
|---|
| 474 | echo -n "(" >> $geometry_var |
|---|
| 475 | for i in `grep -n "^$j$" $parmetisfile | cut -d : -f 1`; do |
|---|
| 476 | echo -n $((i-1)), |
|---|
| 477 | done | sed s/.$// >> $geometry_var |
|---|
| 478 | echo -n ")" >> $geometry_var |
|---|
| 479 | done; |
|---|
| 480 | echo '}"' >> $geometry_var |
|---|
| 481 | |
|---|
| 482 | #cat $geometry_var |
|---|
| 483 | |
|---|
| 484 | |
|---|
| 485 | ############################################################# |
|---|
| 486 | #writing the geometry for BG/Q architecture |
|---|
| 487 | ############################################################# |
|---|
| 488 | |
|---|
| 489 | # each row contains the position of the process in the torus |
|---|
| 490 | # the shape of the torus is defined by the 6 parameters A, B, C, D, E, T |
|---|
| 491 | |
|---|
| 492 | tor_a=2 |
|---|
| 493 | tor_b=2 |
|---|
| 494 | tor_c=4 |
|---|
| 495 | tor_d=4 |
|---|
| 496 | tor_e=2 |
|---|
| 497 | tor_t=16 |
|---|
| 498 | |
|---|
| 499 | cat -n $parmetisfile | awk '{print $2 " " $1}'| sort -n | cat -n | awk -v val=$tor_t '{print $3 " " $2 " " ($1-1) % val}' | sort -n | awk -v val_a=$tor_a -v val_b=$tor_b -v val_c=$tor_c -v val_d=$tor_d -v val_e=$val_e '{print int($2/(val_b*val_c*val_d*val_e)) " " int($2/(val_c*val_d*val_e)) % val_b " " int($2/(val_d*val_e)) % val_c " " int($2/val_e) % val_d " " $2 % val_e " " $3}' > $bg_geometry |
|---|
| 500 | |
|---|
| 501 | |
|---|
| 502 | ############################################################# |
|---|
| 503 | #writing svg |
|---|
| 504 | ############################################################# |
|---|
| 505 | #load the mapping file |
|---|
| 506 | |
|---|
| 507 | i=1 |
|---|
| 508 | for node in `cat $parmetisfile`; do |
|---|
| 509 | map[i]=$node |
|---|
| 510 | ((i++)) |
|---|
| 511 | done; |
|---|
| 512 | |
|---|
| 513 | #loading of the number of ocean points and memory usage |
|---|
| 514 | decomp=`grep "Decomp" $gridfile | cut -d : -f 3` |
|---|
| 515 | k=0 |
|---|
| 516 | for oceanpt in `grep -A ${#map[*]} "$decomp" $layoutfile | grep -v Total | awk '{print $2}'` ; do |
|---|
| 517 | oceanptvett[k]=$oceanpt |
|---|
| 518 | ((k++)) |
|---|
| 519 | done; |
|---|
| 520 | k=0 |
|---|
| 521 | for mempt in `grep -A ${#map[*]} "$decomp" $layoutfile | grep -v Total | awk '{print $3}'` ; do |
|---|
| 522 | memptvett[k]=$mempt |
|---|
| 523 | ((k++)) |
|---|
| 524 | done; |
|---|
| 525 | |
|---|
| 526 | #creation of the color map |
|---|
| 527 | |
|---|
| 528 | num_col=$npart |
|---|
| 529 | for((i=0;i<$num_col;i++)) ; do |
|---|
| 530 | colourvett[i]="rgb($(($RANDOM%256)),$(($RANDOM%256)),$(($RANDOM%256)))" |
|---|
| 531 | done; |
|---|
| 532 | |
|---|
| 533 | outfile=$maptype.display.$npart.svg |
|---|
| 534 | |
|---|
| 535 | #extract the grid dimension |
|---|
| 536 | width=0 ; height=0; |
|---|
| 537 | for i in `grep -A4 "# [ 0-9]" $gridfile | grep -v \- | grep -v \# | sed "s/^[ \t]*//;s/[ \t]*$//" | sed "s/ */ /g" |sed "s/ /,/g" | cut -f 1 -d ,`; do |
|---|
| 538 | if [[ $i -gt $width ]]; then |
|---|
| 539 | width=$i |
|---|
| 540 | fi |
|---|
| 541 | done; |
|---|
| 542 | for i in `grep -A4 "# [ 0-9]" $gridfile | grep -v \- | grep -v \# | sed "s/^[ \t]*//;s/[ \t]*$//" | sed "s/ */ /g" |sed "s/ /,/g" | cut -f 2 -d ,`; do |
|---|
| 543 | if [[ $i -gt $height ]]; then |
|---|
| 544 | height=$i |
|---|
| 545 | fi |
|---|
| 546 | done; |
|---|
| 547 | |
|---|
| 548 | #writing the header |
|---|
| 549 | echo "<?xml version=\"1.0\" encoding=\"iso-8859-1\" standalone=\"no\"?> " > $outfile |
|---|
| 550 | echo "<!DOCTYPE svg PUBLIC \"-//W3C//Dtd SVG 1.1//EN\" \"http://www.w3.org/Graphics/SVG/1.1/Dtd/svg11.dtd\">" >> $outfile |
|---|
| 551 | echo "<svg width=\"$(($width+150))\" height=\"$(($height+100))\" version=\"1.1\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">" >> $outfile |
|---|
| 552 | |
|---|
| 553 | #background image. It can be derived from the bathimetry |
|---|
| 554 | echo "<image x=\"0\" y=\"0\" width=\"$width\" height=\"$height\" xlink:href=\"background.png\"/>" >> $outfile |
|---|
| 555 | |
|---|
| 556 | #writing the subdomain rectangles with additional information about process ID and color |
|---|
| 557 | j=0; |
|---|
| 558 | for i in `grep -A4 "# [ 0-9]" $gridfile | grep -v \- | sed "s/#/ /g" | sed "s/^[ \t]*//;s/[ \t]*$//" | sed "s/ */ /g" |sed "s/ /,/g" | sed '1d'`; do |
|---|
| 559 | case $j in |
|---|
| 560 | 0) |
|---|
| 561 | proc=$(($i-1)); |
|---|
| 562 | nodeid=node_$proc |
|---|
| 563 | col_index=${map[$i]} |
|---|
| 564 | echo -n "<polygon id=\"$nodeid\" points=\"" >> $outfile |
|---|
| 565 | ((j++)); |
|---|
| 566 | printf "\rGenerating SVG ... %3d%% " $(( (($proc + 1 ) * 100) / $dim )) |
|---|
| 567 | ;; |
|---|
| 568 | 1) |
|---|
| 569 | ((j++)); |
|---|
| 570 | x=$(echo $i | cut -f 1 -d ,); |
|---|
| 571 | y=$(($height-$(echo $i | cut -f 2 -d ,))); |
|---|
| 572 | xtxt=$(($x+3)) |
|---|
| 573 | ytxt=$(($y-3)) |
|---|
| 574 | echo -n "$x,$y " >> $outfile |
|---|
| 575 | ;; |
|---|
| 576 | [2-3]) |
|---|
| 577 | ((j++)); |
|---|
| 578 | x=$(echo $i | cut -f 1 -d ,); |
|---|
| 579 | y=$(($height-$(echo $i | cut -f 2 -d ,))); |
|---|
| 580 | echo -n "$x,$y " >> $outfile |
|---|
| 581 | ;; |
|---|
| 582 | 4) |
|---|
| 583 | j=0; |
|---|
| 584 | x=$(echo $i | cut -f 1 -d ,); |
|---|
| 585 | y=$(($height-$(echo $i | cut -f 2 -d ,))); |
|---|
| 586 | echo "$x,$y\" style=\"stroke:white;fill:${colourvett[$col_index]};fill-opacity:0.7\" />" >> $outfile |
|---|
| 587 | echo "<text x=\"$xtxt\" y=\"$ytxt\" fill=\"white\" >$proc</text>" >> $outfile |
|---|
| 588 | echo "<g visibility=\"hidden\" font-family =\"sans-serif\" fill=\"black\" font-size = \"12\">" >> $outfile |
|---|
| 589 | echo "<rect x=\"$xtxt\" y=\"$ytxt\" rx=\"2\" ry=\"2\" width=\"175\" height=\"95\" style=\"stroke:black;fill:lightgray\">" >> $outfile |
|---|
| 590 | echo "<set attributeName=\"visibility\" from=\"hidden\" to=\"visible\" begin=\"$nodeid.mousedown\" end=\"$nodeid.mouseout\"/></rect>" >> $outfile |
|---|
| 591 | echo "<text x=\"$(($xtxt+10))\" y=\"$(($ytxt+20))\" font-weight=\"bold\">Ocean Sub-domain : $proc<set attributeName=\"visibility\" from=\"hidden\" to=\"visible\" begin=\"$nodeid.mousedown\" end=\"$nodeid.mouseout\"/></text> " >> $outfile |
|---|
| 592 | echo "<text x=\"$(($xtxt+10))\" y=\"$(($ytxt+40))\">Total ocepts : ${oceanptvett[$proc]}<set attributeName=\"visibility\" from=\"hidden\" to=\"visible\" begin=\"$nodeid.mousedown\" end=\"$nodeid.mouseout\"/></text> " >> $outfile |
|---|
| 593 | echo "<text x=\"$(($xtxt+10))\" y=\"$(($ytxt+60))\">Memory in Gb : ${memptvett[$proc]}<set attributeName=\"visibility\" from=\"hidden\" to=\"visible\" begin=\"$nodeid.mousedown\" end=\"$nodeid.mouseout\"/></text> " >> $outfile |
|---|
| 594 | echo "<text x=\"$(($xtxt+10))\" y=\"$(($ytxt+80))\">Color : $col_index<set attributeName=\"visibility\" from=\"hidden\" to=\"visible\" begin=\"$nodeid.mousedown\" end=\"$nodeid.mouseout\"/></text> " >> $outfile |
|---|
| 595 | echo "</g>" >> $outfile |
|---|
| 596 | ;; |
|---|
| 597 | *) |
|---|
| 598 | echo "ERROR" |
|---|
| 599 | ;; |
|---|
| 600 | esac; |
|---|
| 601 | done; |
|---|
| 602 | |
|---|
| 603 | #chiudo il file svg |
|---|
| 604 | echo "</svg>" >> $outfile |
|---|
| 605 | |
|---|
| 606 | printf " Done\n" |
|---|
