Changeset 1152 for XIOS/dev/dev_olga/src
- Timestamp:
- 06/02/17 16:56:46 (7 years ago)
- Location:
- XIOS/dev/dev_olga/src
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/dev/dev_olga/src/client.cpp
r1148 r1152 41 41 if (initialized) is_MPI_Initialized=true ; 42 42 else is_MPI_Initialized=false ; 43 int rank ; 43 44 44 45 // don't use OASIS … … 90 91 { 91 92 if (hashAll[i] == hashString(CXios::xiosCodeId)) 92 // || (hashAll[i] == hashString(CXios::xiosCodeIdPrm))93 // || (hashAll[i] == hashString(CXios::xiosCodeIdSnd)))94 93 { 95 94 CXios::setUsingServer(); … … 98 97 } 99 98 100 myColor=colors[hashClient] ; 99 // myColor=colors[hashClient]; 100 myColor=leaders[hashClient] ; 101 101 MPI_Comm_split(CXios::globalComm,myColor,rank_,&intraComm) ; 102 103 if (CXios::usingServer2) 104 MPI_Allgather(&hashClient, 1, MPI_LONG, hashAll, 1, MPI_LONG, CXios::globalComm) ; 102 105 103 106 if (CXios::usingServer) … … 111 114 <<" intraCommRank :"<<intraCommRank<<" clientLeader "<< serverLeader<<endl ; 112 115 MPI_Intercomm_create(intraComm, 0, CXios::globalComm, serverLeader, 0, &interComm) ; 113 rank_ = intraCommRank;116 //rank_ = intraCommRank; 114 117 } 115 118 else … … 150 153 { 151 154 MPI_Status status ; 152 MPI_Comm_rank(intraComm,&rank _) ;155 MPI_Comm_rank(intraComm,&rank) ; 153 156 154 157 oasis_get_intercomm(interComm,CXios::xiosCodeId) ; 155 if (rank _==0) MPI_Recv(&serverLeader,1, MPI_INT, 0, 0, interComm, &status) ;158 if (rank==0) MPI_Recv(&serverLeader,1, MPI_INT, 0, 0, interComm, &status) ; 156 159 MPI_Bcast(&serverLeader,1,MPI_INT,0,intraComm) ; 157 160 -
XIOS/dev/dev_olga/src/server.cpp
r1150 r1152 20 20 list<MPI_Comm> CServer::interCommLeft ; 21 21 list<MPI_Comm> CServer::interCommRight ; 22 // list<MPI_Comm> CServer::interComm ;23 22 std::list<MPI_Comm> CServer::contextInterComms; 24 23 std::list<MPI_Comm> CServer::contextIntraComms; 25 24 int CServer::serverLevel = 0 ; 26 int CServer::serverLeader_ = 0;27 int CServer::serverSize_ = 0;28 int CServer::nbPools = 0;29 int CServer::poolId = 0;30 25 int CServer::nbContexts = 0; 31 26 bool CServer::isRoot = false ; … … 34 29 StdOFStream CServer::m_errorStream; 35 30 map<string,CContext*> CServer::contextList ; 31 vector<int> CServer::sndServerGlobalRanks; 36 32 bool CServer::finished=false ; 37 33 bool CServer::is_MPI_Initialized ; … … 55 51 if (initialized) is_MPI_Initialized=true ; 56 52 else is_MPI_Initialized=false ; 53 int rank ; 57 54 58 55 // Not using OASIS … … 70 67 71 68 unsigned long* hashAll ; 72 73 // int rank ; 69 unsigned long* srvLevelAll ; 70 74 71 int size ; 75 72 int myColor ; 76 73 int i,c ; 77 MPI_Comm newComm , serversComm;74 MPI_Comm newComm; 78 75 79 76 MPI_Comm_size(CXios::globalComm, &size) ; … … 88 85 89 86 int nbSrv = 0; 87 int srvNodeSize = 1, srvNodeLeader = 0; 88 89 // (1) Establish client leaders, distribute processes between two server levels 90 90 for(i=0,c=0;i<size;i++) 91 91 { … … 96 96 c++ ; 97 97 } 98 if (hashAll[i] == hashServer) ++serverSize_; 98 if (CXios::usingServer2) 99 { 100 if (hashAll[i] == hashServer) 101 { 102 if (hashAll[i-1] != hashServer || i == 0) 103 { 104 srvNodeLeader = i; 105 } 106 if (hashAll[i+1] == hashServer) 107 { 108 ++srvNodeSize; 109 } 110 else 111 { 112 if ( (rank_-srvNodeLeader) >= 0 && 113 (rank_-srvNodeLeader) < (1.- CXios::ratioServer2/100.)*srvNodeSize ) 114 { 115 serverLevel = 1; 116 } 117 if ( (rank_-srvNodeLeader) >= (1.- CXios::ratioServer2/100.)*srvNodeSize && 118 (rank_ - srvNodeLeader) < srvNodeSize ) 119 { 120 serverLevel = 2; 121 } 122 srvNodeSize = 1; 123 } 124 } 125 } 99 126 } 100 127 101 // Setting the number of secondary pools 102 myColor = colors[hashServer]; 128 // (2) Create intraComm 129 myColor = (serverLevel == 2) ? rank_ : colors[hashServer]; 130 MPI_Comm_split(CXios::globalComm, myColor, rank_, &intraComm) ; 131 132 // (3) Create interComm 103 133 if (CXios::usingServer2) 104 134 { 105 int serverRank = rank_ - leaders[hashServer]; // server proc rank starting 0 106 nbPools = serverSize_ * CXios::ratioServer2 / 100; 107 if ( serverRank < (serverSize_ - nbPools) ) 108 { 109 serverLevel = 1; 110 } 111 else 112 { 113 serverLevel = 2; 114 poolId = serverRank - serverSize_ + nbPools; 115 myColor = rank_ + size; // + size to make sure that myColor is unique among not only servers but also clients. It's only a temporary solution 116 } 135 MPI_Allgather(&serverLevel, 1, MPI_LONG, hashAll, 1, MPI_LONG, CXios::globalComm) ; 136 for (i=0; i<size; i++) 137 if (hashAll[i] == 2) 138 sndServerGlobalRanks.push_back(i); 117 139 } 118 119 MPI_Comm_split(CXios::globalComm, myColor, rank_, &intraComm) ;120 140 121 141 if (serverLevel == 0) … … 142 162 int clientLeader, srvSndLeader; 143 163 int srvPrmLeader ; 164 144 165 for (it=leaders.begin();it!=leaders.end();it++) 145 166 { … … 155 176 interCommLeft.push_back(newComm) ; 156 177 } 157 else 158 serverLeader_ = it->second; 159 } 160 161 for (int i = 0; i < nbPools; ++i) 162 { 163 srvSndLeader = serverLeader_ + serverSize_ - nbPools + i; 178 } 179 180 for (int i = 0; i < sndServerGlobalRanks.size(); ++i) 181 { 164 182 int intraCommSize, intraCommRank ; 165 183 MPI_Comm_size(intraComm, &intraCommSize) ; 166 184 MPI_Comm_rank(intraComm, &intraCommRank) ; 167 185 info(50)<<"intercommCreate::client (server level 1) "<<rank_<<" intraCommSize : "<<intraCommSize 168 <<" intraCommRank :"<<intraCommRank<<" clientLeader "<< s rvSndLeader<<endl ;169 MPI_Intercomm_create(intraComm, 0, CXios::globalComm, s rvSndLeader, 1, &newComm) ;186 <<" intraCommRank :"<<intraCommRank<<" clientLeader "<< sndServerGlobalRanks[i]<<endl ; 187 MPI_Intercomm_create(intraComm, 0, CXios::globalComm, sndServerGlobalRanks[i], 1, &newComm) ; 170 188 interCommRight.push_back(newComm) ; 171 189 } … … 191 209 else 192 210 { 193 int size , rank;211 int size; 194 212 int myColor; 195 213 if (!is_MPI_Initialized) oasis_init(CXios::xiosCodeId); … … 204 222 else 205 223 { 206 MPI_Comm_rank(localComm,&rank) ; 207 MPI_Comm_size(localComm,&serverSize_) ; 208 nbPools = serverSize_ * CXios::ratioServer2 / 100; 209 if ( rank < (serverSize_ - nbPools) ) 224 MPI_Comm_rank(localComm,&rank_) ; 225 MPI_Comm_size(localComm,&size) ; 226 227 for (int i=size*CXios::ratioServer2/100; i<size; i++) 228 sndServerGlobalRanks.push_back(i); 229 230 if ( rank_ < (size - sndServerGlobalRanks.size()) ) 210 231 { 211 232 serverLevel = 1; … … 215 236 { 216 237 serverLevel = 2; 217 poolId = rank - serverSize_ + nbPools; 218 myColor = rank; 219 } 220 MPI_Comm_split(localComm, myColor, rank, &intraComm) ; 238 myColor = rank_; 239 } 240 MPI_Comm_split(localComm, myColor, rank_, &intraComm) ; 221 241 222 242 } 223 MPI_Comm_rank(intraComm,&rank_) ;224 243 MPI_Comm_size(intraComm,&size) ; 225 226 244 string codesId=CXios::getin<string>("oasis_codes_id") ; 227 245 … … 233 251 int globalRank ; 234 252 MPI_Comm_rank(CXios::globalComm,&globalRank); 235 if (rank_ == 0)236 serverLeader_ = globalRank;237 253 238 254 for(it=splitted.begin();it!=splitted.end();it++) … … 251 267 interCommLeft.push_back(newComm) ; 252 268 if (rank_==0) MPI_Send(&globalRank,1,MPI_INT,0,0,newComm) ; 253 for (int i = 0; i < nbPools; ++i)269 for (int i = 0; i < sndServerGlobalRanks.size(); ++i) 254 270 { 255 int srvSndLeader = s erverLeader_ + serverSize_ - nbPools + i;271 int srvSndLeader = sndServerGlobalRanks[i]; 256 272 info(50)<<"intercommCreate::client (server level 1) "<<globalRank<<" intraCommSize : "<<size 257 273 <<" intraCommRank :"<<rank_<<" clientLeader "<< srvSndLeader<<endl ; … … 273 289 } 274 290 275 MPI_Comm_rank(intraComm, &rank _) ;276 if (rank _==0) isRoot=true;291 MPI_Comm_rank(intraComm, &rank) ; 292 if (rank==0) isRoot=true; 277 293 else isRoot=false; 278 294 … … 612 628 CBufferOut buffer(buff,messageSize) ; 613 629 buffer<<msg ; 614 int sndServerGloRanks = serverSize_-nbPools+serverLeader_ +i; // the assumption is that there is only one proc per secondary server pool 615 MPI_Send(buff, buffer.count(), MPI_CHAR, sndServerGloRanks, 1, CXios::globalComm) ; 630 MPI_Send(buff, buffer.count(), MPI_CHAR, sndServerGlobalRanks[i], 1, CXios::globalComm) ; 616 631 MPI_Comm_dup(*it, &inter); 617 632 contextInterComms.push_back(inter); … … 668 683 ++numDigit; 669 684 } 670 671 if (!CXios::usingServer2) 672 id = getRank(); 673 else 674 { 675 if (serverLevel == 1) 676 id = rank_; 677 else 678 id = poolId; 679 } 685 id = getRank(); 686 687 // if (!CXios::usingServer2) 688 // id = getRank(); 689 // else 690 // { 691 // if (serverLevel == 1) 692 // id = rank_; 693 // else 694 // id = poolId; 695 // } 680 696 fileNameClient << fileName << "_" << std::setfill('0') << std::setw(numDigit) << id << ext; 681 697 fb->open(fileNameClient.str().c_str(), std::ios::out); -
XIOS/dev/dev_olga/src/server.hpp
r1148 r1152 64 64 65 65 private: 66 static int rank_; 67 static int serverLeader_; //!< Leader of the classical or primary server (needed in case of secondary servers) 68 static int serverSize_; //!< Number of procs dedicated to servers (primary and seconday (if any) combined) 69 static int nbPools; //!< Number of secondary server pools 70 static int poolId; //!< id of a secondary server pool starting from 1 66 static vector<int> sndServerGlobalRanks; //!< Global ranks of secondary server processes 67 static int rank_; //!< If (!oasis) global rank, else rank in the intraComm returned by oasis 71 68 static int nbContexts; //!< Number of contexts registered by server 72 69 static StdOFStream m_infoStream;
Note: See TracChangeset
for help on using the changeset viewer.