1 | #include "globalScopeData.hpp" |
---|
2 | #include "xios_spl.hpp" |
---|
3 | #include "cxios.hpp" |
---|
4 | #include "server.hpp" |
---|
5 | #include "client.hpp" |
---|
6 | #include "type.hpp" |
---|
7 | #include "context.hpp" |
---|
8 | #include "object_template.hpp" |
---|
9 | #include "oasis_cinterface.hpp" |
---|
10 | #include <boost/functional/hash.hpp> |
---|
11 | #include <boost/algorithm/string.hpp> |
---|
12 | #include "mpi.hpp" |
---|
13 | #include "tracer.hpp" |
---|
14 | #include "timer.hpp" |
---|
15 | #include "mem_checker.hpp" |
---|
16 | #include "event_scheduler.hpp" |
---|
17 | #include "string_tools.hpp" |
---|
18 | #include "ressources_manager.hpp" |
---|
19 | #include "services_manager.hpp" |
---|
20 | #include "contexts_manager.hpp" |
---|
21 | #include "servers_ressource.hpp" |
---|
22 | #include "services.hpp" |
---|
23 | #include "pool_node.hpp" |
---|
24 | #include <cstdio> |
---|
25 | #include "workflow_graph.hpp" |
---|
26 | #include "release_static_allocation.hpp" |
---|
27 | #include <sys/stat.h> |
---|
28 | #include <unistd.h> |
---|
29 | |
---|
30 | |
---|
31 | |
---|
32 | namespace xios |
---|
33 | { |
---|
34 | MPI_Comm CServer::intraComm_ ; |
---|
35 | MPI_Comm CServer::serversComm_ ; |
---|
36 | std::list<MPI_Comm> CServer::interCommLeft ; |
---|
37 | std::list<MPI_Comm> CServer::interCommRight ; |
---|
38 | std::list<MPI_Comm> CServer::contextInterComms; |
---|
39 | std::list<MPI_Comm> CServer::contextIntraComms; |
---|
40 | int CServer::serverLevel = 0 ; |
---|
41 | int CServer::nbContexts = 0; |
---|
42 | bool CServer::isRoot = false ; |
---|
43 | int CServer::rank_ = INVALID_RANK; |
---|
44 | StdOFStream CServer::m_infoStream; |
---|
45 | StdOFStream CServer::m_errorStream; |
---|
46 | map<string,CContext*> CServer::contextList ; |
---|
47 | vector<int> CServer::sndServerGlobalRanks; |
---|
48 | bool CServer::finished=false ; |
---|
49 | bool CServer::is_MPI_Initialized ; |
---|
50 | CEventScheduler* CServer::eventScheduler = 0; |
---|
51 | CServersRessource* CServer::serversRessource_=nullptr ; |
---|
52 | CThirdPartyDriver* CServer::driver_ =nullptr ; |
---|
53 | |
---|
54 | |
---|
55 | void CServer::initialize(void) |
---|
56 | { |
---|
57 | |
---|
58 | MPI_Comm serverComm ; |
---|
59 | int initialized ; |
---|
60 | MPI_Initialized(&initialized) ; |
---|
61 | if (initialized) is_MPI_Initialized=true ; |
---|
62 | else is_MPI_Initialized=false ; |
---|
63 | MPI_Comm globalComm=CXios::getGlobalComm() ; |
---|
64 | ///////////////////////////////////////// |
---|
65 | ///////////// PART 1 //////////////////// |
---|
66 | ///////////////////////////////////////// |
---|
67 | // don't use OASIS |
---|
68 | if (!CXios::usingOasis) |
---|
69 | { |
---|
70 | if (!is_MPI_Initialized) MPI_Init(NULL, NULL); |
---|
71 | |
---|
72 | // split the global communicator |
---|
73 | // get hash from all model to attribute a unique color (int) and then split to get client communicator |
---|
74 | // every mpi process of globalComm (MPI_COMM_WORLD) must participate |
---|
75 | |
---|
76 | int commRank, commSize ; |
---|
77 | MPI_Comm_rank(globalComm,&commRank) ; |
---|
78 | MPI_Comm_size(globalComm,&commSize) ; |
---|
79 | |
---|
80 | std::hash<string> hashString ; |
---|
81 | size_t hashServer=hashString(CXios::xiosCodeId) ; |
---|
82 | |
---|
83 | size_t* hashAll = new size_t[commSize] ; |
---|
84 | MPI_Allgather(&hashServer,1,MPI_SIZE_T,hashAll,1,MPI_SIZE_T,globalComm) ; |
---|
85 | |
---|
86 | int color=0 ; |
---|
87 | map<size_t,int> listHash ; |
---|
88 | for(int i=0 ; i<=commSize ; i++) |
---|
89 | if (listHash.count(hashAll[i])==0) |
---|
90 | { |
---|
91 | listHash[hashAll[i]]=color ; |
---|
92 | color=color+1 ; |
---|
93 | } |
---|
94 | color=listHash[hashServer] ; |
---|
95 | delete[] hashAll ; |
---|
96 | |
---|
97 | MPI_Comm_split(globalComm, color, commRank, &serverComm) ; |
---|
98 | } |
---|
99 | else // using OASIS |
---|
100 | { |
---|
101 | if (!is_MPI_Initialized) driver_ = new CThirdPartyDriver(); |
---|
102 | |
---|
103 | driver_->getComponentCommunicator( serverComm ); |
---|
104 | } |
---|
105 | MPI_Comm_dup(serverComm, &intraComm_); |
---|
106 | |
---|
107 | CTimer::get("XIOS").resume() ; |
---|
108 | CTimer::get("XIOS server").resume() ; |
---|
109 | CTimer::get("XIOS initialize").resume() ; |
---|
110 | |
---|
111 | ///////////////////////////////////////// |
---|
112 | ///////////// PART 2 //////////////////// |
---|
113 | ///////////////////////////////////////// |
---|
114 | |
---|
115 | |
---|
116 | // Create the XIOS communicator for every process which is related |
---|
117 | // to XIOS, as well on client side as on server side |
---|
118 | MPI_Comm xiosGlobalComm ; |
---|
119 | string strIds=CXios::getin<string>("clients_code_id","") ; |
---|
120 | vector<string> clientsCodeId=splitRegex(strIds,"\\s*,\\s*") ; |
---|
121 | if (strIds.empty()) |
---|
122 | { |
---|
123 | // no code Ids given, suppose XIOS initialisation is global |
---|
124 | int commRank, commGlobalRank, serverLeader, clientLeader,serverRemoteLeader,clientRemoteLeader ; |
---|
125 | MPI_Comm splitComm,interComm ; |
---|
126 | MPI_Comm_rank(globalComm,&commGlobalRank) ; |
---|
127 | MPI_Comm_split(globalComm, 1, commGlobalRank, &splitComm) ; |
---|
128 | MPI_Comm_rank(splitComm,&commRank) ; |
---|
129 | if (commRank==0) serverLeader=commGlobalRank ; |
---|
130 | else serverLeader=0 ; |
---|
131 | clientLeader=0 ; |
---|
132 | MPI_Allreduce(&clientLeader,&clientRemoteLeader,1,MPI_INT,MPI_SUM,globalComm) ; |
---|
133 | MPI_Allreduce(&serverLeader,&serverRemoteLeader,1,MPI_INT,MPI_SUM,globalComm) ; |
---|
134 | MPI_Intercomm_create(splitComm, 0, globalComm, clientRemoteLeader,1341,&interComm) ; |
---|
135 | MPI_Intercomm_merge(interComm,false,&xiosGlobalComm) ; |
---|
136 | CXios::setXiosComm(xiosGlobalComm) ; |
---|
137 | } |
---|
138 | else |
---|
139 | { |
---|
140 | |
---|
141 | xiosGlobalCommByFileExchange(serverComm) ; |
---|
142 | |
---|
143 | } |
---|
144 | |
---|
145 | ///////////////////////////////////////// |
---|
146 | ///////////// PART 4 //////////////////// |
---|
147 | // create servers intra communicator // |
---|
148 | ///////////////////////////////////////// |
---|
149 | |
---|
150 | int commRank ; |
---|
151 | MPI_Comm_rank(CXios::getXiosComm(), &commRank) ; |
---|
152 | MPI_Comm_split(CXios::getXiosComm(),true,commRank,&serversComm_) ; |
---|
153 | |
---|
154 | CXios::setUsingServer() ; |
---|
155 | |
---|
156 | ///////////////////////////////////////// |
---|
157 | ///////////// PART 5 //////////////////// |
---|
158 | // redirect files output // |
---|
159 | ///////////////////////////////////////// |
---|
160 | |
---|
161 | CServer::openInfoStream(CXios::serverFile); |
---|
162 | CServer::openErrorStream(CXios::serverFile); |
---|
163 | |
---|
164 | CMemChecker::logMem( "CServer::initialize" ); |
---|
165 | |
---|
166 | ///////////////////////////////////////// |
---|
167 | ///////////// PART 4 //////////////////// |
---|
168 | ///////////////////////////////////////// |
---|
169 | |
---|
170 | CXios::launchDaemonsManager(true) ; |
---|
171 | |
---|
172 | ///////////////////////////////////////// |
---|
173 | ///////////// PART 5 //////////////////// |
---|
174 | ///////////////////////////////////////// |
---|
175 | |
---|
176 | // create the services |
---|
177 | |
---|
178 | auto ressourcesManager=CXios::getRessourcesManager() ; |
---|
179 | auto servicesManager=CXios::getServicesManager() ; |
---|
180 | auto contextsManager=CXios::getContextsManager() ; |
---|
181 | auto daemonsManager=CXios::getDaemonsManager() ; |
---|
182 | auto serversRessource=CServer::getServersRessource() ; |
---|
183 | |
---|
184 | int rank; |
---|
185 | MPI_Comm_rank(intraComm_, &rank) ; |
---|
186 | if (rank==0) isRoot=true; |
---|
187 | else isRoot=false; |
---|
188 | |
---|
189 | if (serversRessource->isServerLeader()) |
---|
190 | { |
---|
191 | // creating pool |
---|
192 | CPoolNodeGroup::get("xios","pool_definition")->solveDescInheritance(true) ; |
---|
193 | vector<CPoolNode*> pools = CPoolNodeGroup::get("xios","pool_definition")->getAllChildren(); |
---|
194 | for(auto& pool : pools) pool->allocateRessources() ; |
---|
195 | |
---|
196 | int nbRessources = ressourcesManager->getFreeRessourcesSize() ; |
---|
197 | if (nbRessources>0) |
---|
198 | { |
---|
199 | if (!CXios::usingServer2) |
---|
200 | { |
---|
201 | ressourcesManager->createPool(CXios::defaultPoolId, nbRessources) ; |
---|
202 | ressourcesManager->waitPoolRegistration(CXios::defaultPoolId) ; |
---|
203 | servicesManager->createServices(CXios::defaultPoolId, CXios::defaultWriterId, CServicesManager::WRITER,nbRessources,1) ; |
---|
204 | servicesManager->waitServiceRegistration(CXios::defaultPoolId, CXios::defaultWriterId) ; |
---|
205 | servicesManager->createServicesOnto(CXios::defaultPoolId, CXios::defaultReaderId, CServicesManager::READER, CXios::defaultWriterId) ; |
---|
206 | servicesManager->waitServiceRegistration(CXios::defaultPoolId, CXios::defaultReaderId) ; |
---|
207 | } |
---|
208 | else |
---|
209 | { |
---|
210 | int nprocsServer = nbRessources*CXios::ratioServer2/100.; |
---|
211 | int nprocsGatherer = nbRessources - nprocsServer ; |
---|
212 | |
---|
213 | int nbPoolsServer2 = CXios::nbPoolsServer2 ; |
---|
214 | if (nbPoolsServer2 == 0) nbPoolsServer2 = nprocsServer; |
---|
215 | ressourcesManager->createPool(CXios::defaultPoolId, nbRessources) ; |
---|
216 | ressourcesManager->waitPoolRegistration(CXios::defaultPoolId) ; |
---|
217 | servicesManager->createServices(CXios::defaultPoolId, CXios::defaultGathererId, CServicesManager::GATHERER, nprocsGatherer, 1) ; |
---|
218 | servicesManager->waitServiceRegistration(CXios::defaultPoolId, CXios::defaultGathererId) ; |
---|
219 | servicesManager->createServicesOnto(CXios::defaultPoolId, CXios::defaultReaderId, CServicesManager::READER, CXios::defaultGathererId) ; |
---|
220 | servicesManager->waitServiceRegistration(CXios::defaultPoolId, CXios::defaultReaderId) ; |
---|
221 | servicesManager->createServices(CXios::defaultPoolId, CXios::defaultWriterId, CServicesManager::WRITER, nprocsServer, nbPoolsServer2) ; |
---|
222 | servicesManager->waitServiceRegistration(CXios::defaultPoolId, CXios::defaultWriterId) ; |
---|
223 | } |
---|
224 | } |
---|
225 | // servicesManager->createServices(CXios::defaultPoolId, CXios::defaultServicesId, CServicesManager::ALL_SERVICES, nbRessources, 1) ; |
---|
226 | } |
---|
227 | |
---|
228 | MPI_Request req ; |
---|
229 | MPI_Status status ; |
---|
230 | MPI_Ibarrier(getServersRessource()->getCommunicator(),&req) ; // be sure that all services are created now, could be remove later if more asynchronisity |
---|
231 | int ok=false ; |
---|
232 | while (!ok) |
---|
233 | { |
---|
234 | daemonsManager->eventLoop() ; |
---|
235 | MPI_Test(&req,&ok,&status) ; |
---|
236 | } |
---|
237 | |
---|
238 | |
---|
239 | testingEventScheduler() ; |
---|
240 | /* |
---|
241 | MPI_Request req ; |
---|
242 | MPI_Status status ; |
---|
243 | MPI_Ibarrier(CXios::getXiosComm(),&req) ; // be sure that all services are created now, could be remove later if more asynchronisity |
---|
244 | int ok=false ; |
---|
245 | while (!ok) |
---|
246 | { |
---|
247 | daemonsManager->eventLoop() ; |
---|
248 | MPI_Test(&req,&ok,&status) ; |
---|
249 | } |
---|
250 | */ |
---|
251 | CTimer::get("XIOS initialize").suspend() ; |
---|
252 | |
---|
253 | ///////////////////////////////////////// |
---|
254 | ///////////// PART 5 //////////////////// |
---|
255 | ///////////////////////////////////////// |
---|
256 | // loop on event loop |
---|
257 | |
---|
258 | bool finished=false ; |
---|
259 | CTimer::get("XIOS event loop").resume() ; |
---|
260 | |
---|
261 | while (!finished) |
---|
262 | { |
---|
263 | finished=daemonsManager->eventLoop() ; |
---|
264 | } |
---|
265 | CTimer::get("XIOS event loop").suspend() ; |
---|
266 | |
---|
267 | // Delete CContext |
---|
268 | //CObjectTemplate<CContext>::cleanStaticDataStructure(); |
---|
269 | } |
---|
270 | |
---|
271 | |
---|
272 | void CServer::testingEventScheduler(void) |
---|
273 | { |
---|
274 | CXios::getPoolRessource()->getEventScheduler()->registerEvent(1,10) ; |
---|
275 | CXios::getPoolRessource()->getEventScheduler()->registerEvent(2,10) ; |
---|
276 | if (CXios::getPoolRessource()->hasService(CXios::defaultGathererId,0)) |
---|
277 | { |
---|
278 | CXios::getPoolRessource()->getService(CXios::defaultGathererId,0)-> getEventScheduler()->registerEvent(1,100) ; |
---|
279 | CXios::getPoolRessource()->getService(CXios::defaultGathererId,0)-> getEventScheduler()->registerEvent(2,100) ; |
---|
280 | CXios::getPoolRessource()->getService(CXios::defaultGathererId,0)-> getEventScheduler()->registerEvent(3,100) ; |
---|
281 | } |
---|
282 | if (CXios::getPoolRessource()->hasService(CXios::defaultWriterId,0)) |
---|
283 | { |
---|
284 | CXios::getPoolRessource()->getService(CXios::defaultWriterId,0)-> getEventScheduler()->registerEvent(1,1000) ; |
---|
285 | CXios::getPoolRessource()->getService(CXios::defaultWriterId,0)-> getEventScheduler()->registerEvent(2,1000) ; |
---|
286 | } |
---|
287 | CXios::getPoolRessource()->getEventScheduler()->registerEvent(3,10) ; |
---|
288 | CXios::getPoolRessource()->getEventScheduler()->registerEvent(4,10) ; |
---|
289 | |
---|
290 | if (CXios::getPoolRessource()->hasService(CXios::defaultGathererId,0)) |
---|
291 | { |
---|
292 | CXios::getPoolRessource()->getService(CXios::defaultGathererId,0)-> getEventScheduler()->registerEvent(4,100) ; |
---|
293 | CXios::getPoolRessource()->getService(CXios::defaultGathererId,0)-> getEventScheduler()->registerEvent(5,100) ; |
---|
294 | } |
---|
295 | if (CXios::getPoolRessource()->hasService(CXios::defaultWriterId,0)) |
---|
296 | { |
---|
297 | CXios::getPoolRessource()->getService(CXios::defaultWriterId,0)-> getEventScheduler()->registerEvent(3,1000) ; |
---|
298 | CXios::getPoolRessource()->getService(CXios::defaultWriterId,0)-> getEventScheduler()->registerEvent(4,1000) ; |
---|
299 | CXios::getPoolRessource()->getService(CXios::defaultWriterId,0)-> getEventScheduler()->registerEvent(5,1000) ; |
---|
300 | } |
---|
301 | CXios::getPoolRessource()->getEventScheduler()->registerEvent(5,10) ; |
---|
302 | CXios::getPoolRessource()->getEventScheduler()->registerEvent(6,10) ; |
---|
303 | |
---|
304 | int numEvents=0 ; |
---|
305 | int poolEvent=1 ; |
---|
306 | int gatherEvent=1 ; |
---|
307 | int writerEvent=1 ; |
---|
308 | do |
---|
309 | { |
---|
310 | if (CXios::getPoolRessource()->getEventScheduler()->queryEvent(poolEvent,10)) |
---|
311 | { |
---|
312 | CXios::getPoolRessource()->getEventScheduler()->popEvent() ; |
---|
313 | MPI_Barrier(CXios::getPoolRessource()->getCommunicator()); |
---|
314 | poolEvent++ ; |
---|
315 | numEvents++; |
---|
316 | } |
---|
317 | |
---|
318 | if (CXios::getPoolRessource()->getEventScheduler()->queryEvent(gatherEvent,100)) |
---|
319 | { |
---|
320 | CXios::getPoolRessource()->getEventScheduler()->popEvent() ; |
---|
321 | MPI_Barrier(CXios::getPoolRessource()->getService(CXios::defaultGathererId,0)->getCommunicator()); |
---|
322 | gatherEvent++ ; |
---|
323 | numEvents++; |
---|
324 | } |
---|
325 | |
---|
326 | if (CXios::getPoolRessource()->getEventScheduler()->queryEvent(writerEvent,1000)) |
---|
327 | { |
---|
328 | CXios::getPoolRessource()->getEventScheduler()->popEvent() ; |
---|
329 | MPI_Barrier(CXios::getPoolRessource()->getService(CXios::defaultWriterId,0)->getCommunicator()); |
---|
330 | writerEvent++ ; |
---|
331 | numEvents++; |
---|
332 | } |
---|
333 | |
---|
334 | |
---|
335 | } while (numEvents!=11) ; |
---|
336 | |
---|
337 | } |
---|
338 | |
---|
339 | |
---|
340 | void CServer::xiosGlobalCommByFileExchange(MPI_Comm serverComm) |
---|
341 | { |
---|
342 | |
---|
343 | MPI_Comm globalComm=CXios::getGlobalComm() ; |
---|
344 | MPI_Comm xiosGlobalComm ; |
---|
345 | |
---|
346 | string strIds=CXios::getin<string>("clients_code_id","") ; |
---|
347 | vector<string> clientsCodeId=splitRegex(strIds,"\\s*,\\s*") ; |
---|
348 | |
---|
349 | int commRank, globalRank ; |
---|
350 | MPI_Comm_rank(serverComm, &commRank) ; |
---|
351 | MPI_Comm_rank(globalComm, &globalRank) ; |
---|
352 | string serverFileName("__xios_publisher::"+CXios::xiosCodeId+"__to_remove__") ; |
---|
353 | |
---|
354 | if (commRank==0) // if root process publish name |
---|
355 | { |
---|
356 | std::ofstream ofs (serverFileName, std::ofstream::out); |
---|
357 | ofs<<globalRank ; |
---|
358 | ofs.close(); |
---|
359 | } |
---|
360 | |
---|
361 | vector<int> clientsRank(clientsCodeId.size()) ; |
---|
362 | for(int i=0;i<clientsRank.size();i++) |
---|
363 | { |
---|
364 | std::ifstream ifs ; |
---|
365 | string fileName=("__xios_publisher::"+clientsCodeId[i]+"__to_remove__") ; |
---|
366 | struct stat buffer; |
---|
367 | do { |
---|
368 | } while( stat(fileName.c_str(), &buffer) != 0 ); |
---|
369 | sleep(1); |
---|
370 | ifs.open(fileName, ifstream::in) ; |
---|
371 | ifs>>clientsRank[i] ; |
---|
372 | //cout << "\t\t read: " << clientsRank[i] << " in " << fileName << endl; |
---|
373 | ifs.close() ; |
---|
374 | } |
---|
375 | |
---|
376 | MPI_Comm intraComm ; |
---|
377 | MPI_Comm_dup(serverComm,&intraComm) ; |
---|
378 | MPI_Comm interComm ; |
---|
379 | for(int i=0 ; i<clientsRank.size(); i++) |
---|
380 | { |
---|
381 | MPI_Intercomm_create(intraComm, 0, globalComm, clientsRank[i], 3141, &interComm); |
---|
382 | interCommLeft.push_back(interComm) ; |
---|
383 | MPI_Comm_free(&intraComm) ; |
---|
384 | MPI_Intercomm_merge(interComm,false, &intraComm ) ; |
---|
385 | } |
---|
386 | xiosGlobalComm=intraComm ; |
---|
387 | MPI_Barrier(xiosGlobalComm); |
---|
388 | if (commRank==0) std::remove(serverFileName.c_str()) ; |
---|
389 | MPI_Barrier(xiosGlobalComm); |
---|
390 | |
---|
391 | CXios::setXiosComm(xiosGlobalComm) ; |
---|
392 | |
---|
393 | } |
---|
394 | |
---|
395 | |
---|
396 | void CServer::xiosGlobalCommByPublishing(MPI_Comm serverComm) |
---|
397 | { |
---|
398 | // untested, need to be tested on a true MPI-2 compliant library |
---|
399 | |
---|
400 | // try to discover other client/server |
---|
401 | /* |
---|
402 | // publish server name |
---|
403 | char portName[MPI_MAX_PORT_NAME]; |
---|
404 | int ierr ; |
---|
405 | int commRank ; |
---|
406 | MPI_Comm_rank(serverComm, &commRank) ; |
---|
407 | |
---|
408 | if (commRank==0) // if root process publish name |
---|
409 | { |
---|
410 | MPI_Open_port(MPI_INFO_NULL, portName); |
---|
411 | MPI_Publish_name(CXios::xiosCodeId.c_str(), MPI_INFO_NULL, portName); |
---|
412 | } |
---|
413 | |
---|
414 | MPI_Comm intraComm=serverComm ; |
---|
415 | MPI_Comm interComm ; |
---|
416 | for(int i=0 ; i<clientsCodeId.size(); i++) |
---|
417 | { |
---|
418 | MPI_Comm_accept(portName, MPI_INFO_NULL, 0, intraComm, &interComm); |
---|
419 | MPI_Intercomm_merge(interComm,false, &intraComm ) ; |
---|
420 | } |
---|
421 | */ |
---|
422 | } |
---|
423 | |
---|
424 | /*! |
---|
425 | * Root process is listening for an order sent by client to call "oasis_enddef". |
---|
426 | * The root client of a compound send the order (tag 5). It is probed and received. |
---|
427 | * When the order has been received from each coumpound, the server root process ping the order to the root processes of the secondary levels of servers (if any). |
---|
428 | * After, it also inform (asynchronous call) other processes of the communicator that the oasis_enddef call must be done |
---|
429 | */ |
---|
430 | |
---|
431 | void CServer::listenOasisEnddef(void) |
---|
432 | { |
---|
433 | int flag ; |
---|
434 | MPI_Status status ; |
---|
435 | list<MPI_Comm>::iterator it; |
---|
436 | int msg ; |
---|
437 | static int nbCompound=0 ; |
---|
438 | int size ; |
---|
439 | static bool sent=false ; |
---|
440 | static MPI_Request* allRequests ; |
---|
441 | static MPI_Status* allStatus ; |
---|
442 | |
---|
443 | |
---|
444 | if (sent) |
---|
445 | { |
---|
446 | MPI_Comm_size(intraComm_,&size) ; |
---|
447 | MPI_Testall(size,allRequests, &flag, allStatus) ; |
---|
448 | if (flag==true) |
---|
449 | { |
---|
450 | delete [] allRequests ; |
---|
451 | delete [] allStatus ; |
---|
452 | sent=false ; |
---|
453 | } |
---|
454 | } |
---|
455 | |
---|
456 | |
---|
457 | for(it=interCommLeft.begin();it!=interCommLeft.end();it++) |
---|
458 | { |
---|
459 | MPI_Status status ; |
---|
460 | traceOff() ; |
---|
461 | MPI_Iprobe(0,5,*it,&flag,&status) ; // tags oasis_endded = 5 |
---|
462 | traceOn() ; |
---|
463 | if (flag==true) |
---|
464 | { |
---|
465 | MPI_Recv(&msg,1,MPI_INT,0,5,*it,&status) ; // tags oasis_endded = 5 |
---|
466 | nbCompound++ ; |
---|
467 | if (nbCompound==interCommLeft.size()) |
---|
468 | { |
---|
469 | MPI_Comm_size(intraComm_,&size) ; |
---|
470 | allRequests= new MPI_Request[size] ; |
---|
471 | allStatus= new MPI_Status[size] ; |
---|
472 | for(int i=0;i<size;i++) MPI_Isend(&msg,1,MPI_INT,i,5,intraComm_,&allRequests[i]) ; // tags oasis_endded = 5 |
---|
473 | sent=true ; |
---|
474 | } |
---|
475 | } |
---|
476 | } |
---|
477 | } |
---|
478 | |
---|
479 | /*! |
---|
480 | * Processes probes message from root process if oasis_enddef call must be done. |
---|
481 | * When the order is received it is scheduled to be treated in a synchronized way by all server processes of the communicator |
---|
482 | */ |
---|
483 | void CServer::listenRootOasisEnddef(void) |
---|
484 | { |
---|
485 | int flag ; |
---|
486 | MPI_Status status ; |
---|
487 | const int root=0 ; |
---|
488 | int msg ; |
---|
489 | static bool eventSent=false ; |
---|
490 | |
---|
491 | if (eventSent) |
---|
492 | { |
---|
493 | boost::hash<string> hashString; |
---|
494 | size_t hashId = hashString("oasis_enddef"); |
---|
495 | |
---|
496 | if (CXios::getPoolRessource()->getEventScheduler()->queryEvent(0,hashId)) |
---|
497 | { |
---|
498 | CXios::getPoolRessource()->getEventScheduler()->popEvent() ; |
---|
499 | driver_->endSynchronizedDefinition() ; |
---|
500 | eventSent=false ; |
---|
501 | } |
---|
502 | } |
---|
503 | |
---|
504 | traceOff() ; |
---|
505 | MPI_Iprobe(root,5,intraComm_, &flag, &status) ; |
---|
506 | traceOn() ; |
---|
507 | if (flag==true) |
---|
508 | { |
---|
509 | MPI_Recv(&msg,1,MPI_INT,root,5,intraComm_,&status) ; // tags oasis_endded = 5 |
---|
510 | boost::hash<string> hashString; |
---|
511 | size_t hashId = hashString("oasis_enddef"); |
---|
512 | CXios::getPoolRessource()->getEventScheduler()->registerEvent(0,hashId); |
---|
513 | eventSent=true ; |
---|
514 | } |
---|
515 | } |
---|
516 | |
---|
517 | void CServer::finalize(void) |
---|
518 | { |
---|
519 | CTimer::get("XIOS").suspend() ; |
---|
520 | CTimer::get("XIOS server").suspend() ; |
---|
521 | delete eventScheduler ; |
---|
522 | |
---|
523 | for (std::list<MPI_Comm>::iterator it = contextInterComms.begin(); it != contextInterComms.end(); it++) |
---|
524 | MPI_Comm_free(&(*it)); |
---|
525 | |
---|
526 | for (std::list<MPI_Comm>::iterator it = contextIntraComms.begin(); it != contextIntraComms.end(); it++) |
---|
527 | MPI_Comm_free(&(*it)); |
---|
528 | |
---|
529 | for (std::list<MPI_Comm>::iterator it = interCommRight.begin(); it != interCommRight.end(); it++) |
---|
530 | MPI_Comm_free(&(*it)); |
---|
531 | |
---|
532 | // MPI_Comm_free(&intraComm); |
---|
533 | CXios::finalizeDaemonsManager(); |
---|
534 | finalizeServersRessource(); |
---|
535 | |
---|
536 | CContext::removeAllContexts() ; // free memory for related context |
---|
537 | |
---|
538 | CXios::getMpiGarbageCollector().release() ; // release unfree MPI ressources |
---|
539 | |
---|
540 | CMemChecker::logMem( "CServer::finalize", true ); |
---|
541 | if (!is_MPI_Initialized) |
---|
542 | { |
---|
543 | if (CXios::usingOasis) delete driver_; |
---|
544 | else MPI_Finalize() ; |
---|
545 | } |
---|
546 | report(0)<<"Performance report : Time spent for XIOS : "<<CTimer::get("XIOS server").getCumulatedTime()<<endl ; |
---|
547 | report(0)<<"Performance report : Time spent in processing events : "<<CTimer::get("Process events").getCumulatedTime()<<endl ; |
---|
548 | report(0)<<"Performance report : Ratio : "<<CTimer::get("Process events").getCumulatedTime()/CTimer::get("XIOS server").getCumulatedTime()*100.<<"%"<<endl ; |
---|
549 | report(100)<<CTimer::getAllCumulatedTime()<<endl ; |
---|
550 | if (CXios::reportMemory) |
---|
551 | { |
---|
552 | report(100)<<CMemChecker::getAllCumulatedMem()<<endl ; |
---|
553 | } |
---|
554 | |
---|
555 | CWorkflowGraph::drawWorkFlowGraph_server(); |
---|
556 | xios::releaseStaticAllocation() ; // free memory from static allocation |
---|
557 | } |
---|
558 | |
---|
559 | /*! |
---|
560 | * Open a file specified by a suffix and an extension and use it for the given file buffer. |
---|
561 | * The file name will be suffix+rank+extension. |
---|
562 | * |
---|
563 | * \param fileName[in] protype file name |
---|
564 | * \param ext [in] extension of the file |
---|
565 | * \param fb [in/out] the file buffer |
---|
566 | */ |
---|
567 | void CServer::openStream(const StdString& fileName, const StdString& ext, std::filebuf* fb) |
---|
568 | { |
---|
569 | StdStringStream fileNameServer; |
---|
570 | int numDigit = 0; |
---|
571 | int commSize = 0; |
---|
572 | int commRank ; |
---|
573 | int id; |
---|
574 | |
---|
575 | MPI_Comm_size(CXios::getGlobalComm(), &commSize); |
---|
576 | MPI_Comm_rank(CXios::getGlobalComm(), &commRank); |
---|
577 | |
---|
578 | while (commSize) |
---|
579 | { |
---|
580 | commSize /= 10; |
---|
581 | ++numDigit; |
---|
582 | } |
---|
583 | id = commRank; |
---|
584 | |
---|
585 | fileNameServer << fileName << "_" << std::setfill('0') << std::setw(numDigit) << id << ext; |
---|
586 | fb->open(fileNameServer.str().c_str(), std::ios::out); |
---|
587 | if (!fb->is_open()) |
---|
588 | ERROR("void CServer::openStream(const StdString& fileName, const StdString& ext, std::filebuf* fb)", |
---|
589 | << std::endl << "Can not open <" << fileNameServer.str() << "> file to write the server log(s)."); |
---|
590 | } |
---|
591 | |
---|
592 | /*! |
---|
593 | * \brief Open a file stream to write the info logs |
---|
594 | * Open a file stream with a specific file name suffix+rank |
---|
595 | * to write the info logs. |
---|
596 | * \param fileName [in] protype file name |
---|
597 | */ |
---|
598 | void CServer::openInfoStream(const StdString& fileName) |
---|
599 | { |
---|
600 | std::filebuf* fb = m_infoStream.rdbuf(); |
---|
601 | openStream(fileName, ".out", fb); |
---|
602 | |
---|
603 | info.write2File(fb); |
---|
604 | report.write2File(fb); |
---|
605 | } |
---|
606 | |
---|
607 | //! Write the info logs to standard output |
---|
608 | void CServer::openInfoStream() |
---|
609 | { |
---|
610 | info.write2StdOut(); |
---|
611 | report.write2StdOut(); |
---|
612 | } |
---|
613 | |
---|
614 | //! Close the info logs file if it opens |
---|
615 | void CServer::closeInfoStream() |
---|
616 | { |
---|
617 | if (m_infoStream.is_open()) m_infoStream.close(); |
---|
618 | } |
---|
619 | |
---|
620 | /*! |
---|
621 | * \brief Open a file stream to write the error log |
---|
622 | * Open a file stream with a specific file name suffix+rank |
---|
623 | * to write the error log. |
---|
624 | * \param fileName [in] protype file name |
---|
625 | */ |
---|
626 | void CServer::openErrorStream(const StdString& fileName) |
---|
627 | { |
---|
628 | std::filebuf* fb = m_errorStream.rdbuf(); |
---|
629 | openStream(fileName, ".err", fb); |
---|
630 | |
---|
631 | error.write2File(fb); |
---|
632 | } |
---|
633 | |
---|
634 | //! Write the error log to standard error output |
---|
635 | void CServer::openErrorStream() |
---|
636 | { |
---|
637 | error.write2StdErr(); |
---|
638 | } |
---|
639 | |
---|
640 | //! Close the error log file if it opens |
---|
641 | void CServer::closeErrorStream() |
---|
642 | { |
---|
643 | if (m_errorStream.is_open()) m_errorStream.close(); |
---|
644 | } |
---|
645 | |
---|
646 | void CServer::launchServersRessource(MPI_Comm serverComm) |
---|
647 | { |
---|
648 | serversRessource_ = new CServersRessource(serverComm) ; |
---|
649 | } |
---|
650 | |
---|
651 | void CServer::finalizeServersRessource(void) |
---|
652 | { |
---|
653 | delete serversRessource_; serversRessource_=nullptr ; |
---|
654 | } |
---|
655 | } |
---|