source: XIOS/dev/dev_ym/XIOS_ONE_SIDED/src/context_server.cpp @ 1547

Last change on this file since 1547 was 1547, checked in by ymipsl, 4 years ago

New communication protocol between clients and servers, using hybrid mode of p2p mixt with one_sided communication in order to avoid dead-locking. The constraint of the maximum number of event that can be bufferized on client side is released.

Dev branch is created to be tested before merging.

YM

  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
  • Property svn:eol-style set to native
File size: 11.6 KB
Line 
1#include "context_server.hpp"
2#include "buffer_in.hpp"
3#include "type.hpp"
4#include "context.hpp"
5#include "object_template.hpp"
6#include "group_template.hpp"
7#include "attribute_template.hpp"
8#include "domain.hpp"
9#include "field.hpp"
10#include "file.hpp"
11#include "grid.hpp"
12#include "mpi.hpp"
13#include "tracer.hpp"
14#include "timer.hpp"
15#include "cxios.hpp"
16#include "event_scheduler.hpp"
17#include "server.hpp"
18#include <boost/functional/hash.hpp>
19
20
21
22namespace xios
23{
24
25  CContextServer::CContextServer(CContext* parent,MPI_Comm intraComm_,MPI_Comm interComm_)
26  {
27    context=parent;
28    intraComm=intraComm_;
29    MPI_Comm_size(intraComm,&intraCommSize);
30    MPI_Comm_rank(intraComm,&intraCommRank);
31
32    interComm=interComm_;
33    int flag;
34    MPI_Comm_test_inter(interComm,&flag);
35
36    if (flag) attachedMode=false ;
37    else  attachedMode=true ;
38   
39    if (flag) MPI_Comm_remote_size(interComm,&commSize);
40    else  MPI_Comm_size(interComm,&commSize);
41
42     
43    currentTimeLine=1;
44    scheduled=false;
45    finished=false;
46    boost::hash<string> hashString;
47    if (CServer::serverLevel == 1)
48      hashId=hashString(context->getId() + boost::lexical_cast<string>(context->clientPrimServer.size()));
49    else
50      hashId=hashString(context->getId());
51
52    if (!isAttachedModeEnabled()) MPI_Intercomm_merge(interComm_,true,&interCommMerged) ;
53   
54    MPI_Comm_split(intraComm_,intraCommRank,intraCommRank, &commSelf) ;
55    itLastTimeLine=lastTimeLine.begin() ;
56
57    pureOneSided=CXios::getin<bool>("pure_one_sided",false); // pure one sided communication (for test)
58    if (isAttachedModeEnabled()) pureOneSided=false ; // no one sided in attach mode
59     
60  }
61
62//! Attached mode is used ?
63//! \return true if attached mode is used, false otherwise
64  bool CContextServer::isAttachedModeEnabled() const
65  {
66    return attachedMode ;
67  }
68 
69  void CContextServer::setPendingEvent(void)
70  {
71    pendingEvent=true;
72  }
73
74  bool CContextServer::hasPendingEvent(void)
75  {
76    return pendingEvent;
77  }
78
79  bool CContextServer::hasFinished(void)
80  {
81    return finished;
82  }
83
84  bool CContextServer::eventLoop(bool enableEventsProcessing /*= true*/)
85  {
86//    info(100)<<"CContextServer::eventLoop : listen"<<endl ;
87    listen();
88//    info(100)<<"CContextServer::eventLoop : checkPendingRequest"<<endl ;
89    checkPendingRequest();
90//    info(100)<<"CContextServer::eventLoop : process events"<<endl ;
91    if (enableEventsProcessing)  processEvents();
92//    info(100)<<"CContextServer::eventLoop : finished "<<finished<<endl ;
93    return finished;
94  }
95
96  void CContextServer::listen(void)
97  {
98    int rank;
99    int flag;
100    int count;
101    char * addr;
102    MPI_Status status;
103    map<int,CServerBuffer*>::iterator it;
104    bool okLoop;
105
106    traceOff();
107    MPI_Iprobe(MPI_ANY_SOURCE, 20,interComm,&flag,&status);
108    traceOn();
109
110    if (flag==true)
111    {
112      rank=status.MPI_SOURCE ;
113      okLoop = true;
114      if (pendingRequest.find(rank)==pendingRequest.end())
115        okLoop = !listenPendingRequest(status) ;
116      if (okLoop)
117      {
118        for(rank=0;rank<commSize;rank++)
119        {
120          if (pendingRequest.find(rank)==pendingRequest.end())
121          {
122
123            traceOff();
124            MPI_Iprobe(rank, 20,interComm,&flag,&status);
125            traceOn();
126            if (flag==true) listenPendingRequest(status) ;
127          }
128        }
129      }
130    }
131  }
132
133  bool CContextServer::listenPendingRequest(MPI_Status& status)
134  {
135    int count;
136    char * addr;
137    map<int,CServerBuffer*>::iterator it;
138    int rank=status.MPI_SOURCE ;
139
140    it=buffers.find(rank);
141    if (it==buffers.end()) // Receive the buffer size and allocate the buffer
142    {
143       StdSize buffSize = 0;
144       MPI_Recv(&buffSize, 1, MPI_LONG, rank, 20, interComm, &status);
145       mapBufferSize_.insert(std::make_pair(rank, buffSize));
146       it=(buffers.insert(pair<int,CServerBuffer*>(rank,new CServerBuffer(buffSize)))).first;
147       if (!isAttachedModeEnabled())
148       {
149         MPI_Comm OneSidedInterComm, oneSidedComm ;
150         MPI_Intercomm_create(commSelf, 0, interCommMerged, rank, 0, &OneSidedInterComm );
151         MPI_Intercomm_merge(OneSidedInterComm,true,&oneSidedComm);
152         info(100)<<"DEBUG: before creating windows (server)"<<endl ;
153         buffers[rank]->createWindows(oneSidedComm) ;
154         info(100)<<"DEBUG: before creating windows (server)"<<endl ;
155       }
156       lastTimeLine[rank]=0 ;
157       itLastTimeLine=lastTimeLine.begin() ;
158
159       return true;
160    }
161    else
162    {
163      MPI_Get_count(&status,MPI_CHAR,&count);
164      if (it->second->isBufferFree(count))
165      {
166         addr=(char*)it->second->getBuffer(count);
167         MPI_Irecv(addr,count,MPI_CHAR,rank,20,interComm,&pendingRequest[rank]);
168         bufferRequest[rank]=addr;
169         return true;
170       }
171      else
172        return false;
173    }
174  }
175
176
177  void CContextServer::checkPendingRequest(void)
178  {
179    map<int,MPI_Request>::iterator it;
180    list<int> recvRequest;
181    list<int>::iterator itRecv;
182    int rank;
183    int flag;
184    int count;
185    MPI_Status status;
186
187    for(it=pendingRequest.begin();it!=pendingRequest.end();it++)
188    {
189      rank=it->first;
190      traceOff();
191      MPI_Test(& it->second, &flag, &status);
192      traceOn();
193      if (flag==true)
194      {
195        buffers[rank]->updateCurrentWindows() ;
196        recvRequest.push_back(rank);
197        MPI_Get_count(&status,MPI_CHAR,&count);
198        processRequest(rank,bufferRequest[rank],count);
199      }
200    }
201
202    for(itRecv=recvRequest.begin();itRecv!=recvRequest.end();itRecv++)
203    {
204      pendingRequest.erase(*itRecv);
205      bufferRequest.erase(*itRecv);
206    }
207  }
208
209  void CContextServer::getBufferFromClient(size_t timeLine)
210  {
211    if (!isAttachedModeEnabled()) // one sided desactivated in attached mode
212    { 
213      int rank ;
214      char *buffer ;
215      size_t count ; 
216
217      if (itLastTimeLine==lastTimeLine.end()) itLastTimeLine=lastTimeLine.begin() ;
218      for(;itLastTimeLine!=lastTimeLine.end();++itLastTimeLine)
219      {
220        rank=itLastTimeLine->first ;
221        if (itLastTimeLine->second < timeLine &&  pendingRequest.count(rank)==0)
222        {
223          if (buffers[rank]->getBufferFromClient(timeLine, buffer, count))
224          {
225            info(100)<<"get buffer from client : timeLine "<<timeLine<<endl ;
226            processRequest(rank, buffer, count);
227            break ;
228          }
229        }
230      }
231    }
232  }
233         
234       
235  void CContextServer::processRequest(int rank, char* buff,int count)
236  {
237
238    CBufferIn buffer(buff,count);
239    char* startBuffer,endBuffer;
240    int size, offset;
241    size_t timeLine=0;
242    map<size_t,CEventServer*>::iterator it;
243
244   
245    CTimer::get("Process request").resume();
246    while(count>0)
247    {
248      char* startBuffer=(char*)buffer.ptr();
249      CBufferIn newBuffer(startBuffer,buffer.remain());
250      newBuffer>>size>>timeLine;
251      info(100)<<"new event :   timeLine : "<<timeLine<<"  size : "<<size<<endl ;
252      it=events.find(timeLine);
253      if (it==events.end()) it=events.insert(pair<int,CEventServer*>(timeLine,new CEventServer)).first;
254      it->second->push(rank,buffers[rank],startBuffer,size);
255
256      buffer.advance(size);
257      count=buffer.remain();
258    }
259
260    if (timeLine>0) lastTimeLine[rank]=timeLine ;
261   
262    CTimer::get("Process request").suspend();
263  }
264
265  void CContextServer::processEvents(void)
266  {
267    map<size_t,CEventServer*>::iterator it;
268    CEventServer* event;
269
270    it=events.find(currentTimeLine);
271    if (it!=events.end())
272    {
273      event=it->second;
274
275      if (event->isFull())
276      {
277        if (!scheduled && CServer::eventScheduler) // Skip event scheduling for attached mode and reception on client side
278        {
279          CServer::eventScheduler->registerEvent(currentTimeLine,hashId);
280          scheduled=true;
281        }
282        else if (!CServer::eventScheduler || CServer::eventScheduler->queryEvent(currentTimeLine,hashId) )
283        {
284         // When using attached mode, synchronise the processes to avoid that differents event be scheduled by differents processes
285         // The best way to properly solve this problem will be to use the event scheduler also in attached mode
286         // for now just set up a MPI barrier
287         if (!CServer::eventScheduler && CXios::isServer) MPI_Barrier(intraComm) ;
288
289         CTimer::get("Process events").resume();
290         dispatchEvent(*event);
291         CTimer::get("Process events").suspend();
292         pendingEvent=false;
293         delete event;
294         events.erase(it);
295         currentTimeLine++;
296         scheduled = false;
297        }
298      }
299      else getBufferFromClient(currentTimeLine) ;
300    }
301    else if (pureOneSided) getBufferFromClient(currentTimeLine) ; // if pure one sided check buffer even if no event recorded at current time line
302  }
303
304  CContextServer::~CContextServer()
305  {
306    map<int,CServerBuffer*>::iterator it;
307    for(it=buffers.begin();it!=buffers.end();++it) delete it->second;
308  }
309
310  void CContextServer::releaseBuffers()
311  {
312    map<int,CServerBuffer*>::iterator it;
313    bool out ;
314    do
315    {
316      out=true ;
317      for(it=buffers.begin();it!=buffers.end();++it)
318      {
319        out = out && it->second->freeWindows() ;
320
321      }
322    } while (! out) ; 
323  }
324
325  void CContextServer::dispatchEvent(CEventServer& event)
326  {
327    string contextName;
328    string buff;
329    int MsgSize;
330    int rank;
331    list<CEventServer::SSubEvent>::iterator it;
332    StdString ctxId = context->getId();
333    CContext::setCurrent(ctxId);
334    StdSize totalBuf = 0;
335
336    if (event.classId==CContext::GetType() && event.type==CContext::EVENT_ID_CONTEXT_FINALIZE)
337    {
338      finished=true;
339      info(20)<<" CContextServer: Receive context <"<<context->getId()<<"> finalize."<<endl;
340      releaseBuffers() ;
341      context->finalize();
342      std::map<int, StdSize>::const_iterator itbMap = mapBufferSize_.begin(),
343                           iteMap = mapBufferSize_.end(), itMap;
344      for (itMap = itbMap; itMap != iteMap; ++itMap)
345      {
346        rank = itMap->first;
347        report(10)<< " Memory report : Context <"<<ctxId<<"> : server side : memory used for buffer of each connection to client" << endl
348            << "  +) With client of rank " << rank << " : " << itMap->second << " bytes " << endl;
349        totalBuf += itMap->second;
350      }
351      report(0)<< " Memory report : Context <"<<ctxId<<"> : server side : total memory used for buffer "<<totalBuf<<" bytes"<<endl;
352    }
353    else if (event.classId==CContext::GetType()) CContext::dispatchEvent(event);
354    else if (event.classId==CContextGroup::GetType()) CContextGroup::dispatchEvent(event);
355    else if (event.classId==CCalendarWrapper::GetType()) CCalendarWrapper::dispatchEvent(event);
356    else if (event.classId==CDomain::GetType()) CDomain::dispatchEvent(event);
357    else if (event.classId==CDomainGroup::GetType()) CDomainGroup::dispatchEvent(event);
358    else if (event.classId==CAxis::GetType()) CAxis::dispatchEvent(event);
359    else if (event.classId==CAxisGroup::GetType()) CAxisGroup::dispatchEvent(event);
360    else if (event.classId==CScalar::GetType()) CScalar::dispatchEvent(event);
361    else if (event.classId==CScalarGroup::GetType()) CScalarGroup::dispatchEvent(event);
362    else if (event.classId==CGrid::GetType()) CGrid::dispatchEvent(event);
363    else if (event.classId==CGridGroup::GetType()) CGridGroup::dispatchEvent(event);
364    else if (event.classId==CField::GetType()) CField::dispatchEvent(event);
365    else if (event.classId==CFieldGroup::GetType()) CFieldGroup::dispatchEvent(event);
366    else if (event.classId==CFile::GetType()) CFile::dispatchEvent(event);
367    else if (event.classId==CFileGroup::GetType()) CFileGroup::dispatchEvent(event);
368    else if (event.classId==CVariable::GetType()) CVariable::dispatchEvent(event);
369    else
370    {
371      ERROR("void CContextServer::dispatchEvent(CEventServer& event)",<<" Bad event class Id"<<endl);
372    }
373  }
374}
Note: See TracBrowser for help on using the repository browser.