source: XIOS/dev/dev_ym/XIOS_ONE_SIDED/src/context_server.cpp @ 1757

Last change on this file since 1757 was 1757, checked in by ymipsl, 3 years ago

Implement one sided communication in client/server protocol to avoid dead-lock when some buffer are full.

YM

  • Property copyright set to
    Software name : XIOS (Xml I/O Server)
    http://forge.ipsl.jussieu.fr/ioserver
    Creation date : January 2009
    Licence : CeCCIL version2
    see license file in root directory : Licence_CeCILL_V2-en.txt
    or http://www.cecill.info/licences/Licence_CeCILL_V2-en.html
    Holder : CEA/LSCE (Laboratoire des Sciences du CLimat et de l'Environnement)
    CNRS/IPSL (Institut Pierre Simon Laplace)
    Project Manager : Yann Meurdesoif
    yann.meurdesoif@cea.fr
  • Property svn:eol-style set to native
File size: 12.4 KB
Line 
1#include "context_server.hpp"
2#include "buffer_in.hpp"
3#include "type.hpp"
4#include "context.hpp"
5#include "object_template.hpp"
6#include "group_template.hpp"
7#include "attribute_template.hpp"
8#include "domain.hpp"
9#include "field.hpp"
10#include "file.hpp"
11#include "grid.hpp"
12#include "mpi.hpp"
13#include "tracer.hpp"
14#include "timer.hpp"
15#include "cxios.hpp"
16#include "event_scheduler.hpp"
17#include "server.hpp"
18#include <boost/functional/hash.hpp>
19
20
21
22namespace xios
23{
24
25  CContextServer::CContextServer(CContext* parent,MPI_Comm intraComm_,MPI_Comm interComm_)
26  {
27    context=parent;
28    intraComm=intraComm_;
29    MPI_Comm_size(intraComm,&intraCommSize);
30    MPI_Comm_rank(intraComm,&intraCommRank);
31
32    interComm=interComm_;
33    int flag;
34    MPI_Comm_test_inter(interComm,&flag);
35
36    if (flag) attachedMode=false ;
37    else  attachedMode=true ;
38   
39    if (flag) MPI_Comm_remote_size(interComm,&commSize);
40    else  MPI_Comm_size(interComm,&commSize);
41
42     
43    currentTimeLine=1;
44    scheduled=false;
45    finished=false;
46    boost::hash<string> hashString;
47    if (CServer::serverLevel == 1)
48      hashId=hashString(context->getId() + boost::lexical_cast<string>(context->clientPrimServer.size()));
49    else
50      hashId=hashString(context->getId());
51
52    if (!isAttachedModeEnabled())
53    {
54      MPI_Intercomm_merge(interComm_,true,&interCommMerged) ;
55// create windows for one sided comm
56      int interCommMergedRank;
57      MPI_Comm winComm ;
58      MPI_Comm_rank(intraComm, &interCommMergedRank);
59      windows.resize(2) ;
60      for(int rank=commSize; rank<commSize+intraCommSize; rank++)
61      {
62        if (rank==commSize+interCommMergedRank) 
63        {
64          MPI_Comm_split(interCommMerged, interCommMergedRank, rank, &winComm);
65          int myRank ;
66          MPI_Comm_rank(winComm,&myRank);
67          MPI_Win_create_dynamic(MPI_INFO_NULL, winComm, &windows[0]);
68          MPI_Win_create_dynamic(MPI_INFO_NULL, winComm, &windows[1]);     
69        }
70        else MPI_Comm_split(interCommMerged, interCommMergedRank, rank, &winComm);
71        MPI_Comm_free(&winComm) ;
72      }
73    }
74    else 
75    {
76      windows.resize(2) ;
77      windows[0]=MPI_WIN_NULL ;
78      windows[1]=MPI_WIN_NULL ;
79    }
80
81
82   
83    MPI_Comm_split(intraComm_,intraCommRank,intraCommRank, &commSelf) ;
84    itLastTimeLine=lastTimeLine.begin() ;
85
86    pureOneSided=CXios::getin<bool>("pure_one_sided",false); // pure one sided communication (for test)
87    if (isAttachedModeEnabled()) pureOneSided=false ; // no one sided in attach mode
88     
89  }
90
91//! Attached mode is used ?
92//! \return true if attached mode is used, false otherwise
93  bool CContextServer::isAttachedModeEnabled() const
94  {
95    return attachedMode ;
96  }
97 
98  void CContextServer::setPendingEvent(void)
99  {
100    pendingEvent=true;
101  }
102
103  bool CContextServer::hasPendingEvent(void)
104  {
105    return pendingEvent;
106  }
107
108  bool CContextServer::hasFinished(void)
109  {
110    return finished;
111  }
112
113  bool CContextServer::eventLoop(bool enableEventsProcessing /*= true*/)
114  {
115    listen();
116    checkPendingRequest();
117    if (enableEventsProcessing)  processEvents();
118    return finished;
119  }
120
121  void CContextServer::listen(void)
122  {
123    int rank;
124    int flag;
125    int count;
126    char * addr;
127    MPI_Status status;
128    map<int,CServerBuffer*>::iterator it;
129    bool okLoop;
130
131    traceOff();
132    MPI_Iprobe(MPI_ANY_SOURCE, 20,interComm,&flag,&status);
133    traceOn();
134
135    if (flag==true)
136    {
137      rank=status.MPI_SOURCE ;
138      okLoop = true;
139      if (pendingRequest.find(rank)==pendingRequest.end())
140        okLoop = !listenPendingRequest(status) ;
141      if (okLoop)
142      {
143        for(rank=0;rank<commSize;rank++)
144        {
145          if (pendingRequest.find(rank)==pendingRequest.end())
146          {
147
148            traceOff();
149            MPI_Iprobe(rank, 20,interComm,&flag,&status);
150            traceOn();
151            if (flag==true) listenPendingRequest(status) ;
152          }
153        }
154      }
155    }
156  }
157
158  bool CContextServer::listenPendingRequest(MPI_Status& status)
159  {
160    int count;
161    char * addr;
162    map<int,CServerBuffer*>::iterator it;
163    int rank=status.MPI_SOURCE ;
164
165    it=buffers.find(rank);
166    if (it==buffers.end()) // Receive the buffer size and allocate the buffer
167    {
168       MPI_Aint recvBuff[3] ;
169       MPI_Recv(recvBuff, 3, MPI_AINT, rank, 20, interComm, &status);
170       StdSize buffSize = recvBuff[0];
171       vector<MPI_Aint> winAdress(2) ;
172       winAdress[0]=recvBuff[1] ; winAdress[1]=recvBuff[2] ;
173       mapBufferSize_.insert(std::make_pair(rank, buffSize));
174       it=(buffers.insert(pair<int,CServerBuffer*>(rank,new CServerBuffer(windows, winAdress, rank, buffSize)))).first;
175      /*
176       if (!isAttachedModeEnabled())
177       {
178         MPI_Comm OneSidedInterComm, oneSidedComm ;
179         MPI_Intercomm_create(commSelf, 0, interCommMerged, rank, 0, &OneSidedInterComm );
180         MPI_Intercomm_merge(OneSidedInterComm,true,&oneSidedComm);
181         buffers[rank]->createWindows(oneSidedComm) ;
182       }
183       */
184       lastTimeLine[rank]=0 ;
185       itLastTimeLine=lastTimeLine.begin() ;
186
187       return true;
188    }
189    else
190    {
191      MPI_Get_count(&status,MPI_CHAR,&count);
192      if (it->second->isBufferFree(count))
193      {
194         addr=(char*)it->second->getBuffer(count);
195         MPI_Irecv(addr,count,MPI_CHAR,rank,20,interComm,&pendingRequest[rank]);
196         bufferRequest[rank]=addr;
197         return true;
198       }
199      else
200        return false;
201    }
202  }
203
204
205  void CContextServer::checkPendingRequest(void)
206  {
207    map<int,MPI_Request>::iterator it;
208    list<int> recvRequest;
209    list<int>::iterator itRecv;
210    int rank;
211    int flag;
212    int count;
213    MPI_Status status;
214
215    for(it=pendingRequest.begin();it!=pendingRequest.end();it++)
216    {
217      rank=it->first;
218      traceOff();
219      MPI_Test(& it->second, &flag, &status);
220      traceOn();
221      if (flag==true)
222      {
223        buffers[rank]->updateCurrentWindows() ;
224        recvRequest.push_back(rank);
225        MPI_Get_count(&status,MPI_CHAR,&count);
226        processRequest(rank,bufferRequest[rank],count);
227      }
228    }
229
230    for(itRecv=recvRequest.begin();itRecv!=recvRequest.end();itRecv++)
231    {
232      pendingRequest.erase(*itRecv);
233      bufferRequest.erase(*itRecv);
234    }
235  }
236
237  void CContextServer::getBufferFromClient(size_t timeLine)
238  {
239    if (!isAttachedModeEnabled()) // one sided desactivated in attached mode
240    { 
241      int rank ;
242      char *buffer ;
243      size_t count ; 
244
245      if (itLastTimeLine==lastTimeLine.end()) itLastTimeLine=lastTimeLine.begin() ;
246      for(;itLastTimeLine!=lastTimeLine.end();++itLastTimeLine)
247      {
248        rank=itLastTimeLine->first ;
249        if (itLastTimeLine->second < timeLine &&  pendingRequest.count(rank)==0)
250        {
251          if (buffers[rank]->getBufferFromClient(timeLine, buffer, count))
252          {
253            processRequest(rank, buffer, count);
254            break ;
255          }
256        }
257      }
258    }
259  }
260         
261       
262  void CContextServer::processRequest(int rank, char* buff,int count)
263  {
264
265    CBufferIn buffer(buff,count);
266    char* startBuffer,endBuffer;
267    int size, offset;
268    size_t timeLine=0;
269    map<size_t,CEventServer*>::iterator it;
270
271   
272    CTimer::get("Process request").resume();
273    while(count>0)
274    {
275      char* startBuffer=(char*)buffer.ptr();
276      CBufferIn newBuffer(startBuffer,buffer.remain());
277      newBuffer>>size>>timeLine;
278      it=events.find(timeLine);
279      if (it==events.end()) it=events.insert(pair<int,CEventServer*>(timeLine,new CEventServer)).first;
280      it->second->push(rank,buffers[rank],startBuffer,size);
281
282      buffer.advance(size);
283      count=buffer.remain();
284    }
285
286    if (timeLine>0) lastTimeLine[rank]=timeLine ;
287   
288    CTimer::get("Process request").suspend();
289  }
290
291  void CContextServer::processEvents(void)
292  {
293    map<size_t,CEventServer*>::iterator it;
294    CEventServer* event;
295
296    it=events.find(currentTimeLine);
297    if (it!=events.end())
298    {
299      event=it->second;
300
301      if (event->isFull())
302      {
303        if (!scheduled && CServer::eventScheduler) // Skip event scheduling for attached mode and reception on client side
304        {
305          CServer::eventScheduler->registerEvent(currentTimeLine,hashId);
306          scheduled=true;
307        }
308        else if (!CServer::eventScheduler || CServer::eventScheduler->queryEvent(currentTimeLine,hashId) )
309        {
310         // When using attached mode, synchronise the processes to avoid that differents event be scheduled by differents processes
311         // The best way to properly solve this problem will be to use the event scheduler also in attached mode
312         // for now just set up a MPI barrier
313         if (!CServer::eventScheduler && CXios::isServer) MPI_Barrier(intraComm) ;
314
315         CTimer::get("Process events").resume();
316         dispatchEvent(*event);
317         CTimer::get("Process events").suspend();
318         pendingEvent=false;
319         delete event;
320         events.erase(it);
321         currentTimeLine++;
322         scheduled = false;
323        }
324      }
325      else getBufferFromClient(currentTimeLine) ;
326    }
327    else if (pureOneSided) getBufferFromClient(currentTimeLine) ; // if pure one sided check buffer even if no event recorded at current time line
328  }
329
330  CContextServer::~CContextServer()
331  {
332    map<int,CServerBuffer*>::iterator it;
333    for(it=buffers.begin();it!=buffers.end();++it) delete it->second;
334  }
335
336  void CContextServer::releaseBuffers()
337  {
338    map<int,CServerBuffer*>::iterator it;
339    bool out ;
340    do
341    {
342      out=true ;
343      for(it=buffers.begin();it!=buffers.end();++it)
344      {
345//        out = out && it->second->freeWindows() ;
346
347      }
348    } while (! out) ; 
349  }
350
351  void CContextServer::notifyClientsFinalize(void)
352  {
353    for(auto it=buffers.begin();it!=buffers.end();++it)
354    {
355      it->second->notifyClientFinalize() ;
356    }
357  }
358
359  void CContextServer::dispatchEvent(CEventServer& event)
360  {
361    string contextName;
362    string buff;
363    int MsgSize;
364    int rank;
365    list<CEventServer::SSubEvent>::iterator it;
366    StdString ctxId = context->getId();
367    CContext::setCurrent(ctxId);
368    StdSize totalBuf = 0;
369
370    if (event.classId==CContext::GetType() && event.type==CContext::EVENT_ID_CONTEXT_FINALIZE)
371    {
372      finished=true;
373      info(20)<<" CContextServer: Receive context <"<<context->getId()<<"> finalize."<<endl;
374//      releaseBuffers() ;
375      notifyClientsFinalize() ;
376      context->finalize();
377
378/* don't know where release windows
379      MPI_Win_free(&windows[0]) ;
380      MPI_Win_free(&windows[1]) ;
381*/     
382      std::map<int, StdSize>::const_iterator itbMap = mapBufferSize_.begin(),
383                           iteMap = mapBufferSize_.end(), itMap;
384      for (itMap = itbMap; itMap != iteMap; ++itMap)
385      {
386        rank = itMap->first;
387        report(10)<< " Memory report : Context <"<<ctxId<<"> : server side : memory used for buffer of each connection to client" << endl
388            << "  +) With client of rank " << rank << " : " << itMap->second << " bytes " << endl;
389        totalBuf += itMap->second;
390      }
391      report(0)<< " Memory report : Context <"<<ctxId<<"> : server side : total memory used for buffer "<<totalBuf<<" bytes"<<endl;
392    }
393    else if (event.classId==CContext::GetType()) CContext::dispatchEvent(event);
394    else if (event.classId==CContextGroup::GetType()) CContextGroup::dispatchEvent(event);
395    else if (event.classId==CCalendarWrapper::GetType()) CCalendarWrapper::dispatchEvent(event);
396    else if (event.classId==CDomain::GetType()) CDomain::dispatchEvent(event);
397    else if (event.classId==CDomainGroup::GetType()) CDomainGroup::dispatchEvent(event);
398    else if (event.classId==CAxis::GetType()) CAxis::dispatchEvent(event);
399    else if (event.classId==CAxisGroup::GetType()) CAxisGroup::dispatchEvent(event);
400    else if (event.classId==CScalar::GetType()) CScalar::dispatchEvent(event);
401    else if (event.classId==CScalarGroup::GetType()) CScalarGroup::dispatchEvent(event);
402    else if (event.classId==CGrid::GetType()) CGrid::dispatchEvent(event);
403    else if (event.classId==CGridGroup::GetType()) CGridGroup::dispatchEvent(event);
404    else if (event.classId==CField::GetType()) CField::dispatchEvent(event);
405    else if (event.classId==CFieldGroup::GetType()) CFieldGroup::dispatchEvent(event);
406    else if (event.classId==CFile::GetType()) CFile::dispatchEvent(event);
407    else if (event.classId==CFileGroup::GetType()) CFileGroup::dispatchEvent(event);
408    else if (event.classId==CVariable::GetType()) CVariable::dispatchEvent(event);
409    else
410    {
411      ERROR("void CContextServer::dispatchEvent(CEventServer& event)",<<" Bad event class Id"<<endl);
412    }
413  }
414}
Note: See TracBrowser for help on using the repository browser.