Changeset 2246

Timestamp:

10/11/21 14:41:56 (3 years ago)

Author:

ymipsl

Message:

• Update of the tranfer protocol using one sided communication
• Introduce MPI_Improb/MPI_mrecv to listen incomming request
• Introducing latency when looping over managers

YM

Location:

XIOS/dev/dev_ym/XIOS_COUPLING/src

Files:

• buffer_client.cpp (modified) (18 diffs)
• buffer_client.hpp (modified) (5 diffs)
• buffer_cs_base.hpp (added)
• buffer_server.cpp (modified) (9 diffs)
• buffer_server.hpp (modified) (4 diffs)
• context_client.cpp (modified) (4 diffs)
• context_client.hpp (modified) (2 diffs)
• context_server.cpp (modified) (11 diffs)
• context_server.hpp (modified) (2 diffs)
• distribution/distribution_type.hpp (modified) (1 diff)
• manager/contexts_manager.cpp (modified) (10 diffs)
• manager/contexts_manager.hpp (modified) (1 diff)
• manager/daemons_manager.cpp (modified) (1 diff)
• manager/pool_ressource.cpp (modified) (6 diffs)
• manager/pool_ressource.hpp (modified) (1 diff)
• manager/ressources_manager.cpp (modified) (8 diffs)
• manager/ressources_manager.hpp (modified) (1 diff)
• manager/server_context.cpp (modified) (8 diffs)
• manager/server_context.hpp (modified) (1 diff)
• manager/servers_ressource.cpp (modified) (3 diffs)
• manager/servers_ressource.hpp (modified) (1 diff)
• manager/services.cpp (modified) (12 diffs)
• manager/services.hpp (modified) (1 diff)
• manager/services_manager.cpp (modified) (7 diffs)
• manager/services_manager.hpp (modified) (1 diff)
• manager/window_manager.hpp (modified) (4 diffs)

XIOS/dev/dev_ym/XIOS_COUPLING/src/buffer_client.cpp

@@ -8 +8 @@
 #include "tracer.hpp"
 #include "timeline_events.hpp"
+#include "timer.hpp"
 
 namespace xios
@@ -29 +30 @@
     else hasWindows=true ;
 
-      MPI_Alloc_mem(bufferSize+headerSize, MPI_INFO_NULL, &bufferHeader[0]) ;
-      MPI_Alloc_mem(bufferSize+headerSize, MPI_INFO_NULL, &bufferHeader[1]) ;
-      buffer[0] = bufferHeader[0]+headerSize ;
-      buffer[1] = bufferHeader[1]+headerSize ;
-      firstTimeLine[0]=(size_t*)bufferHeader[0] ;
-      firstTimeLine[1]=(size_t*)bufferHeader[1] ;
-      bufferCount[0]=(size_t*)bufferHeader[0] +1 ;
-      bufferCount[1]=(size_t*)bufferHeader[1] +1 ;
-      control[0]=(size_t*)bufferHeader[0] +2 ;
-      control[1]=(size_t*)bufferHeader[1] +2 ;
-      finalize[0]=(size_t*)bufferHeader[0] +3 ;
-      finalize[1]=(size_t*)bufferHeader[1] +3 ;
+      MPI_Alloc_mem(bufferSize+headerSize_, MPI_INFO_NULL, &bufferHeader[0]) ;
+      MPI_Alloc_mem(bufferSize+headerSize_, MPI_INFO_NULL, &bufferHeader[1]) ;
+      buffer[0] = bufferHeader[0]+headerSize_ ;
+      buffer[1] = bufferHeader[1]+headerSize_ ;
+      firstTimeLine[0]=(size_t*)bufferHeader[0] + timeLineOffset_ ;
+      firstTimeLine[1]=(size_t*)bufferHeader[1] + timeLineOffset_ ;
+      bufferCount[0]=(size_t*)bufferHeader[0] + countOffset_ ;
+      bufferCount[1]=(size_t*)bufferHeader[1] + countOffset_ ;
+      control[0]=(size_t*)bufferHeader[0] + controlOffset_ ;
+      control[1]=(size_t*)bufferHeader[1] + controlOffset_ ;
+      notify[0]=(size_t*)bufferHeader[0] + notifyOffset_ ;
+      notify[1]=(size_t*)bufferHeader[1] + notifyOffset_ ;
 
       *firstTimeLine[0]=0 ;
@@ -48 +49 @@
       *control[0]=0 ;
       *control[1]=0 ;
-      *finalize[0]=0 ;
-      *finalize[1]=0 ;
+      *notify[0]=notifyNothing_ ;
+      *notify[1]=notifyNothing_ ;
       winState[0]=false ;
       winState[1]=false ;
@@ -57 +58 @@
     {  
     
-      MPI_Aint buffSize=bufferSize+headerSize ;
+      MPI_Aint buffSize=bufferSize+headerSize_ ;
       MPI_Win_attach(windows_[0], bufferHeader[0], buffSize) ;
       MPI_Win_attach(windows_[1], bufferHeader[1], buffSize) ;
@@ -106 +107 @@
   }
 
-/*  void CClientBuffer::createWindows(MPI_Comm oneSidedComm)
-  {
-    MPI_Barrier(oneSidedComm) ;
-    MPI_Win_create(bufferHeader[0], bufferSize+headerSize, 1, MPI_INFO_NULL, oneSidedComm, &(windows[0])) ;
-    MPI_Win_create(bufferHeader[1], bufferSize+headerSize, 1, MPI_INFO_NULL, oneSidedComm, &(windows[1])) ;
-
-    MPI_Win_lock(MPI_LOCK_EXCLUSIVE, 0, 0, windows[0]) ;
-    *firstTimeLine[0]=0 ;
-    *bufferCount[0]=0 ;
-    *control[0]=0 ;
-    MPI_Win_unlock(0, windows[0]) ;
-
-    MPI_Win_lock(MPI_LOCK_EXCLUSIVE, 0, 0, windows[1]) ;
-    *firstTimeLine[1]=0 ;
-    *bufferCount[1]=0 ;
-    *control[1]=0 ;
-    MPI_Win_unlock(0, windows[1]) ;
-    winState[0]=false ;
-    winState[1]=false ;
-    MPI_Barrier(oneSidedComm) ;
-    hasWindows=true ;
-  }
-*/
-
-/*  
-  void CClientBuffer::freeWindows()
-  {
-    if (hasWindows)
-    {
-      MPI_Win_lock(MPI_LOCK_EXCLUSIVE, 0, 0, windows_[0]) ;
-      MPI_Win_lock(MPI_LOCK_EXCLUSIVE, 0, 0, windows_[1]) ;
-      *control[0]=2 ;
-      *control[1]=2 ;
-      MPI_Win_unlock(0, windows_[1]) ;
-      MPI_Win_unlock(0, windows_[0]) ;
-      
-      MPI_Win_free(&windows_[0]) ;
-      MPI_Win_free(&windows_[1]) ;
-      hasWindows=false ;
-    }
-  }
-*/ 
   void CClientBuffer::lockBuffer(void)
   {
-    if (hasWindows)
-    {
-   //   MPI_Win_lock(MPI_LOCK_EXCLUSIVE, clientRank_, 0, windows_[current]) ;
-      long long int lock=1 ;
-      long long int zero=0, one=1 ;
-     
+    CTimer::get("lock buffer").resume();
+    if (hasWindows)
+    {
       MPI_Win_lock(MPI_LOCK_EXCLUSIVE,clientRank_, 0, windows_[current]) ;
-     
-      while(lock!=0)
-      {
-        MPI_Compare_and_swap(&one, &zero, &lock, MPI_LONG_LONG_INT, clientRank_, MPI_Aint_add(getWinAddress(current),2*sizeof(size_t)),
-                             windows_[current]) ;
-        MPI_Win_flush(clientRank_, windows_[current]) ;
-      }
-
-//      info(100)<<"Buffer locked "<<&windows_<<"  "<<current<<endl ;
       winState[current]=true ;
     }
+    CTimer::get("lock buffer").suspend();
   }
 
   void CClientBuffer::unlockBuffer(void)
   {
-    if (hasWindows)
-    {
-      long long int lock=1 ;
-      long long int zero=0, one=1 ;
-
-      MPI_Compare_and_swap(&zero, &one, &lock, MPI_LONG_LONG_INT, clientRank_, MPI_Aint_add(getWinAddress(current),2*sizeof(size_t)),
-                             windows_[current]) ;
+    CTimer::get("unlock buffer").resume();
+    if (hasWindows)
+    {
       MPI_Win_unlock(clientRank_, windows_[current]) ;
-
- //     info(100)<<"Buffer unlocked "<<&windows_<<"  "<<current<<endl ;
       winState[current]=false ;
     }
+    CTimer::get("unlock buffer").suspend();
   }
 
@@ -193 +136 @@
   bool CClientBuffer::isBufferFree(StdSize size)
   {
-//    bool loop=true ;
-//    while (loop) 
-//    {
-//      lockBuffer();
-//      if (*control[current]==0) loop=false ; // attemp to read from server ?
-//      else unlockBuffer() ;
-//    }
   
     lockBuffer();
@@ -208 +144 @@
     if (size > bufferSize)
     {
-      // ERROR("bool CClientBuffer::isBufferFree(StdSize size)",
-      //      << "The requested size (" << size << " bytes) is too big to fit the buffer (" << bufferSize << " bytes), please increase the client buffer size." << endl);
       resizingBufferStep_=1 ;
+      *firstTimeLine[current]=0 ;
       newBufferSize_=size ;
       return false ;
@@ -231 +166 @@
       {
         resizingBufferStep_ = 1 ;
+        *firstTimeLine[current]=0 ;
         newBufferSize_ = (count+size)*growFactor_ ;
       }  
@@ -247 +183 @@
       if (*firstTimeLine[current]==0) *firstTimeLine[current]=timeLine ;
       *bufferCount[current]=count ;
-/*      info(50)<<"CClientBuffer::getBuffer "<<" clientRank_ "<<clientRank_<<" serverRank "<<serverRank <<" current "<<current
-              <<" size "<<size<<" timeLine "<< timeLine <<" firstTimeLine "<<*firstTimeLine[current]<<" count "<<*bufferCount[current]<<endl ;
-      if (!winState[current]) info(40)<<"CClientBuffer::getBuffer "<<" Windows Not Locked... "<<" clientRank_ "<<clientRank_<<" serverRank "<<serverRank <<" current "<<current
-              <<" size "<<size<<" timeLine "<< timeLine <<" firstTimeLine "<<*firstTimeLine[current]<<" count "<<*bufferCount[current]<<endl ;*/
       return retBuffer;
     }
@@ -284 +216 @@
     MPI_Status status;
     int flag;
+    
+    MPI_Win_lock(MPI_LOCK_EXCLUSIVE, clientRank_, 0, windows_[0]) ;
+    MPI_Win_unlock(clientRank_, windows_[0]) ;
+
+    MPI_Win_lock(MPI_LOCK_EXCLUSIVE, clientRank_, 0, windows_[1]) ;
+    MPI_Win_unlock(clientRank_, windows_[1]) ;
 
     if (pending)
@@ -299 +237 @@
       if (count > 0)
       {
-        lockBuffer() ;
- //       if (*control[current]==0 && bufferCount[current] > 0)
-        if (*bufferCount[current] > 0)
+        double time=MPI_Wtime() ;
+        if (time - lastCheckedWithNothing_ > latency_)
         {
-          MPI_Issend(buffer[current], count, MPI_CHAR, serverRank, 20, interComm, &request);
-          if (resizingBufferStep_==3) resizingBufferStep_=0 ;
-          pending = true;
-//          *control[current]=0 ;
-          *firstTimeLine[current]=0 ;
-          *bufferCount[current]=0 ;
-
-           unlockBuffer() ;
-
-          if (current == 1) current = 0;
-          else current = 1;
-          count = 0;
-        }
-        else 
-        {
-          unlockBuffer() ;
+          lockBuffer() ;
+          if (*bufferCount[current] > 0)
+          {
+            MPI_Issend(buffer[current], count, MPI_CHAR, serverRank, 20, interComm, &request);
+            if (resizingBufferStep_==4) resizingBufferStep_=0 ;
+            pending = true;
+            *firstTimeLine[current]=0 ;
+            *bufferCount[current]=0 ;
+
+             unlockBuffer() ;
+
+            if (current == 1) current = 0;
+            else current = 1;
+            count = 0;
+          }
+          else 
+          {
+            unlockBuffer() ;
+            lastCheckedWithNothing_ = time ;
+          }
         }
       }
       else
       {
-        if (resizingBufferStep_==2) resizeBuffer(newBufferSize_) ;
         if (resizingBufferStep_==1) resizeBufferNotify() ;
+        else if (resizingBufferStep_==2) isNotifiedChangeBufferSize() ;
+        else if (resizingBufferStep_==3) resizeBuffer(newBufferSize_) ;
       }
     }
@@ -345 +287 @@
   void CClientBuffer::resizeBuffer(size_t newSize)
   {
+
     if (hasWindows)
     { 
@@ -354 +297 @@
 
     bufferSize=newSize ;
-    MPI_Alloc_mem(bufferSize+headerSize, MPI_INFO_NULL, &bufferHeader[0]) ;
-    MPI_Alloc_mem(bufferSize+headerSize, MPI_INFO_NULL, &bufferHeader[1]) ;
-    buffer[0] = bufferHeader[0]+headerSize ;
-    buffer[1] = bufferHeader[1]+headerSize ;
-    firstTimeLine[0]=(size_t*)bufferHeader[0] ;
-    firstTimeLine[1]=(size_t*)bufferHeader[1] ;
-    bufferCount[0]=(size_t*)bufferHeader[0] +1 ;
-    bufferCount[1]=(size_t*)bufferHeader[1] +1 ;
-    control[0]=(size_t*)bufferHeader[0] +2 ;
-    control[1]=(size_t*)bufferHeader[1] +2 ;
-    finalize[0]=(size_t*)bufferHeader[0] +3 ;
-    finalize[1]=(size_t*)bufferHeader[1] +3 ;
+    MPI_Alloc_mem(bufferSize+headerSize_, MPI_INFO_NULL, &bufferHeader[0]) ;
+    MPI_Alloc_mem(bufferSize+headerSize_, MPI_INFO_NULL, &bufferHeader[1]) ;
+    buffer[0] = bufferHeader[0]+headerSize_ ;
+    buffer[1] = bufferHeader[1]+headerSize_ ;
+    firstTimeLine[0]=(size_t*)bufferHeader[0] + timeLineOffset_;
+    firstTimeLine[1]=(size_t*)bufferHeader[1] + timeLineOffset_;
+    bufferCount[0]=(size_t*)bufferHeader[0] + countOffset_ ;
+    bufferCount[1]=(size_t*)bufferHeader[1] + countOffset_ ;
+    control[0]=(size_t*)bufferHeader[0] + controlOffset_ ;  // control=0 => nothing ; control=1 => changeBufferSize
+    control[1]=(size_t*)bufferHeader[1] + controlOffset_ ;
+    notify[0]=(size_t*)bufferHeader[0] + notifyOffset_ ;
+    notify[1]=(size_t*)bufferHeader[1] + notifyOffset_ ;
 
     *firstTimeLine[0]=0 ;
@@ -373 +316 @@
     *control[0]=0 ;
     *control[1]=0 ;
-    *finalize[0]=0 ;
-    *finalize[1]=0 ;
+    *notify[0] = notifyNothing_ ;
+    *notify[1] = notifyNothing_ ;
     winState[0]=false ;
     winState[1]=false ;
@@ -382 +325 @@
     {  
     
-      MPI_Win_attach(windows_[0], bufferHeader[0], bufferSize+headerSize) ;
-      MPI_Win_attach(windows_[1], bufferHeader[1], bufferSize+headerSize) ;
+      MPI_Win_attach(windows_[0], bufferHeader[0], bufferSize+headerSize_) ;
+      MPI_Win_attach(windows_[1], bufferHeader[1], bufferSize+headerSize_) ;
           
       MPI_Win_lock(MPI_LOCK_EXCLUSIVE, clientRank_, 0, windows_[0]) ;
@@ -402 +345 @@
     bufOut->put(this->getWinAddress(1));
 
-    resizingBufferStep_=3;
-    unlockBuffer() ;
+    resizingBufferStep_=4;
+    unlockBuffer() ;
+    info(100)<<"CClientBuffer::resizeBuffer(size_t newSize) : resizing buffer of server "<<serverRank<<" ; new size : "<<newSize<<" ; winAdress[0] "<<this->getWinAddress(0)<<" winAdress[1] "<<this->getWinAddress(1)<<endl;
   }
 
@@ -416 +360 @@
   }
 
-  bool CClientBuffer::isNotifiedFinalized(void)
+  bool CClientBuffer::isNotifiedChangeBufferSize(void)
   {
    
     bool ret ;
     lockBuffer() ;
-    ret=*finalize[current] == 1 ? true : false ;
+    ret=*notify[current] == notifyResizeBuffer_ ? true : false ;
+    if (ret) 
+    {
+      *notify[current] = notifyNothing_ ;
+      resizingBufferStep_=3;  
+    }
     unlockBuffer() ;
 
@@ -427 +376 @@
   }
 
+  bool CClientBuffer::isNotifiedFinalized(void)
+  {
+   
+    bool ret ;
+    lockBuffer() ;
+    ret=*notify[current] == notifyFinalize_ ? true : false ;
+    unlockBuffer() ;
+
+    return ret;
+  }
+
 }

XIOS/dev/dev_ym/XIOS_COUPLING/src/buffer_client.hpp

@@ -2 +2 @@
 #define __BUFFER_CLIENT_HPP__
 
+#include "buffer_cs_base.hpp"
 #include "xios_spl.hpp"
 #include "buffer_out.hpp"
@@ -9 +10 @@
 namespace xios
 {
-  class CClientBuffer
+  class CClientBuffer : public CBufferClientServerBase
   {
     public:
@@ -35 +36 @@
        void resizeBuffer(size_t newSize) ;
        void resizeBufferNotify(void) ;
+       bool isNotifiedChangeBufferSize(void) ;
 
 
@@ -42 +44 @@
       size_t* bufferCount[2] ;
       size_t* control[2] ;
-      size_t* finalize[2] ;
+      size_t* notify[2] ;
       bool winState[2] ;
       int current;
@@ -67 +69 @@
       std::vector<MPI_Win> windows_ ;
       bool hasWindows ;
-      static const int headerSize=4*sizeof(size_t);
+
+      double latency_=1e-2 ;
+      double lastCheckedWithNothing_=0 ;
   };
 }

XIOS/dev/dev_ym/XIOS_COUPLING/src/buffer_server.cpp

@@ -2 +2 @@
 #include "exception.hpp"
 #include "buffer_server.hpp"
+#include "timer.hpp"
 
 
@@ -31 +32 @@
   }
 
-/*
-  void CServerBuffer::createWindows(MPI_Comm oneSidedComm)
-  {
-    MPI_Barrier(oneSidedComm) ;
-    MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, oneSidedComm, &(windows[0])) ;
-    MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, oneSidedComm, &(windows[1])) ;
-    hasWindows=true ;
-    updateCurrentWindows() ;
-    MPI_Barrier(oneSidedComm) ;
-  }
-*/
-
-/*
-  bool CServerBuffer::freeWindows()
-  {
-    if (hasWindows)
-    {
-      size_t header[3] ;
-      size_t& control=header[2] ;
-      MPI_Win_lock(MPI_LOCK_EXCLUSIVE,0,0,windows_[0]) ;
-      MPI_Get(&control, 1, MPI_LONG_LONG_INT, windowsRank , 2*sizeof(size_t), 1, MPI_LONG_LONG_INT,windows[0]) ;
-      MPI_Win_unlock(0,windows[0]) ;
-      if (control==2)  // ok for free windows
-      {
-        MPI_Win_free( &(windows[0])) ;
-        MPI_Win_free( &(windows[1])) ;
-        hasWindows=false ;
-        return true ;
-      }
-      else return false ;
-    }
-    else return true ;
-  }
-*/
 
   bool CServerBuffer::isBufferFree(size_t count)
@@ -222 +189 @@
   bool CServerBuffer::getBufferFromClient(size_t timeLine, char*& buffer, size_t& count)
   {
+    count = -1 ;
     if (!hasWindows || resizingBuffer_) return false ;
-
-    
-    size_t header[3] ;
-    size_t& clientTimeline=header[0] ;
-    size_t& clientCount=header[1] ;
-    size_t& control=header[2] ;
+    double time=MPI_Wtime() ;
+    if (time-bufferFromClientTime_ < bufferFromClientLatency_ ) return false;
+    bufferFromClientTime_ = time ;
+    CTimer::get("getBufferFromClient").resume() ;   
+    size_t clientTimeline ;
+    size_t clientCount ;
     bool ok=false ;
     
@@ -238 +206 @@
     
     lockBuffer(); 
-
+    CTimer::get("getBufferFromClient_locked").resume() ;   
 // lock is acquired
 
-    MPI_Get(&clientTimeline, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows],0), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
-    MPI_Get(&clientCount, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows],sizeof(size_t)), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
+    MPI_Get(&clientTimeline, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows],timeLineOffset_*sizeof(size_t)), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
+    MPI_Get(&clientCount, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows],countOffset_*sizeof(size_t)), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
     MPI_Win_flush(windowsRank_, windows_[currentWindows]) ;
 
-//    control=1 ;
-//    MPI_Put(&control, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows],2*sizeof(size_t)), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
-   
-//    MPI_Win_unlock(windowsRank_, windows_[currentWindows]) ;
-    MPI_Win_flush(windowsRank_, windows_[currentWindows]) ;
-//    info(100)<<"getBufferFromClient : windowsRank "<<windowsRank_<<" timeline "<<timeLine<<" clientTimeline "<<clientTimeline<<" clientCount "<<clientCount<<endl ;
     if (timeLine==clientTimeline)
     {
-//      info(50)<<"getBufferFromClient timeLine==clientTimeLine: windowsRank "<<windowsRank_<<" timeline "<<timeLine<<" clientTimeline "<<clientTimeline<<" clientCount "<<clientCount<<endl ;
- 
-//      MPI_Win_lock(MPI_LOCK_EXCLUSIVE,windowsRank_,0,windows_[currentWindows]) ;
       buffer=(char*)getBuffer(clientCount) ;
       count=clientCount ;
@@ -261 +220 @@
       clientTimeline = 0 ;
       clientCount = 0 ;
-//      control=0 ;
-      MPI_Put(&header[0], 2, MPI_LONG_LONG_INT, windowsRank_, MPI_Aint_add(winAddress_[currentWindows],0) , 2, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
+      MPI_Put(&clientTimeline, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows],timeLineOffset_*sizeof(size_t)), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
+      MPI_Put(&clientCount, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows],countOffset_*sizeof(size_t)), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
 
 // release lock
-     unlockBuffer() ;
+      CTimer::get("getBufferFromClient_locked").suspend() ;   
+      unlockBuffer() ;
 
       ok=true ;
@@ -283 +243 @@
     else
     {
-      //MPI_Win_lock(MPI_LOCK_EXCLUSIVE,windowsRank_,0,windows_[currentWindows]) ;
-      //control=0 ;
-      //MPI_Put(&control, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows],2*sizeof(size_t)), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
+      count=0 ;
  
  // release lock
+      CTimer::get("getBufferFromClient_locked").suspend() ; 
       unlockBuffer() ;
     }
-
+    CTimer::get("getBufferFromClient").suspend() ;   
     if (ok) return true ;
 
@@ -299 +258 @@
   {
     if (!hasWindows) return ;
-
-    long long int lock=1 ;
-    long long int zero=0, one=1 ;
-//    control=1 ;
     MPI_Win_lock(MPI_LOCK_EXCLUSIVE,windowsRank_,0,windows_[currentWindows]) ;
-    while(lock!=0)
-    {
-      MPI_Compare_and_swap(&one, &zero, &lock, MPI_LONG_LONG_INT, windowsRank_, MPI_Aint_add(winAddress_[currentWindows],2*sizeof(size_t)),
-                           windows_[currentWindows]) ;
-      MPI_Win_flush(windowsRank_, windows_[currentWindows]) ;
-    }
   }
 
@@ -315 +264 @@
   {
     if (!hasWindows) return ;
-    long long int lock=1 ;
-    long long int zero=0, one=1 ;
-    
-    MPI_Compare_and_swap(&zero, &one, &lock, MPI_LONG_LONG_INT, windowsRank_, MPI_Aint_add(winAddress_[currentWindows],2*sizeof(size_t)),
-                          windows_[currentWindows]) ;
-    MPI_Win_flush(windowsRank_, windows_[currentWindows]) ; 
     MPI_Win_unlock(windowsRank_,windows_[currentWindows]) ;
   }
@@ -327 +270 @@
   {
     if (!hasWindows) return ;
-    size_t finalize=1 ;
+    size_t notify=notifyFinalize_ ;
     lockBuffer(); 
 // lock is acquired
-    MPI_Put(&finalize, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows],3*sizeof(size_t)), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
+    MPI_Put(&notify, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows], notifyOffset_*sizeof(size_t)), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
     unlockBuffer() ;
   }
+
+  void CServerBuffer::notifyBufferResizing(void)
+  {
+    resizingBuffer_=true ;
+    if (!hasWindows) return ;
+    size_t notify=notifyResizeBuffer_ ;
+    lockBuffer(); 
+// lock is acquired
+    MPI_Put(&notify, 1, MPI_LONG_LONG_INT, windowsRank_ , MPI_Aint_add(winAddress_[currentWindows], notifyOffset_*sizeof(size_t)), 1, MPI_LONG_LONG_INT,windows_[currentWindows]) ;
+    unlockBuffer() ;
+  }
 }

XIOS/dev/dev_ym/XIOS_COUPLING/src/buffer_server.hpp

@@ -2 +2 @@
 #define __BUFFER_SERVER_HPP__
 
+#include "buffer_cs_base.hpp"
 #include "xios_spl.hpp"
 #include "buffer.hpp"
@@ -9 +10 @@
 namespace xios
 {
-  class CServerBuffer
+  class CServerBuffer : public CBufferClientServerBase
   {
     public:
@@ -26 +27 @@
       void unlockBuffer(void) ;
       void notifyClientFinalize(void) ;
-      void notifyBufferResizing(void) { resizingBuffer_=true ;}
+      void notifyBufferResizing(void) ;
     private:
       char* buffer;
@@ -40 +41 @@
       bool hasWindows ;
       int windowsRank_ ;
+      double bufferFromClientLatency_=1e-1 ;
+      double bufferFromClientTime_ = 0;
+
   };
 }

XIOS/dev/dev_ym/XIOS_COUPLING/src/context_client.cpp

@@ -50 +50 @@
       computeLeader(clientRank, clientSize, serverSize, ranksServerLeader, ranksServerNotLeader);
 
-      if (flag) 
-      {
-        MPI_Intercomm_merge(interComm_,false, &interCommMerged) ;
-        int interCommMergedRank;
-        MPI_Comm_rank(interComm_, &interCommMergedRank);
-        MPI_Comm_rank(interCommMerged, &interCommMergedRank);
-        MPI_Comm_rank(intraComm, &interCommMergedRank);
-      }
+      if (flag) MPI_Intercomm_merge(interComm_,false, &interCommMerged) ;
       
       if (!isAttachedModeEnabled())
       {  
+
+        CTimer::get("create Windows").resume() ;
+
+        // We create dummy pair of intercommunicator between clients and server
+        // Why ? Just because on openMPI, it reduce the creation time of windows otherwhise which increase quadratically
+        // We don't know the reason
+      
+        MPI_Comm commSelf ;
+        MPI_Comm_split(intraComm_,clientRank,clientRank, &commSelf) ;
+        vector<MPI_Comm> dummyComm(serverSize) ;
+        for(int rank=0; rank<serverSize; rank++) MPI_Intercomm_create(commSelf, 0, interCommMerged, clientSize+rank, 0, &dummyComm[rank]) ;
+
+        // create windows for one-sided
         windows.resize(serverSize) ;
         MPI_Comm winComm ;
@@ -67 +73 @@
           windows[rank].resize(2) ;
           MPI_Comm_split(interCommMerged, rank, clientRank, &winComm);
-          int myRank ;
-          MPI_Comm_rank(winComm,&myRank);
           MPI_Win_create_dynamic(MPI_INFO_NULL, winComm, &windows[rank][0]);
           MPI_Win_create_dynamic(MPI_INFO_NULL, winComm, &windows[rank][1]);
 //       ym : Warning : intelMPI doesn't support that communicator of windows be deallocated before the windows deallocation, crash at MPI_Win_lock
 //            Bug or not ?          
-//        MPI_Comm_free(&winComm) ;
-        }
-      }
-
-      MPI_Comm_split(intraComm_,clientRank,clientRank, &commSelf) ;
+//          MPI_Comm_free(&winComm) ;
+        }
+        
+        // free dummy intercommunicator => take times ?
+        for(int rank=0; rank<serverSize; rank++)  MPI_Comm_free(&dummyComm[rank]) ;
+        MPI_Comm_free(&commSelf) ;
+
+        CTimer::get("create Windows").resume() ;
+     }
 
       auto time=chrono::system_clock::now().time_since_epoch().count() ;
@@ -281 +289 @@
       }
 
+      double lastTimeBuffersNotFree=0. ;
+      double time ;
+      bool doUnlockBuffers ;
       CTimer::get("Blocking time").resume();
       do
       {
         areBuffersFree = true;
-        for (itBuffer = bufferList.begin(), itSize = sizeList.begin(); itBuffer != bufferList.end(); itBuffer++, itSize++)
-        {
-          areBuffersFree &= (*itBuffer)->isBufferFree(*itSize);
-        }
+        doUnlockBuffers=false ;
+        time=MPI_Wtime() ;
+        if (time-lastTimeBuffersNotFree > latency_)
+        {
+          for (itBuffer = bufferList.begin(), itSize = sizeList.begin(); itBuffer != bufferList.end(); itBuffer++, itSize++)
+          {
+            areBuffersFree &= (*itBuffer)->isBufferFree(*itSize);
+          }
+          if (!areBuffersFree)
+          {
+            lastTimeBuffersNotFree = time ;
+            doUnlockBuffers=true ;
+          }          
+        }
+        else areBuffersFree = false ;
 
         if (!areBuffersFree)
         {
-          for (itBuffer = bufferList.begin(); itBuffer != bufferList.end(); itBuffer++) (*itBuffer)->unlockBuffer();
+          if (doUnlockBuffers) for (itBuffer = bufferList.begin(); itBuffer != bufferList.end(); itBuffer++) (*itBuffer)->unlockBuffer();
           checkBuffers();
-/*          
-          context->server->listen();
-
-          if (context->serverPrimServer.size()>0)
-          {
-            for (int i = 0; i < context->serverPrimServer.size(); ++i)  context->serverPrimServer[i]->listen();
- //ym           CServer::contextEventLoop(false) ; // avoid dead-lock at finalize...
-            context->globalEventLoop() ;
-          }
-*/
-           context_->globalEventLoop() ;
+
+          context_->globalEventLoop() ;
         }
 
@@ -383 +396 @@
         }
       } 
-
    }
 

XIOS/dev/dev_ym/XIOS_COUPLING/src/context_client.hpp

@@ -94 +94 @@
       MPI_Comm intraComm; //!< Communicator of client group
 
-      MPI_Comm commSelf; //!< Communicator of the client alone. Needed to create a new communicator between 1 proc client and 1 proc server for one sided communication
-
       map<int,CClientBuffer*> buffers; //!< Buffers for connection to servers
 
@@ -126 +124 @@
       CContextServer* associatedServer_ ; //!< The server associated to the pair client/server
       bool isGrowableBuffer_ = true ;
+
+      double latency_=1e-2 ;
   };
 }

XIOS/dev/dev_ym/XIOS_COUPLING/src/context_server.cpp

@@ -46 +46 @@
     else  attachedMode=true ;
     
-    if (flag) MPI_Comm_remote_size(interComm,&commSize);
-    else  MPI_Comm_size(interComm,&commSize);
+    int clientSize ;
+    if (flag) MPI_Comm_remote_size(interComm,&clientSize);
+    else  MPI_Comm_size(interComm,&clientSize);
 
    
@@ -75 +76 @@
     if (!isAttachedModeEnabled())
     {
+      CTimer::get("create Windows").resume() ;
+
       MPI_Intercomm_merge(interComm_,true,&interCommMerged) ;
-// create windows for one sided comm
-      int interCommMergedRank;
+
+      // We create dummy pair of intercommunicator between clients and server
+      // Why ? Just because on openMPI, it reduce the creation time of windows otherwhise which increase quadratically
+      // We don't know the reason
+      MPI_Comm commSelf ;
+      MPI_Comm_split(intraComm_, intraCommRank, intraCommRank, &commSelf) ;
+      vector<MPI_Comm> dummyComm(clientSize) ;
+      for(int rank=0; rank<clientSize ; rank++) MPI_Intercomm_create(commSelf, 0, interCommMerged, rank, 0 , &dummyComm[rank]) ;
+
+      // create windows for one sided comm
       MPI_Comm winComm ;
-      MPI_Comm_rank(intraComm, &interCommMergedRank);
       windows.resize(2) ;
-      for(int rank=commSize; rank<commSize+intraCommSize; rank++)
-      {
-        if (rank==commSize+interCommMergedRank) 
+      for(int rank=clientSize; rank<clientSize+intraCommSize; rank++)
+      {
+        if (rank==clientSize+intraCommRank) 
         {
-          MPI_Comm_split(interCommMerged, interCommMergedRank, rank, &winComm);
-          int myRank ;
-          MPI_Comm_rank(winComm,&myRank);
+          MPI_Comm_split(interCommMerged, intraCommRank, rank, &winComm);
           MPI_Win_create_dynamic(MPI_INFO_NULL, winComm, &windows[0]);
-          MPI_Win_create_dynamic(MPI_INFO_NULL, winComm, &windows[1]);      
+          MPI_Win_create_dynamic(MPI_INFO_NULL, winComm, &windows[1]);   
         }
-        else MPI_Comm_split(interCommMerged, interCommMergedRank, rank, &winComm);
-//       ym : Warning : intelMPI doesn't support that communicator of windows be deallocated before the windows deallocation, crash at MPI_Win_lock
-//            Bug or not ?          
-        // MPI_Comm_free(&winComm) ;
-      }
+        else MPI_Comm_split(interCommMerged, intraCommRank, rank, &winComm);
+        //       ym : Warning : intelMPI doesn't support that communicator of windows be deallocated before the windows deallocation, crash at MPI_Win_lock
+        //            Bug or not ?          
+        //         MPI_Comm_free(&winComm) ;
+      }
+      
+      // free dummy intercommunicator
+      for(int rank=0; rank<clientSize ; rank++)  MPI_Comm_free(&dummyComm[rank]) ;
+      MPI_Comm_free(&commSelf) ;
+      CTimer::get("create Windows").suspend() ;
     }
     else 
@@ -103 +116 @@
       windows[1]=MPI_WIN_NULL ;
     }
-
-
     
-    MPI_Comm_split(intraComm_,intraCommRank,intraCommRank, &commSelf) ;
     itLastTimeLine=lastTimeLine.begin() ;
 
@@ -138 +148 @@
   bool CContextServer::eventLoop(bool enableEventsProcessing /*= true*/)
   {
+    CTimer::get("listen request").resume();
     listen();
+    CTimer::get("listen request").suspend();
+    CTimer::get("check pending request").resume();
     checkPendingRequest();
+    checkPendingProbe() ;
+    CTimer::get("check pending request").suspend();
+    CTimer::get("check event process").resume();
     if (enableEventsProcessing)  processEvents();
+    CTimer::get("check event process").suspend();
     return finished;
   }
-
+/*
   void CContextServer::listen(void)
   {
@@ -221 +238 @@
     }
   }
+*/
+
+ void CContextServer::listen(void)
+  {
+    int rank;
+    int flag;
+    int count;
+    char * addr;
+    MPI_Status status;
+    MPI_Message message ;
+    map<int,CServerBuffer*>::iterator it;
+    bool okLoop;
+
+    traceOff();
+    MPI_Improbe(MPI_ANY_SOURCE, 20,interComm,&flag,&message, &status);
+    traceOn();
+    if (flag==true) listenPendingRequest(message, status) ;
+  }
+
+  bool CContextServer::listenPendingRequest( MPI_Message &message, MPI_Status& status)
+  {
+    int count;
+    char * addr;
+    map<int,CServerBuffer*>::iterator it;
+    int rank=status.MPI_SOURCE ;
+
+    it=buffers.find(rank);
+    if (it==buffers.end()) // Receive the buffer size and allocate the buffer
+    {
+       MPI_Aint recvBuff[4] ;
+       MPI_Mrecv(recvBuff, 4, MPI_AINT,  &message, &status);
+       remoteHashId_ = recvBuff[0] ;
+       StdSize buffSize = recvBuff[1];
+       vector<MPI_Aint> winAdress(2) ;
+       winAdress[0]=recvBuff[2] ; winAdress[1]=recvBuff[3] ;
+       mapBufferSize_.insert(std::make_pair(rank, buffSize));
+       it=(buffers.insert(pair<int,CServerBuffer*>(rank,new CServerBuffer(windows, winAdress, rank, buffSize)))).first;
+       lastTimeLine[rank]=0 ;
+       itLastTimeLine=lastTimeLine.begin() ;
+       return true;
+    }
+    else
+    {
+        std::pair<MPI_Message,MPI_Status> mypair(message,status) ;
+        pendingProbe[rank].push_back(mypair) ;
+        return false;
+    }
+  }
+
+  void CContextServer::checkPendingProbe(void)
+  {
+    
+    list<int> recvProbe ;
+    list<int>::iterator itRecv ;
+    map<int, list<std::pair<MPI_Message,MPI_Status> > >::iterator itProbe;
+
+    for(itProbe=pendingProbe.begin();itProbe!=pendingProbe.end();itProbe++)
+    {
+      int rank=itProbe->first ;
+      if (pendingRequest.count(rank)==0)
+      {
+        MPI_Message& message = itProbe->second.front().first ;
+        MPI_Status& status = itProbe->second.front().second ;
+        int count ;
+        MPI_Get_count(&status,MPI_CHAR,&count);
+        map<int,CServerBuffer*>::iterator it = buffers.find(rank);
+        if (it->second->isBufferFree(count))
+        {
+          char * addr;
+          addr=(char*)it->second->getBuffer(count);
+          MPI_Imrecv(addr,count,MPI_CHAR, &message, &pendingRequest[rank]);
+          bufferRequest[rank]=addr;
+          recvProbe.push_back(rank) ;
+          itProbe->second.pop_front() ;
+        }
+      }
+    }
+
+    for(itRecv=recvProbe.begin(); itRecv!=recvProbe.end(); itRecv++) if (pendingProbe[*itRecv].empty()) pendingProbe.erase(*itRecv) ;
+  }
 
 
@@ -232 +329 @@
     int count;
     MPI_Status status;
+   
+    if (!pendingRequest.empty()) CTimer::get("receiving requests").resume();
+    else CTimer::get("receiving requests").suspend();
 
     for(it=pendingRequest.begin();it!=pendingRequest.end();it++)
@@ -257 +357 @@
   void CContextServer::getBufferFromClient(size_t timeLine)
   {
+    CTimer::get("CContextServer::getBufferFromClient").resume() ;
     if (!isAttachedModeEnabled()) // one sided desactivated in attached mode
     {  
@@ -267 +368 @@
       {
         rank=itLastTimeLine->first ;
-        if (itLastTimeLine->second < timeLine &&  pendingRequest.count(rank)==0)
+        if (itLastTimeLine->second < timeLine &&  pendingRequest.count(rank)==0 && buffers[rank]->isBufferEmpty())
         {
-          if (buffers[rank]->getBufferFromClient(timeLine, buffer, count))
-          {
-            processRequest(rank, buffer, count);
-            break ;
-          }
+          if (buffers[rank]->getBufferFromClient(timeLine, buffer, count)) processRequest(rank, buffer, count);
+          if (count >= 0) break ;
         }
       }
     }
+    CTimer::get("CContextServer::getBufferFromClient").suspend() ;
   }
          
@@ -388 +487 @@
         }
       }
-      else getBufferFromClient(currentTimeLine) ;
+      else if (pendingRequest.empty()) getBufferFromClient(currentTimeLine) ;
     }
     else if (pureOneSided) getBufferFromClient(currentTimeLine) ; // if pure one sided check buffer even if no event recorded at current time line
@@ -441 +540 @@
 //      releaseBuffers() ;
       notifyClientsFinalize() ;
+      CTimer::get("receiving requests").suspend();
       context->finalize();
 
@@ -446 +546 @@
       MPI_Win_free(&windows[0]) ;
       MPI_Win_free(&windows[1]) ;
-     
+
       std::map<int, StdSize>::const_iterator itbMap = mapBufferSize_.begin(),
                            iteMap = mapBufferSize_.end(), itMap;

XIOS/dev/dev_ym/XIOS_COUPLING/src/context_server.hpp

@@ -19 +19 @@
     bool eventLoop(bool enableEventsProcessing = true);
     void listen(void) ;
-    bool listenPendingRequest(MPI_Status& status) ;
+//    bool listenPendingRequest(MPI_Status& status) ;
+    bool listenPendingRequest(MPI_Message &message, MPI_Status& status) ;
+    void checkPendingProbe(void) ;
     void checkPendingRequest(void) ;
     void getBufferFromClient(size_t timeLine) ;
@@ -42 +44 @@
     MPI_Comm interCommMerged; //!< Communicator of the client group + server group (intraCommunicator) needed for one sided communication.
 
-    MPI_Comm commSelf; //!< Communicator of the server alone. Needed to create a new communicator between 1 proc client and 1 proc server for one sided communication
-
     map<int,CServerBuffer*> buffers ;
     map<int,size_t> lastTimeLine ; //!< last event time line for a processed request
     map<int,size_t>::iterator itLastTimeLine ; //!< iterator on lastTimeLine
+    map<int, list<std::pair<MPI_Message,MPI_Status> > > pendingProbe;
     map<int,MPI_Request> pendingRequest ;
     map<int,char*> bufferRequest ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/distribution/distribution_type.hpp

@@ -4 +4 @@
 namespace xios
 {
-  enum class EDistributionType  { NONE=0, ROOT, BANDS, BLOCKS} ;
+  enum class EDistributionType  { NONE=0, ROOT, BANDS, BLOCKS, COLUMNS} ;
 }
 

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/contexts_manager.cpp

@@ -7 +7 @@
 #include "servers_ressource.hpp"
 #include "server.hpp"
+#include "timer.hpp"
 #include <functional>
 
@@ -48 +49 @@
     int serviceLeader ;
     auto servicesManager = CXios::getServicesManager() ;
-    
+   
     bool ok=servicesManager->getServiceLeader(poolId, serviceId, partitionId, serviceLeader) ;
 
+    info(40)<<"CContextsManager::createServerContext : waiting for service leader ;  serviceId : "<<serviceId<<endl ;
     if (wait)
     {
@@ -64 +66 @@
       notifyType_=NOTIFY_CREATE_CONTEXT ;
       notifyCreateContext_=make_tuple(poolId, serviceId, partitionId, contextId) ;
+      info(40)<<"CContextsManager::createServerContext : notification create_context to service leader "<<serviceLeader<<", serviceId : "<<serviceId<<", contextId "<<contextId<<endl ;
       sendNotification(serviceLeader) ;
       return true ;
@@ -80 +83 @@
     
     int type ;
+    info(40)<<"CContextsManager::createServerContextIntercomm : waiting for context leader ;  contextId : "<<contextId<<endl ;
     ok=CXios::getServicesManager()->getServiceType(poolId,serviceId, 0, type) ;
     if (ok) ok=getContextLeader(getServerContextName(poolId, serviceId, partitionId, type, contextId), contextLeader) ;
@@ -96 +100 @@
       notifyType_=NOTIFY_CREATE_INTERCOMM ;
       notifyCreateIntercomm_=make_tuple(poolId, serviceId, partitionId, contextId, remoteLeader, sourceContext) ;
+      info(40)<<"CContextsManager::createServerContextIntercomm : notification create_intercomm to context leader : "<<contextLeader<<", contextId :"<<contextId<<endl ;
       sendNotification(contextLeader) ;
       return true ;
@@ -149 +154 @@
   void CContextsManager::eventLoop(void)
   {
-    checkNotifications() ;
+    CTimer::get("CContextsManager::eventLoop").resume();
+    double time=MPI_Wtime() ;
+    if (time-lastEventLoop_ > eventLoopLatency_) 
+    {
+      checkNotifications() ;
+      lastEventLoop_=time ;
+    }
+    CTimer::get("CContextsManager::eventLoop").suspend();
   }
   
@@ -166 +178 @@
   void CContextsManager::createServerContext(void)
   {
+    info(40)<<"CContextsManager::createServerContext : receive create server context notification"<<endl ;
     auto arg=notifyCreateContext_ ;
     CXios::getPoolRessource()->getService(get<1>(arg), get<2>(arg))
@@ -174 +187 @@
   void CContextsManager::createServerContextIntercomm(void)
   {
+    info(40)<<"CContextsManager::createServerContext : receive create intercomm context notification"<<endl ;
     auto arg=notifyCreateIntercomm_ ;
     CXios::getPoolRessource()->getService(get<1>(arg), get<2>(arg))
@@ -194 +208 @@
   void CContextsManager::registerContext(const string& fullContextId, const SRegisterContextInfo& contextInfo)
   {
-    winContexts_->lockWindow(managerGlobalLeader_,0) ;
-    winContexts_->updateFromWindow(managerGlobalLeader_, this, &CContextsManager::contextsDumpIn) ;
+    winContexts_->lockWindowExclusive(managerGlobalLeader_) ;
+    winContexts_->updateFromLockedWindow(managerGlobalLeader_, this, &CContextsManager::contextsDumpIn) ;
+    winContexts_->flushWindow(managerGlobalLeader_) ;
     contexts_[fullContextId] = contextInfo ;
-    winContexts_->updateToWindow(managerGlobalLeader_, this, &CContextsManager::contextsDumpOut) ;
-    winContexts_->unlockWindow(managerGlobalLeader_,0) ;    
+    winContexts_->updateToLockedWindow(managerGlobalLeader_, this, &CContextsManager::contextsDumpOut) ;
+    winContexts_->unlockWindow(managerGlobalLeader_) ;
   }
 
@@ -210 +225 @@
     {
 
-      winContexts_->lockWindow(managerGlobalLeader_,0) ;
-      winContexts_->updateFromWindow(managerGlobalLeader_, this, &CContextsManager::contextsDumpIn) ;
-      winContexts_->unlockWindow(managerGlobalLeader_,0) ;
+      winContexts_->lockWindowShared(managerGlobalLeader_) ;
+      winContexts_->updateFromLockedWindow(managerGlobalLeader_, this, &CContextsManager::contextsDumpIn) ;
+      winContexts_->unlockWindow(managerGlobalLeader_) ;
 
       auto it=contexts_.find(fullContextId) ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/contexts_manager.hpp

@@ -82 +82 @@
 
     int managerGlobalLeader_ ;
-    
+
+    const double eventLoopLatency_=1e-2; 
+    double lastEventLoop_=0. ;
+
     friend class CWindowManager ;
  

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/daemons_manager.cpp

@@ -41 +41 @@
   bool CDaemonsManager::eventLoop(void)
   {
+    
     CXios::getRessourcesManager()->eventLoop() ;
     CXios::getServicesManager()->eventLoop() ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/pool_ressource.cpp

@@ -6 +6 @@
 #include "type.hpp"
 #include "cxios.hpp"
+#include "timer.hpp"
 
 namespace xios
@@ -16 +17 @@
     MPI_Comm_rank(poolComm, &commRank) ;
     MPI_Comm_size(poolComm, &commSize) ;
-
-
+    info(40)<<"CPoolRessource::CPoolRessource  : creating new pool : "<<Id<<endl ;
     if (commRank==localLeader_)
     {
@@ -51 +51 @@
     occupancy_.erase(occupancy_.begin(),it) ;
     occupancy_.insert(procs_update.begin(),procs_update.end()) ;
-
+    
+    info(40)<<"CPoolRessource::createService  : notify createService to all pool members ; serviceId : "<<serviceId<<endl ;
     for(int rank=0; rank<commSize; rank++)
     {
@@ -102 +103 @@
   bool CPoolRessource::eventLoop(bool serviceOnly)
   {
-    checkCreateServiceNotification() ;
+    CTimer::get("CPoolRessource::eventLoop").resume();
+   
+    double time=MPI_Wtime() ;
+    if (time-lastEventLoop_ > eventLoopLatency_) 
+    {
+      checkCreateServiceNotification() ;
+      lastEventLoop_=time ;
+    }
+    
     for (auto it=services_.begin(); it!=services_.end() ; ++it) 
     {
@@ -112 +121 @@
       }
     }
-
+    CTimer::get("CPoolRessource::eventLoop").suspend();
     if (services_.empty() && finalizeSignal_) return true ;
     else return false ;
@@ -137 +146 @@
   void CPoolRessource::createNewService(const std::string& serviceId, int type, int size, int nbPartitions, bool in)
   {
+     
+     info(40)<<"CPoolRessource::createNewService  : receive createService notification ; serviceId : "<<serviceId<<endl ;
      MPI_Comm serviceComm, newServiceComm ;
      int commRank ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/pool_ressource.hpp

@@ -44 +44 @@
     std::string Id_ ;
     bool finalizeSignal_ ;
-
+    
+    const double eventLoopLatency_=1e-2; 
+    double lastEventLoop_=0. ;
   };
 

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/ressources_manager.cpp

@@ -2 +2 @@
 #include "server.hpp"
 #include "servers_ressource.hpp"
+#include "timer.hpp"
 
 
@@ -43 +44 @@
   void CRessourcesManager::createPool(const string& poolId, int size)
   {
+    info(40)<<"CRessourcesManager::createPool : calling createPool : "<<poolId<<"  of size"<<size<<endl ;
+    info(40)<<"send notification to leader : "<<serverLeader_<<endl ;
     winRessources_->lockWindow(managerGlobalLeader_,0) ;
     winRessources_->updateFromWindow(managerGlobalLeader_, this, &CRessourcesManager::ressourcesDumpIn) ;
@@ -49 +52 @@
     notifyType_=NOTIFY_CREATE_POOL ;
     notifyCreatePool_=make_tuple(poolId, size) ;
+    info(40)<<"CRessourcesManager::createPool : send notification creating pool to server leader "<<serverLeader_<<endl ;
     sendNotification(serverLeader_) ; 
   }
@@ -61 +65 @@
     {
       notifyType_=NOTIFY_FINALIZE ;
+      info(40)<<"CRessourcesManager::finalize : send notification finalize to server leader "<<serverLeader_<<endl ;
       sendNotification(serverLeader_) ;
     } 
@@ -107 +112 @@
   void CRessourcesManager::eventLoop(void)
   {
-    checkNotifications() ;
+    CTimer::get("CRessourcesManager::eventLoop").resume();
+    double time=MPI_Wtime() ;
+    if (time-lastEventLoop_ > eventLoopLatency_) 
+    {
+      checkNotifications() ;
+      lastEventLoop_=time ;
+    }
+
+    CTimer::get("CRessourcesManager::eventLoop").suspend();
   }
   
@@ -114 +127 @@
     int commRank ;
     MPI_Comm_rank(xiosComm_, &commRank) ;
+    CTimer::get("CRessourcesManager::checkNotifications lock").resume();
     winNotify_->lockWindow(commRank,0) ;
+    CTimer::get("CRessourcesManager::checkNotifications lock").suspend();
+    CTimer::get("CRessourcesManager::checkNotifications pop").resume();
     winNotify_->popFromWindow(commRank, this, &CRessourcesManager::notificationsDumpIn) ;
+    CTimer::get("CRessourcesManager::checkNotifications pop").suspend();
+    CTimer::get("CRessourcesManager::checkNotifications unlock").resume();
     winNotify_->unlockWindow(commRank,0) ;
+    CTimer::get("CRessourcesManager::checkNotifications unlock").suspend();
     if (notifyType_==NOTIFY_CREATE_POOL) createPool() ;
     else if (notifyType_==NOTIFY_FINALIZE) finalizeSignal() ;
@@ -123 +142 @@
   void CRessourcesManager::createPool(void)
   {
+    
     auto& arg=notifyCreatePool_ ;
     string poolId=get<0>(arg) ;
     int size=get<1>(arg) ;
+    info(40)<<"CRessourcesManager::createPool : receive create pool notification : "<< poolId<<"  of size "<<size<<endl ;
     CServer::getServersRessource()->createPool(poolId,size) ;
   } 
@@ -131 +152 @@
   void CRessourcesManager::finalizeSignal(void)
   {
+    info(40)<<"CRessourcesManager::createPool : receive finalize notification"<<endl ;
     CServer::getServersRessource()->finalize() ;
   }

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/ressources_manager.hpp

@@ -72 +72 @@
     int freeRessourcesSize_ ;
 
+    const double eventLoopLatency_=1e-2; 
+    double lastEventLoop_=0. ;
+
     friend class CWindowManager ;
   } ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/server_context.cpp

@@ -6 +6 @@
 #include "register_context_info.hpp"
 #include "services.hpp"
+#include "timer.hpp"
 
 
@@ -18 +19 @@
                                  hasNotification_(false)
   {
+   info(40)<<"CCServerContext::CServerContext  : new context creation ; contextId : "<<contextId<<endl ;
    int localRank, globalRank, commSize ;
 
@@ -58 +60 @@
                                        const MPI_Comm& intraComm, MPI_Comm& interCommClient, MPI_Comm& interCommServer, bool wait)
   {
+    info(40)<<"CServerContext::createIntercomm  : context intercomm creation ; contextId : "<<contextId<<endl ;
     int intraCommRank ;
     MPI_Comm_rank(intraComm, &intraCommRank) ;
@@ -145 +148 @@
      int commSize ;
      MPI_Comm_size(contextComm_,&commSize) ;
+     info(40)<<"CServerContext::createIntercomm  : notify createContextIntercomm to all context members ; sourceContext : "<<sourceContext<<endl ;
+    
      for(int rank=0; rank<commSize; rank++)
      {
@@ -191 +196 @@
     if (!hasNotification_)
     {
-      int commRank ;
-      MPI_Comm_rank(contextComm_, &commRank) ;
-      winNotify_->lockWindow(commRank,0) ;
-      winNotify_->popFromWindow(commRank, this, &CServerContext::notificationsDumpIn) ;
-      winNotify_->unlockWindow(commRank,0) ;
+      double time=MPI_Wtime() ;
+      if (time-lastEventLoop_ > eventLoopLatency_) 
+      {
+        int commRank ;
+        MPI_Comm_rank(contextComm_, &commRank) ;
+        winNotify_->lockWindow(commRank,0) ;
+        winNotify_->popFromWindow(commRank, this, &CServerContext::notificationsDumpIn) ;
+        winNotify_->unlockWindow(commRank,0) ;
       
-      if (notifyInType_!= NOTIFY_NOTHING)
-      {
-        hasNotification_=true ;
-        auto eventScheduler=parentService_->getEventScheduler() ;
-        std::hash<string> hashString ;
-        size_t hashId = hashString(name_) ;
-        size_t currentTimeLine=0 ;
-        eventScheduler->registerEvent(currentTimeLine,hashId); 
+        if (notifyInType_!= NOTIFY_NOTHING)
+        {
+          hasNotification_=true ;
+          auto eventScheduler=parentService_->getEventScheduler() ;
+          std::hash<string> hashString ;
+          size_t hashId = hashString(name_) ;
+          size_t currentTimeLine=0 ;
+          eventScheduler->registerEvent(currentTimeLine,hashId); 
+        }
+        lastEventLoop_=time ;
       }
     }
@@ -225 +235 @@
   bool CServerContext::eventLoop(bool serviceOnly)
   {
+    CTimer::get("CServerContext::eventLoop").resume();
     bool finished=false ;
-    if (winNotify_!=nullptr) checkNotifications() ;
+    
+//    double time=MPI_Wtime() ;
+//    if (time-lastEventLoop_ > eventLoopLatency_) 
+//    {
+      if (winNotify_!=nullptr) checkNotifications() ;
+//      lastEventLoop_=time ;
+//    }
+
+
     if (!serviceOnly && context_!=nullptr)  
     {
@@ -235 +254 @@
       }
     }
-
+    CTimer::get("CServerContext::eventLoop").suspend();
     if (context_==nullptr && finalizeSignal_) finished=true ;
     return finished ;
@@ -242 +261 @@
   void CServerContext::createIntercomm(void)
   {
+    info(40)<<"CServerContext::createIntercomm  : received createIntercomm notification"<<endl ;
+
      MPI_Comm interCommServer, interCommClient ;
      auto& arg=notifyInCreateIntercomm_ ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/server_context.hpp

@@ -61 +61 @@
     bool isAttachedMode_ ;
 
+    const double eventLoopLatency_=1e-2; 
+    double lastEventLoop_=0. ;
+
     friend class CWindowManager ;
   } ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/servers_ressource.cpp

@@ -5 +5 @@
 #include "cxios.hpp"
 #include "mpi.hpp"
+#include "timer.hpp"
 #include <vector>
 #include <string>
@@ -115 +116 @@
   bool CServersRessource::eventLoop(bool serviceOnly)
   {
-    checkNotifications() ;
+    CTimer::get("CServersRessource::eventLoop").resume();
+    double time=MPI_Wtime() ;
+    if (time-lastEventLoop_ > eventLoopLatency_) 
+    {
+      checkNotifications() ;
+      lastEventLoop_=time ;
+    }
+
     if (poolRessource_!=nullptr) 
     {
@@ -124 +132 @@
       } 
     }
-
+    CTimer::get("CServersRessource::eventLoop").suspend();
     if (poolRessource_==nullptr && finalizeSignal_) return true ;
     else return false ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/servers_ressource.hpp

@@ -50 +50 @@
     bool finalizeSignal_ ;
 
+    const double eventLoopLatency_=1e-2; 
+    double lastEventLoop_=0. ;
+
     friend class CWindowManager ;
   } ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/services.cpp

@@ -5 +5 @@
 #include "server_context.hpp"
 #include "event_scheduler.hpp"
+#include "timer.hpp"
 
 namespace xios
@@ -14 +15 @@
 
   {
+    info(40)<<"CService::CService  : new service created ; serviceId : "<<serviceId<<endl ;
+   
     int localRank, globalRank, commSize ;
 
@@ -44 +47 @@
     int commSize ;
     MPI_Comm_size(serviceComm_, &commSize) ;
-    
+    info(40)<<"CService::createContext  : notify CreateContext to all services members ; serviceId : "<<serviceId<<" ; contextId : "<<contextId<<endl ;
+
     for(int rank=0; rank<commSize; rank++) 
     {
@@ -51 +55 @@
       sendNotification(rank) ;
     }
+    info(40)<<"CService::createContext  : notify CreateContext to all services members : DONE "<<endl ;
   }
 /*
@@ -99 +104 @@
   {
     //checkCreateContextNotification() ;
-    checkNotifications() ;
+    CTimer::get("CService::eventLoop").resume();
+    
+//    double time=MPI_Wtime() ;
+//    if (time-lastEventLoop_ > eventLoopLatency_) 
+//    {
+      checkNotifications() ;
+//      lastEventLoop_=time ;
+//    }
+
 
     eventScheduler_->checkEvent() ;
@@ -111 +124 @@
       } ;
     }
-
+    CTimer::get("CService::eventLoop").suspend();
     if (contexts_.empty() && finalizeSignal_) return true ;
     else return false ;
@@ -144 +157 @@
       if (notifyInType_==NOTIFY_CREATE_CONTEXT)
       {
-        info(10)<<"NotifyDumpOut"<<endl ;
         auto& arg=notifyInCreateContext_ ;
         buffer >> std::get<0>(arg)>> std::get<1>(arg) >> std::get<2>(arg)>> std::get<3>(arg);
@@ -158 +170 @@
     if (!hasNotification_)
     {
-      int commRank ;
-      MPI_Comm_rank(serviceComm_, &commRank) ;
-      winNotify_->lockWindow(commRank,0) ;
-      winNotify_->popFromWindow(commRank, this, &CService::notificationsDumpIn) ;
-      winNotify_->unlockWindow(commRank,0) ;
+      double time=MPI_Wtime() ;
+      if (time-lastEventLoop_ > eventLoopLatency_) 
+      {
+        int commRank ;
+        MPI_Comm_rank(serviceComm_, &commRank) ;
+        winNotify_->lockWindow(commRank,0) ;
+        winNotify_->popFromWindow(commRank, this, &CService::notificationsDumpIn) ;
+        winNotify_->unlockWindow(commRank,0) ;
       
-      if (notifyInType_!= NOTIFY_NOTHING)
-      {
-        hasNotification_=true ;
-        std::hash<string> hashString ;
-        size_t hashId = hashString(name_) ;
-        size_t currentTimeLine=0 ;
-        eventScheduler_->registerEvent(currentTimeLine,hashId); 
+        if (notifyInType_!= NOTIFY_NOTHING)
+        {
+          hasNotification_=true ;
+          std::hash<string> hashString ;
+          size_t hashId = hashString(name_) ;
+          size_t currentTimeLine=0 ;
+          info(40)<<"CService::checkNotifications(void) : receive notification => event scheduler"<<endl ;
+          eventScheduler_->registerEvent(currentTimeLine,hashId); 
+        }
+        lastEventLoop_=time ;
       }
     }
@@ -179 +197 @@
       size_t hashId = hashString(name_) ;
       size_t currentTimeLine=0 ;
+      info(40)<<"CService::checkNotifications(void) : receive notification => event scheduler : eventIsReceived ?"<<endl ;
       if (eventScheduler_->queryEvent(currentTimeLine,hashId))
       {
         eventScheduler_->popEvent() ;
+        info(40)<<"CService::checkNotifications(void) : receive notification => event scheduler : RECEIVED"<<endl ;
         if (notifyInType_==NOTIFY_CREATE_CONTEXT) createContext() ;
         hasNotification_=false ;
@@ -190 +210 @@
 
 
-
+//ym not use any more
   void CService::checkCreateContextNotification(void)
   {
@@ -210 +230 @@
   void CService::createContext(void)
    {
+     info(40)<<"CService::createContext(void)  : receive createContext notification"<<endl ;
      auto& arg=notifyInCreateContext_ ;
      string poolId = get<0>(arg) ;
@@ -218 +239 @@
    }
 
-   //to remove
+   //to remove, not used anymore
    void CService::createNewContext(const std::string& poolId, const std::string& serviceId, const int& partitionId, const std::string& contextId)
    {

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/services.hpp

@@ -71 +71 @@
     int nbPartitions_ ;
 
+    const double eventLoopLatency_=1e-2; 
+    double lastEventLoop_=0. ;
+
   };
 

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/services_manager.cpp

@@ -7 +7 @@
 #include "server.hpp"
 #include "servers_ressource.hpp"
+#include "timer.hpp"
 
 namespace xios
@@ -55 +56 @@
     int poolSize ;
     
+    info(40)<<"CServicesManager : waiting for pool info : "<<poolId<<endl ; ;
     bool ok=CXios::getRessourcesManager()->getPoolInfo(poolId, poolSize, leader) ;
     if (wait)
@@ -67 +69 @@
     if (ok) 
     {
+      info(40)<<"CServicesManager : create service notification to leader "<<leader<<", serviceId : "<<serviceId<<", size : "<<size<<endl ;
       createServicesNotify(leader, serviceId, type, size, nbPartitions) ;
       return true ;
@@ -94 +97 @@
     {
       auto info = notifications_.front() ;
+      xios::info(40)<<"CServicesManager : receive create service notification : "<<get<0>(info)<<endl ;
       CServer::getServersRessource()->getPoolRessource()->createService(get<0>(info), get<1>(info), get<2>(info), get<3>(info)) ;
       notifications_.pop_front() ;
@@ -104 +108 @@
   void CServicesManager::eventLoop(void)
   {
-    checkCreateServicesNotification() ;
+    CTimer::get("CServicesManager::eventLoop").resume();
+    double time=MPI_Wtime() ;
+    if (time-lastEventLoop_ > eventLoopLatency_) 
+    {
+      checkCreateServicesNotification() ;
+      lastEventLoop_=time ;
+    }
+    CTimer::get("CServicesManager::eventLoop").suspend();
   }
 
@@ -176 +187 @@
   {
     
+    info(40)<<"CServicesManager : registering service, poolId : "<<poolId<<", serviceId : "<<serviceId<<endl ; ;
+
+    winServices_->lockWindowExclusive(managerGlobalLeader_) ;
+    winServices_->updateFromLockedWindow(managerGlobalLeader_, this, &CServicesManager::servicesDumpIn) ;
+    winServices_->flushWindow(managerGlobalLeader_) ;
+    services_[std::tuple<std::string, std::string,int>(poolId,serviceId,partitionId)]=std::make_tuple(type,size,nbPartitions,leader) ;
+    winServices_->updateToLockedWindow(managerGlobalLeader_, this, &CServicesManager::servicesDumpOut) ;
+    winServices_->unlockWindow(managerGlobalLeader_) ;
+
+/*
     winServices_->lockWindow(managerGlobalLeader_,0) ;
     winServices_->updateFromWindow(managerGlobalLeader_, this, &CServicesManager::servicesDumpIn) ;
     services_[std::tuple<std::string, std::string,int>(poolId,serviceId,partitionId)]=std::make_tuple(type,size,nbPartitions,leader) ;
     winServices_->updateToWindow(managerGlobalLeader_, this, &CServicesManager::servicesDumpOut) ;
-    winServices_->unlockWindow(managerGlobalLeader_,0) ;
+    winServices_->unlockWindow(managerGlobalLeader_,0) ;*/
   }
 
@@ -186 +207 @@
                                         int& size, int& nbPartitions, int& leader)
   {
+    
+    winServices_->lockWindowShared(managerGlobalLeader_) ;
+    winServices_->updateFromLockedWindow(managerGlobalLeader_, this, &CServicesManager::servicesDumpIn) ;
+    winServices_->unlockWindow(managerGlobalLeader_) ;
+/*
     winServices_->lockWindow(managerGlobalLeader_,0) ;
     winServices_->updateFromWindow(managerGlobalLeader_, this, &CServicesManager::servicesDumpIn) ;
-    winServices_->unlockWindow(managerGlobalLeader_,0) ;
+    winServices_->unlockWindow(managerGlobalLeader_,0) ;*/
 
     auto it=services_.find(std::tuple<std::string,std::string,int>(poolId,serviceId,partitionId)) ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/services_manager.hpp

@@ -59 +59 @@
 
     int managerGlobalLeader_ ;
+
+    const double eventLoopLatency_=1e-2; 
+    double lastEventLoop_=0. ;
     
     friend class CWindowManager ;

XIOS/dev/dev_ym/XIOS_COUPLING/src/manager/window_manager.hpp

@@ -25 +25 @@
     MPI_Win window_ ;
     void * winBuffer_ ;
+    map<int,double> lastTimeLock_ ;
+    const double latency_=0e-2 ; 
 
     public :
@@ -46 +48 @@
     {
       int lock=state ;
-          
+      double time ;
+      auto it=lastTimeLock_.find(rank) ;
+      if (it == lastTimeLock_.end()) 
+      { 
+        lastTimeLock_[rank] = 0. ; 
+        it=lastTimeLock_.find(rank) ;
+      }
+      double& lastTime = it->second ;
+
       do 
       {
+        time=MPI_Wtime() ;
+        while(time-lastTime < latency_) time=MPI_Wtime() ;
         MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, window_) ;
         MPI_Compare_and_swap(&WINDOWS_LOCKED, &state, &lock, MPI_INT, rank, OFFSET_LOCK, window_) ;
         MPI_Win_unlock(rank, window_) ;
+        lastTime=MPI_Wtime() ;
       } while (lock!=state) ;
-
-      
-    }
-
+      
+      
+    }
+
+    void lockWindowExclusive(int rank, int state )
+    {
+      int lock=state ;
+      double time ;
+      auto it=lastTimeLock_.find(rank) ;
+      if (it == lastTimeLock_.end()) 
+      { 
+        lastTimeLock_[rank] = 0. ; 
+        it=lastTimeLock_.find(rank) ;
+      }
+      double& lastTime = it->second ;
+
+      do 
+      {
+        time=MPI_Wtime() ;
+        while(time-lastTime < latency_) time=MPI_Wtime() ;
+        MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, window_) ;
+        MPI_Compare_and_swap(&WINDOWS_LOCKED, &state, &lock, MPI_INT, rank, OFFSET_LOCK, window_) ;
+        MPI_Win_unlock(rank, window_) ;
+        lastTime=MPI_Wtime() ;
+      } while (lock!=state) ;
+    }
+
+    void lockWindowExclusive(int rank)
+    {
+      MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, window_) ;
+    }
+
+    void lockWindowShared(int rank)
+    {
+      MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, window_) ;
+    }
+
+    void unlockWindow(int rank)
+    {
+      MPI_Win_unlock(rank, window_) ;
+    }
+
+    void flushWindow(int rank)
+    {
+      MPI_Win_flush(rank, window_) ;
+    }
 
     void unlockWindow(int rank, int state )
@@ -77 +132 @@
       MPI_Win_unlock(rank, window_) ;
     }
-    
+
+    template< class T >
+    void updateToLockedWindow(int rank, T* object, void (T::*dumpOut)(CBufferOut&) )
+    {
+      CBufferOut buffer ;
+      (object->*dumpOut)(buffer) ;
+      size_t size=buffer.count() ;
+//      MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, window_) ;
+      MPI_Put(&size, SIZE_BUFFER_SIZE, MPI_CHAR, rank, OFFSET_BUFFER_SIZE, SIZE_BUFFER_SIZE, MPI_CHAR, window_) ;
+      MPI_Put(buffer.start(), size, MPI_CHAR, rank,OFFSET_BUFFER, size, MPI_CHAR, window_) ;
+//      MPI_Win_unlock(rank, window_) ;
+    }
+
     template< typename T >
     void updateFromWindow(int rank, T* object, void (T::*dumpIn)(CBufferIn&) ) 
@@ -90 +157 @@
       (object->*dumpIn)(buffer) ;
     }
+
+    template< typename T >
+    void updateFromLockedWindow(int rank, T* object, void (T::*dumpIn)(CBufferIn&) ) 
+    {
+      size_t size ;
+//      MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, window_) ;
+      MPI_Get(&size, SIZE_BUFFER_SIZE, MPI_CHAR, rank, OFFSET_BUFFER_SIZE, SIZE_BUFFER_SIZE, MPI_CHAR, window_) ;
+      MPI_Win_flush(rank,window_) ;
+      CBufferIn buffer(size) ;
+      MPI_Get(buffer.start(), size, MPI_CHAR, rank,OFFSET_BUFFER, size, MPI_CHAR, window_) ;
+//      MPI_Win_unlock(rank, window_) ;
+      MPI_Win_flush(rank, window_) ;
+      (object->*dumpIn)(buffer) ;
+    }
+
 
     template< class T >