Changeset 1522

Timestamp:

06/05/18 19:14:40 (5 years ago)

Author:

yushan

Message:

Test_client 6*8 clients 2 servers OK. TO DO : intercomm->intracomm, comm_free_intercomm

Location:

XIOS/dev/branch_openmp/extern/ep_dev

Files:

• ep_intercomm_kernel.cpp (modified) (1 diff)
• ep_lib.hpp (modified) (1 diff)
• ep_lib_mpi.hpp (modified) (1 diff)
• ep_merge.cpp (modified) (2 diffs)
• ep_message.cpp (modified) (5 diffs)
• ep_probe.cpp (modified) (2 diffs)
• ep_recv.cpp (modified) (2 diffs)
• ep_test.cpp (modified) (4 diffs)
• ep_wait.cpp (modified) (5 diffs)
• main.cpp (modified) (4 diffs)

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_kernel.cpp

@@ -535 +535 @@
 
       (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map = new EP_RANK_MAP;
+#ifdef _showinfo
       printf("new local_rank_map = %p\n", (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map);
+#endif
 
       for(std::map<int, std::pair<int, int> >::iterator it = local_comm->ep_rank_map->begin(); it != local_comm->ep_rank_map->end(); it++)

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.hpp

@@ -82 +82 @@
 
   int Request_Check();
+  
+  void show_EP_PendingRequests(std::list< ep_lib::MPI_Request* > * EP_PendingRequest);
 
   int test_sendrecv(MPI_Comm comm);

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_mpi.hpp

@@ -18 +18 @@
   int MPI_Irecv_mpi      (void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request);
 
-
+  int MPI_Iprobe_mpi(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status);
+  
   int MPI_Comm_rank_mpi(MPI_Comm comm, int* rank);
   int MPI_Comm_size_mpi(MPI_Comm comm, int* size);

XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp

@@ -7 +7 @@
 
 
-namespace ep_lib {
+namespace ep_lib 
+{
 
   int MPI_Intercomm_merge_unique_leader(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm)
@@ -198 +199 @@
     
     delete my_triple_list;
- 
-
-    return 0;
-
-
-
-    Debug("intercomm_merge kernel\n");
-
-    //int ep_rank_loc;
-    //int num_ep;
-
-    ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first;
-    num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second;
-
-
-
-    //int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size();
-
-
-    MPI_Barrier(inter_comm);
-
-
-    ::MPI_Comm *mpi_intracomm = new ::MPI_Comm;
-    MPI_Comm *ep_intracomm;
-
-    if(ep_rank_loc == 0)
-    {
-
-      ::MPI_Comm mpi_comm = to_mpi_comm(inter_comm->ep_comm_ptr->intercomm->mpi_inter_comm);
-
-      ::MPI_Intercomm_merge(mpi_comm, high, mpi_intracomm);
-      MPI_Info info;
-      MPI_Comm_create_endpoints(mpi_intracomm, num_ep, info, ep_intracomm);
-
-      inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge = ep_intracomm;
-
-    }
-
-    MPI_Barrier_local(inter_comm);
-
-    int inter_rank;
-    MPI_Comm_rank(inter_comm, &inter_rank);
-    
-    int my_ep_rank = high? inter_rank+remote_ep_size : inter_rank;
-    int my_ep_rank_loc;// = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first;
-    int my_num_ep_loc;// = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second;
-    int my_num_ep_total = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[0]->ep_comm_ptr->size_rank_info[1].second;
-    int my_ep_size = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[0]->ep_comm_ptr->size_rank_info[0].second;
-
-    int tmp_intra_ep_rank_loc = high?my_ep_rank_loc+my_num_ep_total-my_num_ep_loc: my_ep_rank_loc;
-
-
-    *newintracomm = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[tmp_intra_ep_rank_loc];
-
-    int newintracomm_ep_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first;
-    int newintracomm_ep_rank_loc = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first;
-    int newintracomm_mpi_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first;
-    int newintracomm_ep_size = (*newintracomm)->ep_comm_ptr->size_rank_info[0].second;
-    int newintracomm_num_ep = (*newintracomm)->ep_comm_ptr->size_rank_info[1].second;
-    int newintracomm_mpi_size = (*newintracomm)->ep_comm_ptr->size_rank_info[2].second;
-
-
-    int buf[3];
-    buf[0] = my_ep_rank;
-    buf[1] = tmp_intra_ep_rank_loc;
-    buf[2] = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first;
-
-    // printf("my_ep_rank = %d, tmp_intra_ep_rank_loc = %d, mpi_rank = %d\n", my_ep_rank, tmp_intra_ep_rank_loc, newintracomm->ep_comm_ptr->size_rank_info[2].first);
-
-    int *rankmap_buf;
-    rankmap_buf = new int [3*my_ep_size];
-
-    MPI_Allgather(buf, 3, MPI_INT, rankmap_buf, 3, MPI_INT, *newintracomm);
-
-    
-    // printf(" ID = %d : rankmap_buf = (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d), (%d %d %d)\n", newintracomm_ep_rank,
-    //                     rankmap_buf[0], rankmap_buf[1], rankmap_buf[2], rankmap_buf[3], rankmap_buf[4], rankmap_buf[5], rankmap_buf[6], rankmap_buf[7], rankmap_buf[8], rankmap_buf[9],
-    //                     rankmap_buf[10], rankmap_buf[11], rankmap_buf[12], rankmap_buf[13], rankmap_buf[14], rankmap_buf[15], rankmap_buf[16], rankmap_buf[17], rankmap_buf[18], rankmap_buf[19], 
-    //                     rankmap_buf[20], rankmap_buf[21], rankmap_buf[22], rankmap_buf[23], rankmap_buf[24], rankmap_buf[25], rankmap_buf[26], rankmap_buf[27], rankmap_buf[28], rankmap_buf[29],
-    //                     rankmap_buf[30], rankmap_buf[31], rankmap_buf[32], rankmap_buf[33], rankmap_buf[34], rankmap_buf[35], rankmap_buf[36], rankmap_buf[37], rankmap_buf[38], rankmap_buf[39],
-    //                     rankmap_buf[40], rankmap_buf[41], rankmap_buf[42], rankmap_buf[43], rankmap_buf[44], rankmap_buf[45], rankmap_buf[46], rankmap_buf[47]);
-
-
-    for(int i=0; i<newintracomm_ep_size; i++)
-    {
-      (*newintracomm)->ep_rank_map->at(rankmap_buf[3*i]).first  = rankmap_buf[3*i+1];
-      (*newintracomm)->ep_rank_map->at(rankmap_buf[3*i]).second = rankmap_buf[3*i+2];
-    }
-
-
-    (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = my_ep_rank;
-    (*newintracomm)->ep_comm_ptr->size_rank_info[1].first = tmp_intra_ep_rank_loc;
-    
-
-    delete[] rankmap_buf;
-
-    
-    return MPI_SUCCESS;
-
   }
-
-
+  
 }

XIOS/dev/branch_openmp/extern/ep_dev/ep_message.cpp

@@ -23 +23 @@
     
     if(EP_PendingRequests->size() == 0) return 0;
+    
+    show_EP_PendingRequests(EP_PendingRequests);
     
     MPI_Status status;
@@ -30 +32 @@
     std::list<MPI_Request* >::iterator it;
     
+//    show_EP_PendingRequests(EP_PendingRequests);
+    
     
     for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); it++)
     { 
+      if(*(*it) == 0)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
+      if((*(*it))->state == 2)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
       Message_Check(((*(*it))->comm));
     }
+    
+    
 
 
     for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); )
     {
+      if(*(*it) == 0)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
+      if((*(*it))->state == 2)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
       if((*(*it))->state == 0)
       {
@@ -81 +123 @@
       }
       
-      if((*(*it))->state == 2)
-      {
-        int ep_rank = ((*(*it))->comm)->ep_comm_ptr->size_rank_info[0].first;
-        memcheck("delete "<< (*(*it)) <<" : in ep_lib::Request_Check, delete (*(*it))");
-        delete (*(*it));
-        
-        EP_PendingRequests->erase(it);
-        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
-        it = EP_PendingRequests->begin();
-        continue;
-      }
-      else it++;
-    }
+      it++;
+    }
+    
+    show_EP_PendingRequests(EP_PendingRequests);
   }
   
@@ -170 +203 @@
     return MPI_SUCCESS;
   }
-
-
-  
-  
-  
 
 
@@ -241 +269 @@
     return MPI_SUCCESS;
   }
+  
+  
+  
+  void show_EP_PendingRequests(std::list< ep_lib::MPI_Request* > * EP_PendingRequest)
+  {
+#ifdef _showinfo
+    std::list<MPI_Request* >::iterator it;
+    
+    int i=0;
+    
+    for(it = EP_PendingRequests->begin(); it!=EP_PendingRequests->end(); it++)
+    { 
+      if(*(*it) == 0)
+      {
+        EP_PendingRequests->erase(it);
+            
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
+      if((*(*it))->state == 2)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+          
+      if((*(*it))->ep_src>6)
+        printf("EP_PendingRequests[%d] : ep_src = %d, ep_tag = %d, type = %d, state = %d, comm = %d\n", i, (*(*it))->ep_src, (*(*it))->ep_tag, (*(*it))->type, (*(*it))->state, to_mpi_comm(((*(*it))->comm)->mpi_comm));
+      i++;
+    }
+#endif
+
+  }
 
   

XIOS/dev/branch_openmp/extern/ep_dev/ep_probe.cpp

@@ -6 +6 @@
 namespace ep_lib
 {
+  int MPI_Iprobe_mpi(int src, int tag, MPI_Comm comm, int *flag, MPI_Status *status)
+  {
+    status->ep_src = src;
+    status->ep_tag = tag;
+    return ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, to_mpi_status_ptr(*status));
+  }
 
   int MPI_Iprobe(int src, int tag, MPI_Comm comm, int *flag, MPI_Status *status)
   {
-    *flag = false;
-
     if(!comm->is_ep)
     {
-      Debug("calling MPI_Iprobe MPI\n");
-      ::MPI_Status mpi_status;
-      ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_status);
+      Debug("MPI_Iprobe with MPI\n");
+      return MPI_Iprobe_mpi(src, tag, comm, flag, status);
+    }
+    
+    else
+    {
+      Debug("MPI_Iprobe with EP\n");
+      
+      *flag = false;
+    
+      Message_Check(comm);
 
-      status->mpi_status = &mpi_status;
-      status->ep_src = src;
-      status->ep_tag = tag;
-      return 0;
-    }
+      #pragma omp flush
 
-    Debug("calling MPI_Iprobe EP\n");
-    Message_Check(comm);
-
-    #pragma omp flush
-
-    #pragma omp critical (_query)
-    if(!comm->ep_comm_ptr->message_queue->empty())
-    {
+      #pragma omp critical (_query)
       for(Message_list::iterator it = comm->ep_comm_ptr->message_queue->begin(); it!= comm->ep_comm_ptr->message_queue->end(); ++it)
       {
@@ -39 +40 @@
         {
           Debug("find message\n");
-          *flag = true;
+          
 
-          ::MPI_Status mpi_status = *(static_cast< ::MPI_Status *>((*it)->mpi_status));
-
-          status->mpi_status = (*it)->mpi_status;
+          status->mpi_status = new ::MPI_Status(*static_cast< ::MPI_Status*>((*it)->mpi_status));
           status->ep_src = (*it)->ep_src;
           status->ep_tag = (*it)->ep_tag;
 
+          *flag = true;
           break;
         }
       }
     }
-
-    return 0;
   }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_recv.cpp

@@ -64 +64 @@
 
     EP_PendingRequests->push_back(request);  
-    int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first;
+    
     memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());    
+    
+    show_EP_PendingRequests(EP_PendingRequests);
 
 #ifdef _showinfo
@@ -115 +117 @@
     (*request)->ep_tag = (*message)->ep_tag;
     (*request)->ep_src = (*message)->ep_src;
+    
+    (*request)->state = 1;
         
     ::MPI_Imrecv(buf, count, to_mpi_type(datatype), to_mpi_message_ptr(*message), to_mpi_request_ptr(*request));                

XIOS/dev/branch_openmp/extern/ep_dev/ep_test.cpp

@@ -9 +9 @@
 #include <mpi.h>
 #include "ep_declaration.hpp"
+#include "ep_mpi.hpp"
 
 using namespace std;
@@ -18 +19 @@
   {
     Debug("MPI_Test with EP");
-
-    *flag = false;
     
-
-    if((*request)->type == 1)      // isend
+    if(*request==0) return *flag=true;
+    
+    if((*request)->type !=1 && (*request)->type !=2 && (*request)->type !=3) 
     {
-      ::MPI_Status mpi_status;
-      ::MPI_Test(static_cast< ::MPI_Request*>((*request)->mpi_request), flag, &mpi_status);
-      
-      if(*flag) 
-      {
-        status->mpi_status = new ::MPI_Status(mpi_status);
-        status->ep_src = (*request)->ep_src;
-        status->ep_tag = (*request)->ep_tag;
-        status->ep_datatype = (*request)->ep_datatype;
-        delete (*request)->mpi_request;
-        delete request;
-      }
-
-      return 0;
+      printf("MPI_Test : Error in request type\n");
+ 
+      exit(1);
     }
 
     if((*request)->type == 2)   // irecv message not probed
     {
-      Request_Check();
+      *flag = false;
+      return Request_Check();
+    }
+    
+    else //(*request)->type == 1 || (*request)->type == 3      // isend or imrecv
+    {
       
-      #pragma omp flush
-      
-    }
-
-    if((*request)->type == 3)  // imrecv
-    {
       ::MPI_Status mpi_status;
       
-      ::MPI_Test(static_cast< ::MPI_Request*>((*request)->mpi_request), flag, &mpi_status);
       
+      ::MPI_Test(to_mpi_request_ptr(*request), flag, &mpi_status);
       
-      if(*flag)
+      if(*flag) 
       {
-        status->mpi_status = new ::MPI_Status(mpi_status);
+
+        status->mpi_status=new ::MPI_Status(mpi_status);
         status->ep_src = (*request)->ep_src;
         status->ep_tag = (*request)->ep_tag;
         status->ep_datatype = (*request)->ep_datatype;
+        
+        (*request)->state = 2;
+                
+        memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Test, delete (*request)->mpi_request");
         delete (*request)->mpi_request;
-        delete request;
-        //int count;
-        //MPI_Get_count(status, request->ep_datatype, &count);
-        //check_sum_recv(request->buf, count, request->ep_datatype, request->ep_src, request->ep_tag, request->comm, 2);
-      }  
-
-      return 0;
+        delete *request;
+        *request=0;
+      }
     }
+    
+    return Request_Check();
 
   }
@@ -77 +68 @@
   {
     Debug("MPI_Testall with EP");
-
+    *flag = false;
+   
+    int rank = array_of_requests[0]->comm->ep_comm_ptr->size_rank_info[0].first;
+    
+    for(int i=0; i<count; i++)
+    {
+      if(array_of_requests[i]->type == 2)
+      {
+        *flag=false;
+        return Request_Check();
+      }
+    }
+    
     ::MPI_Request* mpi_request = new ::MPI_Request[count];
     ::MPI_Status* mpi_status = new ::MPI_Status[count];
@@ -89 +92 @@
     ::MPI_Testall(count, mpi_request, flag, mpi_status);
 
-    if(flag)
+    if(*flag)
     {
       for(int i=0; i<count; i++)
       {
-        array_of_statuses[i].mpi_status = &mpi_status[i];
+        array_of_statuses[i].mpi_status = new ::MPI_Status(mpi_status[i]);
         array_of_statuses[i].ep_src = array_of_requests[i]->ep_src;
         array_of_statuses[i].ep_tag = array_of_requests[i]->ep_tag;
         array_of_statuses[i].ep_datatype = array_of_requests[i]->ep_datatype;
+        
+        array_of_requests[i]->state = 2;
       
-        memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]->mpi_request");
+        memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Testall, array_of_requests["<<i<<"]->mpi_request");
         delete array_of_requests[i]->mpi_request;
-        delete array_of_requests[i]; 
+        delete array_of_requests[i];
+        array_of_requests[i]=0;
+        
       }
       
     }
+    
+    return Request_Check();
 
   }

XIOS/dev/branch_openmp/extern/ep_dev/ep_wait.cpp

@@ -25 +25 @@
     if((*request)->type !=1 && (*request)->type !=2 && (*request)->type !=3) 
     {
-      printf("Error in request type\n");
+      printf("MPI_Wait : Error in request type\n");
  
       exit(1);
@@ -32 +32 @@
     while((*request)->type == 2) Request_Check();
     
-    ::MPI_Wait(to_mpi_request_ptr(*request), to_mpi_status_ptr(status));
+    ::MPI_Status mpi_status;
+    
+    ::MPI_Wait(to_mpi_request_ptr(*request), &mpi_status);
 
-    
+    status->mpi_status = new ::MPI_Status(mpi_status);
     status->ep_src = (*request)->ep_src;
     status->ep_tag = (*request)->ep_tag;
@@ -42 +44 @@
 
     memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Wait, delete (*request)->mpi_request");
+
     delete (*request)->mpi_request;
-    Request_Check();
-
-    return MPI_SUCCESS;
+    delete *request;
+    *request=0;
+    
+    return Request_Check();
 
   }   /*end of mpi_wait*/
@@ -86 +90 @@
     for(int i=0; i<count; i++)
     {
-      array_of_statuses[i].mpi_status = &mpi_status[i];
+      array_of_statuses[i].mpi_status = new ::MPI_Status(mpi_status[i]);
       array_of_statuses[i].ep_src = array_of_requests[i]->ep_src;
       array_of_statuses[i].ep_tag = array_of_requests[i]->ep_tag;
       array_of_statuses[i].ep_datatype = array_of_requests[i]->ep_datatype;
+      
       array_of_requests[i]->state = 2;
+      
       memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]->mpi_request");
       delete array_of_requests[i]->mpi_request;
-      if(array_of_requests[i]->type == 1) 
-      {
-        memcheck("delete "<< array_of_requests[i] <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]");
-        delete array_of_requests[i];
-      }
+      delete array_of_requests[i];
+      array_of_requests[i] = 0;
     }
 
@@ -103 +106 @@
     delete[] mpi_status;
     
-    Request_Check();
-    
-    
-    return MPI_SUCCESS;
+    return Request_Check();
   }  /* end of mpi_waitall*/
 

XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

@@ -852 +852 @@
 
       if(rank == 0) printf("            \t Test iP2P for intercomm \t OK\n");
+      
+      
 
 
@@ -888 +890 @@
       MPI_Comm_size(inter_comm_dup_merged, &inter_comm_dup_merged_size);
 
-      printf("rank = %d, inter_comm_dup_merged_rank = %d, inter_comm_dup_merged_size = %d\n", rank, inter_comm_dup_merged_rank, inter_comm_dup_merged_size);
+      //printf("rank = %d, inter_comm_dup_merged_rank = %d, inter_comm_dup_merged_size = %d\n", rank, inter_comm_dup_merged_rank, inter_comm_dup_merged_size);
+      
+      // TEST OF TESTALL
+      
+        int merged_size, merged_rank;
+        MPI_Comm_rank(inter_comm_dup_merged, &merged_rank);
+        MPI_Comm_size(inter_comm_dup_merged, &merged_size);
+        
+        int left = merged_rank>0? merged_rank-1 : merged_size-1;
+        int right = (merged_rank+1) % merged_size;
+        
+        //printf("rank = %d, left = %d, right = %d\n", merged_rank, left, right);
+        
+        int N=10000;
+        int *left_buf = new int[N];
+        int *right_buf = new int[N];
+                
+        for(int i=0; i<N; i++)
+        {
+          right_buf[i] = merged_rank*100000+i;
+        }
+        
+        MPI_Request request[2];
+        MPI_Status status[2];
+        
+        MPI_Irecv(left_buf, N, MPI_INT, left, 0, inter_comm_dup_merged, &request[0]);
+        MPI_Isend(right_buf, N, MPI_INT, right, 0, inter_comm_dup_merged, &request[1]);
+        
+        
+        int flag;
+        MPI_Testall(2, request, &flag, status);
+      
+        printf("rank = %d, flag = %d\n", merged_rank, flag);
+        
+        while(!flag) MPI_Testall(2, request, &flag, status);
+        //if(!flag)  MPI_Waitall(2, request, status);
+        
+        int recv_count;
+        MPI_Get_count(&status[0], MPI_INT, &recv_count);
+        
+        MPI_Barrier(inter_comm_dup_merged);
+        
+        printf("rank = %d, recv_count = %d, left_buf[5-10] = %d\t%d\t%d\t%d\t%d\t%d\n", merged_rank, recv_count, left_buf[5], left_buf[6], left_buf[7], left_buf[8], left_buf[9], left_buf[10]);
+                        
+        delete[] left_buf;
+        delete[] right_buf;
+      
+      
 
       // TEST OF GATHER
@@ -961 +1010 @@
 
       // TEST OF EXSCAN
-      {
+      /*{
  
         std::vector<int>sendbuf(2, inter_comm_dup_merged_rank);
@@ -988 +1037 @@
         if(inter_comm_dup_merged_rank == 0 && exscan_result)  printf("            \t test MPI_Exscan for merged comm \t OK\n");
         if(inter_comm_dup_merged_rank == 0 && !exscan_result) printf("            \t test MPI_Exscan for merged comm \t FAILED %d\n", exscan_result);
-      }
+      }*/
 
       // TEST OF SCATTERV