Index: XIOS/dev/branch_openmp/extern/src_ep_dev/ep_intercomm_kernel.cpp
===================================================================
--- XIOS/dev/branch_openmp/extern/src_ep_dev/ep_intercomm_kernel.cpp	(revision 1368)
+++ XIOS/dev/branch_openmp/extern/src_ep_dev/ep_intercomm_kernel.cpp	(revision 1520)
@@ -14,569 +14,622 @@
     int ep_size, num_ep, mpi_size;
 
-    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
-    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
-    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
-    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
-    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
-    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
-
-    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
-                                    //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
+    ep_rank     = local_comm->ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank    = local_comm->ep_comm_ptr->size_rank_info[2].first;
+    ep_size     = local_comm->ep_comm_ptr->size_rank_info[0].second;
+    num_ep      = local_comm->ep_comm_ptr->size_rank_info[1].second;
+    mpi_size    = local_comm->ep_comm_ptr->size_rank_info[2].second;
+
+    //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    // step 1 : local leaders exchange ep_size, leader_rank_in_peer, leader_rank_in_peer_mpi, leader_rank_in_world. //
+    //          local leaders bcast results to all ep in local_comm                                                 //
+    //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+    bool is_local_leader = ep_rank==local_leader? true: false;
+
+
+    
+    int local_leader_rank_in_peer;
+    int local_leader_rank_in_peer_mpi;
+    int local_leader_rank_in_world;
+
+    int remote_ep_size;
+    int remote_leader_rank_in_peer;
+    int remote_leader_rank_in_peer_mpi;
+    int remote_leader_rank_in_world;
+
+    int send_quadruple[4];
+    int recv_quadruple[4];
+
+
+    if(is_local_leader)
+    {
+      MPI_Comm_rank(peer_comm, &local_leader_rank_in_peer);
+      ::MPI_Comm_rank(to_mpi_comm(peer_comm->mpi_comm), &local_leader_rank_in_peer_mpi);
+      ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &local_leader_rank_in_world);
+
+      send_quadruple[0] = ep_size;
+      send_quadruple[1] = local_leader_rank_in_peer;
+      send_quadruple[2] = local_leader_rank_in_peer_mpi;
+      send_quadruple[3] = local_leader_rank_in_world;
+
+      MPI_Request request;
+      MPI_Status status;
+
+
+      if(remote_leader > local_leader_rank_in_peer)
+      {
+        MPI_Isend(send_quadruple, 4, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+       
+        MPI_Irecv(recv_quadruple, 4, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+      else
+      {
+        MPI_Irecv(recv_quadruple, 4, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+          
+        MPI_Isend(send_quadruple, 4, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+
+      remote_ep_size                 = recv_quadruple[0];
+      remote_leader_rank_in_peer     = recv_quadruple[1];
+      remote_leader_rank_in_peer_mpi = recv_quadruple[2];
+      remote_leader_rank_in_world    = recv_quadruple[3];
+#ifdef _showinfo
+      printf("peer_rank = %d, packed exchange OK\n", local_leader_rank_in_peer);
+#endif
+    }
+
+    MPI_Bcast(send_quadruple, 4, MPI_INT, local_leader, local_comm);
+    MPI_Bcast(recv_quadruple, 4, MPI_INT, local_leader, local_comm);
+
+    if(!is_local_leader)
+    {
+      local_leader_rank_in_peer     = send_quadruple[1];
+      local_leader_rank_in_peer_mpi = send_quadruple[2];
+      local_leader_rank_in_world    = send_quadruple[3];
+
+      remote_ep_size                 = recv_quadruple[0];
+      remote_leader_rank_in_peer     = recv_quadruple[1];
+      remote_leader_rank_in_peer_mpi = recv_quadruple[2];
+      remote_leader_rank_in_world    = recv_quadruple[3];
+    }
+
+
+#ifdef _showinfo
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, ep_size = %d, remote_ep_size = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_size, remote_ep_size);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm); 
+#endif
+
+    ///////////////////////////////////////////////////////////////////
+    // step 2 : gather ranks in world for both local and remote comm //
+    ///////////////////////////////////////////////////////////////////
 
     int rank_in_world;
-    int rank_in_local_parent;
-
-    int rank_in_peer_mpi[2];
-
-    int local_ep_size = ep_size;
-    int remote_ep_size;
-
-
-    ::MPI_Comm local_mpi_comm = to_mpi_comm(local_comm.mpi_comm);
-
-    
-    ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD.mpi_comm), &rank_in_world);
-    ::MPI_Comm_rank(local_mpi_comm, &rank_in_local_parent);
-    
-
-    bool is_proc_master = false;
-    bool is_local_leader = false;
-    bool is_final_master = false;
-
-
-    if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
-    if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
-
-
-    int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
-
-    int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
-
-
-    std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
-
-    std::vector<int> new_rank_info[4];
-    std::vector<int> new_ep_info[2];
-
-    std::vector<int> offset;
-
-    if(is_proc_master)
-    {
-
-      size_info[0] = mpi_size;
-
-      rank_info[0].resize(size_info[0]);
-      rank_info[1].resize(size_info[0]);
-
-
-
-      ep_info[0].resize(size_info[0]);
-
-      vector<int> send_buf(6);
-      vector<int> recv_buf(3*size_info[0]);
-
-      send_buf[0] = rank_in_world;
-      send_buf[1] = rank_in_local_parent;
-      send_buf[2] = num_ep;
-
-      ::MPI_Allgather(send_buf.data(), 3, to_mpi_type(MPI_INT), recv_buf.data(), 3, to_mpi_type(MPI_INT), local_mpi_comm);
-
-      for(int i=0; i<size_info[0]; i++)
-      {
-        rank_info[0][i] = recv_buf[3*i];
-        rank_info[1][i] = recv_buf[3*i+1];
-        ep_info[0][i]   = recv_buf[3*i+2];
-      }
+    ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &rank_in_world);
+
+    int *ranks_in_world_local  = new int[ep_size];
+    int *ranks_in_world_remote = new int[remote_ep_size];
+
+    MPI_Allgather(&rank_in_world, 1, MPI_INT, ranks_in_world_local, 1, MPI_INT, local_comm);
+
+    if(is_local_leader)
+    {
+      MPI_Request request;
+      MPI_Status status;
+
+      if(remote_leader > local_leader_rank_in_peer)
+      {
+        MPI_Isend(ranks_in_world_local,  ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+       
+        MPI_Irecv(ranks_in_world_remote, remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+      else
+      {
+        MPI_Irecv(ranks_in_world_remote, remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+          
+        MPI_Isend(ranks_in_world_local,  ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+#ifdef _showinfo
+      printf("peer_rank = %d, ranks_in_world exchange OK\n", local_leader_rank_in_peer);
+#endif
+    }
+
+    MPI_Bcast(ranks_in_world_remote, remote_ep_size, MPI_INT, local_leader, local_comm);
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+    if(remote_leader == 4)
+    {
+      for(int i=0; i<ep_size; i++)
+      {
+        if(ep_rank == i)
+        {
+          printf("peer_rank = %d, ranks_in_world_local = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_local[i]);
+          }
+    
+          printf("\npeer_rank = %d, ranks_in_world_remote = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<remote_ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_remote[i]);
+          }
+          printf("\n");
+         
+        }
+        
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+      }
+    }
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    
+    if(remote_leader == 13)
+    {
+      for(int i=0; i<ep_size; i++)
+      {
+        if(ep_rank == i)
+        {
+          printf("peer_rank = %d, ranks_in_world_local = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_local[i]);
+          }
+    
+          printf("\npeer_rank = %d, ranks_in_world_remote = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<remote_ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_remote[i]);
+          }
+          printf("\n");
+         
+        }
+        
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+      }
+    }
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+    //////////////////////////////////////////////////////////////
+    // step 3 : determine the priority and ownership of each ep //
+    //////////////////////////////////////////////////////////////
+
+    bool priority = local_leader_rank_in_peer > remote_leader_rank_in_peer? true : false;
+
+
+    int ownership;
+
+    if(rank_in_world == ranks_in_world_local[local_leader]) ownership = 1;
+    else if(rank_in_world == remote_leader_rank_in_world)   ownership = 0;
+    else
+    {
+      ownership = 1;
+      for(int i=0; i<remote_ep_size; i++)
+      {
+        if(rank_in_world == ranks_in_world_remote[i])
+        {
+          ownership = priority? 1 : 0;
+          break;
+        }  
+      }
+    }
+
+#ifdef _showinfo
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, priority = %d, local_leader_rank_in_peer = %d, remote_leader_rank_in_peer = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, priority, local_leader_rank_in_peer, remote_leader_rank_in_peer);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+#endif
+
+    
+#ifdef _showinfo
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, priority = %d, ownership = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, priority, ownership);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+#endif
+
+    //////////////////////////////////////////////////////
+    // step 4 : extract local_comm and create intercomm //
+    //////////////////////////////////////////////////////
+
+    bool is_involved = is_local_leader || (!is_local_leader && ep_rank_loc == 0 && rank_in_world != local_leader_rank_in_world);
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, is_involved = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, is_involved);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+    if(is_involved)
+    {
+      ::MPI_Group local_group;
+      ::MPI_Group extracted_group;
+      ::MPI_Comm *extracted_comm = new ::MPI_Comm;
+
+
+      ::MPI_Comm_group(to_mpi_comm(local_comm->mpi_comm), &local_group);
+
+      int *ownership_list = new int[mpi_size];
+      int *mpi_rank_list = new int[mpi_size];
+
+      ::MPI_Allgather(&ownership, 1, to_mpi_type(MPI_INT), ownership_list, 1, to_mpi_type(MPI_INT), to_mpi_comm(local_comm->mpi_comm));
+      ::MPI_Allgather(&mpi_rank,  1, to_mpi_type(MPI_INT), mpi_rank_list,  1, to_mpi_type(MPI_INT), to_mpi_comm(local_comm->mpi_comm));
+
+      
+      int n=0;
+      for(int i=0; i<mpi_size; i++)
+      { 
+        n+=ownership_list[i];
+      }
+
+      int *new_mpi_rank_list = new int[n];
+      int j=0;
+      for(int i=0; i<mpi_size; i++)
+      { 
+        if(ownership_list[i] !=0)
+        {
+          new_mpi_rank_list[j++] = mpi_rank_list[i];
+        }
+      }
+
+
+      ::MPI_Group_incl(local_group, n, new_mpi_rank_list, &extracted_group);
+
+      ::MPI_Comm_create(to_mpi_comm(local_comm->mpi_comm), extracted_group, extracted_comm);
+
+      ::MPI_Comm *mpi_inter_comm = new ::MPI_Comm;
+
+      int local_leader_rank_in_extracted_comm;
 
       if(is_local_leader)
       {
-        leader_info[0] = rank_in_world;
-        leader_info[1] = remote_leader;
-
-        ::MPI_Comm_rank(to_mpi_comm(peer_comm.mpi_comm), &rank_in_peer_mpi[0]);
-
-        send_buf[0] = size_info[0];
-        send_buf[1] = local_ep_size;
-        send_buf[2] = rank_in_peer_mpi[0];
-
-        
-        
-        MPI_Request requests[2];
-        MPI_Status statuses[2];
-        
-        MPI_Isend(send_buf.data(), 3, MPI_INT, remote_leader, tag, peer_comm, &requests[0]);
-        MPI_Irecv(recv_buf.data(), 3, MPI_INT, remote_leader, tag, peer_comm, &requests[1]);
-
-
-        MPI_Waitall(2, requests, statuses);
-        
-        size_info[1] = recv_buf[0];
-        remote_ep_size = recv_buf[1];
-        rank_in_peer_mpi[1] = recv_buf[2];
-
-      }
-
-
-
-      send_buf[0] = size_info[1];
-      send_buf[1] = leader_info[0];
-      send_buf[2] = leader_info[1];
-      send_buf[3] = rank_in_peer_mpi[0];
-      send_buf[4] = rank_in_peer_mpi[1];
-
-      ::MPI_Bcast(send_buf.data(), 5, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
-
-      size_info[1] = send_buf[0];
-      leader_info[0] = send_buf[1];
-      leader_info[1] = send_buf[2];
-      rank_in_peer_mpi[0] = send_buf[3];
-      rank_in_peer_mpi[1] = send_buf[4];
-
-
-      rank_info[2].resize(size_info[1]);
-      rank_info[3].resize(size_info[1]);
-
-      ep_info[1].resize(size_info[1]);
-
-      send_buf.resize(3*size_info[0]);
-      recv_buf.resize(3*size_info[1]);
-
-      if(is_local_leader)
-      {
-        MPI_Request requests[2];
-        MPI_Status statuses[2];
-
-        std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
-        std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
-        std::copy ( ep_info[0].data(),   ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[0] );
-
-        MPI_Isend(send_buf.data(), 3*size_info[0], MPI_INT, remote_leader, tag+1, peer_comm, &requests[0]);
-        MPI_Irecv(recv_buf.data(), 3*size_info[1], MPI_INT, remote_leader, tag+1, peer_comm, &requests[1]);
-        
-        MPI_Waitall(2, requests, statuses);
-      }
-
-      ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
-
-      std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
-      std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin()  );
-      std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() );
-
-
-      offset.resize(size_info[0]);
-
-      if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
-      {
-
-        bool found = false;
-        int ep_local;
-        int ep_remote;
-        for(int i=0; i<size_info[0]; i++)
+        ::MPI_Comm_rank(*extracted_comm, &local_leader_rank_in_extracted_comm);
+      }
+
+      ::MPI_Bcast(&local_leader_rank_in_extracted_comm, 1, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, to_mpi_comm(local_comm->mpi_comm));
+
+
+      if(ownership)
+        ::MPI_Intercomm_create(*extracted_comm, local_leader_rank_in_extracted_comm, to_mpi_comm(peer_comm->mpi_comm), remote_leader_rank_in_peer_mpi, tag, mpi_inter_comm);
+
+      ////////////////////////////////////
+      // step 5 :: determine new num_ep //
+      ////////////////////////////////////
+
+      int num_ep_count=0;
+
+      for(int i=0; i<ep_size; i++)
+      {
+        if(rank_in_world == ranks_in_world_local[i])
+          num_ep_count++;
+      }
+      
+      for(int i=0; i<remote_ep_size; i++)
+      {
+        if(rank_in_world == ranks_in_world_remote[i])
+          num_ep_count++;
+      }
+
+
+      ///////////////////////////////////////////////////
+      // step 6 : create endpoints from extracted_comm //
+      ///////////////////////////////////////////////////
+
+      if(ownership)
+      {
+        MPI_Comm *ep_comm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(extracted_comm, num_ep_count, info, ep_comm);
+      
+#ifdef _showinfo
+        printf("new ep_comm->ep_comm_ptr->intercomm->mpi_inter_comm = %p\n", mpi_inter_comm);
+#endif
+
+        for(int i=0; i<num_ep_count; i++)
         {
-          int target = rank_info[0][i];
-          found = false;
-          for(int j=0; j<size_info[1]; j++)
+          ep_comm[i]->is_intercomm = true;
+          ep_comm[i]->ep_comm_ptr->comm_label = ranks_in_world_local[local_leader];
+          ep_comm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
+#ifdef _showinfo
+          printf("new ep_comm[%d]->ep_comm_ptr->intercomm = %p\n", i, ep_comm[i]->ep_comm_ptr->intercomm);
+#endif
+          ep_comm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter_comm;
+          
+        }
+
+
+        #pragma omp critical (write_to_tag_list)      
+        intercomm_list.push_back(make_pair( make_pair(tag, min(local_leader_rank_in_world, remote_leader_rank_in_world)) , make_pair(ep_comm , make_pair(num_ep_count, 0))));
+        #pragma omp flush
+#ifdef _showinfo
+        for(int i=0; i<num_ep_count; i++)
+          printf("peer_rank = %d, ep_comm = %p, ep_comm[%d] -> new_ep_rank = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_comm, i, ep_comm[i]->ep_comm_ptr->size_rank_info[0].first);
+#endif
+      }
+
+
+      delete ownership_list;
+      delete mpi_rank_list;
+      delete new_mpi_rank_list;
+
+    }
+
+    int repeated=0;
+    for(int i=0; i<remote_ep_size; i++)
+    {
+      if(rank_in_world == ranks_in_world_remote[i])
+        repeated++;
+    }
+
+    int my_turn = ownership? ep_rank_loc : ep_rank_loc+repeated;
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, ep_rank_loc = %d, ownership = %d, repeated = %d, my_turn = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_rank_loc, ownership, repeated, my_turn);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+
+    #pragma omp flush
+    #pragma omp critical (read_from_intercomm_list)
+    {
+      bool flag=true;
+      while(flag)
+      {
+        for(std::list<std::pair<std::pair<int, int> , std::pair<MPI_Comm * , std::pair<int, int> > > >::iterator iter = intercomm_list.begin(); iter!=intercomm_list.end(); iter++)
+        {
+          if(iter->first == make_pair(tag, min(local_leader_rank_in_world, remote_leader_rank_in_world)))
           {
-            if(target == rank_info[2][j])
-            {
-              found = true;
-              ep_local = ep_info[0][j];
-              ep_remote = ep_info[1][j];
-              break;
-            }
-          }
-          if(found)
-          {
-
-            if(target == leader_info[0]) // the leader is doubled in remote
-            {
-              new_rank_info[0].push_back(target);
-              new_rank_info[1].push_back(rank_info[1][i]);
-
-              new_ep_info[0].push_back(ep_local + ep_remote);
-              offset[i] = 0;
-            }
-            else
-            {
-              offset[i] = ep_local;
-            }
-          }
-          else
-          {
-            new_rank_info[0].push_back(target);
-            new_rank_info[1].push_back(rank_info[1][i]);
-
-            new_ep_info[0].push_back(ep_info[0][i]);
-
-            offset[i] = 0;
-          }
-
-        }
-      }
-
-      else // erase rank doubled with remote leader
-      {
-
-        bool found = false;
-        int ep_local;
-        int ep_remote;
-        for(int i=0; i<size_info[0]; i++)
-        {
-          int target = rank_info[0][i];
-          found = false;
-          for(int j=0; j<size_info[1]; j++)
-          {
-
-            if(target == rank_info[2][j])
-            {
-              found = true;
-              ep_local = ep_info[0][j];
-              ep_remote = ep_info[1][j];
-              break;
-            }
-          }
-          if(found)
-          {
-            if(target != leader_info[1])
-            {
-              new_rank_info[0].push_back(target);
-              new_rank_info[1].push_back(rank_info[1][i]);
-
-              new_ep_info[0].push_back(ep_local + ep_remote);
-              offset[i] = 0;
-            }
-            else // found remote leader
-            {
-              offset[i] = ep_remote;
-            }
-          }
-          else
-          {
-            new_rank_info[0].push_back(target);
-            new_rank_info[1].push_back(rank_info[1][i]);
-
-            new_ep_info[0].push_back(ep_info[0][i]);
-            offset[i] = 0;
+            *newintercomm = iter->second.first[my_turn];
+            
+            iter->second.second.second++;
+            
+            if(iter->second.second.first == iter->second.second.second)
+              intercomm_list.erase(iter);
+
+            flag = false;
+            break; 
           }
         }
       }
-
-      if(offset[mpi_rank] == 0)
-      {
-        is_final_master = true;
-      }
-
-
-      //! size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
-
-      if(is_local_leader)
-      {
-        size_info[2] = new_ep_info[0].size();
-        MPI_Request requests[2];
-        MPI_Status statuses[2];
-        MPI_Isend(&size_info[2], 1, MPI_INT, remote_leader, tag+2, peer_comm, &requests[0]);
-        MPI_Irecv(&size_info[3], 1, MPI_INT, remote_leader, tag+2, peer_comm, &requests[1]);
-         
-        MPI_Waitall(2, requests, statuses);
-      }
-
-      ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
-
-      new_rank_info[2].resize(size_info[3]);
-      new_rank_info[3].resize(size_info[3]);
-      new_ep_info[1].resize(size_info[3]);
-
-      send_buf.resize(size_info[2]);
-      recv_buf.resize(size_info[3]);
-
-      if(is_local_leader)
-      {
-        MPI_Request requests[2];
-        MPI_Status statuses[2];
-
-        std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
-        std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
-        std::copy ( new_ep_info[0].data(),   new_ep_info[0].data()   + size_info[0], send_buf.begin() + 2*size_info[2] );
-
-        MPI_Isend(send_buf.data(), 3*size_info[2], MPI_INT, remote_leader, tag+3, peer_comm, &requests[0]);
-        MPI_Irecv(recv_buf.data(), 3*size_info[3], MPI_INT, remote_leader, tag+3, peer_comm, &requests[1]);
-        
-        MPI_Waitall(2, requests, statuses);
-      }
-
-      ::MPI_Bcast(recv_buf.data(),   3*size_info[3], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
-
-      std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
-      std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin()  );
-      std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() );
-
-    }
-
-    
-
-    if(is_proc_master)
-    {
-      //! leader_info[4]: 2-> rank of local leader in new_group generated comm;
-                      // 3-> rank of remote leader in new_group generated comm;
-      ::MPI_Group local_group;
-      ::MPI_Group new_group;
-      ::MPI_Comm *new_comm = new ::MPI_Comm;
-      ::MPI_Comm *intercomm = new ::MPI_Comm;
-
-      ::MPI_Comm_group(local_mpi_comm, &local_group);
-
-      ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
-
-      ::MPI_Comm_create(local_mpi_comm, new_group, new_comm);
-
-
-
-      if(is_local_leader)
-      {
-        ::MPI_Comm_rank(*new_comm, &leader_info[2]);
-      }
-
-      ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
-
-      if(new_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm))
-      {
-
-        ::MPI_Barrier(*new_comm);
-
-        ::MPI_Intercomm_create(*new_comm, leader_info[2], to_mpi_comm(peer_comm.mpi_comm), rank_in_peer_mpi[1], tag, intercomm);
-
-        int id;
-
-        ::MPI_Comm_rank(*new_comm, &id);
-        int my_num_ep = new_ep_info[0][id];
-
-        MPI_Comm *ep_intercomm;
-        MPI_Info info;
-        MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
-
-
-        for(int i= 0; i<my_num_ep; i++)
+    }
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, test_rank = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, (*newintercomm)->ep_comm_ptr->size_rank_info[0].first);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+    //////////////////////////////////////////////////////////
+    // step 7 : create intercomm_rank_map for local leaders //
+    //////////////////////////////////////////////////////////
+    
+    int my_quadruple[4];
+
+    my_quadruple[0] = ep_rank;
+    my_quadruple[1] = (*newintercomm)->ep_comm_ptr->size_rank_info[1].first;
+    my_quadruple[2] = (*newintercomm)->ep_comm_ptr->size_rank_info[2].first;
+    my_quadruple[3] = (*newintercomm)->ep_comm_ptr->comm_label;
+
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, my_quadruple = %d %d %d %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, my_quadruple[0], my_quadruple[1], my_quadruple[2], my_quadruple[3]);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+#endif
+
+    int *local_quadruple_list;
+    int *remote_quadruple_list;
+    if(is_involved)
+    {
+      local_quadruple_list = new int[4*ep_size];
+      remote_quadruple_list = new int[4*remote_ep_size];
+
+    }
+
+    MPI_Gather(my_quadruple, 4, MPI_INT, local_quadruple_list, 4, MPI_INT, local_leader, local_comm);
+
+
+    if(is_local_leader)
+    {
+      MPI_Request request;
+      MPI_Status status;
+
+      if(remote_leader > local_leader_rank_in_peer)
+      {
+        MPI_Isend(local_quadruple_list,  4*ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);        
+
+        MPI_Irecv(remote_quadruple_list, 4*remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+      else
+      {
+        MPI_Irecv(remote_quadruple_list, 4*remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+          
+        MPI_Isend(local_quadruple_list,  4*ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);       
+      }
+#ifdef _showinfo
+      printf("peer_rank = %d, quadruple_list exchange OK\n", local_leader_rank_in_peer);
+#endif
+    }
+
+    if(is_involved)
+    {
+      ::MPI_Bcast(remote_quadruple_list, 4*remote_ep_size, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, to_mpi_comm(local_comm->mpi_comm));
+      (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new INTERCOMM_RANK_MAP;
+#ifdef _showinfo
+      printf("new intercomm_rank_map = %p\n", (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map);
+#endif
+      for(int i=0; i<remote_ep_size; i++)
+      {
+        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->insert(std::pair<int, std::pair< int, std::pair<int, int> > >(remote_quadruple_list[4*i], remote_quadruple_list[4*i+1], remote_quadruple_list[4*i+2], remote_quadruple_list[4*i+3]));
+      }
+    }
+
+    ////////////////////////////////////////////////////////
+    // step 8 : associate intercomm_rank_map to endpoints //
+    ////////////////////////////////////////////////////////
+
+    int *leader_rank_in_world_local_gathered = new int[(*newintercomm)->ep_comm_ptr->size_rank_info[1].second];
+
+    MPI_Allgather_local(&local_leader_rank_in_world, 1, MPI_INT, leader_rank_in_world_local_gathered, *newintercomm);
+
+
+    int new_rank_loc = (*newintercomm)->ep_comm_ptr->size_rank_info[1].first;
+    int *new_rank_loc_local_gathered = new int[(*newintercomm)->ep_comm_ptr->size_rank_info[1].second];
+
+    MPI_Allgather_local(&new_rank_loc, 1, MPI_INT, new_rank_loc_local_gathered, *newintercomm);
+
+    //printf("peer_rank = %d, leader_rank_in_world_local_gathered = %d %d %d %d, new_rank_loc_local_gathered = %d %d %d %d\n", 
+    //  peer_comm->ep_comm_ptr->size_rank_info[0].first, leader_rank_in_world_local_gathered[0], leader_rank_in_world_local_gathered[1], leader_rank_in_world_local_gathered[2], leader_rank_in_world_local_gathered[3],
+    //  new_rank_loc_local_gathered[0], new_rank_loc_local_gathered[1], new_rank_loc_local_gathered[2], new_rank_loc_local_gathered[3]);
+
+    if(is_involved)
+    {
+
+      (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map = new EP_RANK_MAP;
+#ifdef _showinfo
+      printf("new local_rank_map = %p\n", (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map);
+#endif
+
+      for(std::map<int, std::pair<int, int> >::iterator it = local_comm->ep_rank_map->begin(); it != local_comm->ep_rank_map->end(); it++)
+      {
+        (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->insert(std::pair<int, std::pair< int, int > >(it->first, it->second.first, it->second.second));
+      }
+    }
+
+    MPI_Barrier_local(*newintercomm);
+
+    
+    if(!is_involved)
+    {
+      int target;
+      for(int i=0; i<(*newintercomm)->ep_comm_ptr->size_rank_info[1].second; i++)
+      {
+        if(local_leader_rank_in_world == leader_rank_in_world_local_gathered[i])
         {
-          ep_intercomm[i].is_intercomm = true;
-
-          ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
-          ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
-          ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
-        }
-
-
-        #pragma omp critical (write_to_tag_list)
-        tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
-        //printf("tag_list size = %lu\n", tag_list.size());
-      }
-    }
-
-    vector<int> bcast_buf(8);
-    if(is_local_leader)
-    {
-      std::copy(size_info, size_info+4, bcast_buf.begin());
-      std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
-    }
-
-    MPI_Bcast(bcast_buf.data(), 8, MPI_INT, local_leader, local_comm);
-
-    if(!is_local_leader)
-    {
-      std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
-      std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
-    }
-
-    if(!is_local_leader)
-    {
-      new_rank_info[1].resize(size_info[2]);
-      ep_info[1].resize(size_info[1]);
-      offset.resize(size_info[0]);
-    }
-
-    bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
-
-    if(is_local_leader)
-    {
-      bcast_buf[0] = remote_ep_size;
-      std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
-      std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
-      std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
-    }
-
-    MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT, local_leader, local_comm);
-
-    if(!is_local_leader)
-    {
-      remote_ep_size = bcast_buf[0];
-      std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
-      std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
-      std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
-    }
-
-    int my_position = offset[rank_in_local_parent]+ep_rank_loc;
-    
-    MPI_Barrier_local(local_comm);
-    #pragma omp flush
-
-
-    #pragma omp critical (read_from_tag_list)
-    {
-      bool found = false;
-      while(!found)
-      {
-        for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
-        {
-          if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
-          {
-            *newintercomm = iter->second[my_position];
-            found = true;
-            break;
-          }
-        }
-      }
-    }
-
-    MPI_Barrier(local_comm);
-
-    if(is_local_leader)
-    {
-      int local_flag = true;
-      int remote_flag = false;
-      MPI_Request mpi_requests[2];
-      MPI_Status mpi_statuses[2];
-      
-      MPI_Isend(&local_flag, 1, MPI_INT, remote_leader, tag, peer_comm, &mpi_requests[0]);
-      MPI_Irecv(&remote_flag, 1, MPI_INT, remote_leader, tag, peer_comm, &mpi_requests[1]);
-      
-      MPI_Waitall(2, mpi_requests, mpi_statuses);
-    }
-
-
-    MPI_Barrier(local_comm);
-
-    if(is_proc_master)
-    {
-      for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
-      {
-        if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
-        {
-          tag_list.erase(iter);
+          target = i;
+          (*newintercomm)->ep_comm_ptr->intercomm->intercomm_tag = target;
           break;
         }
       }
-    }
-
-    int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
-    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
-
-    intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
-    intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
-    intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
-    intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
-    intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
-    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
-
-    MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm);
-
-    int my_rank_map_elem[2];
-
-    my_rank_map_elem[0] = intercomm_ep_rank;
-    my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
-
-    vector<pair<int, int> > local_rank_map_array;
-    vector<pair<int, int> > remote_rank_map_array;
-
-
-    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
-    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
-
-    MPI_Allgather(my_rank_map_elem, 2, MPI_INT, 
-      (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm);
-
-    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
-    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
-
-    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
-    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
-    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
-
-    int local_intercomm_size = intercomm_ep_size;
-    int remote_intercomm_size;
-
-    int new_bcast_root_0 = 0;
-    int new_bcast_root = 0;
-
-
-    if(is_local_leader)
-    {
-      MPI_Request requests[4];
-      MPI_Status statuses[4];
-      
-      MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, MPI_INT, remote_leader, tag+4, peer_comm, &requests[0]);
-      MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, remote_leader, tag+4, peer_comm, &requests[1]);
-
-      MPI_Isend(&local_intercomm_size, 1, MPI_INT, remote_leader, tag+5, peer_comm, &requests[2]);
-      MPI_Irecv(&remote_intercomm_size, 1, MPI_INT, remote_leader, tag+5, peer_comm, &requests[3]);
-      
-      MPI_Waitall(4, requests, statuses);
-
-      new_bcast_root_0 = intercomm_ep_rank;
-    }
-
-    MPI_Allreduce(&new_bcast_root_0, &new_bcast_root, 1, MPI_INT, MPI_SUM, *newintercomm);
-
-
-    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, local_leader, local_comm);
-    MPI_Bcast(&remote_intercomm_size, 1, MPI_INT, new_bcast_root, *newintercomm);
-
-
-    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
-    (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
-
-
-
-
-    if(is_local_leader)
-    {
-      MPI_Request requests[2];
-      MPI_Status statuses[2];
-      
-      MPI_Isend((*newintercomm).rank_map->data(), 2*local_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[0]);
-      MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, remote_leader, tag+6, peer_comm, &requests[1]);
-      
-      MPI_Waitall(2, requests, statuses);
-    }
-
-    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, new_bcast_root, *newintercomm);
-
-    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
-    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = tag;
+      (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = (*newintercomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->intercomm_rank_map;
+      (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map     = (*newintercomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->local_rank_map;
+    }
+    else
+    {
+      (*newintercomm)->ep_comm_ptr->intercomm->intercomm_tag = -1;
+    }
+
+    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[0] = (*newintercomm)->ep_comm_ptr->size_rank_info[0];
+    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[1] = (*newintercomm)->ep_comm_ptr->size_rank_info[1];
+    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[2] = (*newintercomm)->ep_comm_ptr->size_rank_info[2];
+
+
+    (*newintercomm)->ep_comm_ptr->size_rank_info[0] = local_comm->ep_comm_ptr->size_rank_info[0];
+    (*newintercomm)->ep_comm_ptr->size_rank_info[1] = local_comm->ep_comm_ptr->size_rank_info[1];
+    (*newintercomm)->ep_comm_ptr->size_rank_info[2] = local_comm->ep_comm_ptr->size_rank_info[2];
+
+
+#ifdef _showinfo
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+    printf("peer_rank = %d, size_rank_info = %d %d %d %d %d %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, 
+      (*newintercomm)->ep_comm_ptr->size_rank_info[0].first, (*newintercomm)->ep_comm_ptr->size_rank_info[0].second,
+      (*newintercomm)->ep_comm_ptr->size_rank_info[1].first, (*newintercomm)->ep_comm_ptr->size_rank_info[1].second,
+      (*newintercomm)->ep_comm_ptr->size_rank_info[2].first, (*newintercomm)->ep_comm_ptr->size_rank_info[2].second);
+    
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
 
 /*
-    for(int i=0; i<local_ep_size; i++)
-    if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, local_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
-          (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).second);
-
-    for(int i=0; i<remote_ep_size; i++)
-    if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, remote_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
-          (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->at(i).second);
-
-    for(int i=0; i<remote_intercomm_size; i++)
-    if(local_comm.ep_comm_ptr->comm_label == 0) printf("ep_rank (from EP) = %d, intercomm_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
-          (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first, (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second);
+    if(peer_comm->ep_comm_ptr->size_rank_info[0].first == 5)
+    {
+      int receiver = rand()%10;
+      printf("receiver = %d, intercomm_local_rank = %d, intercomm_mpi_rank = %d, comm_label = %d\n", receiver, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).first, 
+        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.first, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.second);
+    }
+
+    if(peer_comm->ep_comm_ptr->size_rank_info[0].first == 9)
+    {
+      int receiver = rand()%6;
+      printf("receiver = %d, intercomm_local_rank = %d, intercomm_mpi_rank = %d, comm_label = %d\n", receiver, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).first, 
+        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.first, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.second);
+    }
+
+
+    if(peer_comm->ep_comm_ptr->size_rank_info[0].first == 5)
+    {
+      for(int i=0; i<ep_size; i++)
+      {
+        printf("rank_map->at(%d) = %d, %d\n", i, (*newintercomm)->ep_rank_map->at(i).first, (*newintercomm)->ep_rank_map->at(i).second);
+      }
+    }
 */
-
-//    for(int i=0; i<(*newintercomm).rank_map->size(); i++)
-//    if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
-//          (*newintercomm).rank_map->at(i).first, (*newintercomm).rank_map->at(i).second);
-
-//    MPI_Comm *test_comm = newintercomm->ep_comm_ptr->intercomm->local_comm;
-//    int test_rank;
-//    MPI_Comm_rank(*test_comm, &test_rank);
-//    printf("=================test_rank = %d\n", test_rank);
-    
-    
-
-    return MPI_SUCCESS;
+    //////////////
+    // clean up //
+    //////////////
+
+    delete ranks_in_world_local;
+    delete ranks_in_world_remote;
+
+    if(is_involved)
+    {
+      delete local_quadruple_list;
+      delete remote_quadruple_list;
+    }
+
+    delete leader_rank_in_world_local_gathered;
+    delete new_rank_loc_local_gathered;
+
 
   }
@@ -584,220 +637,4 @@
 
 
-
-  int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
-  {
-    //! mpi_size of local comm = 1
-    //! same world rank of leaders
-
-    int ep_rank, ep_rank_loc, mpi_rank;
-    int ep_size, num_ep, mpi_size;
-
-    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
-    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
-    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
-    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
-    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
-    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
-
-
-
-    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
-                                    //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
-
-    int rank_in_world;
-
-    int rank_in_peer_mpi[2];
-
-    ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD.mpi_comm), &rank_in_world);
-
-
-    int local_num_ep = num_ep;
-    int remote_num_ep;
-    int total_num_ep;
-
-    int leader_rank_in_peer[2];
-
-    int my_position;
-    int tag_label[2];
-
-    vector<int> send_buf(4);
-    vector<int> recv_buf(4);
-
-
-    if(ep_rank == local_leader)
-    {
-      MPI_Status status;
-
-
-
-      MPI_Comm_rank(peer_comm, &leader_rank_in_peer[0]);
-
-      send_buf[0] = local_num_ep;
-      send_buf[1] = leader_rank_in_peer[0];
-
-      MPI_Request req_s, req_r;
-
-      MPI_Isend(send_buf.data(), 2, MPI_INT, remote_leader, tag, peer_comm, &req_s);
-      MPI_Irecv(recv_buf.data(), 2, MPI_INT, remote_leader, tag, peer_comm, &req_r);
-
-
-      MPI_Wait(&req_s, &status);
-      MPI_Wait(&req_r, &status);
-
-      recv_buf[2] = leader_rank_in_peer[0];
-
-    }
-
-    MPI_Bcast(recv_buf.data(), 3, MPI_INT, local_leader, local_comm);
-
-    remote_num_ep = recv_buf[0];
-    leader_rank_in_peer[1] = recv_buf[1];
-    leader_rank_in_peer[0] = recv_buf[2];
-
-    total_num_ep = local_num_ep + remote_num_ep;
-
-
-    if(leader_rank_in_peer[0] < leader_rank_in_peer[1])
-    {
-      my_position = ep_rank_loc;
-      //! LEADER create EP
-      if(ep_rank == local_leader)
-      {
-        ::MPI_Comm *mpi_dup = new ::MPI_Comm;
-        
-        ::MPI_Comm_dup(to_mpi_comm(local_comm.mpi_comm), mpi_dup);
-
-        MPI_Comm *ep_intercomm;
-        MPI_Info info;
-        MPI_Comm_create_endpoints(mpi_dup, total_num_ep, info, ep_intercomm);
-
-
-        for(int i=0; i<total_num_ep; i++)
-        {
-          ep_intercomm[i].is_intercomm = true;
-          ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
-          ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = 0;
-
-          ep_intercomm[i].ep_comm_ptr->comm_label = leader_rank_in_peer[0];
-        }
-
-        tag_label[0] = TAG++;
-        tag_label[1] = rank_in_world;
-
-        #pragma omp critical (write_to_tag_list)
-        tag_list.push_back(make_pair( make_pair(tag_label[0], tag_label[1]) , ep_intercomm));
-
-        MPI_Request req_s;
-        MPI_Status sta_s;
-        MPI_Isend(tag_label, 2, MPI_INT, remote_leader, tag, peer_comm, &req_s);
-
-        MPI_Wait(&req_s, &sta_s);
-
-      }
-    }
-    else
-    {
-      //! Wait for EP creation
-      my_position = remote_num_ep + ep_rank_loc;
-      if(ep_rank == local_leader)
-      {
-        MPI_Status status;
-        MPI_Request req_r;
-        MPI_Irecv(tag_label, 2, MPI_INT, remote_leader, tag, peer_comm, &req_r);
-        MPI_Wait(&req_r, &status);
-      }
-    }
-
-    MPI_Bcast(tag_label, 2, MPI_INT, local_leader, local_comm);
-
-
-
-
-    #pragma omp critical (read_from_tag_list)
-    {
-      bool found = false;
-      while(!found)
-      {
-        for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
-        {
-          if((*iter).first == make_pair(tag_label[0], tag_label[1]))
-          {
-            *newintercomm =  iter->second[my_position];
-            found = true;
-            // tag_list.erase(iter);
-            break;
-          }
-        }
-      }
-    }
-
-    MPI_Barrier_local(local_comm);
-
-    if(leader_rank_in_peer[0] < leader_rank_in_peer[1])
-    {
-      for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
-        {
-          if((*iter).first == make_pair(tag_label[0], tag_label[1]))
-          {
-            tag_list.erase(iter);
-            break;
-          }
-        }
-    }
-
-
-
-    int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
-    int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
-
-    intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
-    intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
-    intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
-    intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
-    intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
-    intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
-
-
-
-    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map  = new RANK_MAP;
-    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
-    (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_num_ep);
-    (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_num_ep);
-
-    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
-    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
-    (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
-
-
-
-    int local_rank_map_ele[2];
-    local_rank_map_ele[0] = intercomm_ep_rank;
-    local_rank_map_ele[1] = (*newintercomm).ep_comm_ptr->comm_label;
-
-    MPI_Allgather(local_rank_map_ele, 2, MPI_INT, 
-      (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm);
-
-    if(ep_rank == local_leader)
-    {
-      MPI_Status status;
-      MPI_Request req_s, req_r;
-
-      MPI_Isend((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_num_ep, MPI_INT, remote_leader, tag, peer_comm, &req_s);
-      MPI_Irecv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT, remote_leader, tag, peer_comm, &req_r);
-
-
-      MPI_Wait(&req_s, &status);
-      MPI_Wait(&req_r, &status);
-
-    }
-
-    MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_num_ep, MPI_INT, local_leader, local_comm);
-    (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
-    (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = tag;
-
-
-    return MPI_SUCCESS;
-  }
-
-
+  
 }
