Changeset 1538 for XIOS/dev/branch_openmp/extern

Timestamp:

06/11/18 19:06:45 (6 years ago)

Author:

yushan

Message:

tests in XIOS OK (client, complete, remap, toy)

Location:

XIOS/dev/branch_openmp/extern

Files:

• ep_dev/ep_free.cpp (modified) (2 diffs)
• ep_dev/ep_intercomm.cpp (modified) (3 diffs)
• ep_dev/ep_lib_collective.hpp (modified) (1 diff)
• ep_dev/ep_lib_endpoint.hpp (modified) (1 diff)
• ep_dev/ep_lib_mpi.hpp (modified) (1 diff)
• ep_dev/ep_message.cpp (modified) (3 diffs)
• ep_dev/ep_probe.cpp (modified) (2 diffs)
• ep_dev/ep_recv.cpp (modified) (5 diffs)
• ep_dev/ep_request.hpp (modified) (1 diff)
• ep_dev/ep_test.cpp (modified) (2 diffs)
• ep_dev/ep_wait.cpp (modified) (2 diffs)
• remap/src/mapper.hpp (modified) (1 diff)
• remap/src/mpi_cascade.hpp (modified) (1 diff)
• remap/src/parallel_tree.cpp (modified) (1 diff)

XIOS/dev/branch_openmp/extern/ep_dev/ep_free.cpp

@@ -18 +18 @@
   {
     Debug("MPI_Comm_free with MPI\n");
-
     return ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->mpi_comm));
-    
   }
 
@@ -30 +28 @@
 
     ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first;
-    int ep_rank = (*comm)->ep_comm_ptr->size_rank_info[0].first;
     num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second;
 
-    MPI_Barrier(*comm);
+    MPI_Barrier_local(*comm);
 
     if(ep_rank_loc == 0)

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.cpp

@@ -14 +14 @@
 namespace ep_lib
 {
+
   int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
   {
-    assert(local_comm->is_ep);
-
+    if(!local_comm->is_ep) return MPI_Intercomm_create_mpi(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);
+
+    int ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first;
+    
+    
+    // check if local leaders are in the same mpi proc
+    // by checking their mpi_rank in peer_comm
+    
+    int mpi_rank_of_leader[2];
+    
+    if(ep_rank == local_leader)
+    {
+      mpi_rank_of_leader[0] = peer_comm->ep_comm_ptr->size_rank_info[2].first;
+      mpi_rank_of_leader[1] = peer_comm->ep_rank_map->at(remote_leader).second;      
+    }
+    
+    MPI_Bcast(mpi_rank_of_leader, 2, MPI_INT, local_leader, local_comm);
+    
+    if(mpi_rank_of_leader[0] != mpi_rank_of_leader[1])
+    {
+      Debug("calling MPI_Intercomm_create_kernel\n");
+      return MPI_Intercomm_create_endpoint(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);
+    }
+    else
+    {
+      printf("local leaders are in the same MPI proc. Routine not yet implemented\n");
+      MPI_Abort(local_comm, 0);
+    }
+  }
+  
+  
+  
+  int MPI_Intercomm_create_endpoint(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  {
     int ep_rank, ep_rank_loc, mpi_rank;
     int ep_size, num_ep, mpi_size;
 
-    ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first;
+    ep_rank     = local_comm->ep_comm_ptr->size_rank_info[0].first;
     ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first;
-    mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first;
-    ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second;
-    num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second;
-    mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second;
-
-    int world_rank_and_num_ep[2];
-    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank_and_num_ep[0]);
-    world_rank_and_num_ep[1] = num_ep;
-
-    int remote_mpi_size;
+    mpi_rank    = local_comm->ep_comm_ptr->size_rank_info[2].first;
+    ep_size     = local_comm->ep_comm_ptr->size_rank_info[0].second;
+    num_ep      = local_comm->ep_comm_ptr->size_rank_info[1].second;
+    mpi_size    = local_comm->ep_comm_ptr->size_rank_info[2].second;
+
+    //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    // step 1 : local leaders exchange ep_size, leader_rank_in_peer, leader_rank_in_peer_mpi, leader_rank_in_world. //
+    //          local leaders bcast results to all ep in local_comm                                                 //
+    //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+    bool is_local_leader = ep_rank==local_leader? true: false;
+    
+    
+    int local_leader_rank_in_peer;
+    int local_leader_rank_in_peer_mpi;
+    int local_leader_rank_in_world;
+
     int remote_ep_size;
-
-    int *local_world_rank_and_num_ep;
-    int *remote_world_rank_and_num_ep;
-    int *summed_world_rank_and_num_ep;
-
-
-    bool is_leader = ep_rank==local_leader? true : false;
-    bool is_local_leader = is_leader? true: (ep_rank_loc==0 && mpi_rank!=local_comm->ep_rank_map->at(local_leader).second ? true : false);
-    bool priority;
-
-    if(is_leader)
-    {
-      int leader_mpi_rank_in_peer;
-      MPI_Comm_rank(peer_comm, &leader_mpi_rank_in_peer);
-      if(leader_mpi_rank_in_peer == remote_leader) 
-      {
-        printf("same leader in peer_comm\n");
-        exit(1);
-      }
-      priority = leader_mpi_rank_in_peer<remote_leader? true : false;
-    }
-
-
-    MPI_Bcast(&priority, 1, MPI_INT, local_leader, local_comm);
+    int remote_leader_rank_in_peer;
+    int remote_leader_rank_in_peer_mpi;
+    int remote_leader_rank_in_world;
+
+    int send_quadruple[4];
+    int recv_quadruple[4];
+
 
     if(is_local_leader)
     {
-      local_world_rank_and_num_ep = new int[2*mpi_size];
-      ::MPI_Allgather(world_rank_and_num_ep, 2, to_mpi_type(MPI_INT), local_world_rank_and_num_ep, 2, to_mpi_type(MPI_INT), to_mpi_comm(local_comm->mpi_comm));
-    }
-
-    
-    
-    if(is_leader)
-    {
+      MPI_Comm_rank(peer_comm, &local_leader_rank_in_peer);
+      ::MPI_Comm_rank(to_mpi_comm(peer_comm->mpi_comm), &local_leader_rank_in_peer_mpi);
+      ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &local_leader_rank_in_world);
+
+      send_quadruple[0] = ep_size;
+      send_quadruple[1] = local_leader_rank_in_peer;
+      send_quadruple[2] = local_leader_rank_in_peer_mpi;
+      send_quadruple[3] = local_leader_rank_in_world;
+
       MPI_Request request;
       MPI_Status status;
 
-      if(priority)
-      {
-        MPI_Isend(&mpi_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+
+      if(remote_leader > local_leader_rank_in_peer)
+      {
+        MPI_Isend(send_quadruple, 4, MPI_INT, remote_leader, tag, peer_comm, &request);
         MPI_Wait(&request, &status);
        
-        MPI_Irecv(&remote_mpi_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
-        MPI_Wait(&request, &status);
-
-        MPI_Isend(&ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
-        MPI_Wait(&request, &status);
-       
-        MPI_Irecv(&remote_ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Irecv(recv_quadruple, 4, MPI_INT, remote_leader, tag, peer_comm, &request);
         MPI_Wait(&request, &status);
       }
       else
       {
-        MPI_Irecv(&remote_mpi_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Irecv(recv_quadruple, 4, MPI_INT, remote_leader, tag, peer_comm, &request);
         MPI_Wait(&request, &status);
           
-        MPI_Isend(&mpi_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
-        MPI_Wait(&request, &status);
-
-        MPI_Irecv(&remote_ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
-        MPI_Wait(&request, &status);
-          
-        MPI_Isend(&ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
-        MPI_Wait(&request, &status);
-      }
-    }
-
-    MPI_Bcast(&remote_mpi_size, 1, MPI_INT, local_leader, local_comm);
-    MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm);
-
-    remote_world_rank_and_num_ep = new int[2*remote_mpi_size];
-
-
-    if(is_leader)
+        MPI_Isend(send_quadruple, 4, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+
+      remote_ep_size                 = recv_quadruple[0];
+      remote_leader_rank_in_peer     = recv_quadruple[1];
+      remote_leader_rank_in_peer_mpi = recv_quadruple[2];
+      remote_leader_rank_in_world    = recv_quadruple[3];
+#ifdef _showinfo
+      printf("peer_rank = %d, packed exchange OK\n", local_leader_rank_in_peer);
+#endif
+    }
+
+    MPI_Bcast(send_quadruple, 4, MPI_INT, local_leader, local_comm);
+    MPI_Bcast(recv_quadruple, 4, MPI_INT, local_leader, local_comm);
+
+    if(!is_local_leader)
+    {
+      local_leader_rank_in_peer     = send_quadruple[1];
+      local_leader_rank_in_peer_mpi = send_quadruple[2];
+      local_leader_rank_in_world    = send_quadruple[3];
+
+      remote_ep_size                 = recv_quadruple[0];
+      remote_leader_rank_in_peer     = recv_quadruple[1];
+      remote_leader_rank_in_peer_mpi = recv_quadruple[2];
+      remote_leader_rank_in_world    = recv_quadruple[3];
+    }
+
+
+#ifdef _showinfo
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, ep_size = %d, remote_ep_size = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_size, remote_ep_size);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm); 
+#endif
+
+    ///////////////////////////////////////////////////////////////////
+    // step 2 : gather ranks in world for both local and remote comm //
+    ///////////////////////////////////////////////////////////////////
+
+    int rank_in_world;
+    ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &rank_in_world);
+
+    int *ranks_in_world_local  = new int[ep_size];
+    int *ranks_in_world_remote = new int[remote_ep_size];
+
+    MPI_Allgather(&rank_in_world, 1, MPI_INT, ranks_in_world_local, 1, MPI_INT, local_comm);
+
+    if(is_local_leader)
     {
       MPI_Request request;
       MPI_Status status;
 
-      if(priority)
-      {
-        MPI_Isend(local_world_rank_and_num_ep, 2*mpi_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+      if(remote_leader > local_leader_rank_in_peer)
+      {
+        MPI_Isend(ranks_in_world_local,  ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
         MPI_Wait(&request, &status);
        
-        MPI_Irecv(remote_world_rank_and_num_ep, 2*remote_mpi_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Irecv(ranks_in_world_remote, remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
         MPI_Wait(&request, &status);
       }
       else
       {
-        MPI_Irecv(remote_world_rank_and_num_ep, 2*remote_mpi_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Irecv(ranks_in_world_remote, remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
         MPI_Wait(&request, &status);
           
-        MPI_Isend(local_world_rank_and_num_ep, 2*mpi_size, MPI_INT, remote_leader, tag, peer_comm, &request);
-        MPI_Wait(&request, &status);
-      }
-    }
-
-    
-    MPI_Bcast(remote_world_rank_and_num_ep, 2*remote_mpi_size, MPI_INT, local_leader, local_comm);
-    
-
-
-    bool is_new_leader  = is_local_leader;
-
-    if(is_local_leader && !priority)
-    {
-      for(int i=0; i<remote_mpi_size; i++)
-      {
-        if(world_rank_and_num_ep[0] == remote_world_rank_and_num_ep[2*i])
+        MPI_Isend(ranks_in_world_local,  ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+#ifdef _showinfo
+      printf("peer_rank = %d, ranks_in_world exchange OK\n", local_leader_rank_in_peer);
+#endif
+    }
+
+    MPI_Bcast(ranks_in_world_remote, remote_ep_size, MPI_INT, local_leader, local_comm);
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+    if(remote_leader == 4)
+    {
+      for(int i=0; i<ep_size; i++)
+      {
+        if(ep_rank == i)
         {
-          is_new_leader = false;
-          break;
+          printf("peer_rank = %d, ranks_in_world_local = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_local[i]);
+          }
+    
+          printf("\npeer_rank = %d, ranks_in_world_remote = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<remote_ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_remote[i]);
+          }
+          printf("\n");
+         
         }
-      }
-    } 
-    
-
-    ::MPI_Group *empty_group;
-    ::MPI_Group *local_group;
-    ::MPI_Group union_group;
-
-    if(is_local_leader)
-    {
-
-      int *ranks = new int[mpi_size];
+        
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+      }
+    }
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    
+    if(remote_leader == 13)
+    {
+      for(int i=0; i<ep_size; i++)
+      {
+        if(ep_rank == i)
+        {
+          printf("peer_rank = %d, ranks_in_world_local = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_local[i]);
+          }
+    
+          printf("\npeer_rank = %d, ranks_in_world_remote = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<remote_ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_remote[i]);
+          }
+          printf("\n");
+         
+        }
+        
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+      }
+    }
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+    //////////////////////////////////////////////////////////////
+    // step 3 : determine the priority and ownership of each ep //
+    //////////////////////////////////////////////////////////////
+
+    bool priority = local_leader_rank_in_peer < remote_leader_rank_in_peer? true : false;
+
+
+    int ownership = priority;
+
+    if(rank_in_world == ranks_in_world_local[local_leader]) ownership = 1;
+    if(rank_in_world == remote_leader_rank_in_world)   ownership = 0;
+    
+
+#ifdef _showinfo
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, priority = %d, local_leader_rank_in_peer = %d, remote_leader_rank_in_peer = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, priority, local_leader_rank_in_peer, remote_leader_rank_in_peer);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+#endif
+
+    
+#ifdef _showinfo
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, priority = %d, ownership = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, priority, ownership);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+#endif
+
+    //////////////////////////////////////////////////////
+    // step 4 : extract local_comm and create intercomm //
+    //////////////////////////////////////////////////////
+
+    bool is_involved = is_local_leader || (!is_local_leader && ep_rank_loc == 0 && rank_in_world != local_leader_rank_in_world);
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, is_involved = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, is_involved);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+    if(is_involved)
+    {
+      ::MPI_Group local_group;
+      ::MPI_Group extracted_group;
+      ::MPI_Comm extracted_comm;
+
+
+      ::MPI_Comm_group(to_mpi_comm(local_comm->mpi_comm), &local_group);
+
+      int *ownership_list = new int[mpi_size];
+      int *mpi_rank_list = new int[mpi_size];
+
+      ::MPI_Allgather(&ownership, 1, to_mpi_type(MPI_INT), ownership_list, 1, to_mpi_type(MPI_INT), to_mpi_comm(local_comm->mpi_comm));
+      ::MPI_Allgather(&mpi_rank,  1, to_mpi_type(MPI_INT), mpi_rank_list,  1, to_mpi_type(MPI_INT), to_mpi_comm(local_comm->mpi_comm));
+
+      
+      int n=0;
       for(int i=0; i<mpi_size; i++)
-      {
-        ranks[i] = local_world_rank_and_num_ep[2*i];
-      }
-
-      local_group = new ::MPI_Group;
-      ::MPI_Group_incl(MPI_GROUP_WORLD, mpi_size, ranks, local_group);
-
-      delete[] ranks;
-
+      { 
+        n+=ownership_list[i];
+      }
+
+      int *new_mpi_rank_list = new int[n];
+      int j=0;
+      for(int i=0; i<mpi_size; i++)
+      { 
+        if(ownership_list[i] !=0)
+        {
+          new_mpi_rank_list[j++] = mpi_rank_list[i];
+        }
+      }
+
+
+      ::MPI_Group_incl(local_group, n, new_mpi_rank_list, &extracted_group);
+
+      ::MPI_Comm_create(to_mpi_comm(local_comm->mpi_comm), extracted_group, &extracted_comm);
+
+      ::MPI_Comm mpi_inter_comm;
+
+      int local_leader_rank_in_extracted_comm;
+
+      if(is_local_leader)
+      {
+        ::MPI_Comm_rank(extracted_comm, &local_leader_rank_in_extracted_comm);
+      }
+
+      ::MPI_Bcast(&local_leader_rank_in_extracted_comm, 1, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, to_mpi_comm(local_comm->mpi_comm));
+
+      ::MPI_Comm *intracomm = new ::MPI_Comm;
+      bool is_real_involved = ownership && extracted_comm != to_mpi_comm(MPI_COMM_NULL->mpi_comm);
+
+      if(is_real_involved)
+      {
+        ::MPI_Intercomm_create(extracted_comm, local_leader_rank_in_extracted_comm, to_mpi_comm(peer_comm->mpi_comm), remote_leader_rank_in_peer_mpi, tag, &mpi_inter_comm);
+        ::MPI_Intercomm_merge(mpi_inter_comm, !priority, intracomm);
+      }
+        
+        
+
+      ////////////////////////////////////
+      // step 5 :: determine new num_ep //
+      ////////////////////////////////////
+
+      int num_ep_count=0;
+
+      for(int i=0; i<ep_size; i++)
+      {
+        if(rank_in_world == ranks_in_world_local[i])
+          num_ep_count++;
+      }
       
-      #pragma omp flush
-      #pragma omp critical (write_to_tag_group_map)
-      {
-        if(tag_group_map == 0) tag_group_map = new map< std::pair<int, int>, ::MPI_Group * >;
-        
-        tag_group_map->insert(std::make_pair(std::make_pair(tag, priority? 1 : 2), local_group));       
-      }
-    }
-
-    MPI_Barrier(local_comm);
-
-    if(is_leader)
-    {
-      MPI_Request request;
-      MPI_Status status;
-
-      int send_signal=0;
-      int recv_signal;
-
-      if(priority)
-      {
-        MPI_Isend(&send_signal, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
-        MPI_Wait(&request, &status);
-       
-        MPI_Irecv(&recv_signal, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
-        MPI_Wait(&request, &status);
-      }
-      else
-      {
-        MPI_Irecv(&recv_signal, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
-        MPI_Wait(&request, &status);
-          
-        MPI_Isend(&send_signal, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
-        MPI_Wait(&request, &status);
-      }
-    }
-
-
-    MPI_Barrier(local_comm);
-
-    if(is_new_leader)
-    {
-      ::MPI_Group *group1;
-      ::MPI_Group *group2;
-
-      empty_group = new ::MPI_Group;
-      *empty_group = MPI_GROUP_EMPTY;
-
-      #pragma omp flush
-      #pragma omp critical (read_from_tag_group_map)
-      {
-        group1 = tag_group_map->find(make_pair(tag, 1)) != tag_group_map->end()? tag_group_map->at(std::make_pair(tag, 1)) : empty_group;
-        group2 = tag_group_map->find(make_pair(tag, 2)) != tag_group_map->end()? tag_group_map->at(std::make_pair(tag, 2)) : empty_group;
-      }
-
+      for(int i=0; i<remote_ep_size; i++)
+      {
+        if(rank_in_world == ranks_in_world_remote[i])
+          num_ep_count++;
+      }
+
+
+      ///////////////////////////////////////////////////
+      // step 6 : create endpoints from extracted_comm //
+      ///////////////////////////////////////////////////
+
+      if(is_real_involved)
+      {
+        MPI_Comm *ep_comm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(intracomm, num_ep_count, info, ep_comm);
       
 #ifdef _showinfo
-
-      int group1_rank, group1_size;
-      int group2_rank, group2_size;
-      ::MPI_Group_rank(*group1, &group1_rank);
-      ::MPI_Group_size(*group1, &group1_size);
-      ::MPI_Group_rank(*group2, &group2_rank);
-      ::MPI_Group_size(*group2, &group2_size);
-
-#endif
-
-      ::MPI_Group_union(*group1, *group2, &union_group);
-
-
-      #pragma omp critical (read_from_tag_group_map)
-      {
-        tag_group_map->erase(make_pair(tag, 1));
-        tag_group_map->erase(make_pair(tag, 2));
-      }
-
-#ifdef _showinfo
-
-      int group_rank, group_size;
-      ::MPI_Group_rank(union_group, &group_rank);
-      ::MPI_Group_size(union_group, &group_size);
-      printf("rank = %d : map = %p, group1_rank/size = %d/%d, group2_rank/size = %d/%d, union_rank/size = %d/%d\n", ep_rank, tag_group_map, group1_rank, group1_size, group2_rank, group2_size, group_rank, group_size);
-#endif
-
-    }
-
-    int summed_world_rank_and_num_ep_size=mpi_size;
-    summed_world_rank_and_num_ep = new int[2*(mpi_size+remote_mpi_size)];
-
-
-    if(is_leader)
-    {
-      
-      for(int i=0; i<mpi_size; i++)
-      {
-        summed_world_rank_and_num_ep[2*i] = local_world_rank_and_num_ep[2*i];
-        summed_world_rank_and_num_ep[2*i+1] = local_world_rank_and_num_ep[2*i+1]; 
-      }
-
-      for(int i=0; i<remote_mpi_size; i++)
-      {
-        bool found=false;
-        for(int j=0; j<mpi_size; j++)
+        printf("new ep_comm->ep_comm_ptr->intercomm->mpi_inter_comm = %p\n", mpi_inter_comm);
+#endif
+
+        #pragma omp critical (write_to_tag_list)      
+        intercomm_list.push_back(make_pair( make_pair(tag, min(local_leader_rank_in_world, remote_leader_rank_in_world)) , make_pair(ep_comm , make_pair(num_ep_count, 0))));
+        #pragma omp flush
+#ifdef _showinfo
+        for(int i=0; i<num_ep_count; i++)
+          printf("peer_rank = %d, ep_comm = %p, ep_comm[%d] -> new_ep_rank = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_comm, i, ep_comm[i]->ep_comm_ptr->size_rank_info[0].first);
+#endif  
+        ::MPI_Comm_free(intracomm);
+        delete intracomm;
+      }
+
+
+      delete ownership_list;
+      delete mpi_rank_list;
+      delete new_mpi_rank_list;
+
+    }
+
+    int repeated=0;
+    for(int i=0; i<remote_ep_size; i++)
+    {
+      if(rank_in_world == ranks_in_world_remote[i])
+        repeated++;
+    }
+
+    int my_turn = ownership==1? ep_rank_loc : ep_rank_loc+repeated;
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, ep_rank_loc = %d, ownership = %d, repeated = %d, my_turn = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_rank_loc, ownership, repeated, my_turn);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+
+    #pragma omp flush
+    #pragma omp critical (read_from_intercomm_list)
+    {
+      bool flag=true;
+      while(flag)
+      {
+        for(std::list<std::pair<std::pair<int, int> , std::pair<MPI_Comm * , std::pair<int, int> > > >::iterator iter = intercomm_list.begin(); iter!=intercomm_list.end(); iter++)
         {
-          if(remote_world_rank_and_num_ep[2*i] == local_world_rank_and_num_ep[2*j])
+          if(iter->first == make_pair(tag, min(local_leader_rank_in_world, remote_leader_rank_in_world)))
           {
-            found=true;
-            summed_world_rank_and_num_ep[2*j+1] += remote_world_rank_and_num_ep[2*i+1]; 
+            *newintercomm = iter->second.first[my_turn];
+            
+            iter->second.second.second++;
+            
+            if(iter->second.second.first == iter->second.second.second)
+              intercomm_list.erase(iter);
+
+            flag = false;
+            break; 
           }
         }
-        if(!found)
-        {
-          summed_world_rank_and_num_ep[2*summed_world_rank_and_num_ep_size] = remote_world_rank_and_num_ep[2*i];
-          summed_world_rank_and_num_ep[2*summed_world_rank_and_num_ep_size+1] = remote_world_rank_and_num_ep[2*i+1];
-          summed_world_rank_and_num_ep_size++;
-        }
-
-      }
-    }
-
-    MPI_Bcast(&summed_world_rank_and_num_ep_size, 1, MPI_INT, local_leader, local_comm);
-
-    MPI_Bcast(summed_world_rank_and_num_ep, 2*summed_world_rank_and_num_ep_size, MPI_INT, local_leader, local_comm);
-    
-    
-
-    int remote_num_ep = 0;
-    for(int i=0; i<remote_mpi_size; i++)
-    {
-      if(remote_world_rank_and_num_ep[2*i] == world_rank_and_num_ep[0])
-      {
-        remote_num_ep = remote_world_rank_and_num_ep[2*i+1];
-        break;
-      }
-    }
-
-    int new_ep_rank_loc = priority? ep_rank_loc : ep_rank_loc+remote_num_ep;
-
-#ifdef _showinfo
-    printf("rank = %d, priority = %d, remote_num_ep = %d, new_ep_rank_loc = %d\n", ep_rank, priority, remote_num_ep, new_ep_rank_loc);
-#endif
-    
-    if(is_new_leader)
-    {
-      int new_num_ep;
-      for(int i=0; i<summed_world_rank_and_num_ep_size; i++)
-      {
-        if(summed_world_rank_and_num_ep[2*i] == world_rank_and_num_ep[0])
-        {
-          new_num_ep = summed_world_rank_and_num_ep[2*i+1];
-          break;
-        }
-      }
-
-      ::MPI_Comm mpi_comm;
-      ::MPI_Comm_create_group(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), union_group, tag, &mpi_comm);
-
-
-      MPI_Comm *ep_comm;
-      MPI_Info info;
-      MPI_Comm_create_endpoints(&mpi_comm, new_num_ep, info, ep_comm);
-
-      #pragma omp critical (write_to_tag_comm_map)
-      {
-        if(tag_comm_map == 0) tag_comm_map = new std::map<int, std::pair<ep_lib::MPI_Comm*, std::pair<int, int> > >;
-        tag_comm_map->insert(std::make_pair(tag, std::make_pair(ep_comm, std::make_pair(new_num_ep, 0))));
-      }
-      #pragma omp flush
-    }
-
-
-    bool found=false;
-    while(!found)
-    {
-      #pragma omp flush
-      #pragma omp critical (read_from_tag_comm_map)
-      {
-        if(tag_comm_map!=0)
-        {
-          if(tag_comm_map->find(tag) != tag_comm_map->end())
-          {              
-            *newintercomm = tag_comm_map->at(tag).first[new_ep_rank_loc];
-            
-            tag_comm_map->at(tag).second.second++;
-            if(tag_comm_map->at(tag).second.second == tag_comm_map->at(tag).second.first)
-            {
-              tag_comm_map->erase(tag_comm_map->find(tag));
-            }
-
-            found=true;
-          }
-        }
-      }  
-    }
+      }
+    }
+    
+
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, test_rank = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, (*newintercomm)->ep_comm_ptr->size_rank_info[0].first);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+    //////////////////////////////////////////////////////////
+    // step 7 : create intercomm_rank_map for local leaders //
+    //////////////////////////////////////////////////////////
 
     (*newintercomm)->is_intercomm = true;
-
-    
-
-
+    
     (*newintercomm)->inter_rank_map = new INTER_RANK_MAP;
-  
-
+    
+    
     int rank_info[2];
     rank_info[0] = ep_rank;
@@ -386 +489 @@
     MPI_Allgather(rank_info, 2, MPI_INT, local_rank_info, 2, MPI_INT, local_comm);
 
-    if(is_leader)
+    if(is_local_leader)
     {
       MPI_Request request;
@@ -415 +518 @@
       (*newintercomm)->inter_rank_map->insert(make_pair(remote_rank_info[2*i], remote_rank_info[2*i+1]));
     }
-
-#ifdef _showinfo
-    if(ep_rank==4 && !priority)
-    {
-      for(std::map<int, int > :: iterator it=(*newintercomm)->inter_rank_map->begin(); it != (*newintercomm)->inter_rank_map->end(); it++)
-      {
-        printf("inter_rank_map[%d] = %d\n", it->first, it->second);
-      }
-    }
-#endif
-
+    
     (*newintercomm)->ep_comm_ptr->size_rank_info[0] = local_comm->ep_comm_ptr->size_rank_info[0];
 
-    if(is_local_leader)
-    {
-      delete[] local_world_rank_and_num_ep;  
-    
-      MPI_Group_free(local_group);
-      delete local_group;
-    }
-
-    if(is_new_leader)
-    {
-      MPI_Group_free(&union_group);
-      delete empty_group;
-    }
-
-    delete[] remote_world_rank_and_num_ep;
-    delete[] summed_world_rank_and_num_ep;
+    
     delete[] local_rank_info;
     delete[] remote_rank_info;
-
-
+    delete[] ranks_in_world_local;
+    delete[] ranks_in_world_remote;
+    /*
+    if((*newintercomm)->ep_comm_ptr->size_rank_info[0].second == 1)
+    {
+      for(INTER_RANK_MAP::iterator it = (*newintercomm)->inter_rank_map->begin(); it != (*newintercomm)->inter_rank_map->end(); it++)
+      {
+        printf("inter_rank_map[%d] = %d\n", it->first, it->second);
+      }
+    }
+    */
+    
+ 
   }
+    
+  
+  
+  int MPI_Intercomm_create_mpi(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+    printf("MPI_Intercomm_create_mpi not yet implemented\n");
+    MPI_Abort(local_comm, 0);
+  }
 
 }

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_collective.hpp

@@ -47 +47 @@
   int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
 
-  //int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
+  
   
   

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_endpoint.hpp

@@ -19 +19 @@
   int MPI_Iprobe_endpoint(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status);
   int MPI_Improbe_endpoint(int source, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status);
+  
+  int MPI_Intercomm_create_endpoint(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
 
 }

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_mpi.hpp

@@ -29 +29 @@
 
   int MPI_Barrier_mpi(MPI_Comm comm);
+  
+  int MPI_Intercomm_create_mpi(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
 
 }

XIOS/dev/branch_openmp/extern/ep_dev/ep_message.cpp

@@ -46 +46 @@
       }
       
+      if((*(*it))->state == 2)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
       Message_Check(((*(*it))->comm));
     }
@@ -55 +64 @@
     {
       if(*(*it) == 0)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
+      
+      if((*(*it))->state == 2)
       {
         EP_PendingRequests->erase(it);
@@ -196 +214 @@
         continue;
       }
+      
+      if((*(*it))->state == 2)
+      {
+        EP_PendingRequests->erase(it);
+        
+        memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());
+        it = EP_PendingRequests->begin();
+        continue;
+      }
           
       if((*(*it))->ep_src>6)

XIOS/dev/branch_openmp/extern/ep_dev/ep_probe.cpp

@@ -56 +56 @@
     if(comm->is_intercomm)
     {
-      src = comm->inter_rank_map->at(src);
+      if(src>=0) src = comm->inter_rank_map->at(src);
     } 
     
@@ -85 +85 @@
         status->ep_src = (*it)->ep_src;
         status->ep_tag = (*it)->ep_tag;
+        
+        if(comm->is_intercomm)
+        {
+          for(INTER_RANK_MAP::iterator iter = comm->inter_rank_map->begin(); iter != comm->inter_rank_map->end(); iter++)
+          {
+            if(iter->second == (*it)->ep_src) status->ep_src=iter->first;
+          }
+        }
 
         *flag = true;

XIOS/dev/branch_openmp/extern/ep_dev/ep_recv.cpp

@@ -43 +43 @@
     if(comm->is_intercomm)
     {
-      src = comm->inter_rank_map->at(src);
-      printf("new src = %d\n", src);
+      if(src>=0) src = comm->inter_rank_map->at(src);
     }
     
@@ -61 +60 @@
     (*request)->type = 2;
     (*request)->probed = false;
-    
+    (*request)->state = 0;
 
     (*request)->ep_src = src;
@@ -69 +68 @@
     if(EP_PendingRequests == 0 ) EP_PendingRequests = new std::list< MPI_Request* >;
 
-    EP_PendingRequests->push_back(request);  
+    EP_PendingRequests->push_back(request); 
     
     memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());    
@@ -127 +126 @@
     
     (*request)->probed = true;
+    (*request)->state = 1;
         
     ::MPI_Imrecv(buf, count, to_mpi_type(datatype), to_mpi_message_ptr(*message), to_mpi_request_ptr(*request));                
@@ -139 +139 @@
 
     
-    return Request_Check();
+    //return Request_Check();
   }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_request.hpp

@@ -18 +18 @@
                 // 3: Imrecv 
                 
-      //int state; // 0: new 
+      int state; // 0: new 
                  // 1: imrecvd
                  // 2: tested or waited

XIOS/dev/branch_openmp/extern/ep_dev/ep_test.cpp

@@ -48 +48 @@
         status->ep_datatype = (*request)->ep_datatype;
         
-        //(*request)->state = 2;
+        (*request)->state = 2;
                 
         memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Test, delete (*request)->mpi_request");
@@ -97 +97 @@
         array_of_statuses[i].ep_datatype = array_of_requests[i]->ep_datatype;
         
-        //array_of_requests[i]->state = 2;
+        array_of_requests[i]->state = 2;
       
         memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Testall, array_of_requests["<<i<<"]->mpi_request");

XIOS/dev/branch_openmp/extern/ep_dev/ep_wait.cpp

@@ -40 +40 @@
     status->ep_datatype = (*request)->ep_datatype;
     
-    //(*request)->state = 2;
+    (*request)->state = 2;
 
     memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Wait, delete (*request)->mpi_request");
@@ -94 +94 @@
       array_of_statuses[i].ep_datatype = array_of_requests[i]->ep_datatype;
       
-      //array_of_requests[i]->state = 2;
+      array_of_requests[i]->state = 2;
       
       memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]->mpi_request");

XIOS/dev/branch_openmp/extern/remap/src/mapper.hpp

@@ -18 +18 @@
 {
 public:
-       //Mapper(ep_lib::MPI_Comm comm) : communicator(comm), verbose(SILENT), neighbourElements(NULL), sstree(comm) {}
-       Mapper(ep_lib::MPI_Comm comm) : verbose(SILENT), neighbourElements(NULL), sstree(comm) 
-       {
-         ep_lib::MPI_Comm_dup(comm, &communicator);
-       }
+       Mapper(ep_lib::MPI_Comm comm) : communicator(comm), verbose(SILENT), neighbourElements(NULL), sstree(comm) {}
 
        ~Mapper();

XIOS/dev/branch_openmp/extern/remap/src/mpi_cascade.hpp

@@ -12 +12 @@
 {
 public:
-  CCascadeLevel(ep_lib::MPI_Comm in_comm) : comm(in_comm)
+  CCascadeLevel(ep_lib::MPI_Comm comm) : comm(comm)
   {
-    ep_lib::MPI_Comm_dup(in_comm, &comm);
     ep_lib::MPI_Comm_size(comm, &size);
     ep_lib::MPI_Comm_rank(comm, &rank);

XIOS/dev/branch_openmp/extern/remap/src/parallel_tree.cpp

@@ -121 +121 @@
 }
 
-//CParallelTree::CParallelTree(MPI_Comm comm) : communicator(comm), cascade(MIN_NODE_SZ*MIN_NODE_SZ*2, comm)
-CParallelTree::CParallelTree(MPI_Comm comm) : cascade(MAX_NODE_SZ*MAX_NODE_SZ*2, comm)
-{
-  MPI_Comm_dup(comm, &communicator);
+CParallelTree::CParallelTree(MPI_Comm comm) : communicator(comm), cascade(MIN_NODE_SZ*MIN_NODE_SZ*2, comm)
+{
   treeCascade.reserve(cascade.num_levels);
   for (int lev = 0; lev < cascade.num_levels; lev++)