Changeset 1520 for XIOS/dev/branch_openmp/extern/src_ep_dev/ep_intercomm.cpp

Timestamp:

06/04/18 19:25:08 (6 years ago)

Author:

yushan

Message:

save dev. TO DO : test with xios

File:

• XIOS/dev/branch_openmp/extern/src_ep_dev/ep_intercomm.cpp (modified) (7 diffs)

XIOS/dev/branch_openmp/extern/src_ep_dev/ep_intercomm.cpp

@@ -10 +10 @@
   int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
   {
-    assert(local_comm.is_ep);
+    assert(local_comm->is_ep);
 
     int ep_rank, ep_rank_loc, mpi_rank;
     int ep_size, num_ep, mpi_size;
 
-    ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
-    ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
-    mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
-    ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
-    num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
-    mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
+    ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second;
 
 
     MPI_Barrier(local_comm);
+
+    int leader_ranks_in_peer[3];  // local_leader_rank_in_peer
+                                  // remote_leader_rank_in_peer
+                                  // size of peer
+
+    if(ep_rank == local_leader)
+    {
+      MPI_Comm_rank(peer_comm, &leader_ranks_in_peer[0]);
+      leader_ranks_in_peer[1] = remote_leader;
+      MPI_Comm_size(peer_comm, &leader_ranks_in_peer[2]);
+    }
+
+    MPI_Bcast(leader_ranks_in_peer, 3, MPI_INT, local_leader, local_comm);
+
+    if(leader_ranks_in_peer[0] != leader_ranks_in_peer[2])
+    {
+      Debug("calling MPI_Intercomm_create_kernel\n");
+      return MPI_Intercomm_create_kernel(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);
+    }
+
+    else
+    {
+      if(leader_ranks_in_peer[2] == 1)
+      {
+        Debug("calling MPI_Intercomm_create_unique\n");
+        return MPI_Intercomm_create_unique_leader(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm); 
+        
+      }
+      else
+      {
+        Debug("calling MPI_Intercomm_create_world\n");
+        return MPI_Intercomm_create_from_world(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);   
+      }
+      
+    }
+
+
 
 
@@ -61 +98 @@
       if( leader_ranks[1] * leader_ranks[4] == 1)
       {
-        if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_unique_leader\n");
-        local_comm.ep_comm_ptr->comm_label = -99;
+        if(ep_rank == local_leader) printf("calling MPI_Intercomm_create_unique_leader\n");
+        local_comm->ep_comm_ptr->comm_label = -99;
 
         return MPI_Intercomm_create_unique_leader(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);
@@ -77 +114 @@
             // change leader
             is_decider = true;
-            int target = local_comm.rank_map->at(local_leader).second;
+            int target = local_comm->ep_rank_map->at(local_leader).second;
             {
               for(int i=0; i<ep_size; i++)
               {
-                if(local_comm.rank_map->at(i).second != target && local_comm.rank_map->at(i).first == 0)
+                if(local_comm->ep_rank_map->at(i).second != target && local_comm->ep_rank_map->at(i).first == 0)
                 {
                   new_local_leader = i;
@@ -100 +137 @@
             // change leader
             is_decider = true;
-            int target = local_comm.rank_map->at(local_leader).second;
+            int target = local_comm->ep_rank_map->at(local_leader).second;
             {
               for(int i=0; i<ep_size; i++)
               {
-                if(local_comm.rank_map->at(i).second != target && local_comm.rank_map->at(i).first == 0)
+                if(local_comm->ep_rank_map->at(i).second != target && local_comm->ep_rank_map->at(i).first == 0)
                 {
                   new_local_leader = i;
@@ -146 +183 @@
         if(ep_rank == new_local_leader)
         {
-          ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD.mpi_comm), &leader_in_world[0]);
+          ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &leader_in_world[0]);
         }
 
@@ -165 +202 @@
         MPI_Bcast(&leader_in_world[1], 1, MPI_INT, local_leader, local_comm);
 
-        local_comm.ep_comm_ptr->comm_label = tag;
-
-        if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_from_world\n");
-
-        return MPI_Intercomm_create_from_world(local_comm, new_local_leader, MPI_COMM_WORLD.mpi_comm, leader_in_world[1], new_tag_in_world, newintercomm);
+        local_comm->ep_comm_ptr->comm_label = tag;
+
+        if(ep_rank == local_leader) printf("calling MPI_Intercomm_create_from_world\n");
+
+        return MPI_Intercomm_create_from_world(local_comm, new_local_leader, MPI_COMM_WORLD->mpi_comm, leader_in_world[1], new_tag_in_world, newintercomm);
         
       }
     }
 
-    if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_kernel\n");
-
-    return MPI_Intercomm_create_kernel(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);
+    
 
   }
@@ -183 +218 @@
   {
     *flag = false;
-    if(comm.is_ep)
-    {
-      *flag = comm.is_intercomm;
+    if(comm->is_ep)
+    {
+      *flag = comm->is_intercomm;
       return 0;
     } 
-    else if(comm.mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm))
-    {
-      ::MPI_Comm mpi_comm = to_mpi_comm(comm.mpi_comm);
+    else if(comm->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm))
+    {
+      ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm);
       
       ::MPI_Comm_test_inter(mpi_comm, flag);