Changeset 1525 for XIOS/dev

Timestamp:

06/07/18 19:39:07 (6 years ago)

Author:

yushan

Message:

save dev : intercomm_create via intracomm. To do : modif related intercomm communication routines, barrier, free, rank, size...

Location:

XIOS/dev/branch_openmp/extern/ep_dev

Files:

• ep_barrier.cpp (modified) (1 diff)
• ep_comm.hpp (modified) (2 diffs)
• ep_free.cpp (modified) (1 diff)
• ep_init.cpp (modified) (2 diffs)
• ep_intercomm.cpp (modified) (3 diffs)
• ep_intercomm.hpp (modified) (1 diff)
• ep_lib.cpp (modified) (3 diffs)
• ep_lib.hpp (modified) (2 diffs)
• ep_lib_collective.hpp (modified) (2 diffs)
• ep_size.cpp (modified) (2 diffs)
• ep_type.hpp (modified) (1 diff)
• main.cpp (modified) (12 diffs)

XIOS/dev/branch_openmp/extern/ep_dev/ep_barrier.cpp

@@ -22 +22 @@
 
   int MPI_Barrier_intracomm(MPI_Comm comm)
+  {
+    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;
+
+    MPI_Barrier_local(comm);
+
+    if(ep_rank_loc == 0)
+    {
+      ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm);
+
+      ::MPI_Barrier(mpi_comm);
+    }
+
+    MPI_Barrier_local(comm);
+
+    return 0;
+  }
+
+  int MPI_Barrier2(MPI_Comm comm)
   {
     int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;

XIOS/dev/branch_openmp/extern/ep_dev/ep_comm.hpp

@@ -10 +10 @@
 typedef std::vector< std::pair<int, int> > RANK_MAP;  // at(ep_rank) = <ep_rank_local, mpi_rank>
 typedef std::map<int, std::pair<int, int> > EP_RANK_MAP;  // key(ep_rank) = <ep_rank_local, mpi_rank>
-//typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP;
 typedef std::list<ep_lib::MPI_Message > Message_list;
+typedef std::map<int, int > INTER_RANK_MAP;
 
 namespace ep_lib
@@ -54 +54 @@
     EP_RANK_MAP   *ep_rank_map;      // for intercomm : = ep_rank_map of newcomm
     
-
+    INTER_RANK_MAP *inter_rank_map;
 
     void* mpi_comm;

XIOS/dev/branch_openmp/extern/ep_dev/ep_free.cpp

@@ -103 +103 @@
 
 
-
+  int MPI_Comm_free2(MPI_Comm *comm)
+  {
+    Debug("MPI_Comm_free with EP_intracomm\n");
+
+    int ep_rank_loc, num_ep;
+
+    ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first;
+    num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second;
+
+    MPI_Barrier2(*comm);
+
+    if(ep_rank_loc == 0)
+    {
+
+#ifdef _showinfo
+      printf("delete (*comm)->my_buffer = %p\n", (*comm)->my_buffer);
+#endif
+      delete (*comm)->my_buffer;
+
+
+      
+#ifdef _showinfo        
+      printf("delete (*comm)->ep_barrier = %p\n", (*comm)->ep_barrier);
+#endif
+      delete (*comm)->ep_barrier;
+      
+
+
+      (*comm)->ep_rank_map->clear();
+#ifdef _showinfo
+      printf("delete (*comm)->ep_rank_map = %p\n", (*comm)->ep_rank_map);
+#endif
+      delete (*comm)->ep_rank_map;
+      
+
+      for(int i=0; i<num_ep; i++)
+      {
+        if((*comm)->is_intercomm)
+        {
+          (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map->clear();
+#ifdef _showinfo
+          printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->inter_rank_map = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map);
+#endif
+          delete (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map;
+        }
+
+        (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue->clear();
+#ifdef _showinfo
+        printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr->message_queue = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue);
+#endif
+        delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue;
+          
+
+#ifdef _showinfo
+        printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr);
+#endif
+        delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr;
+
+
+#ifdef _showinfo
+        printf("delete (*comm)->ep_comm_ptr->comm_list[%d] = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]);
+#endif
+        delete (*comm)->ep_comm_ptr->comm_list[i];
+      }
+
+#ifdef _showinfo
+      printf("delete (*comm)->mpi_comm = %p\n", (*comm)->mpi_comm);
+#endif
+      ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->mpi_comm));
+      
+#ifdef _showinfo
+      printf("delete (*comm)->ep_comm_ptr->comm_list = %p\n", (*comm)->ep_comm_ptr->comm_list);
+#endif
+      delete[] (*comm)->ep_comm_ptr->comm_list;
+    }
+  }
 
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_init.cpp

@@ -2 +2 @@
 #include <mpi.h>
 #include "ep_declaration.hpp"
+#include "ep_mpi.hpp"
 
+extern MPI_Group MPI_GROUP_WORLD;
 
 namespace ep_lib
@@ -16 +18 @@
     {
       ::MPI_Init_thread(argc, argv, required, provided);
+      ::MPI_Comm_group(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &MPI_GROUP_WORLD);
     }
   }

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.cpp

@@ -6 +6 @@
 using namespace std;
 
+extern std::map<std::pair<int, int>, MPI_Group* > * tag_group_map;
+
+extern std::map<int, std::pair<ep_lib::MPI_Comm*, std::pair<int, int> > > * tag_comm_map;
+
+extern MPI_Group MPI_GROUP_WORLD;
+
 namespace ep_lib
 {
-  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  int MPI_Intercomm_create2(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
   {
     assert(local_comm->is_ep);
@@ -22 +28 @@
     mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second;
 
+    int world_rank_and_num_ep[2];
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank_and_num_ep[0]);
+    world_rank_and_num_ep[1] = num_ep;
+
+    int remote_mpi_size;
+    int remote_ep_size;
+
+    int *local_world_rank_and_num_ep;
+    int *remote_world_rank_and_num_ep;
+    int *summed_world_rank_and_num_ep;
+
+
+    bool is_leader = ep_rank==local_leader? true : false;
+    bool is_local_leader = is_leader? true: (ep_rank_loc==0 && mpi_rank!=local_comm->ep_rank_map->at(local_leader).second ? true : false);
+    bool priority;
+
+    if(is_leader)
+    {
+      int leader_mpi_rank_in_peer;
+      MPI_Comm_rank(peer_comm, &leader_mpi_rank_in_peer);
+      if(leader_mpi_rank_in_peer == remote_leader) 
+      {
+        printf("same leader in peer_comm\n");
+        exit(1);
+      }
+      priority = leader_mpi_rank_in_peer<remote_leader? true : false;
+    }
+
+
+    MPI_Bcast(&priority, 1, MPI_INT, local_leader, local_comm);
+
+    if(is_local_leader)
+    {
+      local_world_rank_and_num_ep = new int[2*mpi_size];
+      ::MPI_Allgather(world_rank_and_num_ep, 2, to_mpi_type(MPI_INT), local_world_rank_and_num_ep, 2, to_mpi_type(MPI_INT), to_mpi_comm(local_comm->mpi_comm));
+    }
+
+    
+    
+    if(is_leader)
+    {
+      MPI_Request request;
+      MPI_Status status;
+
+      if(priority)
+      {
+        MPI_Isend(&mpi_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+       
+        MPI_Irecv(&remote_mpi_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+
+        MPI_Isend(&ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+       
+        MPI_Irecv(&remote_ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+      else
+      {
+        MPI_Irecv(&remote_mpi_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+          
+        MPI_Isend(&mpi_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+
+        MPI_Irecv(&remote_ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+          
+        MPI_Isend(&ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+    }
+
+    MPI_Bcast(&remote_mpi_size, 1, MPI_INT, local_leader, local_comm);
+    MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm);
+
+    remote_world_rank_and_num_ep = new int[2*remote_mpi_size];
+
+
+    if(is_leader)
+    {
+      MPI_Request request;
+      MPI_Status status;
+
+      if(priority)
+      {
+        MPI_Isend(local_world_rank_and_num_ep, 2*mpi_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+       
+        MPI_Irecv(remote_world_rank_and_num_ep, 2*remote_mpi_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+      else
+      {
+        MPI_Irecv(remote_world_rank_and_num_ep, 2*remote_mpi_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+          
+        MPI_Isend(local_world_rank_and_num_ep, 2*mpi_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+    }
+
+    
+    MPI_Bcast(remote_world_rank_and_num_ep, 2*remote_mpi_size, MPI_INT, local_leader, local_comm);
+    
+
+
+    bool is_new_leader  = is_local_leader;
+
+    if(is_local_leader && !priority)
+    {
+      for(int i=0; i<remote_mpi_size; i++)
+      {
+        if(world_rank_and_num_ep[0] == remote_world_rank_and_num_ep[2*i])
+        {
+          is_new_leader = false;
+          break;
+        }
+      }
+    } 
+    
+
+    ::MPI_Group *empty_group;
+    ::MPI_Group *local_group;
+    ::MPI_Group union_group;
+
+    if(is_local_leader)
+    {
+
+      int *ranks = new int[mpi_size];
+      for(int i=0; i<mpi_size; i++)
+      {
+        ranks[i] = local_world_rank_and_num_ep[2*i];
+      }
+
+      local_group = new ::MPI_Group;
+      ::MPI_Group_incl(MPI_GROUP_WORLD, mpi_size, ranks, local_group);
+
+      delete[] ranks;
+
+      
+      #pragma omp flush
+      #pragma omp critical (write_to_tag_group_map)
+      {
+        if(tag_group_map == 0) tag_group_map = new map< std::pair<int, int>, ::MPI_Group * >;
+        
+        tag_group_map->insert(std::make_pair(std::make_pair(tag, priority? 1 : 2), local_group));       
+      }
+    }
+
+    MPI_Barrier(local_comm);
+
+    if(is_leader)
+    {
+      MPI_Request request;
+      MPI_Status status;
+
+      int send_signal=0;
+      int recv_signal;
+
+      if(priority)
+      {
+        MPI_Isend(&send_signal, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+       
+        MPI_Irecv(&recv_signal, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+      else
+      {
+        MPI_Irecv(&recv_signal, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+          
+        MPI_Isend(&send_signal, 1, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+    }
+
+
+    MPI_Barrier(local_comm);
+
+    if(is_new_leader)
+    {
+      ::MPI_Group *group1;
+      ::MPI_Group *group2;
+
+      empty_group = new ::MPI_Group;
+      *empty_group = MPI_GROUP_EMPTY;
+
+      #pragma omp flush
+      #pragma omp critical (read_from_tag_group_map)
+      {
+        group1 = tag_group_map->find(make_pair(tag, 1)) != tag_group_map->end()? tag_group_map->at(std::make_pair(tag, 1)) : empty_group;
+        group2 = tag_group_map->find(make_pair(tag, 2)) != tag_group_map->end()? tag_group_map->at(std::make_pair(tag, 2)) : empty_group;
+      }
+
+      
+#ifdef _showinfo
+
+      int group1_rank, group1_size;
+      int group2_rank, group2_size;
+      ::MPI_Group_rank(*group1, &group1_rank);
+      ::MPI_Group_size(*group1, &group1_size);
+      ::MPI_Group_rank(*group2, &group2_rank);
+      ::MPI_Group_size(*group2, &group2_size);
+
+#endif
+
+      ::MPI_Group_union(*group1, *group2, &union_group);
+
+
+      #pragma omp critical (read_from_tag_group_map)
+      {
+        tag_group_map->erase(make_pair(tag, 1));
+        tag_group_map->erase(make_pair(tag, 2));
+      }
+
+#ifdef _showinfo
+
+      int group_rank, group_size;
+      ::MPI_Group_rank(union_group, &group_rank);
+      ::MPI_Group_size(union_group, &group_size);
+      printf("rank = %d : map = %p, group1_rank/size = %d/%d, group2_rank/size = %d/%d, union_rank/size = %d/%d\n", ep_rank, tag_group_map, group1_rank, group1_size, group2_rank, group2_size, group_rank, group_size);
+#endif
+
+    }
+
+    int summed_world_rank_and_num_ep_size=mpi_size;
+    summed_world_rank_and_num_ep = new int[2*(mpi_size+remote_mpi_size)];
+
+
+    if(is_leader)
+    {
+      
+      for(int i=0; i<mpi_size; i++)
+      {
+        summed_world_rank_and_num_ep[2*i] = local_world_rank_and_num_ep[2*i];
+        summed_world_rank_and_num_ep[2*i+1] = local_world_rank_and_num_ep[2*i+1]; 
+      }
+
+      for(int i=0; i<remote_mpi_size; i++)
+      {
+        bool found=false;
+        for(int j=0; j<mpi_size; j++)
+        {
+          if(remote_world_rank_and_num_ep[2*i] == local_world_rank_and_num_ep[2*j])
+          {
+            found=true;
+            summed_world_rank_and_num_ep[2*j+1] += remote_world_rank_and_num_ep[2*i+1]; 
+          }
+        }
+        if(!found)
+        {
+          summed_world_rank_and_num_ep[2*summed_world_rank_and_num_ep_size] = remote_world_rank_and_num_ep[2*i];
+          summed_world_rank_and_num_ep[2*summed_world_rank_and_num_ep_size+1] = remote_world_rank_and_num_ep[2*i+1];
+          summed_world_rank_and_num_ep_size++;
+        }
+
+      }
+    }
+
+    MPI_Bcast(&summed_world_rank_and_num_ep_size, 1, MPI_INT, local_leader, local_comm);
+
+    MPI_Bcast(summed_world_rank_and_num_ep, 2*summed_world_rank_and_num_ep_size, MPI_INT, local_leader, local_comm);
+    
+    
+
+    int remote_num_ep;
+    for(int i=0; i<remote_mpi_size; i++)
+    {
+      if(remote_world_rank_and_num_ep[2*i] == world_rank_and_num_ep[0])
+      {
+        remote_num_ep = remote_world_rank_and_num_ep[2*i+1];
+        break;
+      }
+    }
+
+    int new_ep_rank_loc = priority? ep_rank_loc : ep_rank_loc+remote_num_ep;
+
+#ifdef _showinfo
+    printf("rank = %d, priority = %d, remote_num_ep = %d, new_ep_rank_loc = %d\n", ep_rank, priority, remote_num_ep, new_ep_rank_loc);
+#endif
+    
+    if(is_new_leader)
+    {
+      int new_num_ep;
+      for(int i=0; i<summed_world_rank_and_num_ep_size; i++)
+      {
+        if(summed_world_rank_and_num_ep[2*i] == world_rank_and_num_ep[0])
+        {
+          new_num_ep = summed_world_rank_and_num_ep[2*i+1];
+          break;
+        }
+      }
+
+      ::MPI_Comm mpi_comm;
+      ::MPI_Comm_create_group(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), union_group, tag, &mpi_comm);
+
+
+      MPI_Comm *ep_comm;
+      MPI_Info info;
+      MPI_Comm_create_endpoints(&mpi_comm, new_num_ep, info, ep_comm);
+
+      #pragma omp critical (write_to_tag_comm_map)
+      {
+        if(tag_comm_map == 0) tag_comm_map = new std::map<int, std::pair<ep_lib::MPI_Comm*, std::pair<int, int> > >;
+        tag_comm_map->insert(std::make_pair(tag, std::make_pair(ep_comm, std::make_pair(new_num_ep, 0))));
+      }
+      #pragma omp flush
+    }
+
+
+    bool found=false;
+    while(!found)
+    {
+      #pragma omp flush
+      #pragma omp critical (read_from_tag_comm_map)
+      {
+        if(tag_comm_map!=0)
+        {
+          if(tag_comm_map->find(tag) != tag_comm_map->end())
+          {              
+            *newintercomm = tag_comm_map->at(tag).first[new_ep_rank_loc];
+            
+            tag_comm_map->at(tag).second.second++;
+            if(tag_comm_map->at(tag).second.second == tag_comm_map->at(tag).second.first)
+            {
+              tag_comm_map->erase(tag_comm_map->find(tag));
+            }
+
+            found=true;
+          }
+        }
+      }  
+    }
+
+    (*newintercomm)->is_intercomm = true;
+
+    
+
+
+    (*newintercomm)->inter_rank_map = new INTER_RANK_MAP;
+
+    int rank_info[2];
+    rank_info[0] = ep_rank;
+    rank_info[1] = (*newintercomm)->ep_comm_ptr->size_rank_info[0].first;
+
+#ifdef _showinfo
+    printf("priority = %d, ep_rank = %d, new_ep_rank = %d\n", priority, rank_info[0], rank_info[1]);
+#endif
+
+    int *local_rank_info = new int[2*ep_size];
+    int *remote_rank_info = new int[2*remote_ep_size];
+
+    MPI_Allgather(rank_info, 2, MPI_INT, local_rank_info, 2, MPI_INT, local_comm);
+
+    if(is_leader)
+    {
+      MPI_Request request;
+      MPI_Status status;
+
+      if(priority)
+      {
+        MPI_Isend(local_rank_info, 2*ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+       
+        MPI_Irecv(remote_rank_info, 2*remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+      else
+      {
+        MPI_Irecv(remote_rank_info, 2*remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+          
+        MPI_Isend(local_rank_info, 2*ep_size, MPI_INT, remote_leader, tag, peer_comm, &request);
+        MPI_Wait(&request, &status);
+      }
+    }
+
+    MPI_Bcast(remote_rank_info, 2*remote_ep_size, MPI_INT, local_leader, local_comm);
+
+    for(int i=0; i<remote_ep_size; i++)
+    {
+      (*newintercomm)->inter_rank_map->insert(make_pair(remote_rank_info[2*i], remote_rank_info[2*i+1]));
+    }
+
+#ifdef _showinfo
+    if(ep_rank==4 && !priority)
+    {
+      for(std::map<int, int > :: iterator it=(*newintercomm)->inter_rank_map->begin(); it != (*newintercomm)->inter_rank_map->end(); it++)
+      {
+        printf("inter_rank_map[%d] = %d\n", it->first, it->second);
+      }
+    }
+#endif
+
+    (*newintercomm)->ep_comm_ptr->size_rank_info[0] = local_comm->ep_comm_ptr->size_rank_info[0];
+
+    if(is_local_leader)
+    {
+      delete[] local_world_rank_and_num_ep;  
+    
+      MPI_Group_free(local_group);
+      delete local_group;
+    }
+
+    if(is_new_leader)
+    {
+      MPI_Group_free(&union_group);
+      delete empty_group;
+    }
+
+    delete[] remote_world_rank_and_num_ep;
+    delete[] summed_world_rank_and_num_ep;
+    delete[] local_rank_info;
+    delete[] remote_rank_info;
+
+
+  }
+
+
+  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+    assert(local_comm->is_ep);
+
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second;
+
 
     MPI_Barrier(local_comm);
@@ -60 +502 @@
     }
 
-
-
-
-
-    int leader_ranks[6]; //! 0: rank in world, 1: mpi_size, 2: rank_in_peer.
-                         //! 3, 4, 5 : remote
-
-    bool is_decider = false;
-
-
-    if(ep_rank == local_leader)
-    {
-      MPI_Comm_rank(MPI_COMM_WORLD, &leader_ranks[0]);
-
-      leader_ranks[1] = mpi_size;
-      MPI_Comm_rank(peer_comm, &leader_ranks[2]);
-
-      MPI_Request request[2];
-      MPI_Status status[2];
-
-      MPI_Isend(&leader_ranks[0], 3, MPI_INT, remote_leader, tag, peer_comm, &request[0]);
-      MPI_Irecv(&leader_ranks[3], 3, MPI_INT, remote_leader, tag, peer_comm, &request[1]);
-
-      MPI_Waitall(2, request, status);
-
-    }
-
-
-    MPI_Bcast(leader_ranks, 6, MPI_INT, local_leader, local_comm);
-
-    
-    MPI_Barrier(local_comm);
-    
-
-    if(leader_ranks[0] == leader_ranks[3])
-    {
-      if( leader_ranks[1] * leader_ranks[4] == 1)
-      {
-        if(ep_rank == local_leader) printf("calling MPI_Intercomm_create_unique_leader\n");
-        local_comm->ep_comm_ptr->comm_label = -99;
-
-        return MPI_Intercomm_create_unique_leader(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);
-      }
-      else // leader_ranks[1] * leader_ranks[4] != 1
-      {
-        // change leader
-        int new_local_leader;
-
-        if(leader_ranks[2] < leader_ranks[5])
-        {
-          if(leader_ranks[1] > 1) //! change leader
-          {
-            // change leader
-            is_decider = true;
-            int target = local_comm->ep_rank_map->at(local_leader).second;
-            {
-              for(int i=0; i<ep_size; i++)
-              {
-                if(local_comm->ep_rank_map->at(i).second != target && local_comm->ep_rank_map->at(i).first == 0)
-                {
-                  new_local_leader = i;
-                  break;
-                }
-              }
-            }
-          }
-          else
-          {
-            new_local_leader = local_leader;
-          }
-        }
-        else
-        {
-          if(leader_ranks[4] == 1)
-          {
-            // change leader
-            is_decider = true;
-            int target = local_comm->ep_rank_map->at(local_leader).second;
-            {
-              for(int i=0; i<ep_size; i++)
-              {
-                if(local_comm->ep_rank_map->at(i).second != target && local_comm->ep_rank_map->at(i).first == 0)
-                {
-                  new_local_leader = i;
-                  break;
-                }
-              }
-            }
-          }
-          else
-          {
-            new_local_leader = local_leader;
-          }
-        }
-
-
-        int new_tag_in_world;
-
-        int leader_in_world[2];
-
-
-        if(is_decider)
-        {
-          if(ep_rank == new_local_leader)
-          {
-            new_tag_in_world = TAG++;
-          }
-          MPI_Bcast(&new_tag_in_world, 1, MPI_INT, new_local_leader, local_comm);
-          if(ep_rank == local_leader) MPI_Send(&new_tag_in_world, 1, MPI_INT, remote_leader, tag, peer_comm);
-        }
-        else
-        {
-          if(ep_rank == local_leader)
-          {
-            MPI_Status status;
-            MPI_Recv(&new_tag_in_world, 1, MPI_INT, remote_leader, tag, peer_comm, &status);
-          }
-          MPI_Bcast(&new_tag_in_world, 1, MPI_INT, new_local_leader, local_comm);
-        }
-
-
-        if(ep_rank == new_local_leader)
-        {
-          ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &leader_in_world[0]);
-        }
-
-        MPI_Bcast(&leader_in_world[0], 1, MPI_INT, new_local_leader, local_comm);
-
-
-        if(ep_rank == local_leader)
-        {
-          MPI_Request request[2];
-          MPI_Status status[2];
-
-          MPI_Isend(&leader_in_world[0], 1, MPI_INT, remote_leader, tag, peer_comm, &request[0]);
-          MPI_Irecv(&leader_in_world[1], 1, MPI_INT, remote_leader, tag, peer_comm, &request[1]);
-
-          MPI_Waitall(2, request, status);
-        }
-
-        MPI_Bcast(&leader_in_world[1], 1, MPI_INT, local_leader, local_comm);
-
-        local_comm->ep_comm_ptr->comm_label = tag;
-
-        if(ep_rank == local_leader) printf("calling MPI_Intercomm_create_from_world\n");
-
-        return MPI_Intercomm_create_from_world(local_comm, new_local_leader, MPI_COMM_WORLD->mpi_comm, leader_in_world[1], new_tag_in_world, newintercomm);
-        
-      }
-    }
-
-    
-
   }
 
   int MPI_Comm_test_inter(MPI_Comm comm, int *flag)
   {
-    *flag = false;
     if(comm->is_ep)
     {
-      *flag = comm->is_intercomm;
-      return 0;
+      return *flag = comm->is_intercomm;
     } 
-    else if(comm->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm))
-    {
-      ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm);
-      
-      ::MPI_Comm_test_inter(mpi_comm, flag);
-      return 0;  
-    }
-    return 0;
+    else
+    {
+      return ::MPI_Comm_test_inter(to_mpi_comm(comm->mpi_comm), flag);
+    }
   }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.hpp

@@ -6 +6 @@
 typedef std::map<int, std::pair<int, int> > EP_RANK_MAP;  // key(ep_rank) = <ep_rank_local, mpi_rank>
 typedef std::map<int, std::pair< int, std::pair<int, int> > > INTERCOMM_RANK_MAP;
+typedef std::map<int, int > INTER_RANK_MAP;
 
 namespace ep_lib

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.cpp

@@ -12 +12 @@
 
 
+std::map<std::pair<int, int>, MPI_Group* > * tag_group_map = 0;
+
+std::map<int, std::pair<ep_lib::MPI_Comm*, std::pair<int, int> > > * tag_comm_map = 0;
+
+
+MPI_Group MPI_GROUP_WORLD;
 
 namespace ep_lib
@@ -27 +33 @@
   int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank)
   {
-    if(comm->is_intercomm)
+    /*if(comm->is_intercomm)
     {
       for(std::map<int, std::pair< int, std::pair<int, int> > >::iterator it = comm->ep_comm_ptr->intercomm->intercomm_rank_map->begin(); it != comm->ep_comm_ptr->intercomm->intercomm_rank_map->end(); it++)
@@ -40 +46 @@
       int err;
       return MPI_Abort(comm, err);
-    }
+    }*/
     
     for(std::map<int, std::pair<int, int> >::iterator it = comm->ep_rank_map->begin(); it != comm->ep_rank_map->end(); it++)

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.hpp

@@ -30 +30 @@
 
   int MPI_Comm_free(MPI_Comm* comm);
+  int MPI_Comm_free2(MPI_Comm* comm);
 
   int MPI_Finalize();
@@ -40 +41 @@
   
   int MPI_Comm_remote_size(MPI_Comm comm, int *size);
+  int MPI_Comm_remote_size2(MPI_Comm comm, int *size);
+
 
   int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm);

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_collective.hpp

@@ -10 +10 @@
 
   int MPI_Barrier(MPI_Comm comm);
+  int MPI_Barrier2(MPI_Comm comm);
   
 
@@ -46 +47 @@
 
   int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
+  int MPI_Intercomm_create2(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);
 
   int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);

XIOS/dev/branch_openmp/extern/ep_dev/ep_size.cpp

@@ -36 +36 @@
   }
 
+  int MPI_Comm_remote_size2(MPI_Comm comm, int *size)
+  {
+    if(comm->is_ep)
+    {
+      if(comm->is_intercomm)
+      {
+        Debug("MPI_Comm_remote_size with EP_intercomm");
+        return *size = comm->inter_rank_map->size();
+      }
+      return *size=0;
+    }
+    return MPI_Comm_remote_size_mpi(comm, size);
+  }
+
 
   int MPI_Comm_size_mpi(MPI_Comm comm, int* size)
@@ -49 +63 @@
   }
 
-
 }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_type.hpp

@@ -86 +86 @@
   static std::map<std::pair<int, int>, MPI_Comm >  fc_comm_map;
 
+  
+
 
 }

XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

@@ -717 +717 @@
       MPI_Barrier(comm);
       
-      if(rank == 0) printf("            \t tMPI_Comm_split \t OK\n");
+      if(rank == 0) printf("            \t MPI_Comm_split \t OK\n");
       
       MPI_Barrier(comm);
@@ -746 +746 @@
       MPI_Barrier(split_comm2);
       MPI_Comm_free(&split_comm2);
+
       
       
@@ -825 +826 @@
       MPI_Barrier(comm);
       MPI_Barrier(comm);
+
 
       if(color==2 && split_rank==0)
@@ -878 +880 @@
 
 
-      //printf("rank = %d, high = %d, inter_comm_dup_size = %d, inter_comm_dup_remote_size = %d\n", rank, high, inter_comm_dup_size, inter_comm_dup_remote_size);
+      printf("rank = %d, high = %d, inter_comm_dup_size = %d, inter_comm_dup_remote_size = %d\n", rank, high, inter_comm_dup_size, inter_comm_dup_remote_size);
 
       MPI_Comm inter_comm_dup_merged;
@@ -892 +894 @@
       //printf("rank = %d, inter_comm_dup_merged_rank = %d, inter_comm_dup_merged_size = %d\n", rank, inter_comm_dup_merged_rank, inter_comm_dup_merged_size);
       
-      // TEST OF TESTALL
-      
-        int merged_size, merged_rank;
-        MPI_Comm_rank(inter_comm_dup_merged, &merged_rank);
-        MPI_Comm_size(inter_comm_dup_merged, &merged_size);
-        
-        int left = merged_rank>0? merged_rank-1 : merged_size-1;
-        int right = (merged_rank+1) % merged_size;
-        
-        //printf("rank = %d, left = %d, right = %d\n", merged_rank, left, right);
-        
-        int N=10000;
-        int *left_buf = new int[N];
-        int *right_buf = new int[N];
-                
-        for(int i=0; i<N; i++)
-        {
-          right_buf[i] = merged_rank*100000+i;
-        }
-        
-        MPI_Request request[2];
-        MPI_Status status[2];
-        
-        MPI_Irecv(left_buf, N, MPI_INT, left, 0, inter_comm_dup_merged, &request[0]);
-        MPI_Isend(right_buf, N, MPI_INT, right, 0, inter_comm_dup_merged, &request[1]);
-        
-        
-        int flag;
-        MPI_Testall(2, request, &flag, status);
-      
-        printf("rank = %d, flag = %d\n", merged_rank, flag);
-        
-        while(!flag) MPI_Testall(2, request, &flag, status);
-        //if(!flag)  MPI_Waitall(2, request, status);
-        
-        int recv_count;
-        MPI_Get_count(&status[0], MPI_INT, &recv_count);
-        
-        MPI_Barrier(inter_comm_dup_merged);
-        
-        printf("rank = %d, recv_count = %d, left_buf[5-10] = %d\t%d\t%d\t%d\t%d\t%d\n", merged_rank, recv_count, left_buf[5], left_buf[6], left_buf[7], left_buf[8], left_buf[9], left_buf[10]);
-                        
-        delete[] left_buf;
-        delete[] right_buf;
-      
-      
+      
+      MPI_Barrier(inter_comm_dup_merged);
 
       // TEST OF GATHER
       {
+        
+
         int gather_root = 99;
   
@@ -976 +936 @@
       }
 
+      MPI_Barrier(inter_comm_dup_merged);
+
       // TEST OF ALLREDUCE
       {
@@ -1009 +971 @@
       }
 
+      MPI_Barrier(inter_comm_dup_merged);
+
       // TEST OF EXSCAN
-      /*{
+      {
  
         std::vector<int>sendbuf(2, inter_comm_dup_merged_rank);
@@ -1029 +993 @@
           exscan_test = 0; 
         }
-
-        //printf(" ID=%d : sendbuf = (%d, %d), recvbuf = (%d, %d), exscan_test = %d \n", inter_comm_dup_merged_rank, sendbuf[0], sendbuf[1], recvbuf[0], recvbuf[1], exscan_test); 
         
         int exscan_result;
@@ -1037 +999 @@
         if(inter_comm_dup_merged_rank == 0 && exscan_result)  printf("            \t test MPI_Exscan for merged comm \t OK\n");
         if(inter_comm_dup_merged_rank == 0 && !exscan_result) printf("            \t test MPI_Exscan for merged comm \t FAILED %d\n", exscan_result);
-      }*/
+      }
+
+      MPI_Barrier(inter_comm_dup_merged);
 
       // TEST OF SCATTERV
@@ -1067 +1031 @@
         MPI_Scatterv(sendbuf.data(), sendcounts.data(), displs.data(), MPI_INT, recvbuf.data(), 2, MPI_INT, scatterv_root, inter_comm_dup_merged);
 
-        //printf("ID = %d : %d %d\n", inter_comm_dup_merged_rank, recvbuf[0], recvbuf[1]);
-
         int scatterv_test = 1;
 
@@ -1085 +1047 @@
       }
 
-
-
+      MPI_Barrier(inter_comm_dup_merged);
+
+      // TEST OF TESTALL
+      {
+        int merged_size, merged_rank;
+        MPI_Comm_rank(inter_comm_dup_merged, &merged_rank);
+        MPI_Comm_size(inter_comm_dup_merged, &merged_size);
+        
+        int left = merged_rank>0? merged_rank-1 : merged_size-1;
+        int right = (merged_rank+1) % merged_size;
+                
+        int N=10000;
+        int *left_buf = new int[N];
+        int *right_buf = new int[N];
+                
+        for(int i=0; i<N; i++)
+        {
+          right_buf[i] = merged_rank*100000+i;
+        }
+        
+        MPI_Request request[2];
+        MPI_Status status[2];
+        
+        MPI_Irecv(left_buf, N, MPI_INT, left, 0, inter_comm_dup_merged, &request[0]);
+        MPI_Isend(right_buf, N, MPI_INT, right, 0, inter_comm_dup_merged, &request[1]);
+        
+        
+        int flag;
+        MPI_Testall(2, request, &flag, status);
+      
+        printf("rank = %d, flag = %d\n", merged_rank, flag);
+        
+        while(!flag) MPI_Testall(2, request, &flag, status);
+        
+        int recv_count;
+        MPI_Get_count(&status[0], MPI_INT, &recv_count);
+        
+        MPI_Barrier(inter_comm_dup_merged);
+        
+        printf("rank = %d, recv_count = %d, left_buf[5-10] = %d\t%d\t%d\t%d\t%d\t%d\n", merged_rank, recv_count, left_buf[5], left_buf[6], left_buf[7], left_buf[8], left_buf[9], left_buf[10]);
+                        
+        delete[] left_buf;
+        delete[] right_buf;
+      }
+      
+      MPI_Barrier(inter_comm_dup_merged);
+
+      // TEST WAITALL
+      {
+        int merged_rank;
+        int merged_size;
+
+        MPI_Comm_rank(inter_comm_dup_merged, &merged_rank);
+        MPI_Comm_size(inter_comm_dup_merged, &merged_size);
+
+        int left = merged_rank>0? merged_rank-1 : merged_size-1;
+        int right = (merged_rank+1)%merged_size;
+
+        printf("merged_rank = %d, left = %d, right = %d\n", merged_rank, left, right);
+
+        int NN=10000;
+
+        double *left_buf = new double[NN];
+        double *right_buf = new double[NN];
+
+        for(int i=0; i<NN; i++)
+        {
+          right_buf[i] = merged_rank*1000000 + i;
+        }
+
+        MPI_Request request[2];
+        MPI_Status  status[2];
+
+        MPI_Irecv(left_buf,  NN, MPI_DOUBLE, left,  0, inter_comm_dup_merged, &request[0]);
+        MPI_Isend(right_buf, NN, MPI_DOUBLE, right, 0, inter_comm_dup_merged, &request[1]);
+        
+
+        MPI_Waitall(2, request, status);
+
+        printf("merged_rank = %d, left_buf[0-4] = %lf\t%lf\t%lf\t%lf\t%lf\n", merged_rank, left_buf[0], left_buf[1], left_buf[2], left_buf[3], left_buf[4]);
+
+
+        delete left_buf;
+        delete right_buf;
+      }
 
       MPI_Barrier(comm);
@@ -1108 +1153 @@
       MPI_Comm_free(&inter_comm);
       
-
-      /*
-      MPI_Barrier(comm);
-      if(rank == 0) printf("\tMPI_Intercomm_create OK\n");
-      MPI_Barrier(comm);
-
-    
-
-      int high=color;
-      MPI_Comm intra_comm;
-      MPI_Intercomm_merge(inter_comm, high, &intra_comm);
-    
-      int intra_rank, intra_size;
-      MPI_Comm_rank(intra_comm, &intra_rank);
-      MPI_Comm_size(intra_comm, &intra_size);
-
-      MPI_Barrier(comm);
-      if(rank == 0) printf("\tMPI_Intercomm_merge OK\n");
-      MPI_Barrier(comm);
-    
-    
-
-      //check_test_gatherv(comm);
-      
-      MPI_Barrier(comm);
-      MPI_Comm_free(&inter_comm);
-      */
+    }
+
+    // TESE OF INTERCOMM_CREATE
+    {
+      MPI_Barrier(comm);
+      
+      int rank, size;
+      MPI_Comm_rank(comm, &rank);
+      MPI_Comm_size(comm, &size);
+
+      int config = 1;
+      if(config=1)
+      {
+        assert(size == 16 && omp_get_num_threads()==4);
+      }
+      
+      int tab_color[16] = {2, 2, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1}; // used for config 1
+      int tab_key[16]   = {2, 1, 4, 1, 0, 4, 3, 3, 4, 3, 4, 4, 4, 0, 0, 2}; // used for config 1
+
+      int color = tab_color[rank];
+      int key = tab_key[rank];
+      
+  
+      MPI_Comm split_comm;
+      MPI_Comm_split(comm, color, key, &split_comm);
+
+    
+      int split_rank, split_size;
+      MPI_Comm_rank(split_comm, &split_rank);
+      MPI_Comm_size(split_comm, &split_size);
+
+
+      
+
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+      
+      if(rank == 0) printf("            \t MPI_Comm_split \t OK\n");
+      
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+      
+            
+      int local_leader = 5;
+      int remote_leader = color==2? 5: 2; // used for config 1
+
+      MPI_Comm peer_comm = comm;
+
+      MPI_Comm inter_comm;
+      MPI_Intercomm_create2(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm);
+
+      int inter_rank;
+      int inter_size;
+      MPI_Comm_rank(inter_comm, &inter_rank);
+      MPI_Comm_size(inter_comm, &inter_size);
+
+      printf("split_rank = %d, inter_rank = %d / %d\n", split_rank, inter_rank, inter_size);
+
+      MPI_Barrier(split_comm);
+      MPI_Comm_free(&split_comm);
+
+      MPI_Barrier2(inter_comm);
+      MPI_Comm_free2(&inter_comm);
+      
     }