Changeset 1504

Timestamp:

05/28/18 23:25:35 (7 years ago)

Author:

yushan

Message:

MPI_split can deal with discontinuous ranking within a process

Location:

XIOS/dev/branch_openmp/extern/ep_dev

Files:

• ep_allgather.cpp (modified) (1 diff)
• ep_create.cpp (modified) (2 diffs)
• ep_lib_local.hpp (modified) (1 diff)
• ep_split.cpp (modified) (6 diffs)
• main.cpp (modified) (2 diffs)

XIOS/dev/branch_openmp/extern/ep_dev/ep_allgather.cpp

@@ -16 +16 @@
 namespace ep_lib
 {
+
+  int MPI_Allgather_local(const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm)
+  {
+    assert(valid_type(datatype));
+
+    ::MPI_Aint datasize, lb;
+    ::MPI_Type_get_extent(to_mpi_type(datatype), &lb, &datasize);
+
+    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;
+    int num_ep = comm->ep_comm_ptr->size_rank_info[1].second;
+
+    #pragma omp critical (write_buffer)
+    comm->my_buffer->void_buffer[ep_rank_loc] = const_cast< void* >(sendbuf);
+
+    MPI_Barrier_local(comm);
+
+    #pragma omp critical (read_buffer)
+    {
+      for(int i=0; i<num_ep; i++)
+        memcpy(recvbuf + datasize * i * count, comm->my_buffer->void_buffer[i], datasize * count);
+    }
+  }
 
   int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm)

XIOS/dev/branch_openmp/extern/ep_dev/ep_create.cpp

@@ -64 +64 @@
     out_comm_hdls[0]->my_buffer = new BUFFER;
 
-    //out_comm_hdls[0]->rank_map = new RANK_MAP;
-    //out_comm_hdls[0]->rank_map->resize(ep_size);
-
     out_comm_hdls[0]->ep_rank_map = new EP_RANK_MAP;
-    //out_comm_hdls[0]->ep_rank_map->resize(ep_size);
 
 
@@ -75 +71 @@
       out_comm_hdls[i]->ep_barrier = out_comm_hdls[0]->ep_barrier;
       out_comm_hdls[i]->my_buffer  = out_comm_hdls[0]->my_buffer;
-      //out_comm_hdls[i]->rank_map   = out_comm_hdls[0]->rank_map;
       out_comm_hdls[i]->ep_rank_map= out_comm_hdls[0]->ep_rank_map;
     }

XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_local.hpp

@@ -18 +18 @@
   
   int MPI_Gather_local      (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, int local_root, MPI_Comm comm);
+  
+  int MPI_Allgather_local      (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf, MPI_Comm comm);
 
   int MPI_Gatherv_local       (const void *sendbuf, int count, MPI_Datatype datatype, void *recvbuf,

XIOS/dev/branch_openmp/extern/ep_dev/ep_split.cpp

@@ -30 +30 @@
     }
     inout_vector->swap(out_vec);
+  }
+  
+  void vec_simplify(std::vector<int> *in_vector, std::vector<int> *out_vector)
+  {
+    int found=false;
+    for(std::vector<int>::iterator it_in = in_vector->begin() ; it_in != in_vector->end(); ++it_in)
+    {
+      for(std::vector<int>::iterator it = out_vector->begin() ; it != out_vector->end(); ++it)
+      {
+        if(*it_in == *it)
+        {
+          found=true;
+          break;
+        }
+        else found=false;
+      }
+      if(found == false)
+      {
+        out_vector->push_back(*it_in);
+      }
+    }
   }
 
@@ -52 +73 @@
     vector<int> matched_number;
     vector<int> matched_number_loc;
-
-
+    
     vector<int> all_color(ep_size);
     vector<int> all_color_loc(num_ep);
 
-    MPI_Gather_local(&color, 1, MPI_INT, all_color_loc.data(), 0, comm);
-    MPI_Bcast_local(all_color_loc.data(), num_ep, MPI_INT, 0, comm);
-
-
     MPI_Allgather(&color, 1, MPI_INT, all_color.data(), 1, MPI_INT, comm);
+    MPI_Allgather_local(&color, 1, MPI_INT, all_color_loc.data(), comm);
 
     list<int> color_list(all_color.begin(), all_color.end());
     list<int> color_list_loc(all_color_loc.begin(), all_color_loc.end());
 
-    vec_simplify(&all_color);
-
-    matched_number.resize(all_color.size(), 0);
-    matched_number_loc.resize(all_color.size(), 0);
+    vector<int> all_color_simplified;
+    vec_simplify(&all_color, &all_color_simplified);
+    int number_of_color;
+    for(int i=0; i<all_color_simplified.size(); i++)
+    {
+      if(color == all_color_simplified[i])
+      {
+        number_of_color = i;
+        break;
+      }
+    }
+
+    matched_number.resize(all_color_simplified.size(), 0);
+    matched_number_loc.resize(all_color_simplified.size(), 0);
 
 
@@ -93 +120 @@
       num_color++;
     }
-
-
+        
+
+    vector<int> all_key(ep_size);
+    vector<int> all_key_loc(num_ep);
+    
+    vector<int> colored_key[num_color];
     vector<int> colored_key_loc[num_color];
-    vector<int> key_loc(num_ep);
-
-    MPI_Gather_local(&key, 1, MPI_INT, key_loc.data(), 0, comm);
-    MPI_Bcast_local(key_loc.data(), num_ep, MPI_INT, 0, comm);
-    
-    std::sort(key_loc.begin(), key_loc.end());
+    
+
+    MPI_Allgather(&key, 1, MPI_INT, all_key.data(),1, MPI_INT, comm);
+    MPI_Allgather_local(&key, 1, MPI_INT, all_key_loc.data(), comm);
 
     for(int i=0; i<num_ep; i++)
@@ -107 +136 @@
       for(int j = 0; j<num_color; j++)
       {
-        if(all_color_loc[i] == all_color[j])
-        {
-          colored_key_loc[j].push_back(key_loc[i]);
-        }
-      }
-    }
-
+        if(all_color_loc[i] == all_color_simplified[j])
+        {
+          colored_key_loc[j].push_back(all_key_loc[i]);
+        }
+      }
+    }
+    
+    for(int i=0; i<ep_size; i++)
+    {
+      for(int j = 0; j<num_color; j++)
+      {
+        if(all_color[i] == all_color_simplified[j])
+        {
+          colored_key[j].push_back(all_key[i]);
+        }
+      }
+    }
+    
+    for(int i=0; i<num_color; i++)
+    {
+      std::sort(colored_key[i].begin(), colored_key[i].end());
+      std::sort(colored_key_loc[i].begin(), colored_key_loc[i].end());
+    }
+
+    int new_ep_rank;
+    
+    for(int i=0; i<colored_key[number_of_color].size(); i++)
+    {
+      if(key == colored_key[number_of_color][i])
+      {
+        new_ep_rank = i;
+        break;
+      }
+    }
+    
     int new_ep_rank_loc;
-
-    for(int i=0; i<num_color; i++)
-    {
-      if(colored_key_loc[i].size()>1)
-      {
-        std::sort(colored_key_loc[i].begin(), colored_key_loc[i].end());
-      }
-      if(color == all_color[i])
-      {
-        for(int j=0; j<colored_key_loc[i].size(); j++)
-        {
-          if(key == colored_key_loc[i][j])
-          {
-            new_ep_rank_loc = j;
-            break;
-          }
-        }
-      }
-    }
+    
+    for(int i=0; i<colored_key_loc[number_of_color].size(); i++)
+    {
+      if(key == colored_key_loc[number_of_color][i])
+      {
+        new_ep_rank_loc = i;
+        break;
+      }
+    }
+    
 
     ::MPI_Comm **split_mpi_comm;
@@ -151 +199 @@
         comm->ep_comm_ptr->comm_list[0]->mpi_bridge = split_mpi_comm[j];
       }
+      
       MPI_Barrier_local(comm);
+      
       int num_new_ep = 0;
 
-      if(new_ep_rank_loc == 0 && color == all_color[j])
+      if(new_ep_rank_loc == 0 && color == all_color_simplified[j])
       {
         num_new_ep = matched_number_loc[j];
@@ -162 +212 @@
         MPI_Comm_create_endpoints(comm->ep_comm_ptr->comm_list[0]->mpi_bridge, num_new_ep, info, ep_comm);
 
-
         comm->ep_comm_ptr->comm_list[0]->mem_bridge = ep_comm;
-      }
+        
+        (*ep_comm)->ep_rank_map->clear();
+        
+        memcheck("in MPI_Split ep_rank="<< ep_rank <<" : *ep_comm = "<< *ep_comm);
+      }
+      
       MPI_Barrier_local(comm);
-      if(color == all_color[j])
+      
+      if(color == all_color_simplified[j])
       {
         *newcomm = comm->ep_comm_ptr->comm_list[0]->mem_bridge[new_ep_rank_loc];
+        memcheck("in MPI_Split ep_rank="<< ep_rank <<" : *newcomm = "<< *newcomm);
 
         (*newcomm)->ep_comm_ptr->comm_label = color;
-      }
-    }
-
-
-
-
+        
+        (*newcomm)->ep_comm_ptr->size_rank_info[0].first = new_ep_rank;
+        (*newcomm)->ep_comm_ptr->size_rank_info[1].first = new_ep_rank_loc;
+        
+        int my_triple[3];
+        vector<int> my_triple_vector;
+        vector<int> my_triple_vector_recv;
+        my_triple[0] = new_ep_rank;
+        my_triple[1] = new_ep_rank_loc;
+        my_triple[2] = (*newcomm)->ep_comm_ptr->size_rank_info[2].first; // new_mpi_rank
+        
+        int new_ep_size = (*newcomm)->ep_comm_ptr->size_rank_info[0].second;
+        int new_num_ep  = (*newcomm)->ep_comm_ptr->size_rank_info[1].second;
+        
+        int new_mpi_size = (*newcomm)->ep_comm_ptr->size_rank_info[2].second;
+        
+        if(new_ep_rank_loc == 0) my_triple_vector.resize(3*new_ep_size);
+        if(new_ep_rank_loc == 0) my_triple_vector_recv.resize(3*new_ep_size);
+        
+        MPI_Gather_local(my_triple, 3, MPI_INT, my_triple_vector.data(), 0, *newcomm);
+        
+        if(new_ep_rank_loc == 0)
+        {
+          int *recvcounts = new int[new_mpi_size];
+          int *displs = new int[new_mpi_size];
+          int new_num_epx3 = new_num_ep * 3;
+          ::MPI_Allgather(&new_num_epx3, 1, to_mpi_type(MPI_INT), recvcounts, 1, to_mpi_type(MPI_INT), to_mpi_comm((*newcomm)->mpi_comm));
+          displs[0]=0;
+          for(int i=1; i<new_mpi_size; i++)
+            displs[i] = displs[i-1] + recvcounts[i-1];
+             
+          ::MPI_Allgatherv(my_triple_vector.data(), 3*new_num_ep, to_mpi_type(MPI_INT), my_triple_vector_recv.data(), recvcounts, displs, to_mpi_type(MPI_INT), to_mpi_comm((*newcomm)->mpi_comm));
+          
+          for(int i=0; i<new_ep_size; i++)
+          {
+            (*newcomm)->ep_comm_ptr->comm_list[0]->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(my_triple_vector_recv[3*i], my_triple_vector_recv[3*i+1], my_triple_vector_recv[3*i+2]));
+          }
+          
+          (*newcomm)->ep_rank_map = (*newcomm)->ep_comm_ptr->comm_list[0]->ep_rank_map;
+          
+          delete recvcounts;
+          delete displs;
+        } 
+      }
+    }
+    
+    /*for(int i=0; i<ep_size; i++)
+    {
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+      if(ep_rank==i) 
+      {
+        printf("ep_rank_map for endpoint %d = \n", ep_rank);
+        for(std::map<int, std::pair<int, int> > :: iterator it = (*newcomm)->ep_rank_map->begin(); it != (*newcomm)->ep_rank_map->end(); it++)
+        {
+          printf("\t\t\t %d %d %d\n", it->first, it->second.first, it->second.second);
+        }
+        printf("\n");
+      }
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+    }*/   
+    
     return 0;
   }

XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

@@ -687 +687 @@
 
     //int color = rank%2;
-    int color, remote_leader;
-    if(rank<size-2) {color = 1; remote_leader = size-2;}
-    else {color = 0; remote_leader = 0;}
-    
-    printf("rank = %d, color = %d, remote_leader = %d\n", rank, color, remote_leader);
-  
-    MPI_Comm sub_comm;
-    MPI_Comm_split(comm, color, rank, &sub_comm);
-
-
-    
-    int sub_rank;
-    MPI_Comm_rank(sub_comm, &sub_rank);
+    int tab_color[16]={2,2,2,3,0,1,1,3,2,1,3,0,0,2,0,0};
+    int tab_key[16]={3,11,10,5,6,8,15,7,2,1,9,13,4,14,12,0};
+    int color = tab_color[rank];
+    int key = tab_key[rank];
+  
+    MPI_Comm split_comm;
+    MPI_Comm_split(comm, color, key, &split_comm);
+
+
+    
+    int split_rank;
+    MPI_Comm_rank(split_comm, &split_rank);
+    
+    printf("rank = %d, color = %d, key = %d, split_rank = %d\n", rank, color, key, split_rank);
 
 
@@ -733 +734 @@
 */
     MPI_Barrier(comm);
-    MPI_Comm_free(&sub_comm);
+    MPI_Comm_free(&split_comm);