Changeset 1520 for XIOS/dev/branch_openmp/extern/src_ep_dev/ep_split.cpp
- Timestamp:
- 06/04/18 19:25:08 (6 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/dev/branch_openmp/extern/src_ep_dev/ep_split.cpp
r1354 r1520 31 31 inout_vector->swap(out_vec); 32 32 } 33 34 void vec_simplify(std::vector<int> *in_vector, std::vector<int> *out_vector) 35 { 36 int found=false; 37 for(std::vector<int>::iterator it_in = in_vector->begin() ; it_in != in_vector->end(); ++it_in) 38 { 39 for(std::vector<int>::iterator it = out_vector->begin() ; it != out_vector->end(); ++it) 40 { 41 if(*it_in == *it) 42 { 43 found=true; 44 break; 45 } 46 else found=false; 47 } 48 if(found == false) 49 { 50 out_vector->push_back(*it_in); 51 } 52 } 53 } 33 54 34 55 … … 39 60 int ep_size, num_ep, mpi_size; 40 61 41 ep_rank = comm .ep_comm_ptr->size_rank_info[0].first;42 ep_rank_loc = comm .ep_comm_ptr->size_rank_info[1].first;43 mpi_rank = comm .ep_comm_ptr->size_rank_info[2].first;44 ep_size = comm .ep_comm_ptr->size_rank_info[0].second;45 num_ep = comm .ep_comm_ptr->size_rank_info[1].second;46 mpi_size = comm .ep_comm_ptr->size_rank_info[2].second;62 ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; 63 ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 64 mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; 65 ep_size = comm->ep_comm_ptr->size_rank_info[0].second; 66 num_ep = comm->ep_comm_ptr->size_rank_info[1].second; 67 mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; 47 68 48 69 int num_color = 0; … … 52 73 vector<int> matched_number; 53 74 vector<int> matched_number_loc; 54 55 75 56 76 vector<int> all_color(ep_size); 57 77 vector<int> all_color_loc(num_ep); 58 78 59 MPI_Gather_local(&color, 1, MPI_INT, all_color_loc.data(), 0, comm);60 MPI_Bcast_local(all_color_loc.data(), num_ep, MPI_INT, 0, comm);61 62 63 79 MPI_Allgather(&color, 1, MPI_INT, all_color.data(), 1, MPI_INT, comm); 80 MPI_Allgather_local(&color, 1, MPI_INT, all_color_loc.data(), comm); 64 81 65 82 list<int> color_list(all_color.begin(), all_color.end()); 66 83 list<int> color_list_loc(all_color_loc.begin(), all_color_loc.end()); 67 84 68 vec_simplify(&all_color); 69 70 matched_number.resize(all_color.size(), 0); 71 matched_number_loc.resize(all_color.size(), 0); 85 vector<int> all_color_simplified; 86 vec_simplify(&all_color, &all_color_simplified); 87 int number_of_color; 88 for(int i=0; i<all_color_simplified.size(); i++) 89 { 90 if(color == all_color_simplified[i]) 91 { 92 number_of_color = i; 93 break; 94 } 95 } 96 97 matched_number.resize(all_color_simplified.size(), 0); 98 matched_number_loc.resize(all_color_simplified.size(), 0); 72 99 73 100 … … 93 120 num_color++; 94 121 } 95 96 122 123 124 vector<int> all_key(ep_size); 125 vector<int> all_key_loc(num_ep); 126 127 vector<int> colored_key[num_color]; 97 128 vector<int> colored_key_loc[num_color]; 98 vector<int> key_loc(num_ep); 99 100 MPI_Gather_local(&key, 1, MPI_INT, key_loc.data(), 0, comm); 101 MPI_Bcast_local(key_loc.data(), num_ep, MPI_INT, 0, comm); 102 103 std::sort(key_loc.begin(), key_loc.end()); 129 130 131 MPI_Allgather(&key, 1, MPI_INT, all_key.data(),1, MPI_INT, comm); 132 MPI_Allgather_local(&key, 1, MPI_INT, all_key_loc.data(), comm); 133 134 vector<int> all_ep_rank(ep_size); 135 vector<int> all_ep_rank_loc(num_ep); 136 137 vector<int> colored_ep_rank[num_color]; 138 vector<int> colored_ep_rank_loc[num_color]; 139 140 MPI_Allgather(&ep_rank, 1, MPI_INT, all_ep_rank.data(),1, MPI_INT, comm); 141 MPI_Allgather_local(&ep_rank, 1, MPI_INT, all_ep_rank_loc.data(), comm); 104 142 105 143 for(int i=0; i<num_ep; i++) … … 107 145 for(int j = 0; j<num_color; j++) 108 146 { 109 if(all_color_loc[i] == all_color[j]) 110 { 111 colored_key_loc[j].push_back(key_loc[i]); 112 } 113 } 114 } 115 147 if(all_color_loc[i] == all_color_simplified[j]) 148 { 149 colored_key_loc[j].push_back(all_key_loc[i]); 150 colored_ep_rank_loc[j].push_back(all_ep_rank_loc[i]); 151 } 152 } 153 } 154 155 for(int i=0; i<ep_size; i++) 156 { 157 for(int j = 0; j<num_color; j++) 158 { 159 if(all_color[i] == all_color_simplified[j]) 160 { 161 colored_key[j].push_back(all_key[i]); 162 colored_ep_rank[j].push_back(all_ep_rank[i]); 163 } 164 } 165 } 166 167 int my_offset=0; 168 for(int i=0; i<colored_key[number_of_color].size(); i++) 169 { 170 if(key == colored_key[number_of_color][i]) 171 { 172 if(ep_rank != colored_ep_rank[number_of_color][i]) 173 { 174 my_offset++; 175 } 176 else 177 break; 178 } 179 } 180 181 int my_offset_loc=0; 182 for(int i=0; i<colored_key_loc[number_of_color].size(); i++) 183 { 184 if(key == colored_key_loc[number_of_color][i]) 185 { 186 if(ep_rank != colored_ep_rank_loc[number_of_color][i]) 187 { 188 my_offset_loc++; 189 } 190 else 191 break; 192 } 193 } 194 195 196 197 for(int i=0; i<num_color; i++) 198 { 199 std::sort(colored_key[i].begin(), colored_key[i].end()); 200 std::sort(colored_key_loc[i].begin(), colored_key_loc[i].end()); 201 } 202 203 int new_ep_rank; 204 205 for(int i=0; i<colored_key[number_of_color].size(); i++) 206 { 207 if(key == colored_key[number_of_color][i]) 208 { 209 new_ep_rank = i+my_offset; 210 break; 211 } 212 } 213 116 214 int new_ep_rank_loc; 117 118 for(int i=0; i<num_color; i++) 119 { 120 if(colored_key_loc[i].size()>1) 121 { 122 std::sort(colored_key_loc[i].begin(), colored_key_loc[i].end()); 123 } 124 if(color == all_color[i]) 125 { 126 for(int j=0; j<colored_key_loc[i].size(); j++) 127 { 128 if(key == colored_key_loc[i][j]) 129 { 130 new_ep_rank_loc = j; 131 break; 132 } 133 } 134 } 135 } 215 216 for(int i=0; i<colored_key_loc[number_of_color].size(); i++) 217 { 218 if(key == colored_key_loc[number_of_color][i]) 219 { 220 new_ep_rank_loc = i+my_offset_loc; 221 break; 222 } 223 } 224 225 #ifdef _showinfo 226 printf("ep_rank = %d, color = %d, key = %d, new_ep_rank = %d, new_ep_rank_loc = %d\n", ep_rank, color, key, new_ep_rank, new_ep_rank_loc); 227 #endif 228 229 MPI_Barrier(comm); 230 MPI_Barrier(comm); 231 MPI_Barrier(comm); 232 136 233 137 234 ::MPI_Comm **split_mpi_comm; … … 147 244 if(matched_number_loc[j] == 0) master_color = MPI_UNDEFINED; 148 245 149 ::MPI_Comm_split(to_mpi_comm(comm.mpi_comm), master_color, mpi_rank, split_mpi_comm[j]); 150 151 comm.ep_comm_ptr->comm_list->mpi_bridge = split_mpi_comm[j]; 152 } 246 ::MPI_Comm_split(to_mpi_comm(comm->mpi_comm), master_color, mpi_rank, split_mpi_comm[j]); 247 248 comm->ep_comm_ptr->comm_list[0]->mpi_bridge = split_mpi_comm[j]; 249 } 250 153 251 MPI_Barrier_local(comm); 252 154 253 int num_new_ep = 0; 155 254 156 if(new_ep_rank_loc == 0 && color == all_color [j])255 if(new_ep_rank_loc == 0 && color == all_color_simplified[j]) 157 256 { 158 257 num_new_ep = matched_number_loc[j]; … … 160 259 MPI_Comm *ep_comm; 161 260 162 MPI_Comm_create_endpoints(comm.ep_comm_ptr->comm_list->mpi_bridge, num_new_ep, info, ep_comm); 163 164 165 comm.ep_comm_ptr->comm_list->mem_bridge = ep_comm; 166 } 261 MPI_Comm_create_endpoints(comm->ep_comm_ptr->comm_list[0]->mpi_bridge, num_new_ep, info, ep_comm); 262 263 comm->ep_comm_ptr->comm_list[0]->mem_bridge = ep_comm; 264 265 (*ep_comm)->ep_rank_map->clear(); 266 267 memcheck("in MPI_Split ep_rank="<< ep_rank <<" : *ep_comm = "<< *ep_comm); 268 } 269 167 270 MPI_Barrier_local(comm); 168 if(color == all_color[j]) 169 { 170 *newcomm = comm.ep_comm_ptr->comm_list->mem_bridge[new_ep_rank_loc]; 171 172 (*newcomm).ep_comm_ptr->comm_label = color; 173 } 174 } 175 176 177 178 271 272 if(color == all_color_simplified[j]) 273 { 274 *newcomm = comm->ep_comm_ptr->comm_list[0]->mem_bridge[new_ep_rank_loc]; 275 memcheck("in MPI_Split ep_rank="<< ep_rank <<" : *newcomm = "<< *newcomm); 276 277 (*newcomm)->ep_comm_ptr->comm_label = color; 278 279 (*newcomm)->ep_comm_ptr->size_rank_info[0].first = new_ep_rank; 280 (*newcomm)->ep_comm_ptr->size_rank_info[1].first = new_ep_rank_loc; 281 282 int my_triple[3]; 283 vector<int> my_triple_vector; 284 vector<int> my_triple_vector_recv; 285 my_triple[0] = new_ep_rank; 286 my_triple[1] = new_ep_rank_loc; 287 my_triple[2] = (*newcomm)->ep_comm_ptr->size_rank_info[2].first; // new_mpi_rank 288 289 int new_ep_size = (*newcomm)->ep_comm_ptr->size_rank_info[0].second; 290 int new_num_ep = (*newcomm)->ep_comm_ptr->size_rank_info[1].second; 291 292 int new_mpi_size = (*newcomm)->ep_comm_ptr->size_rank_info[2].second; 293 294 if(new_ep_rank_loc == 0) my_triple_vector.resize(3*new_ep_size); 295 if(new_ep_rank_loc == 0) my_triple_vector_recv.resize(3*new_ep_size); 296 297 MPI_Gather_local(my_triple, 3, MPI_INT, my_triple_vector.data(), 0, *newcomm); 298 299 if(new_ep_rank_loc == 0) 300 { 301 int *recvcounts = new int[new_mpi_size]; 302 int *displs = new int[new_mpi_size]; 303 int new_num_epx3 = new_num_ep * 3; 304 ::MPI_Allgather(&new_num_epx3, 1, to_mpi_type(MPI_INT), recvcounts, 1, to_mpi_type(MPI_INT), to_mpi_comm((*newcomm)->mpi_comm)); 305 displs[0]=0; 306 for(int i=1; i<new_mpi_size; i++) 307 displs[i] = displs[i-1] + recvcounts[i-1]; 308 309 ::MPI_Allgatherv(my_triple_vector.data(), 3*new_num_ep, to_mpi_type(MPI_INT), my_triple_vector_recv.data(), recvcounts, displs, to_mpi_type(MPI_INT), to_mpi_comm((*newcomm)->mpi_comm)); 310 311 for(int i=0; i<new_ep_size; i++) 312 { 313 (*newcomm)->ep_comm_ptr->comm_list[0]->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(my_triple_vector_recv[3*i], my_triple_vector_recv[3*i+1], my_triple_vector_recv[3*i+2])); 314 } 315 316 (*newcomm)->ep_rank_map = (*newcomm)->ep_comm_ptr->comm_list[0]->ep_rank_map; 317 318 delete recvcounts; 319 delete displs; 320 } 321 } 322 } 323 324 #ifdef _Memory_check 325 for(int i=0; i<ep_size; i++) 326 { 327 MPI_Barrier(comm); 328 MPI_Barrier(comm); 329 if(ep_rank==i) 330 { 331 printf("ep_rank_map for endpoint %d = \n", ep_rank); 332 for(std::map<int, std::pair<int, int> > :: iterator it = (*newcomm)->ep_rank_map->begin(); it != (*newcomm)->ep_rank_map->end(); it++) 333 { 334 printf("\t\t\t %d %d %d\n", it->first, it->second.first, it->second.second); 335 } 336 printf("\n"); 337 } 338 MPI_Barrier(comm); 339 MPI_Barrier(comm); 340 } 341 #endif 342 179 343 return 0; 180 344 }
Note: See TracChangeset
for help on using the changeset viewer.