Context Navigation

source: XIOS/dev/branch_openmp/extern/src_ep_dev/ep_intercomm_world.cpp @ 1287

Last change on this file since 1287 was 1287, checked in by yushan, 7 years ago
EP updated
File size: 35.3 KB

Line
1	#include "ep_lib.hpp"
2	#include <mpi.h>
3	#include "ep_declaration.hpp"
4
5	using namespace std;
6
7	namespace ep_lib
8	{
9
10	#ifdef _openmpi
11
12	int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
13	{
14
15	int ep_rank, ep_rank_loc, mpi_rank;
16	int ep_size, num_ep, mpi_size;
17
18	ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
19	ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
20	mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
21	ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
22	num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
23	mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
24
25
26	std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm
27	//! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
28
29	int rank_in_world;
30	int rank_in_local_parent;
31
32	int local_ep_size = ep_size;
33	int remote_ep_size;
34
35	::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
36	::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
37
38	::MPI_Comm_rank(peer_comm, &rank_in_world);
39
40	::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
41
42	bool is_proc_master = false;
43	bool is_local_leader = false;
44	bool is_final_master = false;
45
46
47	if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
48	if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
49
50
51	int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
52
53	int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
54
55
56	std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
57
58	std::vector<int> new_rank_info[4];
59	std::vector<int> new_ep_info[2];
60
61	std::vector<int> offset;
62
63	if(is_proc_master)
64	{
65
66	size_info[0] = mpi_size;
67
68	rank_info[0].resize(size_info[0]);
69	rank_info[1].resize(size_info[0]);
70
71
72
73	ep_info[0].resize(size_info[0]);
74
75	vector<int> send_buf(6);
76	vector<int> recv_buf(3*size_info[0]);
77
78	send_buf[0] = rank_in_world;
79	send_buf[1] = rank_in_local_parent;
80	send_buf[2] = num_ep;
81
82	::MPI_Allgather(send_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), recv_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), local_mpi_comm);
83
84	for(int i=0; i<size_info[0]; i++)
85	{
86	rank_info[0][i] = recv_buf[3*i];
87	rank_info[1][i] = recv_buf[3*i+1];
88	ep_info[0][i] = recv_buf[3*i+2];
89	}
90
91	if(is_local_leader)
92	{
93	leader_info[0] = rank_in_world;
94	leader_info[1] = mpi_remote_leader;
95
96	::MPI_Status mpi_status;
97
98	send_buf[0] = size_info[0];
99	send_buf[1] = local_ep_size;
100
101	::MPI_Send(send_buf.data(), 2, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
102
103	::MPI_Recv(recv_buf.data(), 2, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status);
104
105	recv_buf[2] = rank_in_world;
106	recv_buf[3] = mpi_remote_leader;
107
108	}
109
110	::MPI_Bcast(recv_buf.data(), 4, static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
111
112	size_info[1] = recv_buf[0];
113	remote_ep_size = recv_buf[1];
114	leader_info[0] = recv_buf[2];
115	leader_info[1] = recv_buf[3];
116
117	rank_info[2].resize(size_info[1]);
118	rank_info[3].resize(size_info[1]);
119
120	ep_info[1].resize(size_info[1]);
121
122	send_buf.resize(3*size_info[0]);
123	recv_buf.resize(3*size_info[1]);
124
125	if(is_local_leader)
126	{
127	::MPI_Status mpi_status;
128
129
130	std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
131	std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
132	std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] );
133
134	::MPI_Send(send_buf.data(), 3*size_info[0], static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
135
136	::MPI_Recv(recv_buf.data(), 3*size_info[1], static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status);
137
138	}
139
140	::MPI_Bcast(recv_buf.data(), 3*size_info[1], static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
141
142	std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
143	std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() );
144	std::copy ( recv_buf.data() + 2size_info[1], recv_buf.data() + 3size_info[1], ep_info[1].begin() );
145
146	offset.resize(size_info[0]);
147
148	if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
149	{
150
151	bool found = false;
152	int ep_tmp;
153	int ep_local;
154	int ep_remote;
155	for(int i=0; i<size_info[0]; i++)
156	{
157	int target = rank_info[0][i];
158	found = false;
159	for(int j=0; j<size_info[1]; j++)
160	{
161	if(target == rank_info[2][j])
162	{
163	found = true;
164	ep_tmp = ep_info[1][j];
165	ep_local = ep_info[0][j];
166	ep_remote = ep_info[1][j];
167	break;
168	}
169	}
170	if(found)
171	{
172
173	if(target == leader_info[0]) // the leader is doubled in remote
174	{
175	new_rank_info[0].push_back(target);
176	new_rank_info[1].push_back(rank_info[1][i]);
177
178	new_ep_info[0].push_back(ep_local + ep_remote);
179	offset[i] = 0;
180	}
181	else
182	{
183	offset[i] = ep_local;
184	}
185	}
186	else
187	{
188	new_rank_info[0].push_back(target);
189	new_rank_info[1].push_back(rank_info[1][i]);
190
191	new_ep_info[0].push_back(ep_info[0][i]);
192
193	offset[i] = 0;
194	}
195
196	}
197	}
198
199	else // erase rank doubled with remote leader
200	{
201
202	bool found = false;
203	int ep_tmp;
204	int ep_local;
205	int ep_remote;
206	for(int i=0; i<size_info[0]; i++)
207	{
208	int target = rank_info[0][i];
209	found = false;
210	for(int j=0; j<size_info[1]; j++)
211	{
212
213	if(target == rank_info[2][j])
214	{
215	found = true;
216	ep_tmp = ep_info[1][j];
217	ep_local = ep_info[0][j];
218	ep_remote = ep_info[1][j];
219	break;
220	}
221	}
222	if(found)
223	{
224	if(target != leader_info[1])
225	{
226	new_rank_info[0].push_back(target);
227	new_rank_info[1].push_back(rank_info[1][i]);
228
229	new_ep_info[0].push_back(ep_local + ep_remote);
230	offset[i] = 0;
231	}
232	else // found remote leader
233	{
234	offset[i] = ep_remote;
235	}
236	}
237	else
238	{
239	new_rank_info[0].push_back(target);
240	new_rank_info[1].push_back(rank_info[1][i]);
241
242	new_ep_info[0].push_back(ep_info[0][i]);
243	offset[i] = 0;
244	}
245	}
246	}
247
248	if(offset[mpi_rank] == 0)
249	{
250	is_final_master = true;
251	}
252
253
254	// size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
255
256	if(is_local_leader)
257	{
258	size_info[2] = new_ep_info[0].size();
259	::MPI_Status mpi_status;
260
261	::MPI_Send(&size_info[2], 1, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
262
263	::MPI_Recv(&size_info[3], 1, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status);
264	}
265
266	::MPI_Bcast(&size_info[2], 2, static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
267
268	new_rank_info[2].resize(size_info[3]);
269	new_rank_info[3].resize(size_info[3]);
270	new_ep_info[1].resize(size_info[3]);
271
272	send_buf.resize(size_info[2]);
273	recv_buf.resize(size_info[3]);
274
275	if(is_local_leader)
276	{
277	::MPI_Status mpi_status;
278
279	std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
280	std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
281	std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] );
282
283	::MPI_Send(send_buf.data(), 3*size_info[2], static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
284	::MPI_Recv(recv_buf.data(), 3*size_info[3], static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status);
285
286	}
287
288	::MPI_Bcast(recv_buf.data(), 3*size_info[3], static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
289
290	std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
291	std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() );
292	std::copy ( recv_buf.data() + 2size_info[3], recv_buf.data() + 3size_info[3], new_ep_info[1].begin() );
293
294	}
295
296
297
298	if(is_proc_master)
299	{
300	// leader_info[4]: 2-> rank of local leader in new_group generated comm;
301	// 3-> rank of remote leader in new_group generated comm;
302	::MPI_Group local_group;
303	::MPI_Group new_group;
304	::MPI_Comm new_comm;
305	::MPI_Comm intercomm;
306
307	::MPI_Comm_group(local_mpi_comm, &local_group);
308
309	::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
310
311	::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
312
313
314
315	if(is_local_leader)
316	{
317	::MPI_Comm_rank(new_comm, &leader_info[2]);
318	}
319
320	::MPI_Bcast(&leader_info[2], 1, static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
321
322	if(new_comm != static_cast< ::MPI_Comm>(MPI_COMM_NULL.mpi_comm))
323	{
324	::MPI_Barrier(new_comm);
325
326	::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
327
328	int id;
329	MPI_Comm_rank(new_comm, &id);
330	int my_num_ep = new_ep_info[0][id];
331
332	MPI_Comm *ep_intercomm;
333	MPI_Info info;
334	MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
335
336
337
338	for(int i= 0; i<my_num_ep; i++)
339	{
340	ep_intercomm[i].is_intercomm = true;
341
342	ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
343	ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
344	ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
345	}
346
347	#pragma omp critical (write_to_tag_list)
348	tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
349	}
350
351
352	}
353
354
355	MPI_Barrier_local(local_comm);
356
357	vector<int> bcast_buf(8);
358	if(is_local_leader)
359	{
360	std::copy(size_info, size_info+4, bcast_buf.begin());
361	std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
362	}
363
364	MPI_Bcast(bcast_buf.data(), 8, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm);
365
366	if(!is_local_leader)
367	{
368	std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
369	std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
370	}
371
372	if(!is_local_leader)
373	{
374	new_rank_info[1].resize(size_info[2]);
375	ep_info[1].resize(size_info[1]);
376	offset.resize(size_info[0]);
377	}
378
379	bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
380
381	if(is_local_leader)
382	{
383	bcast_buf[0] = remote_ep_size;
384	std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
385	std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
386	std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
387	}
388
389	MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm);
390
391	if(!is_local_leader)
392	{
393	remote_ep_size = bcast_buf[0];
394	std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
395	std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
396	std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
397	}
398
399
400	int my_position = offset[rank_in_local_parent]+ep_rank_loc;
401
402
403	MPI_Barrier_local(local_comm);
404	#pragma omp flush
405
406
407	#pragma omp critical (read_from_tag_list)
408	{
409	bool found = false;
410	while(!found)
411	{
412	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
413	{
414	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
415	{
416	*newintercomm = iter->second[my_position];
417
418	found = true;
419	//tag_list.erase(iter);
420	break;
421	}
422	}
423	}
424	}
425
426	MPI_Barrier(local_comm);
427
428	if(is_local_leader)
429	{
430	int local_flag = true;
431	int remote_flag = false;
432	::MPI_Status mpi_status;
433
434	::MPI_Send(&local_flag, 1, static_cast< ::MPI_Datatype>(MPI_INT), mpi_remote_leader, tag, peer_comm);
435
436	::MPI_Recv(&remote_flag, 1, static_cast< ::MPI_Datatype>(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status);
437	}
438
439	MPI_Barrier(local_comm);
440
441	if(is_proc_master)
442	{
443	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
444	{
445	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
446	{
447	tag_list.erase(iter);
448	break;
449	}
450	}
451	}
452
453
454
455	int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
456	int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
457
458	intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
459	intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
460	intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
461	intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
462	intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
463	intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
464
465	MPI_Bcast(&remote_ep_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm);
466
467	int my_rank_map_elem[2];
468
469
470	my_rank_map_elem[0] = intercomm_ep_rank;
471
472	my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
473
474	vector<pair<int, int> > local_rank_map_array;
475	vector<pair<int, int> > remote_rank_map_array;
476
477
478	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
479	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
480
481	MPI_Allgather2(my_rank_map_elem, 2, MPI_INT, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm);
482
483	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
484	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
485
486	int local_intercomm_size = intercomm_ep_size;
487	int remote_intercomm_size;
488
489
490
491
492	if(is_local_leader)
493	{
494	::MPI_Status status;
495
496	::MPI_Send((newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2local_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
497
498	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &status);
499
500	::MPI_Send(&local_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
501
502	::MPI_Recv(&remote_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &status);
503	}
504
505	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm);
506	MPI_Bcast(&remote_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), 0, *newintercomm);
507
508
509	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
510	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
511
512	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
513	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
514	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
515
516
517	if(is_local_leader)
518	{
519	::MPI_Status status;
520
521	::MPI_Send((newintercomm).rank_map->data(), 2local_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
522
523	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &status);
524	}
525
526	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), 0, *newintercomm);
527
528	(*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
529	(*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
530
531
532	return MPI_SUCCESS;
533
534	}
535
536
537
538	#elif _intelmpi
539	int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, int peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
540	{
541	int ep_rank, ep_rank_loc, mpi_rank;
542	int ep_size, num_ep, mpi_size;
543
544	ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
545	ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
546	mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
547	ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
548	num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
549	mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
550
551	std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm
552	//! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
553
554	int rank_in_world;
555	int rank_in_local_parent;
556
557	int local_ep_size = ep_size;
558	int remote_ep_size;
559
560	::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
561	::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
562
563	::MPI_Comm_rank(peer_comm, &rank_in_world);
564
565	::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
566
567	bool is_proc_master = false;
568	bool is_local_leader = false;
569	bool is_final_master = false;
570
571
572	if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
573	if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
574
575
576	int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
577
578	int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
579
580
581	std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
582
583	std::vector<int> new_rank_info[4];
584	std::vector<int> new_ep_info[2];
585
586	std::vector<int> offset;
587
588	if(is_proc_master)
589	{
590
591	size_info[0] = mpi_size;
592
593	rank_info[0].resize(size_info[0]);
594	rank_info[1].resize(size_info[0]);
595
596	ep_info[0].resize(size_info[0]);
597
598	vector<int> send_buf(6);
599	vector<int> recv_buf(3*size_info[0]);
600
601	send_buf[0] = rank_in_world;
602	send_buf[1] = rank_in_local_parent;
603	send_buf[2] = num_ep;
604
605	::MPI_Allgather(send_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), recv_buf.data(), 3, static_cast< ::MPI_Datatype> (MPI_INT), local_mpi_comm);
606
607	for(int i=0; i<size_info[0]; i++)
608	{
609	rank_info[0][i] = recv_buf[3*i];
610	rank_info[1][i] = recv_buf[3*i+1];
611	ep_info[0][i] = recv_buf[3*i+2];
612	}
613
614
615	if(is_local_leader)
616	{
617	leader_info[0] = rank_in_world;
618	leader_info[1] = mpi_remote_leader;
619
620	::MPI_Status mpi_status;
621
622	send_buf[0] = size_info[0];
623	send_buf[1] = local_ep_size;
624
625	::MPI_Send(send_buf.data(), 2, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
626
627	::MPI_Recv(recv_buf.data(), 2, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status);
628
629	recv_buf[2] = rank_in_world;
630	recv_buf[3] = mpi_remote_leader;
631
632	}
633
634	::MPI_Bcast(recv_buf.data(), 4, static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
635
636	size_info[1] = recv_buf[0];
637	remote_ep_size = recv_buf[1];
638	leader_info[0] = recv_buf[2];
639	leader_info[1] = recv_buf[3];
640
641	rank_info[2].resize(size_info[1]);
642	rank_info[3].resize(size_info[1]);
643
644	ep_info[1].resize(size_info[1]);
645
646	send_buf.resize(3*size_info[0]);
647	recv_buf.resize(3*size_info[1]);
648
649	if(is_local_leader)
650	{
651	::MPI_Status mpi_status;
652
653
654	std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
655	std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
656	std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] );
657
658	::MPI_Send(send_buf.data(), 3*size_info[0], static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
659
660	::MPI_Recv(recv_buf.data(), 3*size_info[1], static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status);
661
662	}
663
664	::MPI_Bcast(recv_buf.data(), 3*size_info[1], static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
665
666	std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
667	std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() );
668	std::copy ( recv_buf.data() + 2size_info[1], recv_buf.data() + 3size_info[1], ep_info[1].begin() );
669
670	offset.resize(size_info[0]);
671
672	if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
673	{
674
675	bool found = false;
676	int ep_tmp;
677	int ep_local;
678	int ep_remote;
679	for(int i=0; i<size_info[0]; i++)
680	{
681	int target = rank_info[0][i];
682	found = false;
683	for(int j=0; j<size_info[1]; j++)
684	{
685	if(target == rank_info[2][j])
686	{
687	found = true;
688	ep_tmp = ep_info[1][j];
689	ep_local = ep_info[0][j];
690	ep_remote = ep_info[1][j];
691	break;
692	}
693	}
694	if(found)
695	{
696
697	if(target == leader_info[0]) // the leader is doubled in remote
698	{
699	new_rank_info[0].push_back(target);
700	new_rank_info[1].push_back(rank_info[1][i]);
701
702	new_ep_info[0].push_back(ep_local + ep_remote);
703	offset[i] = 0;
704	}
705	else
706	{
707	offset[i] = ep_local;
708	}
709	}
710	else
711	{
712	new_rank_info[0].push_back(target);
713	new_rank_info[1].push_back(rank_info[1][i]);
714
715	new_ep_info[0].push_back(ep_info[0][i]);
716
717	offset[i] = 0;
718	}
719
720	}
721	}
722
723	else // erase rank doubled with remote leader
724	{
725
726	bool found = false;
727	int ep_tmp;
728	int ep_local;
729	int ep_remote;
730	for(int i=0; i<size_info[0]; i++)
731	{
732	int target = rank_info[0][i];
733	found = false;
734	for(int j=0; j<size_info[1]; j++)
735	{
736
737	if(target == rank_info[2][j])
738	{
739	found = true;
740	ep_tmp = ep_info[1][j];
741	ep_local = ep_info[0][j];
742	ep_remote = ep_info[1][j];
743	break;
744	}
745	}
746	if(found)
747	{
748	if(target != leader_info[1])
749	{
750	new_rank_info[0].push_back(target);
751	new_rank_info[1].push_back(rank_info[1][i]);
752
753	new_ep_info[0].push_back(ep_local + ep_remote);
754	offset[i] = 0;
755	}
756	else // found remote leader
757	{
758	offset[i] = ep_remote;
759	}
760	}
761	else
762	{
763	new_rank_info[0].push_back(target);
764	new_rank_info[1].push_back(rank_info[1][i]);
765
766	new_ep_info[0].push_back(ep_info[0][i]);
767	offset[i] = 0;
768	}
769	}
770	}
771
772	if(offset[mpi_rank] == 0)
773	{
774	is_final_master = true;
775	}
776
777
778	// size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
779
780	if(is_local_leader)
781	{
782	size_info[2] = new_ep_info[0].size();
783	::MPI_Status mpi_status;
784
785	::MPI_Send(&size_info[2], 1, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
786
787	::MPI_Recv(&size_info[3], 1, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status);
788	}
789
790	::MPI_Bcast(&size_info[2], 2, static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
791
792	new_rank_info[2].resize(size_info[3]);
793	new_rank_info[3].resize(size_info[3]);
794	new_ep_info[1].resize(size_info[3]);
795
796	send_buf.resize(size_info[2]);
797	recv_buf.resize(size_info[3]);
798
799	if(is_local_leader)
800	{
801	::MPI_Status mpi_status;
802
803	std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
804	std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
805	std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] );
806
807	::MPI_Send(send_buf.data(), 3*size_info[2], static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
808
809	::MPI_Recv(recv_buf.data(), 3*size_info[3], static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status);
810	}
811
812	::MPI_Bcast(recv_buf.data(), 3*size_info[3], static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
813
814	std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
815	std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() );
816	std::copy ( recv_buf.data() + 2size_info[3], recv_buf.data() + 3size_info[3], new_ep_info[1].begin() );
817
818	}
819
820
821
822	if(is_proc_master)
823	{
824	// leader_info[4]: 2-> rank of local leader in new_group generated comm;
825	// 3-> rank of remote leader in new_group generated comm;
826	::MPI_Group local_group;
827	::MPI_Group new_group;
828	::MPI_Comm new_comm;
829	::MPI_Comm intercomm;
830
831	::MPI_Comm_group(local_mpi_comm, &local_group);
832
833	::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
834
835	::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
836
837
838
839	if(is_local_leader)
840	{
841	::MPI_Comm_rank(new_comm, &leader_info[2]);
842	}
843
844	::MPI_Bcast(&leader_info[2], 1, static_cast< ::MPI_Datatype> (MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm);
845
846	if(new_comm != static_cast< ::MPI_Comm >(MPI_COMM_NULL.mpi_comm))
847	{
848	::MPI_Barrier(new_comm);
849
850	::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
851
852	int id;
853	::MPI_Comm_rank(new_comm, &id);
854	int my_num_ep = new_ep_info[0][id];
855
856	MPI_Comm *ep_intercomm;
857	MPI_Info info;
858	MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
859
860	for(int i= 0; i<my_num_ep; i++)
861	{
862	ep_intercomm[i].is_intercomm = true;
863
864	ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
865	ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
866	ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
867	}
868
869
870	#pragma omp critical (write_to_tag_list)
871	tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
872
873
874	}
875
876
877	}
878
879
880	vector<int> bcast_buf(8);
881	if(is_local_leader)
882	{
883	std::copy(size_info, size_info+4, bcast_buf.begin());
884	std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
885	}
886
887
888
889	MPI_Bcast(bcast_buf.data(), 8, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm);
890
891
892	if(!is_local_leader)
893	{
894	std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
895	std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
896	}
897
898
899
900	if(!is_local_leader)
901	{
902	new_rank_info[1].resize(size_info[2]);
903	ep_info[1].resize(size_info[1]);
904	offset.resize(size_info[0]);
905	}
906
907	bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
908
909	if(is_local_leader)
910	{
911	bcast_buf[0] = remote_ep_size;
912	std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
913	std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
914	std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
915	}
916
917	MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm);
918
919
920	if(!is_local_leader)
921	{
922	remote_ep_size = bcast_buf[0];
923	std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
924	std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
925	std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
926	}
927
928
929	int my_position = offset[rank_in_local_parent]+ep_rank_loc;
930
931
932	#pragma omp critical (read_from_tag_list)
933	{
934	bool found = false;
935	while(!found)
936	{
937	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
938	{
939	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
940	{
941	*newintercomm = iter->second[my_position];
942
943	found = true;
944	break;
945	}
946	}
947	}
948	}
949
950	MPI_Barrier(local_comm);
951
952	if(is_local_leader)
953	{
954	int local_flag = true;
955	int remote_flag = false;
956	::MPI_Status mpi_status;
957
958	::MPI_Send(&local_flag, 1, static_cast< ::MPI_Datatype>(MPI_INT), mpi_remote_leader, tag, peer_comm);
959
960	::MPI_Recv(&remote_flag, 1, static_cast< ::MPI_Datatype>(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status);
961	}
962
963	MPI_Barrier(local_comm);
964
965	if(is_proc_master)
966	{
967	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
968	{
969	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
970	{
971	tag_list.erase(iter);
972	break;
973	}
974	}
975	}
976
977	int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
978	int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
979
980	intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
981	intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
982	intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
983	intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
984	intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
985	intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
986
987
988	MPI_Bcast(&remote_ep_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm);
989
990	int my_rank_map_elem[2];
991
992
993	my_rank_map_elem[0] = intercomm_ep_rank;
994
995	my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
996
997	vector<pair<int, int> > local_rank_map_array;
998	vector<pair<int, int> > remote_rank_map_array;
999
1000
1001	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
1002	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
1003
1004	MPI_Allgather(my_rank_map_elem, 2, MPI_INT, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm);
1005
1006	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
1007	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
1008
1009	int local_intercomm_size = intercomm_ep_size;
1010	int remote_intercomm_size;
1011
1012
1013	if(is_local_leader)
1014	{
1015	::MPI_Status status;
1016
1017	::MPI_Send((newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2local_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
1018
1019	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &status);
1020
1021	::MPI_Send(&local_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
1022
1023	::MPI_Recv(&remote_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &status);
1024	}
1025
1026	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, static_cast< ::MPI_Datatype> (MPI_INT), local_leader, local_comm);
1027	MPI_Bcast(&remote_intercomm_size, 1, static_cast< ::MPI_Datatype> (MPI_INT), 0, *newintercomm);
1028
1029
1030	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
1031	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
1032
1033	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
1034	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
1035	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
1036
1037
1038	if(is_local_leader)
1039	{
1040	::MPI_Status status;
1041
1042	::MPI_Send((newintercomm).rank_map->data(), 2local_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm);
1043
1044	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), mpi_remote_leader, tag, peer_comm, &status);
1045	}
1046
1047	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, static_cast< ::MPI_Datatype> (MPI_INT), 0, *newintercomm);
1048
1049	(*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
1050	(*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
1051
1052	return MPI_SUCCESS;
1053
1054	}
1055
1056	#endif
1057
1058	}
1059

Note: See TracBrowser for help on using the repository browser.

Download in other formats: