Context Navigation

source: XIOS/trunk/extern/src_ep/ep_intercomm_world.cpp @ 1034

Last change on this file since 1034 was 1034, checked in by yushan, 7 years ago
adding src_ep into extern folder
File size: 33.4 KB

Line
1	#include "ep_lib.hpp"
2	#include <mpi.h>
3	#include "ep_declaration.hpp"
4
5	using namespace std;
6
7	namespace ep_lib
8	{
9
10	#ifdef _openmpi
11
12	int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
13	{
14
15	int ep_rank, ep_rank_loc, mpi_rank;
16	int ep_size, num_ep, mpi_size;
17
18	ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
19	ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
20	mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
21	ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
22	num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
23	mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
24
25
26	std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm
27	//! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
28
29	int rank_in_world;
30	int rank_in_local_parent;
31
32	int local_ep_size = ep_size;
33	int remote_ep_size;
34
35	::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
36	::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
37
38	::MPI_Comm_rank(peer_comm, &rank_in_world);
39
40	::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
41
42	bool is_proc_master = false;
43	bool is_local_leader = false;
44	bool is_final_master = false;
45
46
47	if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
48	if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
49
50
51	int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
52
53	int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
54
55
56	std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
57
58	std::vector<int> new_rank_info[4];
59	std::vector<int> new_ep_info[2];
60
61	std::vector<int> offset;
62
63	if(is_proc_master)
64	{
65
66	size_info[0] = mpi_size;
67
68	rank_info[0].resize(size_info[0]);
69	rank_info[1].resize(size_info[0]);
70
71
72
73	ep_info[0].resize(size_info[0]);
74
75	vector<int> send_buf(6);
76	vector<int> recv_buf(3*size_info[0]);
77
78	send_buf[0] = rank_in_world;
79	send_buf[1] = rank_in_local_parent;
80	send_buf[2] = num_ep;
81
82	::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
83
84	for(int i=0; i<size_info[0]; i++)
85	{
86	rank_info[0][i] = recv_buf[3*i];
87	rank_info[1][i] = recv_buf[3*i+1];
88	ep_info[0][i] = recv_buf[3*i+2];
89	}
90
91	if(is_local_leader)
92	{
93	leader_info[0] = rank_in_world;
94	leader_info[1] = mpi_remote_leader;
95
96	::MPI_Status mpi_status;
97
98	send_buf[0] = size_info[0];
99	send_buf[1] = local_ep_size;
100
101	::MPI_Send(send_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
102
103	::MPI_Recv(recv_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
104
105	recv_buf[2] = rank_in_world;
106	recv_buf[3] = mpi_remote_leader;
107
108	}
109
110	::MPI_Bcast(recv_buf.data(), 4, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
111
112	size_info[1] = recv_buf[0];
113	remote_ep_size = recv_buf[1];
114	leader_info[0] = recv_buf[2];
115	leader_info[1] = recv_buf[3];
116
117	rank_info[2].resize(size_info[1]);
118	rank_info[3].resize(size_info[1]);
119
120	ep_info[1].resize(size_info[1]);
121
122	send_buf.resize(3*size_info[0]);
123	recv_buf.resize(3*size_info[1]);
124
125	if(is_local_leader)
126	{
127	::MPI_Status mpi_status;
128
129
130	std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
131	std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
132	std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] );
133
134	::MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
135
136	::MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
137
138	}
139
140	::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
141
142	std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
143	std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() );
144	std::copy ( recv_buf.data() + 2size_info[1], recv_buf.data() + 3size_info[1], ep_info[1].begin() );
145
146	offset.resize(size_info[0]);
147
148	if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
149	{
150
151	bool found = false;
152	int ep_tmp;
153	int ep_local;
154	int ep_remote;
155	for(int i=0; i<size_info[0]; i++)
156	{
157	int target = rank_info[0][i];
158	found = false;
159	for(int j=0; j<size_info[1]; j++)
160	{
161	if(target == rank_info[2][j])
162	{
163	found = true;
164	ep_tmp = ep_info[1][j];
165	ep_local = ep_info[0][j];
166	ep_remote = ep_info[1][j];
167	break;
168	}
169	}
170	if(found)
171	{
172
173	if(target == leader_info[0]) // the leader is doubled in remote
174	{
175	new_rank_info[0].push_back(target);
176	new_rank_info[1].push_back(rank_info[1][i]);
177
178	new_ep_info[0].push_back(ep_local + ep_remote);
179	offset[i] = 0;
180	}
181	else
182	{
183	offset[i] = ep_local;
184	}
185	}
186	else
187	{
188	new_rank_info[0].push_back(target);
189	new_rank_info[1].push_back(rank_info[1][i]);
190
191	new_ep_info[0].push_back(ep_info[0][i]);
192
193	offset[i] = 0;
194	}
195
196	}
197	}
198
199	else // erase rank doubled with remote leader
200	{
201
202	bool found = false;
203	int ep_tmp;
204	int ep_local;
205	int ep_remote;
206	for(int i=0; i<size_info[0]; i++)
207	{
208	int target = rank_info[0][i];
209	found = false;
210	for(int j=0; j<size_info[1]; j++)
211	{
212
213	if(target == rank_info[2][j])
214	{
215	found = true;
216	ep_tmp = ep_info[1][j];
217	ep_local = ep_info[0][j];
218	ep_remote = ep_info[1][j];
219	break;
220	}
221	}
222	if(found)
223	{
224	if(target != leader_info[1])
225	{
226	new_rank_info[0].push_back(target);
227	new_rank_info[1].push_back(rank_info[1][i]);
228
229	new_ep_info[0].push_back(ep_local + ep_remote);
230	offset[i] = 0;
231	}
232	else // found remote leader
233	{
234	offset[i] = ep_remote;
235	}
236	}
237	else
238	{
239	new_rank_info[0].push_back(target);
240	new_rank_info[1].push_back(rank_info[1][i]);
241
242	new_ep_info[0].push_back(ep_info[0][i]);
243	offset[i] = 0;
244	}
245	}
246	}
247
248	if(offset[mpi_rank] == 0)
249	{
250	is_final_master = true;
251	}
252
253
254	// size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
255
256	if(is_local_leader)
257	{
258	size_info[2] = new_ep_info[0].size();
259	::MPI_Status mpi_status;
260
261	::MPI_Send(&size_info[2], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
262
263	::MPI_Recv(&size_info[3], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
264	}
265
266	::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
267
268	new_rank_info[2].resize(size_info[3]);
269	new_rank_info[3].resize(size_info[3]);
270	new_ep_info[1].resize(size_info[3]);
271
272	send_buf.resize(size_info[2]);
273	recv_buf.resize(size_info[3]);
274
275	if(is_local_leader)
276	{
277	::MPI_Status mpi_status;
278
279	std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
280	std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
281	std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] );
282
283	MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
284	MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
285
286	}
287
288	::MPI_Bcast(recv_buf.data(), 3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
289
290	std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
291	std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() );
292	std::copy ( recv_buf.data() + 2size_info[3], recv_buf.data() + 3size_info[3], new_ep_info[1].begin() );
293
294	}
295
296
297
298	if(is_proc_master)
299	{
300	// leader_info[4]: 2-> rank of local leader in new_group generated comm;
301	// 3-> rank of remote leader in new_group generated comm;
302	::MPI_Group local_group;
303	::MPI_Group new_group;
304	::MPI_Comm new_comm;
305	::MPI_Comm intercomm;
306
307	::MPI_Comm_group(local_mpi_comm, &local_group);
308
309	::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
310
311	::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
312
313
314
315	if(is_local_leader)
316	{
317	::MPI_Comm_rank(new_comm, &leader_info[2]);
318	}
319
320	::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
321
322	if(new_comm != MPI_COMM_NULL_STD)
323	{
324	::MPI_Barrier(new_comm);
325
326	::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
327
328	int id;
329	MPI_Comm_rank(new_comm, &id);
330	int my_num_ep = new_ep_info[0][id];
331
332	MPI_Comm *ep_intercomm;
333	MPI_Info info;
334	MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
335
336
337
338	for(int i= 0; i<my_num_ep; i++)
339	{
340	ep_intercomm[i].is_intercomm = true;
341
342	ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
343	ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
344	ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
345	}
346
347	#pragma omp critical (write_to_tag_list)
348	tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
349	}
350
351
352	}
353
354
355	MPI_Barrier_local(local_comm);
356
357	vector<int> bcast_buf(8);
358	if(is_local_leader)
359	{
360	std::copy(size_info, size_info+4, bcast_buf.begin());
361	std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
362	}
363
364	MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
365
366	if(!is_local_leader)
367	{
368	std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
369	std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
370	}
371
372	if(!is_local_leader)
373	{
374	new_rank_info[1].resize(size_info[2]);
375	ep_info[1].resize(size_info[1]);
376	offset.resize(size_info[0]);
377	}
378
379	bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
380
381	if(is_local_leader)
382	{
383	bcast_buf[0] = remote_ep_size;
384	std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
385	std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
386	std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
387	}
388
389	MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
390
391	if(!is_local_leader)
392	{
393	remote_ep_size = bcast_buf[0];
394	std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
395	std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
396	std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
397	}
398
399
400	int my_position = offset[rank_in_local_parent]+ep_rank_loc;
401
402
403	MPI_Barrier_local(local_comm);
404	#pragma omp flush
405
406
407	#pragma omp critical (read_from_tag_list)
408	{
409	bool found = false;
410	while(!found)
411	{
412	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
413	{
414	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
415	{
416	*newintercomm = iter->second[my_position];
417
418	found = true;
419	break;
420	}
421	}
422	}
423	}
424
425	MPI_Barrier_local(local_comm);
426
427	int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
428	int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
429
430	intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
431	intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
432	intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
433	intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
434	intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
435	intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
436
437	MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
438
439	int my_rank_map_elem[2];
440
441
442	my_rank_map_elem[0] = intercomm_ep_rank;
443
444	my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
445
446	vector<pair<int, int> > local_rank_map_array;
447	vector<pair<int, int> > remote_rank_map_array;
448
449
450	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
451	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
452
453	MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
454
455	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
456	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
457
458	int local_intercomm_size = intercomm_ep_size;
459	int remote_intercomm_size;
460
461
462
463
464	if(is_local_leader)
465	{
466	::MPI_Status status;
467
468	::MPI_Send((newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2local_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
469
470	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
471
472	::MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
473
474	::MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
475	}
476
477	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, local_leader, local_comm);
478	MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, 0, *newintercomm);
479
480
481	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
482	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
483
484	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
485	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
486	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
487
488
489	if(is_local_leader)
490	{
491	::MPI_Status status;
492
493	::MPI_Send((newintercomm).rank_map->data(), 2local_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
494
495	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
496	}
497
498	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, 0, *newintercomm);
499
500	(*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
501	(*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
502
503
504	return MPI_SUCCESS;
505
506	}
507
508
509
510	#elif _intelmpi
511	int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, int peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
512	{
513	int ep_rank, ep_rank_loc, mpi_rank;
514	int ep_size, num_ep, mpi_size;
515
516	ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
517	ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
518	mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
519	ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
520	num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
521	mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
522
523	std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm
524	//! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
525
526	int rank_in_world;
527	int rank_in_local_parent;
528
529	int local_ep_size = ep_size;
530	int remote_ep_size;
531
532	::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
533	::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
534
535	::MPI_Comm_rank(peer_comm, &rank_in_world);
536
537	::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
538
539	bool is_proc_master = false;
540	bool is_local_leader = false;
541	bool is_final_master = false;
542
543
544	if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
545	if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
546
547
548	int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
549
550	int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
551
552
553	std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
554
555	std::vector<int> new_rank_info[4];
556	std::vector<int> new_ep_info[2];
557
558	std::vector<int> offset;
559
560	if(is_proc_master)
561	{
562
563	size_info[0] = mpi_size;
564
565	rank_info[0].resize(size_info[0]);
566	rank_info[1].resize(size_info[0]);
567
568	ep_info[0].resize(size_info[0]);
569
570	vector<int> send_buf(6);
571	vector<int> recv_buf(3*size_info[0]);
572
573	send_buf[0] = rank_in_world;
574	send_buf[1] = rank_in_local_parent;
575	send_buf[2] = num_ep;
576
577	::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
578
579	for(int i=0; i<size_info[0]; i++)
580	{
581	rank_info[0][i] = recv_buf[3*i];
582	rank_info[1][i] = recv_buf[3*i+1];
583	ep_info[0][i] = recv_buf[3*i+2];
584	}
585
586
587	if(is_local_leader)
588	{
589	leader_info[0] = rank_in_world;
590	leader_info[1] = mpi_remote_leader;
591
592	::MPI_Status mpi_status;
593
594	send_buf[0] = size_info[0];
595	send_buf[1] = local_ep_size;
596
597	::MPI_Send(send_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
598
599	::MPI_Recv(recv_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
600
601	recv_buf[2] = rank_in_world;
602	recv_buf[3] = mpi_remote_leader;
603
604	}
605
606	::MPI_Bcast(recv_buf.data(), 4, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
607
608	size_info[1] = recv_buf[0];
609	remote_ep_size = recv_buf[1];
610	leader_info[0] = recv_buf[2];
611	leader_info[1] = recv_buf[3];
612
613	rank_info[2].resize(size_info[1]);
614	rank_info[3].resize(size_info[1]);
615
616	ep_info[1].resize(size_info[1]);
617
618	send_buf.resize(3*size_info[0]);
619	recv_buf.resize(3*size_info[1]);
620
621	if(is_local_leader)
622	{
623	::MPI_Status mpi_status;
624
625
626	std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
627	std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
628	std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] );
629
630	::MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
631
632	::MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
633
634	}
635
636	::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
637
638	std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
639	std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() );
640	std::copy ( recv_buf.data() + 2size_info[1], recv_buf.data() + 3size_info[1], ep_info[1].begin() );
641
642	offset.resize(size_info[0]);
643
644	if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
645	{
646
647	bool found = false;
648	int ep_tmp;
649	int ep_local;
650	int ep_remote;
651	for(int i=0; i<size_info[0]; i++)
652	{
653	int target = rank_info[0][i];
654	found = false;
655	for(int j=0; j<size_info[1]; j++)
656	{
657	if(target == rank_info[2][j])
658	{
659	found = true;
660	ep_tmp = ep_info[1][j];
661	ep_local = ep_info[0][j];
662	ep_remote = ep_info[1][j];
663	break;
664	}
665	}
666	if(found)
667	{
668
669	if(target == leader_info[0]) // the leader is doubled in remote
670	{
671	new_rank_info[0].push_back(target);
672	new_rank_info[1].push_back(rank_info[1][i]);
673
674	new_ep_info[0].push_back(ep_local + ep_remote);
675	offset[i] = 0;
676	}
677	else
678	{
679	offset[i] = ep_local;
680	}
681	}
682	else
683	{
684	new_rank_info[0].push_back(target);
685	new_rank_info[1].push_back(rank_info[1][i]);
686
687	new_ep_info[0].push_back(ep_info[0][i]);
688
689	offset[i] = 0;
690	}
691
692	}
693	}
694
695	else // erase rank doubled with remote leader
696	{
697
698	bool found = false;
699	int ep_tmp;
700	int ep_local;
701	int ep_remote;
702	for(int i=0; i<size_info[0]; i++)
703	{
704	int target = rank_info[0][i];
705	found = false;
706	for(int j=0; j<size_info[1]; j++)
707	{
708
709	if(target == rank_info[2][j])
710	{
711	found = true;
712	ep_tmp = ep_info[1][j];
713	ep_local = ep_info[0][j];
714	ep_remote = ep_info[1][j];
715	break;
716	}
717	}
718	if(found)
719	{
720	if(target != leader_info[1])
721	{
722	new_rank_info[0].push_back(target);
723	new_rank_info[1].push_back(rank_info[1][i]);
724
725	new_ep_info[0].push_back(ep_local + ep_remote);
726	offset[i] = 0;
727	}
728	else // found remote leader
729	{
730	offset[i] = ep_remote;
731	}
732	}
733	else
734	{
735	new_rank_info[0].push_back(target);
736	new_rank_info[1].push_back(rank_info[1][i]);
737
738	new_ep_info[0].push_back(ep_info[0][i]);
739	offset[i] = 0;
740	}
741	}
742	}
743
744	if(offset[mpi_rank] == 0)
745	{
746	is_final_master = true;
747	}
748
749
750	// size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
751
752	if(is_local_leader)
753	{
754	size_info[2] = new_ep_info[0].size();
755	::MPI_Status mpi_status;
756
757	::MPI_Send(&size_info[2], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
758
759	::MPI_Recv(&size_info[3], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
760	}
761
762	::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
763
764	new_rank_info[2].resize(size_info[3]);
765	new_rank_info[3].resize(size_info[3]);
766	new_ep_info[1].resize(size_info[3]);
767
768	send_buf.resize(size_info[2]);
769	recv_buf.resize(size_info[3]);
770
771	if(is_local_leader)
772	{
773	::MPI_Status mpi_status;
774
775	std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
776	std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
777	std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] );
778
779	::MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
780
781	::MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
782	}
783
784	::MPI_Bcast(recv_buf.data(), 3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
785
786	std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
787	std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() );
788	std::copy ( recv_buf.data() + 2size_info[3], recv_buf.data() + 3size_info[3], new_ep_info[1].begin() );
789
790	}
791
792
793
794	if(is_proc_master)
795	{
796	// leader_info[4]: 2-> rank of local leader in new_group generated comm;
797	// 3-> rank of remote leader in new_group generated comm;
798	::MPI_Group local_group;
799	::MPI_Group new_group;
800	::MPI_Comm new_comm;
801	::MPI_Comm intercomm;
802
803	::MPI_Comm_group(local_mpi_comm, &local_group);
804
805	::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
806
807	::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
808
809
810
811	if(is_local_leader)
812	{
813	::MPI_Comm_rank(new_comm, &leader_info[2]);
814	}
815
816	::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
817
818	if(new_comm != MPI_COMM_NULL_STD)
819	{
820	::MPI_Barrier(new_comm);
821
822	::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
823
824	int id;
825	::MPI_Comm_rank(new_comm, &id);
826	int my_num_ep = new_ep_info[0][id];
827
828	MPI_Comm *ep_intercomm;
829	MPI_Info info;
830	MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
831
832	for(int i= 0; i<my_num_ep; i++)
833	{
834	ep_intercomm[i].is_intercomm = true;
835
836	ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
837	ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
838	ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
839	}
840
841
842	#pragma omp critical (write_to_tag_list)
843	tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
844
845	}
846
847
848	}
849
850
851	MPI_Barrier_local(local_comm);
852
853
854	vector<int> bcast_buf(8);
855	if(is_local_leader)
856	{
857	std::copy(size_info, size_info+4, bcast_buf.begin());
858	std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
859	}
860
861
862
863	MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
864
865
866	if(!is_local_leader)
867	{
868	std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
869	std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
870	}
871
872
873
874	if(!is_local_leader)
875	{
876	new_rank_info[1].resize(size_info[2]);
877	ep_info[1].resize(size_info[1]);
878	offset.resize(size_info[0]);
879	}
880
881	bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
882
883	if(is_local_leader)
884	{
885	bcast_buf[0] = remote_ep_size;
886	std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
887	std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
888	std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
889	}
890
891	MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
892
893
894	if(!is_local_leader)
895	{
896	remote_ep_size = bcast_buf[0];
897	std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
898	std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
899	std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
900	}
901
902
903	int my_position = offset[rank_in_local_parent]+ep_rank_loc;
904
905
906	MPI_Barrier_local(local_comm);
907	#pragma omp flush
908
909
910	#pragma omp critical (read_from_tag_list)
911	{
912	bool found = false;
913	while(!found)
914	{
915	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
916	{
917	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
918	{
919	*newintercomm = iter->second[my_position];
920
921	found = true;
922	break;
923	}
924	}
925	}
926	}
927
928	MPI_Barrier_local(local_comm);
929
930	int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
931	int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
932
933	intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
934	intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
935	intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
936	intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
937	intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
938	intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
939
940
941	MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
942
943	int my_rank_map_elem[2];
944
945
946	my_rank_map_elem[0] = intercomm_ep_rank;
947
948	my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
949
950	vector<pair<int, int> > local_rank_map_array;
951	vector<pair<int, int> > remote_rank_map_array;
952
953
954	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
955	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
956
957	MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
958
959	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
960	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
961
962	int local_intercomm_size = intercomm_ep_size;
963	int remote_intercomm_size;
964
965
966	if(is_local_leader)
967	{
968	::MPI_Status status;
969
970	::MPI_Send((newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2local_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
971
972	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
973
974	::MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
975
976	::MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
977	}
978
979	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, local_leader, local_comm);
980	MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, 0, *newintercomm);
981
982
983	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
984	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
985
986	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
987	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
988	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
989
990
991	if(is_local_leader)
992	{
993	::MPI_Status status;
994
995	::MPI_Send((newintercomm).rank_map->data(), 2local_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
996
997	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
998	}
999
1000	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, 0, *newintercomm);
1001
1002	(*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
1003	(*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
1004
1005
1006	// for(int i=0; i<local_ep_size; i++)
1007	// if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, remote_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
1008	// (newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).first, (newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).second);
1009
1010	// for(int i=0; i<remote_intercomm_size; i++)
1011	// if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, intercomm_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
1012	// (newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first, (newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second);
1013
1014	// for(int i=0; i<(*newintercomm).rank_map->size(); i++)
1015	// if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
1016	// (newintercomm).rank_map->at(i).first, (newintercomm).rank_map->at(i).second);
1017
1018
1019
1020
1021
1022	return MPI_SUCCESS;
1023
1024	}
1025
1026	#endif
1027
1028	}
1029

Note: See TracBrowser for help on using the repository browser.

Download in other formats: