Context Navigation

source: XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_intercomm_world.cpp @ 1134

Last change on this file since 1134 was 1134, checked in by yushan, 7 years ago
branch merged with trunk r1130
File size: 34.1 KB

Line
1	#include "ep_lib.hpp"
2	#include <mpi.h>
3	#include "ep_declaration.hpp"
4
5	using namespace std;
6
7	namespace ep_lib
8	{
9
10	#ifdef _openmpi
11
12	int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
13	{
14
15	int ep_rank, ep_rank_loc, mpi_rank;
16	int ep_size, num_ep, mpi_size;
17
18	ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
19	ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
20	mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
21	ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
22	num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
23	mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
24
25
26	std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm
27	//! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
28
29	int rank_in_world;
30	int rank_in_local_parent;
31
32	int local_ep_size = ep_size;
33	int remote_ep_size;
34
35	::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
36	::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
37
38	::MPI_Comm_rank(peer_comm, &rank_in_world);
39
40	::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
41
42	bool is_proc_master = false;
43	bool is_local_leader = false;
44	bool is_final_master = false;
45
46
47	if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
48	if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
49
50
51	int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
52
53	int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
54
55
56	std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
57
58	std::vector<int> new_rank_info[4];
59	std::vector<int> new_ep_info[2];
60
61	std::vector<int> offset;
62
63	if(is_proc_master)
64	{
65
66	size_info[0] = mpi_size;
67
68	rank_info[0].resize(size_info[0]);
69	rank_info[1].resize(size_info[0]);
70
71
72
73	ep_info[0].resize(size_info[0]);
74
75	vector<int> send_buf(6);
76	vector<int> recv_buf(3*size_info[0]);
77
78	send_buf[0] = rank_in_world;
79	send_buf[1] = rank_in_local_parent;
80	send_buf[2] = num_ep;
81
82	::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
83
84	for(int i=0; i<size_info[0]; i++)
85	{
86	rank_info[0][i] = recv_buf[3*i];
87	rank_info[1][i] = recv_buf[3*i+1];
88	ep_info[0][i] = recv_buf[3*i+2];
89	}
90
91	if(is_local_leader)
92	{
93	leader_info[0] = rank_in_world;
94	leader_info[1] = mpi_remote_leader;
95
96	::MPI_Status mpi_status;
97
98	send_buf[0] = size_info[0];
99	send_buf[1] = local_ep_size;
100
101	::MPI_Send(send_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
102
103	::MPI_Recv(recv_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
104
105	recv_buf[2] = rank_in_world;
106	recv_buf[3] = mpi_remote_leader;
107
108	}
109
110	::MPI_Bcast(recv_buf.data(), 4, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
111
112	size_info[1] = recv_buf[0];
113	remote_ep_size = recv_buf[1];
114	leader_info[0] = recv_buf[2];
115	leader_info[1] = recv_buf[3];
116
117	rank_info[2].resize(size_info[1]);
118	rank_info[3].resize(size_info[1]);
119
120	ep_info[1].resize(size_info[1]);
121
122	send_buf.resize(3*size_info[0]);
123	recv_buf.resize(3*size_info[1]);
124
125	if(is_local_leader)
126	{
127	::MPI_Status mpi_status;
128
129
130	std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
131	std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
132	std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] );
133
134	::MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
135
136	::MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
137
138	}
139
140	::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
141
142	std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
143	std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() );
144	std::copy ( recv_buf.data() + 2size_info[1], recv_buf.data() + 3size_info[1], ep_info[1].begin() );
145
146	offset.resize(size_info[0]);
147
148	if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
149	{
150
151	bool found = false;
152	int ep_tmp;
153	int ep_local;
154	int ep_remote;
155	for(int i=0; i<size_info[0]; i++)
156	{
157	int target = rank_info[0][i];
158	found = false;
159	for(int j=0; j<size_info[1]; j++)
160	{
161	if(target == rank_info[2][j])
162	{
163	found = true;
164	ep_tmp = ep_info[1][j];
165	ep_local = ep_info[0][j];
166	ep_remote = ep_info[1][j];
167	break;
168	}
169	}
170	if(found)
171	{
172
173	if(target == leader_info[0]) // the leader is doubled in remote
174	{
175	new_rank_info[0].push_back(target);
176	new_rank_info[1].push_back(rank_info[1][i]);
177
178	new_ep_info[0].push_back(ep_local + ep_remote);
179	offset[i] = 0;
180	}
181	else
182	{
183	offset[i] = ep_local;
184	}
185	}
186	else
187	{
188	new_rank_info[0].push_back(target);
189	new_rank_info[1].push_back(rank_info[1][i]);
190
191	new_ep_info[0].push_back(ep_info[0][i]);
192
193	offset[i] = 0;
194	}
195
196	}
197	}
198
199	else // erase rank doubled with remote leader
200	{
201
202	bool found = false;
203	int ep_tmp;
204	int ep_local;
205	int ep_remote;
206	for(int i=0; i<size_info[0]; i++)
207	{
208	int target = rank_info[0][i];
209	found = false;
210	for(int j=0; j<size_info[1]; j++)
211	{
212
213	if(target == rank_info[2][j])
214	{
215	found = true;
216	ep_tmp = ep_info[1][j];
217	ep_local = ep_info[0][j];
218	ep_remote = ep_info[1][j];
219	break;
220	}
221	}
222	if(found)
223	{
224	if(target != leader_info[1])
225	{
226	new_rank_info[0].push_back(target);
227	new_rank_info[1].push_back(rank_info[1][i]);
228
229	new_ep_info[0].push_back(ep_local + ep_remote);
230	offset[i] = 0;
231	}
232	else // found remote leader
233	{
234	offset[i] = ep_remote;
235	}
236	}
237	else
238	{
239	new_rank_info[0].push_back(target);
240	new_rank_info[1].push_back(rank_info[1][i]);
241
242	new_ep_info[0].push_back(ep_info[0][i]);
243	offset[i] = 0;
244	}
245	}
246	}
247
248	if(offset[mpi_rank] == 0)
249	{
250	is_final_master = true;
251	}
252
253
254	// size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
255
256	if(is_local_leader)
257	{
258	size_info[2] = new_ep_info[0].size();
259	::MPI_Status mpi_status;
260
261	::MPI_Send(&size_info[2], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
262
263	::MPI_Recv(&size_info[3], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
264	}
265
266	::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
267
268	new_rank_info[2].resize(size_info[3]);
269	new_rank_info[3].resize(size_info[3]);
270	new_ep_info[1].resize(size_info[3]);
271
272	send_buf.resize(size_info[2]);
273	recv_buf.resize(size_info[3]);
274
275	if(is_local_leader)
276	{
277	::MPI_Status mpi_status;
278
279	std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
280	std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
281	std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] );
282
283	::MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
284	::MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
285
286	}
287
288	::MPI_Bcast(recv_buf.data(), 3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
289
290	std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
291	std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() );
292	std::copy ( recv_buf.data() + 2size_info[3], recv_buf.data() + 3size_info[3], new_ep_info[1].begin() );
293
294	}
295
296
297
298	if(is_proc_master)
299	{
300	// leader_info[4]: 2-> rank of local leader in new_group generated comm;
301	// 3-> rank of remote leader in new_group generated comm;
302	::MPI_Group local_group;
303	::MPI_Group new_group;
304	::MPI_Comm new_comm;
305	::MPI_Comm intercomm;
306
307	::MPI_Comm_group(local_mpi_comm, &local_group);
308
309	::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
310
311	::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
312
313
314
315	if(is_local_leader)
316	{
317	::MPI_Comm_rank(new_comm, &leader_info[2]);
318	}
319
320	::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
321
322	if(new_comm != MPI_COMM_NULL_STD)
323	{
324	::MPI_Barrier(new_comm);
325
326	::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
327
328	int id;
329	MPI_Comm_rank(new_comm, &id);
330	int my_num_ep = new_ep_info[0][id];
331
332	MPI_Comm *ep_intercomm;
333	MPI_Info info;
334	MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
335
336
337
338	for(int i= 0; i<my_num_ep; i++)
339	{
340	ep_intercomm[i].is_intercomm = true;
341
342	ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
343	ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
344	ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
345	}
346
347	#pragma omp critical (write_to_tag_list)
348	tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
349	}
350
351
352	}
353
354
355	MPI_Barrier_local(local_comm);
356
357	vector<int> bcast_buf(8);
358	if(is_local_leader)
359	{
360	std::copy(size_info, size_info+4, bcast_buf.begin());
361	std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
362	}
363
364	MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
365
366	if(!is_local_leader)
367	{
368	std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
369	std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
370	}
371
372	if(!is_local_leader)
373	{
374	new_rank_info[1].resize(size_info[2]);
375	ep_info[1].resize(size_info[1]);
376	offset.resize(size_info[0]);
377	}
378
379	bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
380
381	if(is_local_leader)
382	{
383	bcast_buf[0] = remote_ep_size;
384	std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
385	std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
386	std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
387	}
388
389	MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
390
391	if(!is_local_leader)
392	{
393	remote_ep_size = bcast_buf[0];
394	std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
395	std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
396	std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
397	}
398
399
400	int my_position = offset[rank_in_local_parent]+ep_rank_loc;
401
402
403	MPI_Barrier_local(local_comm);
404	#pragma omp flush
405
406
407	#pragma omp critical (read_from_tag_list)
408	{
409	bool found = false;
410	while(!found)
411	{
412	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
413	{
414	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
415	{
416	*newintercomm = iter->second[my_position];
417
418	found = true;
419	//tag_list.erase(iter);
420	break;
421	}
422	}
423	}
424	}
425
426	MPI_Barrier_local(local_comm);
427
428	if(is_proc_master)
429	{
430	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
431	{
432	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
433	{
434	tag_list.erase(iter);
435	break;
436	}
437	}
438	}
439
440
441
442	int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
443	int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
444
445	intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
446	intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
447	intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
448	intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
449	intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
450	intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
451
452	MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
453
454	int my_rank_map_elem[2];
455
456
457	my_rank_map_elem[0] = intercomm_ep_rank;
458
459	my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
460
461	vector<pair<int, int> > local_rank_map_array;
462	vector<pair<int, int> > remote_rank_map_array;
463
464
465	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
466	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
467
468	MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
469
470	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
471	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
472
473	int local_intercomm_size = intercomm_ep_size;
474	int remote_intercomm_size;
475
476
477
478
479	if(is_local_leader)
480	{
481	::MPI_Status status;
482
483	::MPI_Send((newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2local_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
484
485	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
486
487	::MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
488
489	::MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
490	}
491
492	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, local_leader, local_comm);
493	MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, 0, *newintercomm);
494
495
496	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
497	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
498
499	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
500	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
501	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
502
503
504	if(is_local_leader)
505	{
506	::MPI_Status status;
507
508	::MPI_Send((newintercomm).rank_map->data(), 2local_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
509
510	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
511	}
512
513	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, 0, *newintercomm);
514
515	(*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
516	(*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
517
518
519	return MPI_SUCCESS;
520
521	}
522
523
524
525	#elif _intelmpi
526	int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, int peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
527	{
528	int ep_rank, ep_rank_loc, mpi_rank;
529	int ep_size, num_ep, mpi_size;
530
531	ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
532	ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
533	mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
534	ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
535	num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
536	mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
537
538	std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm
539	//! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
540
541	int rank_in_world;
542	int rank_in_local_parent;
543
544	int local_ep_size = ep_size;
545	int remote_ep_size;
546
547	::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
548	::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
549
550	::MPI_Comm_rank(peer_comm, &rank_in_world);
551
552	::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
553
554	bool is_proc_master = false;
555	bool is_local_leader = false;
556	bool is_final_master = false;
557
558
559	if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
560	if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
561
562
563	int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
564
565	int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
566
567
568	std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
569
570	std::vector<int> new_rank_info[4];
571	std::vector<int> new_ep_info[2];
572
573	std::vector<int> offset;
574
575	if(is_proc_master)
576	{
577
578	size_info[0] = mpi_size;
579
580	rank_info[0].resize(size_info[0]);
581	rank_info[1].resize(size_info[0]);
582
583	ep_info[0].resize(size_info[0]);
584
585	vector<int> send_buf(6);
586	vector<int> recv_buf(3*size_info[0]);
587
588	send_buf[0] = rank_in_world;
589	send_buf[1] = rank_in_local_parent;
590	send_buf[2] = num_ep;
591
592	::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
593
594	for(int i=0; i<size_info[0]; i++)
595	{
596	rank_info[0][i] = recv_buf[3*i];
597	rank_info[1][i] = recv_buf[3*i+1];
598	ep_info[0][i] = recv_buf[3*i+2];
599	}
600
601
602	if(is_local_leader)
603	{
604	leader_info[0] = rank_in_world;
605	leader_info[1] = mpi_remote_leader;
606
607	::MPI_Status mpi_status;
608
609	send_buf[0] = size_info[0];
610	send_buf[1] = local_ep_size;
611
612	::MPI_Send(send_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
613
614	::MPI_Recv(recv_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
615
616	recv_buf[2] = rank_in_world;
617	recv_buf[3] = mpi_remote_leader;
618
619	}
620
621	::MPI_Bcast(recv_buf.data(), 4, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
622
623	size_info[1] = recv_buf[0];
624	remote_ep_size = recv_buf[1];
625	leader_info[0] = recv_buf[2];
626	leader_info[1] = recv_buf[3];
627
628	rank_info[2].resize(size_info[1]);
629	rank_info[3].resize(size_info[1]);
630
631	ep_info[1].resize(size_info[1]);
632
633	send_buf.resize(3*size_info[0]);
634	recv_buf.resize(3*size_info[1]);
635
636	if(is_local_leader)
637	{
638	::MPI_Status mpi_status;
639
640
641	std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
642	std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
643	std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] );
644
645	::MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
646
647	::MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
648
649	}
650
651	::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
652
653	std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
654	std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() );
655	std::copy ( recv_buf.data() + 2size_info[1], recv_buf.data() + 3size_info[1], ep_info[1].begin() );
656
657	offset.resize(size_info[0]);
658
659	if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
660	{
661
662	bool found = false;
663	int ep_tmp;
664	int ep_local;
665	int ep_remote;
666	for(int i=0; i<size_info[0]; i++)
667	{
668	int target = rank_info[0][i];
669	found = false;
670	for(int j=0; j<size_info[1]; j++)
671	{
672	if(target == rank_info[2][j])
673	{
674	found = true;
675	ep_tmp = ep_info[1][j];
676	ep_local = ep_info[0][j];
677	ep_remote = ep_info[1][j];
678	break;
679	}
680	}
681	if(found)
682	{
683
684	if(target == leader_info[0]) // the leader is doubled in remote
685	{
686	new_rank_info[0].push_back(target);
687	new_rank_info[1].push_back(rank_info[1][i]);
688
689	new_ep_info[0].push_back(ep_local + ep_remote);
690	offset[i] = 0;
691	}
692	else
693	{
694	offset[i] = ep_local;
695	}
696	}
697	else
698	{
699	new_rank_info[0].push_back(target);
700	new_rank_info[1].push_back(rank_info[1][i]);
701
702	new_ep_info[0].push_back(ep_info[0][i]);
703
704	offset[i] = 0;
705	}
706
707	}
708	}
709
710	else // erase rank doubled with remote leader
711	{
712
713	bool found = false;
714	int ep_tmp;
715	int ep_local;
716	int ep_remote;
717	for(int i=0; i<size_info[0]; i++)
718	{
719	int target = rank_info[0][i];
720	found = false;
721	for(int j=0; j<size_info[1]; j++)
722	{
723
724	if(target == rank_info[2][j])
725	{
726	found = true;
727	ep_tmp = ep_info[1][j];
728	ep_local = ep_info[0][j];
729	ep_remote = ep_info[1][j];
730	break;
731	}
732	}
733	if(found)
734	{
735	if(target != leader_info[1])
736	{
737	new_rank_info[0].push_back(target);
738	new_rank_info[1].push_back(rank_info[1][i]);
739
740	new_ep_info[0].push_back(ep_local + ep_remote);
741	offset[i] = 0;
742	}
743	else // found remote leader
744	{
745	offset[i] = ep_remote;
746	}
747	}
748	else
749	{
750	new_rank_info[0].push_back(target);
751	new_rank_info[1].push_back(rank_info[1][i]);
752
753	new_ep_info[0].push_back(ep_info[0][i]);
754	offset[i] = 0;
755	}
756	}
757	}
758
759	if(offset[mpi_rank] == 0)
760	{
761	is_final_master = true;
762	}
763
764
765	// size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
766
767	if(is_local_leader)
768	{
769	size_info[2] = new_ep_info[0].size();
770	::MPI_Status mpi_status;
771
772	::MPI_Send(&size_info[2], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
773
774	::MPI_Recv(&size_info[3], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
775	}
776
777	::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
778
779	new_rank_info[2].resize(size_info[3]);
780	new_rank_info[3].resize(size_info[3]);
781	new_ep_info[1].resize(size_info[3]);
782
783	send_buf.resize(size_info[2]);
784	recv_buf.resize(size_info[3]);
785
786	if(is_local_leader)
787	{
788	::MPI_Status mpi_status;
789
790	std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
791	std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
792	std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] );
793
794	::MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
795
796	::MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
797	}
798
799	::MPI_Bcast(recv_buf.data(), 3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
800
801	std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
802	std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() );
803	std::copy ( recv_buf.data() + 2size_info[3], recv_buf.data() + 3size_info[3], new_ep_info[1].begin() );
804
805	}
806
807
808
809	if(is_proc_master)
810	{
811	// leader_info[4]: 2-> rank of local leader in new_group generated comm;
812	// 3-> rank of remote leader in new_group generated comm;
813	::MPI_Group local_group;
814	::MPI_Group new_group;
815	::MPI_Comm new_comm;
816	::MPI_Comm intercomm;
817
818	::MPI_Comm_group(local_mpi_comm, &local_group);
819
820	::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
821
822	::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
823
824
825
826	if(is_local_leader)
827	{
828	::MPI_Comm_rank(new_comm, &leader_info[2]);
829	}
830
831	::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
832
833	if(new_comm != MPI_COMM_NULL_STD)
834	{
835	::MPI_Barrier(new_comm);
836
837	::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
838
839	int id;
840	::MPI_Comm_rank(new_comm, &id);
841	int my_num_ep = new_ep_info[0][id];
842
843	MPI_Comm *ep_intercomm;
844	MPI_Info info;
845	MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
846
847	for(int i= 0; i<my_num_ep; i++)
848	{
849	ep_intercomm[i].is_intercomm = true;
850
851	ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
852	ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
853	ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
854	}
855
856
857	#pragma omp critical (write_to_tag_list)
858	tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
859
860	}
861
862
863	}
864
865
866	MPI_Barrier_local(local_comm);
867
868
869	vector<int> bcast_buf(8);
870	if(is_local_leader)
871	{
872	std::copy(size_info, size_info+4, bcast_buf.begin());
873	std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
874	}
875
876
877
878	MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
879
880
881	if(!is_local_leader)
882	{
883	std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
884	std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
885	}
886
887
888
889	if(!is_local_leader)
890	{
891	new_rank_info[1].resize(size_info[2]);
892	ep_info[1].resize(size_info[1]);
893	offset.resize(size_info[0]);
894	}
895
896	bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
897
898	if(is_local_leader)
899	{
900	bcast_buf[0] = remote_ep_size;
901	std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
902	std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
903	std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
904	}
905
906	MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
907
908
909	if(!is_local_leader)
910	{
911	remote_ep_size = bcast_buf[0];
912	std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
913	std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
914	std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
915	}
916
917
918	int my_position = offset[rank_in_local_parent]+ep_rank_loc;
919
920
921	MPI_Barrier_local(local_comm);
922	#pragma omp flush
923
924
925	#pragma omp critical (read_from_tag_list)
926	{
927	bool found = false;
928	while(!found)
929	{
930	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
931	{
932	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
933	{
934	*newintercomm = iter->second[my_position];
935
936	found = true;
937	//tag_list.erase(iter);
938	break;
939	}
940	}
941	}
942	}
943
944	MPI_Barrier_local(local_comm);
945
946	if(is_proc_master)
947	{
948	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
949	{
950	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
951	{
952	tag_list.erase(iter);
953	break;
954	}
955	}
956	}
957
958	int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
959	int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
960
961	intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
962	intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
963	intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
964	intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
965	intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
966	intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
967
968
969	MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
970
971	int my_rank_map_elem[2];
972
973
974	my_rank_map_elem[0] = intercomm_ep_rank;
975
976	my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
977
978	vector<pair<int, int> > local_rank_map_array;
979	vector<pair<int, int> > remote_rank_map_array;
980
981
982	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
983	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
984
985	MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
986
987	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
988	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
989
990	int local_intercomm_size = intercomm_ep_size;
991	int remote_intercomm_size;
992
993
994	if(is_local_leader)
995	{
996	::MPI_Status status;
997
998	::MPI_Send((newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2local_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
999
1000	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
1001
1002	::MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
1003
1004	::MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
1005	}
1006
1007	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, local_leader, local_comm);
1008	MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, 0, *newintercomm);
1009
1010
1011	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
1012	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
1013
1014	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
1015	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
1016	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
1017
1018
1019	if(is_local_leader)
1020	{
1021	::MPI_Status status;
1022
1023	::MPI_Send((newintercomm).rank_map->data(), 2local_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
1024
1025	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
1026	}
1027
1028	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, 0, *newintercomm);
1029
1030	(*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
1031	(*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
1032
1033
1034	// for(int i=0; i<local_ep_size; i++)
1035	// if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, remote_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
1036	// (newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).first, (newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).second);
1037
1038	// for(int i=0; i<remote_intercomm_size; i++)
1039	// if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, intercomm_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
1040	// (newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first, (newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second);
1041
1042	// for(int i=0; i<(*newintercomm).rank_map->size(); i++)
1043	// if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
1044	// (newintercomm).rank_map->at(i).first, (newintercomm).rank_map->at(i).second);
1045
1046
1047
1048
1049
1050	return MPI_SUCCESS;
1051
1052	}
1053
1054	#endif
1055
1056	}
1057

Note: See TracBrowser for help on using the repository browser.

Download in other formats: