Context Navigation

source: XIOS/dev/branch_yushan/extern/src_ep_dev/ep_intercomm_world.cpp @ 1068

Last change on this file since 1068 was 1067, checked in by yushan, 7 years ago
server mode OK tested with test_complete
File size: 34.2 KB

Line
1	#include "ep_lib.hpp"
2	#include <mpi.h>
3	#include "ep_declaration.hpp"
4
5	using namespace std;
6
7	namespace ep_lib
8	{
9
10	#ifdef _openmpi
11
12	int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
13	{
14
15	int ep_rank, ep_rank_loc, mpi_rank;
16	int ep_size, num_ep, mpi_size;
17
18	ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
19	ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
20	mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
21	ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
22	num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
23	mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
24
25
26	std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm
27	//! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
28
29	int rank_in_world;
30	int rank_in_local_parent;
31
32	int local_ep_size = ep_size;
33	int remote_ep_size;
34
35	::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
36	::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
37
38	::MPI_Comm_rank(peer_comm, &rank_in_world);
39
40	::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
41
42	bool is_proc_master = false;
43	bool is_local_leader = false;
44	bool is_final_master = false;
45
46
47	if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
48	if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
49
50
51	int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
52
53	int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
54
55
56	std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
57
58	std::vector<int> new_rank_info[4];
59	std::vector<int> new_ep_info[2];
60
61	std::vector<int> offset;
62
63	if(is_proc_master)
64	{
65
66	size_info[0] = mpi_size;
67
68	rank_info[0].resize(size_info[0]);
69	rank_info[1].resize(size_info[0]);
70
71
72
73	ep_info[0].resize(size_info[0]);
74
75	vector<int> send_buf(6);
76	vector<int> recv_buf(3*size_info[0]);
77
78	send_buf[0] = rank_in_world;
79	send_buf[1] = rank_in_local_parent;
80	send_buf[2] = num_ep;
81
82	::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
83
84	for(int i=0; i<size_info[0]; i++)
85	{
86	rank_info[0][i] = recv_buf[3*i];
87	rank_info[1][i] = recv_buf[3*i+1];
88	ep_info[0][i] = recv_buf[3*i+2];
89	}
90
91	if(is_local_leader)
92	{
93	leader_info[0] = rank_in_world;
94	leader_info[1] = mpi_remote_leader;
95
96	::MPI_Status mpi_status;
97
98	send_buf[0] = size_info[0];
99	send_buf[1] = local_ep_size;
100
101	::MPI_Send(send_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
102
103	::MPI_Recv(recv_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
104
105	recv_buf[2] = rank_in_world;
106	recv_buf[3] = mpi_remote_leader;
107
108	}
109
110	::MPI_Bcast(recv_buf.data(), 4, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
111
112	size_info[1] = recv_buf[0];
113	remote_ep_size = recv_buf[1];
114	leader_info[0] = recv_buf[2];
115	leader_info[1] = recv_buf[3];
116
117	rank_info[2].resize(size_info[1]);
118	rank_info[3].resize(size_info[1]);
119
120	ep_info[1].resize(size_info[1]);
121
122	send_buf.resize(3*size_info[0]);
123	recv_buf.resize(3*size_info[1]);
124
125	if(is_local_leader)
126	{
127	::MPI_Status mpi_status;
128
129
130	std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
131	std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
132	std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] );
133
134	::MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
135
136	::MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
137
138	}
139
140	::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
141
142	std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
143	std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() );
144	std::copy ( recv_buf.data() + 2size_info[1], recv_buf.data() + 3size_info[1], ep_info[1].begin() );
145
146	offset.resize(size_info[0]);
147
148	if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
149	{
150
151	bool found = false;
152	int ep_tmp;
153	int ep_local;
154	int ep_remote;
155	for(int i=0; i<size_info[0]; i++)
156	{
157	int target = rank_info[0][i];
158	found = false;
159	for(int j=0; j<size_info[1]; j++)
160	{
161	if(target == rank_info[2][j])
162	{
163	found = true;
164	ep_tmp = ep_info[1][j];
165	ep_local = ep_info[0][j];
166	ep_remote = ep_info[1][j];
167	break;
168	}
169	}
170	if(found)
171	{
172
173	if(target == leader_info[0]) // the leader is doubled in remote
174	{
175	new_rank_info[0].push_back(target);
176	new_rank_info[1].push_back(rank_info[1][i]);
177
178	new_ep_info[0].push_back(ep_local + ep_remote);
179	offset[i] = 0;
180	}
181	else
182	{
183	offset[i] = ep_local;
184	}
185	}
186	else
187	{
188	new_rank_info[0].push_back(target);
189	new_rank_info[1].push_back(rank_info[1][i]);
190
191	new_ep_info[0].push_back(ep_info[0][i]);
192
193	offset[i] = 0;
194	}
195
196	}
197	}
198
199	else // erase rank doubled with remote leader
200	{
201
202	bool found = false;
203	int ep_tmp;
204	int ep_local;
205	int ep_remote;
206	for(int i=0; i<size_info[0]; i++)
207	{
208	int target = rank_info[0][i];
209	found = false;
210	for(int j=0; j<size_info[1]; j++)
211	{
212
213	if(target == rank_info[2][j])
214	{
215	found = true;
216	ep_tmp = ep_info[1][j];
217	ep_local = ep_info[0][j];
218	ep_remote = ep_info[1][j];
219	break;
220	}
221	}
222	if(found)
223	{
224	if(target != leader_info[1])
225	{
226	new_rank_info[0].push_back(target);
227	new_rank_info[1].push_back(rank_info[1][i]);
228
229	new_ep_info[0].push_back(ep_local + ep_remote);
230	offset[i] = 0;
231	}
232	else // found remote leader
233	{
234	offset[i] = ep_remote;
235	}
236	}
237	else
238	{
239	new_rank_info[0].push_back(target);
240	new_rank_info[1].push_back(rank_info[1][i]);
241
242	new_ep_info[0].push_back(ep_info[0][i]);
243	offset[i] = 0;
244	}
245	}
246	}
247
248	if(offset[mpi_rank] == 0)
249	{
250	is_final_master = true;
251	}
252
253
254	// size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
255
256	if(is_local_leader)
257	{
258	size_info[2] = new_ep_info[0].size();
259	::MPI_Status mpi_status;
260
261	::MPI_Send(&size_info[2], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
262
263	::MPI_Recv(&size_info[3], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
264	}
265
266	::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
267
268	new_rank_info[2].resize(size_info[3]);
269	new_rank_info[3].resize(size_info[3]);
270	new_ep_info[1].resize(size_info[3]);
271
272	send_buf.resize(size_info[2]);
273	recv_buf.resize(size_info[3]);
274
275	if(is_local_leader)
276	{
277	::MPI_Status mpi_status;
278
279	std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
280	std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
281	std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] );
282
283	::MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
284	::MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
285
286	}
287
288	::MPI_Bcast(recv_buf.data(), 3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
289
290	std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
291	std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() );
292	std::copy ( recv_buf.data() + 2size_info[3], recv_buf.data() + 3size_info[3], new_ep_info[1].begin() );
293
294	}
295
296
297
298	if(is_proc_master)
299	{
300	// leader_info[4]: 2-> rank of local leader in new_group generated comm;
301	// 3-> rank of remote leader in new_group generated comm;
302	::MPI_Group local_group;
303	::MPI_Group new_group;
304	::MPI_Comm new_comm;
305	::MPI_Comm intercomm;
306
307	::MPI_Comm_group(local_mpi_comm, &local_group);
308
309	::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
310
311	::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
312
313
314
315	if(is_local_leader)
316	{
317	::MPI_Comm_rank(new_comm, &leader_info[2]);
318	}
319
320	::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
321
322	if(new_comm != MPI_COMM_NULL_STD)
323	{
324	::MPI_Barrier(new_comm);
325
326	::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
327
328	int id;
329	MPI_Comm_rank(new_comm, &id);
330	int my_num_ep = new_ep_info[0][id];
331
332	MPI_Comm *ep_intercomm;
333	MPI_Info info;
334	MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
335
336
337
338	for(int i= 0; i<my_num_ep; i++)
339	{
340	ep_intercomm[i].is_intercomm = true;
341
342	ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
343	ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
344	ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
345	}
346
347	#pragma omp critical (write_to_tag_list)
348	tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
349	}
350
351
352	}
353
354
355	MPI_Barrier_local(local_comm);
356
357	vector<int> bcast_buf(8);
358	if(is_local_leader)
359	{
360	std::copy(size_info, size_info+4, bcast_buf.begin());
361	std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
362	}
363
364	MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
365
366	if(!is_local_leader)
367	{
368	std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
369	std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
370	}
371
372	if(!is_local_leader)
373	{
374	new_rank_info[1].resize(size_info[2]);
375	ep_info[1].resize(size_info[1]);
376	offset.resize(size_info[0]);
377	}
378
379	bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
380
381	if(is_local_leader)
382	{
383	bcast_buf[0] = remote_ep_size;
384	std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
385	std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
386	std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
387	}
388
389	MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
390
391	if(!is_local_leader)
392	{
393	remote_ep_size = bcast_buf[0];
394	std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
395	std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
396	std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
397	}
398
399
400	int my_position = offset[rank_in_local_parent]+ep_rank_loc;
401
402
403	MPI_Barrier_local(local_comm);
404	#pragma omp flush
405
406
407	#pragma omp critical (read_from_tag_list)
408	{
409	bool found = false;
410	while(!found)
411	{
412	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
413	{
414	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
415	{
416	*newintercomm = iter->second[my_position];
417
418	found = true;
419	tag_list.erase(iter);
420	break;
421	}
422	}
423	}
424	}
425
426	MPI_Barrier_local(local_comm);
427
428	// if(is_proc_master)
429	// {
430	// for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
431	// {
432	// if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
433	// {
434	// tag_list.erase(iter);
435	// }
436	// }
437	// }
438
439
440
441	int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
442	int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
443
444	intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
445	intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
446	intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
447	intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
448	intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
449	intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
450
451	MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
452
453	int my_rank_map_elem[2];
454
455
456	my_rank_map_elem[0] = intercomm_ep_rank;
457
458	my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
459
460	vector<pair<int, int> > local_rank_map_array;
461	vector<pair<int, int> > remote_rank_map_array;
462
463
464	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
465	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
466
467	MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
468
469	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
470	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
471
472	int local_intercomm_size = intercomm_ep_size;
473	int remote_intercomm_size;
474
475
476
477
478	if(is_local_leader)
479	{
480	::MPI_Status status;
481
482	::MPI_Send((newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2local_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
483
484	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
485
486	::MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
487
488	::MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
489	}
490
491	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, local_leader, local_comm);
492	MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, 0, *newintercomm);
493
494
495	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
496	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
497
498	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
499	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
500	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
501
502
503	if(is_local_leader)
504	{
505	::MPI_Status status;
506
507	::MPI_Send((newintercomm).rank_map->data(), 2local_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
508
509	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
510	}
511
512	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, 0, *newintercomm);
513
514	(*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
515	(*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
516
517
518	return MPI_SUCCESS;
519
520	}
521
522
523
524	#elif _intelmpi
525	int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, int peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm)
526	{
527	int ep_rank, ep_rank_loc, mpi_rank;
528	int ep_size, num_ep, mpi_size;
529
530	ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first;
531	ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first;
532	mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first;
533	ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second;
534	num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second;
535	mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second;
536
537	std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm
538	//! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm
539
540	int rank_in_world;
541	int rank_in_local_parent;
542
543	int local_ep_size = ep_size;
544	int remote_ep_size;
545
546	::MPI_Comm peer_comm = static_cast< ::MPI_Comm>(peer_comm_ptr);
547	::MPI_Comm local_mpi_comm = static_cast< ::MPI_Comm>(local_comm.mpi_comm);
548
549	::MPI_Comm_rank(peer_comm, &rank_in_world);
550
551	::MPI_Comm_rank(static_cast< ::MPI_Comm>(local_comm.mpi_comm), &rank_in_local_parent);
552
553	bool is_proc_master = false;
554	bool is_local_leader = false;
555	bool is_final_master = false;
556
557
558	if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;}
559	if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true;
560
561
562	int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm
563
564	int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader
565
566
567	std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm
568
569	std::vector<int> new_rank_info[4];
570	std::vector<int> new_ep_info[2];
571
572	std::vector<int> offset;
573
574	if(is_proc_master)
575	{
576
577	size_info[0] = mpi_size;
578
579	rank_info[0].resize(size_info[0]);
580	rank_info[1].resize(size_info[0]);
581
582	ep_info[0].resize(size_info[0]);
583
584	vector<int> send_buf(6);
585	vector<int> recv_buf(3*size_info[0]);
586
587	send_buf[0] = rank_in_world;
588	send_buf[1] = rank_in_local_parent;
589	send_buf[2] = num_ep;
590
591	::MPI_Allgather(send_buf.data(), 3, MPI_INT_STD, recv_buf.data(), 3, MPI_INT_STD, local_mpi_comm);
592
593	for(int i=0; i<size_info[0]; i++)
594	{
595	rank_info[0][i] = recv_buf[3*i];
596	rank_info[1][i] = recv_buf[3*i+1];
597	ep_info[0][i] = recv_buf[3*i+2];
598	}
599
600
601	if(is_local_leader)
602	{
603	leader_info[0] = rank_in_world;
604	leader_info[1] = mpi_remote_leader;
605
606	::MPI_Status mpi_status;
607
608	send_buf[0] = size_info[0];
609	send_buf[1] = local_ep_size;
610
611	::MPI_Send(send_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
612
613	::MPI_Recv(recv_buf.data(), 2, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
614
615	recv_buf[2] = rank_in_world;
616	recv_buf[3] = mpi_remote_leader;
617
618	}
619
620	::MPI_Bcast(recv_buf.data(), 4, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
621
622	size_info[1] = recv_buf[0];
623	remote_ep_size = recv_buf[1];
624	leader_info[0] = recv_buf[2];
625	leader_info[1] = recv_buf[3];
626
627	rank_info[2].resize(size_info[1]);
628	rank_info[3].resize(size_info[1]);
629
630	ep_info[1].resize(size_info[1]);
631
632	send_buf.resize(3*size_info[0]);
633	recv_buf.resize(3*size_info[1]);
634
635	if(is_local_leader)
636	{
637	::MPI_Status mpi_status;
638
639
640	std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() );
641	std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] );
642	std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] );
643
644	::MPI_Send(send_buf.data(), 3*size_info[0], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
645
646	::MPI_Recv(recv_buf.data(), 3*size_info[1], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
647
648	}
649
650	::MPI_Bcast(recv_buf.data(), 3*size_info[1], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
651
652	std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() );
653	std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() );
654	std::copy ( recv_buf.data() + 2size_info[1], recv_buf.data() + 3size_info[1], ep_info[1].begin() );
655
656	offset.resize(size_info[0]);
657
658	if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader
659	{
660
661	bool found = false;
662	int ep_tmp;
663	int ep_local;
664	int ep_remote;
665	for(int i=0; i<size_info[0]; i++)
666	{
667	int target = rank_info[0][i];
668	found = false;
669	for(int j=0; j<size_info[1]; j++)
670	{
671	if(target == rank_info[2][j])
672	{
673	found = true;
674	ep_tmp = ep_info[1][j];
675	ep_local = ep_info[0][j];
676	ep_remote = ep_info[1][j];
677	break;
678	}
679	}
680	if(found)
681	{
682
683	if(target == leader_info[0]) // the leader is doubled in remote
684	{
685	new_rank_info[0].push_back(target);
686	new_rank_info[1].push_back(rank_info[1][i]);
687
688	new_ep_info[0].push_back(ep_local + ep_remote);
689	offset[i] = 0;
690	}
691	else
692	{
693	offset[i] = ep_local;
694	}
695	}
696	else
697	{
698	new_rank_info[0].push_back(target);
699	new_rank_info[1].push_back(rank_info[1][i]);
700
701	new_ep_info[0].push_back(ep_info[0][i]);
702
703	offset[i] = 0;
704	}
705
706	}
707	}
708
709	else // erase rank doubled with remote leader
710	{
711
712	bool found = false;
713	int ep_tmp;
714	int ep_local;
715	int ep_remote;
716	for(int i=0; i<size_info[0]; i++)
717	{
718	int target = rank_info[0][i];
719	found = false;
720	for(int j=0; j<size_info[1]; j++)
721	{
722
723	if(target == rank_info[2][j])
724	{
725	found = true;
726	ep_tmp = ep_info[1][j];
727	ep_local = ep_info[0][j];
728	ep_remote = ep_info[1][j];
729	break;
730	}
731	}
732	if(found)
733	{
734	if(target != leader_info[1])
735	{
736	new_rank_info[0].push_back(target);
737	new_rank_info[1].push_back(rank_info[1][i]);
738
739	new_ep_info[0].push_back(ep_local + ep_remote);
740	offset[i] = 0;
741	}
742	else // found remote leader
743	{
744	offset[i] = ep_remote;
745	}
746	}
747	else
748	{
749	new_rank_info[0].push_back(target);
750	new_rank_info[1].push_back(rank_info[1][i]);
751
752	new_ep_info[0].push_back(ep_info[0][i]);
753	offset[i] = 0;
754	}
755	}
756	}
757
758	if(offset[mpi_rank] == 0)
759	{
760	is_final_master = true;
761	}
762
763
764	// size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote
765
766	if(is_local_leader)
767	{
768	size_info[2] = new_ep_info[0].size();
769	::MPI_Status mpi_status;
770
771	::MPI_Send(&size_info[2], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
772
773	::MPI_Recv(&size_info[3], 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
774	}
775
776	::MPI_Bcast(&size_info[2], 2, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
777
778	new_rank_info[2].resize(size_info[3]);
779	new_rank_info[3].resize(size_info[3]);
780	new_ep_info[1].resize(size_info[3]);
781
782	send_buf.resize(size_info[2]);
783	recv_buf.resize(size_info[3]);
784
785	if(is_local_leader)
786	{
787	::MPI_Status mpi_status;
788
789	std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() );
790	std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] );
791	std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] );
792
793	::MPI_Send(send_buf.data(), 3*size_info[2], MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
794
795	::MPI_Recv(recv_buf.data(), 3*size_info[3], MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &mpi_status);
796	}
797
798	::MPI_Bcast(recv_buf.data(), 3*size_info[3], MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
799
800	std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() );
801	std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() );
802	std::copy ( recv_buf.data() + 2size_info[3], recv_buf.data() + 3size_info[3], new_ep_info[1].begin() );
803
804	}
805
806
807
808	if(is_proc_master)
809	{
810	// leader_info[4]: 2-> rank of local leader in new_group generated comm;
811	// 3-> rank of remote leader in new_group generated comm;
812	::MPI_Group local_group;
813	::MPI_Group new_group;
814	::MPI_Comm new_comm;
815	::MPI_Comm intercomm;
816
817	::MPI_Comm_group(local_mpi_comm, &local_group);
818
819	::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group);
820
821	::MPI_Comm_create(local_mpi_comm, new_group, &new_comm);
822
823
824
825	if(is_local_leader)
826	{
827	::MPI_Comm_rank(new_comm, &leader_info[2]);
828	}
829
830	::MPI_Bcast(&leader_info[2], 1, MPI_INT_STD, local_comm.rank_map->at(local_leader).second, local_mpi_comm);
831
832	if(new_comm != MPI_COMM_NULL_STD)
833	{
834	::MPI_Barrier(new_comm);
835
836	::MPI_Intercomm_create(new_comm, leader_info[2], peer_comm, leader_info[1], tag, &intercomm);
837
838	int id;
839	::MPI_Comm_rank(new_comm, &id);
840	int my_num_ep = new_ep_info[0][id];
841
842	MPI_Comm *ep_intercomm;
843	MPI_Info info;
844	MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm);
845
846	for(int i= 0; i<my_num_ep; i++)
847	{
848	ep_intercomm[i].is_intercomm = true;
849
850	ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
851	ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm;
852	ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0];
853	}
854
855
856	#pragma omp critical (write_to_tag_list)
857	tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm));
858
859	}
860
861
862	}
863
864
865	MPI_Barrier_local(local_comm);
866
867
868	vector<int> bcast_buf(8);
869	if(is_local_leader)
870	{
871	std::copy(size_info, size_info+4, bcast_buf.begin());
872	std::copy(leader_info, leader_info+4, bcast_buf.begin()+4);
873	}
874
875
876
877	MPI_Bcast(bcast_buf.data(), 8, MPI_INT_STD, local_leader, local_comm);
878
879
880	if(!is_local_leader)
881	{
882	std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info);
883	std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info);
884	}
885
886
887
888	if(!is_local_leader)
889	{
890	new_rank_info[1].resize(size_info[2]);
891	ep_info[1].resize(size_info[1]);
892	offset.resize(size_info[0]);
893	}
894
895	bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1);
896
897	if(is_local_leader)
898	{
899	bcast_buf[0] = remote_ep_size;
900	std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1);
901	std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1);
902	std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1);
903	}
904
905	MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT_STD, local_leader, local_comm);
906
907
908	if(!is_local_leader)
909	{
910	remote_ep_size = bcast_buf[0];
911	std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin());
912	std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin());
913	std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin());
914	}
915
916
917	int my_position = offset[rank_in_local_parent]+ep_rank_loc;
918
919
920	MPI_Barrier_local(local_comm);
921	#pragma omp flush
922
923
924	#pragma omp critical (read_from_tag_list)
925	{
926	bool found = false;
927	while(!found)
928	{
929	for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
930	{
931	if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
932	{
933	*newintercomm = iter->second[my_position];
934
935	found = true;
936	tag_list.erase(iter);
937	break;
938	}
939	}
940	}
941	}
942
943	MPI_Barrier_local(local_comm);
944
945	// if(is_proc_master)
946	// {
947	// for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++)
948	// {
949	// if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1])))
950	// {
951	// tag_list.erase(iter);
952	// }
953	// }
954	// }
955
956	int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank;
957	int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size;
958
959	intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first;
960	intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first;
961	intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first;
962	intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second;
963	intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second;
964	intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second;
965
966
967	MPI_Bcast(&remote_ep_size, 1, MPI_INT_STD, local_leader, local_comm);
968
969	int my_rank_map_elem[2];
970
971
972	my_rank_map_elem[0] = intercomm_ep_rank;
973
974	my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label;
975
976	vector<pair<int, int> > local_rank_map_array;
977	vector<pair<int, int> > remote_rank_map_array;
978
979
980	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP;
981	(*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size);
982
983	MPI_Allgather(my_rank_map_elem, 2, MPI_INT_STD, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT_STD, local_comm);
984
985	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP;
986	(*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size);
987
988	int local_intercomm_size = intercomm_ep_size;
989	int remote_intercomm_size;
990
991
992	if(is_local_leader)
993	{
994	::MPI_Status status;
995
996	::MPI_Send((newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2local_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
997
998	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
999
1000	::MPI_Send(&local_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
1001
1002	::MPI_Recv(&remote_intercomm_size, 1, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
1003	}
1004
1005	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2remote_ep_size, MPI_INT_STD, local_leader, local_comm);
1006	MPI_Bcast(&remote_intercomm_size, 1, MPI_INT_STD, 0, *newintercomm);
1007
1008
1009	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
1010	(*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
1011
1012	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0];
1013	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1];
1014	(*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2];
1015
1016
1017	if(is_local_leader)
1018	{
1019	::MPI_Status status;
1020
1021	::MPI_Send((newintercomm).rank_map->data(), 2local_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm);
1022
1023	::MPI_Recv((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, mpi_remote_leader, tag, peer_comm, &status);
1024	}
1025
1026	MPI_Bcast((newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2remote_intercomm_size, MPI_INT_STD, 0, *newintercomm);
1027
1028	(*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]);
1029	(*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label;
1030
1031
1032	// for(int i=0; i<local_ep_size; i++)
1033	// if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, remote_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
1034	// (newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).first, (newintercomm).ep_comm_ptr->intercomm->local_rank_map->at(i).second);
1035
1036	// for(int i=0; i<remote_intercomm_size; i++)
1037	// if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, intercomm_rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
1038	// (newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).first, (newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->at(i).second);
1039
1040	// for(int i=0; i<(*newintercomm).rank_map->size(); i++)
1041	// if(local_comm.ep_comm_ptr->comm_label != 99) printf("ep_rank = %d, rank_map[%d] = (%d,%d)\n", intercomm_ep_rank, i,
1042	// (newintercomm).rank_map->at(i).first, (newintercomm).rank_map->at(i).second);
1043
1044
1045
1046
1047
1048	return MPI_SUCCESS;
1049
1050	}
1051
1052	#endif
1053
1054	}
1055

Note: See TracBrowser for help on using the repository browser.

Download in other formats: