Changeset 1527
- Timestamp:
- 06/08/18 15:59:53 (7 years ago)
- Location:
- XIOS/dev/branch_openmp/extern/ep_dev
- Files:
-
- 1 added
- 2 deleted
- 43 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/dev/branch_openmp/extern/ep_dev/ep_allgather.cpp
r1511 r1527 43 43 { 44 44 45 if(!comm->is_ep && comm->mpi_comm) 46 { 47 return ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 48 } 45 if(!comm->is_ep) return ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 46 if(comm->is_intercomm) return MPI_Allgather_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); 47 49 48 50 49 assert(sendcount == recvcount); … … 119 118 MPI_Bcast_local(recvbuf, count*ep_size, datatype, 0, comm); 120 119 121 MPI_Barrier(comm);122 123 124 120 if(is_master) 125 121 { … … 131 127 } 132 128 129 int MPI_Allgather_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) 130 { 131 printf("MPI_Allgather_intercomm not yet implemented\n"); 132 MPI_Abort(comm, 0); 133 } 133 134 134 135 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_allgatherv.cpp
r1503 r1527 20 20 { 21 21 22 if(!comm->is_ep && comm->mpi_comm) 23 { 24 return ::MPI_Allgatherv(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcounts, displs, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 25 } 26 27 if(!comm->mpi_comm) return 0; 22 if(!comm->is_ep) return ::MPI_Allgatherv(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcounts, displs, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 23 if(comm->is_intercomm) return MPI_Allgatherv_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm); 28 24 29 25 … … 114 110 delete[] tmp_recvbuf; 115 111 } 116 117 118 112 } 119 113 120 114 115 int MPI_Allgatherv_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, MPI_Comm comm) 116 { 117 printf("MPI_Allgatherv_intercomm not yet implemented\n"); 118 MPI_Abort(comm, 0); 119 } 120 121 121 122 122 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_allocate.cpp
r1500 r1527 12 12 { 13 13 ::MPI_Alloc_mem(to_mpi_aint(size), to_mpi_info(info), baseptr); 14 //::MPI_Alloc_mem(size.mpi_aint, MPI_INFO_NULL_STD, baseptr);15 14 return 0; 16 15 } … … 19 18 { 20 19 ::MPI_Alloc_mem(size, *(static_cast< ::MPI_Info*>(info->mpi_info)), baseptr); 21 //::MPI_Alloc_mem(size, MPI_INFO_NULL_STD, baseptr);22 20 return 0; 23 21 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_allreduce.cpp
r1500 r1527 17 17 { 18 18 19 20 21 19 int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 22 20 { 23 if(!comm->is_ep && comm->mpi_comm) 24 { 25 return ::MPI_Allreduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 26 } 27 21 if(!comm->is_ep) return ::MPI_Allreduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 22 if(comm->is_intercomm) return MPI_Allreduce_intercomm(sendbuf, recvbuf, count, datatype, op, comm); 28 23 29 24 … … 66 61 } 67 62 68 MPI_Barrier_local(comm);69 63 } 70 64 71 65 66 int MPI_Allreduce_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 67 { 68 printf("MPI_Allreduce_intercomm not yet implemented\n"); 69 MPI_Abort(comm, 0); 70 } 72 71 73 72 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_alltoall.cpp
r1500 r1527 9 9 int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) 10 10 { 11 if(!comm->is_ep) 12 { 13 return ::MPI_Alltoall(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 14 } 11 if(!comm->is_ep) return ::MPI_Alltoall(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 12 if(comm->is_intercomm) return MPI_Alltoall_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); 15 13 16 14 … … 55 53 delete[] tmp_sendbuf; 56 54 } 57 58 MPI_Barrier(comm);59 60 return 0;61 55 } 62 56 57 58 int MPI_Alltoall_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) 59 { 60 printf("MPI_Alltoall_intercomm not yet implemented\n"); 61 MPI_Abort(comm, 0); 62 } 63 63 } 64 64 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_barrier.cpp
r1525 r1527 9 9 int MPI_Barrier(MPI_Comm comm) 10 10 { 11 if(comm->is_intercomm) return MPI_Barrier_intercomm(comm);12 11 13 12 if(comm->is_ep) 14 13 { 15 return MPI_Barrier_ intracomm(comm);14 return MPI_Barrier_endpoint(comm); 16 15 } 17 16 … … 21 20 } 22 21 23 int MPI_Barrier_ intracomm(MPI_Comm comm)22 int MPI_Barrier_endpoint(MPI_Comm comm) 24 23 { 25 24 int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; … … 35 34 36 35 MPI_Barrier_local(comm); 37 38 return 0;39 36 } 40 37 41 int MPI_Barrier2(MPI_Comm comm)42 {43 int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first;44 45 MPI_Barrier_local(comm);46 47 if(ep_rank_loc == 0)48 {49 ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm);50 51 ::MPI_Barrier(mpi_comm);52 }53 54 MPI_Barrier_local(comm);55 56 return 0;57 }58 59 int MPI_Barrier_intercomm(MPI_Comm comm)60 {61 MPI_Barrier_local(comm);62 63 if(comm->ep_comm_ptr->intercomm->size_rank_info[1].first == 0)64 ::MPI_Barrier(to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm));65 66 MPI_Barrier_local(comm);67 }68 38 69 39 int MPI_Barrier_mpi(MPI_Comm comm) -
XIOS/dev/branch_openmp/extern/ep_dev/ep_bcast.cpp
r1503 r1527 45 45 { 46 46 47 if(!comm->is_ep) 48 { 49 #pragma omp single nowait 50 ::MPI_Bcast(buffer, count, to_mpi_type(datatype), root, to_mpi_comm(comm->mpi_comm)); 51 return 0; 52 } 47 if(!comm->is_ep) return ::MPI_Bcast(buffer, count, to_mpi_type(datatype), root, to_mpi_comm(comm->mpi_comm)); 48 if(comm->is_intercomm) return MPI_Bcast_intercomm(buffer, count, datatype, root, comm); 53 49 54 50 … … 59 55 int root_mpi_rank = comm->ep_rank_map->at(root).second; 60 56 int root_ep_rank_loc = comm->ep_rank_map->at(root).first; 57 58 //printf("ep_rank = %d, root_mpi_rank = %d, root_ep_rank_loc = %d\n", ep_rank, root_mpi_rank, root_ep_rank_loc); 61 59 62 60 … … 69 67 else MPI_Bcast_local(buffer, count, datatype, 0, comm); 70 68 71 return 0;72 69 } 73 70 74 71 75 72 int MPI_Bcast_intercomm(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm) 73 { 74 printf("MPI_Bcast_intercomm not yet implemented\n"); 75 MPI_Abort(comm, 0); 76 } 76 77 77 78 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_comm.hpp
r1525 r1527 3 3 4 4 #include "ep_message.hpp" 5 #include "ep_intercomm.hpp"6 5 #include "ep_barrier.hpp" 7 6 #include "ep_buffer.hpp" … … 26 25 // 2: mpi_rank, mpi_size 27 26 28 // for intercomm : = size_rank_info of local_comm29 30 31 27 ep_comm **comm_list; 32 28 33 29 Message_list *message_queue; 34 30 35 36 int comm_label;37 38 ep_intercomm *intercomm;39 40 31 }; 41 32 … … 52 43 ep_barrier *ep_barrier; 53 44 54 EP_RANK_MAP *ep_rank_map; // for intercomm : = ep_rank_map of newcomm45 EP_RANK_MAP *ep_rank_map; 55 46 56 47 INTER_RANK_MAP *inter_rank_map; -
XIOS/dev/branch_openmp/extern/ep_dev/ep_create.cpp
r1517 r1527 71 71 out_comm_hdls[idx]->mpi_comm = parent_comm; 72 72 out_comm_hdls[idx]->ep_comm_ptr->comm_list = out_comm_hdls; 73 out_comm_hdls[idx]->ep_comm_ptr->comm_label = 0;74 73 } 75 74 … … 130 129 } 131 130 132 return 0;133 134 131 } //MPI_Comm_create_endpoints 135 132 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_dup.cpp
r1518 r1527 7 7 { 8 8 9 int MPI_Comm_dup_mpi(MPI_Comm comm, MPI_Comm *newcomm) 9 10 int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm) 10 11 { 11 newcomm = new MPI_Comm; 12 (*newcomm)->is_ep = false; 12 13 if(!comm->is_ep) 14 { 15 Debug("MPI_Comm_dup with MPI\n"); 16 return MPI_Comm_dup_mpi(comm, newcomm); 17 } 13 18 14 ::MPI_Comm *output = new ::MPI_Comm; 15 16 ::MPI_Comm_dup(to_mpi_comm(comm->mpi_comm), output); 17 18 (*newcomm)->mpi_comm = output; 19 return MPI_Comm_dup_endpoint(comm, newcomm); 20 19 21 } 20 22 21 int MPI_Comm_dup_intracomm(MPI_Comm comm, MPI_Comm *newcomm) 23 24 25 int MPI_Comm_dup_endpoint(MPI_Comm comm, MPI_Comm *newcomm) 22 26 { 23 27 int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; … … 37 41 38 42 *newcomm = (comm->ep_comm_ptr->comm_list[0]->mem_bridge[ep_rank_loc]); 43 44 if(comm->is_intercomm) 45 { 46 (*newcomm)->is_intercomm = true; 47 (*newcomm)->ep_comm_ptr->size_rank_info[0] = comm->ep_comm_ptr->size_rank_info[0]; 48 (*newcomm)->inter_rank_map = new INTER_RANK_MAP; 49 50 for(INTER_RANK_MAP::iterator it = comm->inter_rank_map->begin(); it !=comm->inter_rank_map->end(); it++) 51 { 52 (*newcomm)->inter_rank_map->insert(std::make_pair(it->first, it->second)); 53 } 54 55 } 39 56 40 57 } 41 58 42 int MPI_Comm_dup (MPI_Comm comm, MPI_Comm *newcomm)59 int MPI_Comm_dup_mpi(MPI_Comm comm, MPI_Comm *newcomm) 43 60 { 44 45 if(!comm->is_ep) 46 { 47 Debug("MPI_Comm_dup with MPI\n"); 48 return MPI_Comm_dup_mpi(comm, newcomm); 49 } 61 newcomm = new MPI_Comm; 62 (*newcomm)->is_ep = false; 50 63 51 if(comm->is_intercomm) return MPI_Comm_dup_intercomm(comm, newcomm); 64 ::MPI_Comm *output = new ::MPI_Comm; 65 66 ::MPI_Comm_dup(to_mpi_comm(comm->mpi_comm), output); 52 67 53 54 return MPI_Comm_dup_intracomm(comm, newcomm); 55 56 57 } 58 59 int MPI_Comm_dup_intercomm(MPI_Comm comm, MPI_Comm *newcomm) 60 { 61 62 int newcomm_ep_rank =comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 63 int newcomm_ep_rank_loc = comm->ep_comm_ptr->intercomm->size_rank_info[1].first; 64 int newcomm_num_ep = comm->ep_comm_ptr->intercomm->size_rank_info[1].second; 65 66 67 if(0 == newcomm_ep_rank_loc) 68 { 69 //printf("in dup , newcomm_ep_rank_loc = 0 : ep %d\n", comm->ep_comm_ptr->intercomm->size_rank_info[0].first); 70 71 MPI_Info info; 72 MPI_Comm *out_comm; 73 74 MPI_Comm_create_endpoints(comm->mpi_comm, newcomm_num_ep, info, out_comm); 75 76 ::MPI_Comm *mpi_inter_comm = new ::MPI_Comm; 77 ::MPI_Comm_dup(to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), mpi_inter_comm); 78 79 for(int i=0; i<newcomm_num_ep; i++) 80 { 81 out_comm[i]->is_intercomm = true; 82 out_comm[i]->ep_comm_ptr->comm_label = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->comm_label; 83 out_comm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 84 #ifdef _showinfo 85 printf("new out_comm[%d]->ep_comm_ptr->intercomm = %p\n", i, out_comm[i]->ep_comm_ptr->intercomm); 86 #endif 87 out_comm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter_comm; 88 } 89 90 91 comm->ep_comm_ptr->comm_list[0]->mem_bridge = out_comm; 92 } 93 94 MPI_Barrier_local(comm); 95 96 *newcomm = (comm->ep_comm_ptr->comm_list[0]->mem_bridge[newcomm_ep_rank_loc]); 97 98 (*newcomm)->ep_comm_ptr->size_rank_info[0] = comm->ep_comm_ptr->size_rank_info[0]; 99 (*newcomm)->ep_comm_ptr->size_rank_info[1] = comm->ep_comm_ptr->size_rank_info[1]; 100 (*newcomm)->ep_comm_ptr->size_rank_info[2] = comm->ep_comm_ptr->size_rank_info[2]; 101 102 (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[0] = comm->ep_comm_ptr->intercomm->size_rank_info[0]; 103 (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[1] = comm->ep_comm_ptr->intercomm->size_rank_info[1]; 104 (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[2] = comm->ep_comm_ptr->intercomm->size_rank_info[2]; 105 106 (*newcomm)->ep_comm_ptr->intercomm->intercomm_tag = comm->ep_comm_ptr->intercomm->intercomm_tag; 107 108 109 int ep_rank_loc = (*newcomm)->ep_comm_ptr->size_rank_info[1].first; 110 111 if(ep_rank_loc == 0) 112 { 113 int world_rank; 114 MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 115 116 (*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new INTERCOMM_RANK_MAP; 117 (*newcomm)->ep_comm_ptr->intercomm->local_rank_map = new EP_RANK_MAP; 118 119 *(*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = *comm->ep_comm_ptr->intercomm->intercomm_rank_map; 120 *(*newcomm)->ep_comm_ptr->intercomm->local_rank_map = *comm->ep_comm_ptr->intercomm->local_rank_map; 121 } 122 123 MPI_Barrier_local(comm); 124 125 if(ep_rank_loc !=0 ) 126 { 127 int target = (*newcomm)->ep_comm_ptr->intercomm->intercomm_tag; 128 (*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = (*newcomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->intercomm_rank_map; 129 (*newcomm)->ep_comm_ptr->intercomm->local_rank_map = (*newcomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->local_rank_map; 130 } 131 132 133 134 135 136 68 (*newcomm)->mpi_comm = output; 137 69 } 138 70 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_exscan.cpp
r1503 r1527 228 228 int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 229 229 { 230 if(!comm->is_ep) 231 { 232 return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 233 } 230 if(!comm->is_ep) return ::MPI_Exscan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 231 if(comm->is_intercomm) return MPI_Exscan_intercomm(sendbuf, recvbuf, count, datatype, op, comm); 234 232 235 233 valid_type(datatype); … … 291 289 292 290 if(ep_rank_loc == 0) 291 { 293 292 ::MPI_Exscan(MPI_IN_PLACE, tmp_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 294 295 // printf(" ID=%d : %d %d \n", ep_rank, static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]); 293 } 296 294 297 295 MPI_Exscan_local(tmp_sendbuf, tmp_recvbuf, count, datatype, op, comm); 298 299 // printf(" ID=%d : after local tmp_sendbuf = %d %d ; tmp_recvbuf = %d %d \n", ep_rank, static_cast<int*>(tmp_sendbuf)[0], static_cast<int*>(tmp_sendbuf)[1], static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]);300 301 296 302 297 … … 314 309 315 310 else memcpy(recvbuf, tmp_recvbuf, datasize*count); 316 317 318 319 311 320 312 delete[] tmp_sendbuf; … … 323 315 } 324 316 317 318 int MPI_Exscan_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 319 { 320 printf("MPI_Exscan_intercomm not yet implemented\n"); 321 MPI_Abort(comm, 0); 322 } 323 325 324 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_finalize.cpp
r1517 r1527 9 9 int MPI_Finalize() 10 10 { 11 printf("calling EP Finalize\n"); 12 13 int id = omp_get_thread_num(); 14 15 if(id == 0) 11 #pragma omp master 16 12 { 13 printf("calling EP Finalize\n"); 17 14 ::MPI_Finalize(); 18 15 } 19 return 0;20 16 } 21 17 22 18 int MPI_Abort(MPI_Comm comm, int errorcode) 23 {24 if(!comm->is_ep)25 return MPI_Abort_mpi(comm, errorcode);26 27 else28 {29 if(comm->ep_comm_ptr->size_rank_info[1].first == 0)30 {31 ::MPI_Abort(to_mpi_comm(comm->mpi_comm), errorcode);32 }33 }34 }35 36 int MPI_Abort_mpi(MPI_Comm comm, int errorcode)37 19 { 38 20 return ::MPI_Abort(to_mpi_comm(comm->mpi_comm), errorcode); -
XIOS/dev/branch_openmp/extern/ep_dev/ep_free.cpp
r1525 r1527 9 9 int MPI_Comm_free(MPI_Comm *comm) 10 10 { 11 if(! (*comm)->is_ep) 12 { 13 return MPI_Comm_free_mpi(comm); 14 } 15 16 else 17 { 18 if((*comm)->is_intercomm) 19 return MPI_Comm_free_intercomm(comm); 20 else 21 return MPI_Comm_free_intracomm(comm); 22 } 11 if(! (*comm)->is_ep) return MPI_Comm_free_mpi(comm); 12 else return MPI_Comm_free_endpoint(comm); 23 13 } 24 14 … … 33 23 } 34 24 35 int MPI_Comm_free_ intracomm(MPI_Comm *comm)25 int MPI_Comm_free_endpoint(MPI_Comm *comm) 36 26 { 37 27 Debug("MPI_Comm_free with EP_intracomm\n"); … … 40 30 41 31 ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first; 32 int ep_rank = (*comm)->ep_comm_ptr->size_rank_info[0].first; 42 33 num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second; 43 34 … … 46 37 if(ep_rank_loc == 0) 47 38 { 39 40 if((*comm)->is_intercomm) 41 { 42 for(int i=0; i<num_ep; i++) 43 { 44 (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map->clear(); 45 #ifdef _showinfo 46 printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->inter_rank_map = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map); 47 #endif 48 delete (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map; 49 } 50 } 51 52 48 53 49 54 #ifdef _showinfo … … 103 108 104 109 105 int MPI_Comm_free2(MPI_Comm *comm)106 {107 Debug("MPI_Comm_free with EP_intracomm\n");108 109 int ep_rank_loc, num_ep;110 111 ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first;112 num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second;113 114 MPI_Barrier2(*comm);115 116 if(ep_rank_loc == 0)117 {118 119 #ifdef _showinfo120 printf("delete (*comm)->my_buffer = %p\n", (*comm)->my_buffer);121 #endif122 delete (*comm)->my_buffer;123 124 125 126 #ifdef _showinfo127 printf("delete (*comm)->ep_barrier = %p\n", (*comm)->ep_barrier);128 #endif129 delete (*comm)->ep_barrier;130 131 132 133 (*comm)->ep_rank_map->clear();134 #ifdef _showinfo135 printf("delete (*comm)->ep_rank_map = %p\n", (*comm)->ep_rank_map);136 #endif137 delete (*comm)->ep_rank_map;138 139 140 for(int i=0; i<num_ep; i++)141 {142 if((*comm)->is_intercomm)143 {144 (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map->clear();145 #ifdef _showinfo146 printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->inter_rank_map = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map);147 #endif148 delete (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map;149 }150 151 (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue->clear();152 #ifdef _showinfo153 printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr->message_queue = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue);154 #endif155 delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue;156 157 158 #ifdef _showinfo159 printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr);160 #endif161 delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr;162 163 164 #ifdef _showinfo165 printf("delete (*comm)->ep_comm_ptr->comm_list[%d] = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]);166 #endif167 delete (*comm)->ep_comm_ptr->comm_list[i];168 }169 170 #ifdef _showinfo171 printf("delete (*comm)->mpi_comm = %p\n", (*comm)->mpi_comm);172 #endif173 ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->mpi_comm));174 175 #ifdef _showinfo176 printf("delete (*comm)->ep_comm_ptr->comm_list = %p\n", (*comm)->ep_comm_ptr->comm_list);177 #endif178 delete[] (*comm)->ep_comm_ptr->comm_list;179 }180 }181 182 183 184 int MPI_Comm_free_intercomm(MPI_Comm *comm)185 {186 int ep_rank;187 MPI_Comm_rank(*comm, &ep_rank);188 int ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first;189 int num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second;190 191 int newcomm_ep_rank =(*comm)->ep_comm_ptr->intercomm->size_rank_info[0].first;192 int newcomm_ep_rank_loc = (*comm)->ep_comm_ptr->intercomm->size_rank_info[1].first;193 int newcomm_num_ep = (*comm)->ep_comm_ptr->intercomm->size_rank_info[1].second;194 195 MPI_Barrier(*comm);196 197 if(ep_rank_loc == 0)198 {199 (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map->clear();200 #ifdef _showinfo201 printf("delete (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map = %p\n", (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map);202 #endif203 delete (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map;204 205 (*comm)->ep_comm_ptr->intercomm->local_rank_map->clear();206 #ifdef _showinfo207 printf("delete (*comm)->ep_comm_ptr->intercomm->local_rank_map = %p\n", (*comm)->ep_comm_ptr->intercomm->local_rank_map);208 #endif209 delete (*comm)->ep_comm_ptr->intercomm->local_rank_map;210 }211 212 if(newcomm_ep_rank_loc == 0)213 {214 215 #ifdef _showinfo216 printf("delete (*comm)->my_buffer = %p\n", (*comm)->my_buffer);217 #endif218 delete (*comm)->my_buffer;219 220 221 222 #ifdef _showinfo223 printf("delete (*comm)->ep_barrier = %p\n", (*comm)->ep_barrier);224 #endif225 delete (*comm)->ep_barrier;226 227 228 (*comm)->ep_rank_map->clear();229 #ifdef _showinfo230 printf("delete (*comm)->ep_rank_map = %p\n", (*comm)->ep_rank_map);231 #endif232 delete (*comm)->ep_rank_map;233 234 #ifdef _showinfo235 printf("delete (*comm)->ep_comm_ptr->intercomm->mpi_inter_comm = %p\n", (*comm)->ep_comm_ptr->intercomm->mpi_inter_comm);236 #endif237 ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->ep_comm_ptr->intercomm->mpi_inter_comm));238 239 for(int i=0; i<newcomm_num_ep; i++)240 {241 (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue->clear();242 #ifdef _showinfo243 printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr->message_queue = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue);244 #endif245 delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue;246 247 #ifdef _showinfo248 printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr->intercomm = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm);249 #endif250 delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm;251 252 253 #ifdef _showinfo254 printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr);255 #endif256 delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr;257 258 259 #ifdef _showinfo260 printf("delete (*comm)->ep_comm_ptr->comm_list[%d] = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]);261 #endif262 delete (*comm)->ep_comm_ptr->comm_list[i];263 264 }265 266 #ifdef _showinfo267 printf("delete (*comm)->mpi_comm = %p\n", (*comm)->mpi_comm);268 #endif269 ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->mpi_comm));270 271 #ifdef _showinfo272 printf("delete (*comm)->ep_comm_ptr->comm_list = %p\n", (*comm)->ep_comm_ptr->comm_list);273 #endif274 delete[] (*comm)->ep_comm_ptr->comm_list;275 }276 }277 278 279 110 } 280 111 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_gather.cpp
r1503 r1527 35 35 for(int i=0; i<num_ep; i++) 36 36 memcpy(recvbuf + datasize * i * count, comm->my_buffer->void_buffer[i], datasize * count); 37 38 //printf("local_recvbuf = %d %d \n", static_cast<int*>(recvbuf)[0], static_cast<int*>(recvbuf)[1] );39 37 } 40 38 … … 44 42 int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) 45 43 { 46 if(!comm->is_ep) 47 { 48 return ::MPI_Gather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), 49 root, to_mpi_comm(comm->mpi_comm)); 50 } 44 if(!comm->is_ep) return ::MPI_Gather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), 45 root, to_mpi_comm(comm->mpi_comm)); 46 if(comm->is_intercomm) return MPI_Gather_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm); 51 47 52 48 assert(sendcount == recvcount && sendtype == recvtype); … … 103 99 if(is_root) 104 100 { 105 // printf("tmp_recvbuf = %d %d %d %d %d %d %d %d\n", static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1],106 // static_cast<int*>(tmp_recvbuf)[2], static_cast<int*>(tmp_recvbuf)[3],107 // static_cast<int*>(tmp_recvbuf)[4], static_cast<int*>(tmp_recvbuf)[5],108 // static_cast<int*>(tmp_recvbuf)[6], static_cast<int*>(tmp_recvbuf)[7] );109 110 101 int offset; 111 102 for(int i=0; i<ep_size; i++) … … 128 119 } 129 120 121 122 int MPI_Gather_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) 123 { 124 printf("MPI_Gather_intercomm not yet implemented\n"); 125 MPI_Abort(comm, 0); 126 } 130 127 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_gatherv.cpp
r1503 r1527 46 46 { 47 47 48 if(!comm->is_ep) 49 { 50 return ::MPI_Gatherv(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, const_cast<int*>(input_recvcounts), const_cast<int*>(input_displs), 51 to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 52 } 53 48 if(!comm->is_ep) return ::MPI_Gatherv(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, const_cast<int*>(input_recvcounts), const_cast<int*>(input_displs), 49 to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 50 if(comm->is_intercomm) return MPI_Gatherv_intercomm(sendbuf, sendcount, sendtype, recvbuf, input_recvcounts, input_displs, recvtype, root, comm); 54 51 55 52 assert(sendtype == recvtype); … … 169 166 } 170 167 168 int MPI_Gatherv_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int input_recvcounts[], const int input_displs[], 169 MPI_Datatype recvtype, int root, MPI_Comm comm) 170 { 171 printf("MPI_Gatherv_intercomm not yet implemented\n"); 172 MPI_Abort(comm, 0); 173 } 174 171 175 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_init.cpp
r1525 r1527 15 15 Debug("MPI_Init_thread with EP/MPI\n"); 16 16 17 if(omp_get_thread_num() == 0)17 #pragma omp master 18 18 { 19 19 ::MPI_Init_thread(argc, argv, required, provided); … … 26 26 Debug("MPI_Init with EP/MPI\n"); 27 27 28 if(omp_get_thread_num() == 0)28 #pragma omp master 29 29 { 30 30 ::MPI_Init(argc, argv); -
XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.cpp
r1525 r1527 14 14 namespace ep_lib 15 15 { 16 int MPI_Intercomm_create 2(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)16 int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 17 17 { 18 18 assert(local_comm->is_ep); … … 371 371 372 372 (*newintercomm)->inter_rank_map = new INTER_RANK_MAP; 373 373 374 374 375 int rank_info[2]; … … 449 450 } 450 451 451 452 int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)453 {454 assert(local_comm->is_ep);455 456 int ep_rank, ep_rank_loc, mpi_rank;457 int ep_size, num_ep, mpi_size;458 459 ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first;460 ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first;461 mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first;462 ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second;463 num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second;464 mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second;465 466 467 MPI_Barrier(local_comm);468 469 int leader_ranks_in_peer[3]; // local_leader_rank_in_peer470 // remote_leader_rank_in_peer471 // size of peer472 473 if(ep_rank == local_leader)474 {475 MPI_Comm_rank(peer_comm, &leader_ranks_in_peer[0]);476 leader_ranks_in_peer[1] = remote_leader;477 MPI_Comm_size(peer_comm, &leader_ranks_in_peer[2]);478 }479 480 MPI_Bcast(leader_ranks_in_peer, 3, MPI_INT, local_leader, local_comm);481 482 if(leader_ranks_in_peer[0] != leader_ranks_in_peer[2])483 {484 Debug("calling MPI_Intercomm_create_kernel\n");485 return MPI_Intercomm_create_kernel(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);486 }487 488 else489 {490 if(leader_ranks_in_peer[2] == 1)491 {492 Debug("calling MPI_Intercomm_create_unique\n");493 return MPI_Intercomm_create_unique_leader(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);494 495 }496 else497 {498 Debug("calling MPI_Intercomm_create_world\n");499 return MPI_Intercomm_create_from_world(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);500 }501 502 }503 504 }505 506 int MPI_Comm_test_inter(MPI_Comm comm, int *flag)507 {508 if(comm->is_ep)509 {510 return *flag = comm->is_intercomm;511 }512 else513 {514 return ::MPI_Comm_test_inter(to_mpi_comm(comm->mpi_comm), flag);515 }516 }517 518 519 452 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_kernel.cpp
r1522 r1527 9 9 namespace ep_lib 10 10 { 11 int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)11 /*int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 12 12 { 13 13 int ep_rank, ep_rank_loc, mpi_rank; … … 619 619 // clean up // 620 620 ////////////// 621 621 /* 622 622 delete ranks_in_world_local; 623 623 delete ranks_in_world_remote; … … 635 635 } 636 636 637 637 */ 638 638 639 639 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_unique.cpp
r1515 r1527 10 10 { 11 11 12 12 /* 13 13 int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 14 14 { … … 225 225 return MPI_SUCCESS; 226 226 } 227 227 */ 228 228 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_world.cpp
r1512 r1527 9 9 { 10 10 11 11 /* 12 12 int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm) 13 13 { … … 541 541 542 542 } 543 543 */ 544 544 // #endif 545 545 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.cpp
r1525 r1527 32 32 33 33 int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank) 34 { 35 /*if(comm->is_intercomm) 36 { 37 for(std::map<int, std::pair< int, std::pair<int, int> > >::iterator it = comm->ep_comm_ptr->intercomm->intercomm_rank_map->begin(); it != comm->ep_comm_ptr->intercomm->intercomm_rank_map->end(); it++) 38 { 39 if( ( it->second.first == ep_rank_loc ) 40 && ( it->second.second.first == mpi_rank ) ) 41 { 42 return it->first; 43 } 44 } 45 printf("rank not find for EP_intercomm\n"); 46 int err; 47 return MPI_Abort(comm, err); 48 }*/ 49 34 { 50 35 for(std::map<int, std::pair<int, int> >::iterator it = comm->ep_rank_map->begin(); it != comm->ep_rank_map->end(); it++) 51 36 { … … 57 42 } 58 43 printf("rank not find for EP_intracomm\n"); 59 int err; 60 return MPI_Abort(comm, err); 44 return MPI_Abort(comm, 0); 61 45 } 62 46 … … 70 54 { 71 55 return ::MPI_Wtime(); 72 56 } 57 58 int MPI_Comm_test_inter(MPI_Comm comm, int *flag) 59 { 60 if(comm->is_ep) return *flag = comm->is_intercomm; 61 else return ::MPI_Comm_test_inter(to_mpi_comm(comm->mpi_comm), flag); 73 62 } 74 63 … … 139 128 } 140 129 141 int test_sendrecv(MPI_Comm comm)142 {143 int myRank;144 MPI_Comm_rank(comm, &myRank);145 bool amClient = false;146 bool amServer = false;147 if(myRank<=3) amClient = true;148 else amServer = true;149 150 if(amServer)151 {152 int send_buf[4];153 MPI_Request send_request[8];154 MPI_Status send_status[8];155 156 157 158 for(int j=0; j<4; j++) // 4 buffers159 {160 for(int i=0; i<2; i++)161 {162 send_buf[j] = (myRank+1)*100 + j;163 MPI_Isend(&send_buf[j], 1, MPI_INT, i*2, 9999, comm, &send_request[i*4+j]);164 }165 }166 167 168 MPI_Waitall(8, send_request, send_status);169 }170 171 172 if(amClient&&myRank%2==0) // Clients leaders173 {174 int recv_buf[8];175 MPI_Request recv_request[8];176 MPI_Status recv_status[8];177 178 for(int i=0; i<2; i++) // 2 servers179 {180 for(int j=0; j<4; j++)181 {182 MPI_Irecv(&recv_buf[i*4+j], 1, MPI_INT, i+4, 9999, comm, &recv_request[i*4+j]);183 }184 }185 186 MPI_Waitall(8, recv_request, recv_status);187 printf("============ client %d, recv_buf = %d, %d, %d, %d, %d, %d, %d, %d ================\n",188 myRank, recv_buf[0], recv_buf[1], recv_buf[2], recv_buf[3], recv_buf[4], recv_buf[5], recv_buf[6], recv_buf[7]);189 }190 191 MPI_Barrier(comm);192 193 }194 130 195 131 bool valid_type(MPI_Datatype datatype) -
XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.hpp
r1525 r1527 4 4 #include "ep_type.hpp" 5 5 #include "ep_lib_intercomm.hpp" 6 #include "ep_lib_ intracomm.hpp"6 #include "ep_lib_endpoint.hpp" 7 7 #include "ep_lib_local.hpp" 8 8 #include "ep_lib_collective.hpp" 9 #include "ep_tag.hpp"10 9 #include "ep_lib_fortran.hpp" 11 10 #include "ep_lib_win.hpp" 12 11 #include "ep_lib_mpi.hpp" 13 //#include "ep_mpi.hpp"14 12 15 13 … … 30 28 31 29 int MPI_Comm_free(MPI_Comm* comm); 32 int MPI_Comm_free2(MPI_Comm* comm);33 30 34 31 int MPI_Finalize(); … … 41 38 42 39 int MPI_Comm_remote_size(MPI_Comm comm, int *size); 43 int MPI_Comm_remote_size2(MPI_Comm comm, int *size);44 40 45 41 … … 67 63 68 64 int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr); 69 70 65 int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr); 71 66 … … 93 88 } 94 89 95 //MPI_Datatype to_mpi(ep_lib::MPI_Datatype type);96 97 98 90 #endif // EP_LIB_HPP_INCLUDED -
XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_collective.hpp
r1525 r1527 10 10 11 11 int MPI_Barrier(MPI_Comm comm); 12 int MPI_Barrier2(MPI_Comm comm); 13 12 14 13 15 14 int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm); … … 47 46 48 47 int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 49 int MPI_Intercomm_create2(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);50 48 51 int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);49 //int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 52 50 53 51 54 int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm);52 //int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm); 55 53 56 int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm);54 //int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 57 55 58 56 int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from EP to create endpoints … … 62 60 int MPI_Intercomm_merge(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm); 63 61 64 int MPI_Intercomm_merge_unique_leader(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm);62 //int MPI_Intercomm_merge_unique_leader(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm); 65 63 66 64 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_intercomm.hpp
r1517 r1527 8 8 typedef void* MPI_Op; 9 9 10 int MPI_Send_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 11 int MPI_Ssend_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 12 int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 10 int MPI_Allgather_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm); 11 int MPI_Allgatherv_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, MPI_Comm comm); 13 12 14 int MPI_ Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request);13 int MPI_Allreduce_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm); 15 14 15 int MPI_Alltoall_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm); 16 16 17 int MPI_ Comm_dup_intercomm(MPI_Comm comm, MPI_Comm *newcomm);17 int MPI_Bcast_intercomm(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm); 18 18 19 int MPI_Comm_free_intercomm(MPI_Comm* comm); 19 int MPI_Exscan_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm); 20 int MPI_Scan_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm); 20 21 21 int MPI_ Barrier_intercomm(MPI_Comm comm);22 int MPI_Gather_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm); 22 23 23 int Message_Check_intercomm(MPI_Comm comm); 24 int MPI_Gatherv_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int input_recvcounts[], const int input_displs[], 25 MPI_Datatype recvtype, int root, MPI_Comm comm); 24 26 25 int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 26 int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 27 int MPI_Reduce_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm); 27 28 29 int MPI_Reduce_scatter_intercomm(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm); 28 30 29 int MPI_Iprobe_intercomm(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status); 30 int MPI_Improbe_intercomm(int source, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status); 31 32 33 34 31 int MPI_Scatter_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm); 32 int MPI_Scatterv_intercomm(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount, 33 MPI_Datatype recvtype, int root, MPI_Comm comm); 35 34 } 36 35 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_mpi.hpp
r1522 r1527 19 19 20 20 int MPI_Iprobe_mpi(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status); 21 int MPI_Improbe_mpi(int src, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status); 21 22 22 23 int MPI_Comm_rank_mpi(MPI_Comm comm, int* rank); … … 28 29 29 30 int MPI_Barrier_mpi(MPI_Comm comm); 30 31 int MPI_Abort_mpi(MPI_Comm comm, int errorcode); 31 32 32 } 33 33 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp
r1522 r1527 10 10 { 11 11 12 int MPI_Intercomm_merge _unique_leader(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm)12 int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm) 13 13 { 14 Debug("intercomm_merge with unique leader\n"); 14 15 int ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 16 int ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 17 int ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 18 int num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 19 int mpi_rank = inter_comm->ep_comm_ptr->size_rank_info[2].first; 15 20 21 int remote_ep_size = inter_comm->inter_rank_map->size(); 16 22 17 18 int ep_rank, ep_rank_loc, mpi_rank; 19 int ep_size, num_ep, mpi_size; 20 21 ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 22 ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 23 mpi_rank = inter_comm->ep_comm_ptr->size_rank_info[2].first; 24 ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 25 num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 26 mpi_size = inter_comm->ep_comm_ptr->size_rank_info[2].second; 27 28 int local_high = high; 29 int remote_high; 30 31 int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size(); 32 33 int local_ep_rank, local_ep_rank_loc, local_mpi_rank; 34 int local_ep_size, local_num_ep, local_mpi_size; 35 36 //local_ep_rank = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first; 37 //local_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 38 //local_mpi_rank = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first; 39 //local_ep_size = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second; 40 //local_num_ep = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 41 //local_mpi_size = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second; 42 43 44 if(local_ep_rank == 0) 45 { 46 MPI_Status status[2]; 47 MPI_Request request[2]; 48 MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[0]); 49 MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[1]); 50 51 MPI_Waitall(2, request, status); 52 } 53 54 55 //MPI_Bcast(&remote_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->local_comm); 56 23 int new_ep_rank = high? remote_ep_size + ep_rank : ep_rank; 57 24 58 25 59 26 MPI_Comm_dup(inter_comm, newintracomm); 60 27 61 int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size; 28 (*newintracomm)->is_intercomm = false; 29 (*newintracomm)->inter_rank_map->clear(); 30 delete (*newintracomm)->inter_rank_map; 31 32 (*newintracomm)->ep_comm_ptr->size_rank_info[0].second = ep_size + remote_ep_size; 33 (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = new_ep_rank; 34 35 // modif ep_rank_map 36 int my_triple[3] = {new_ep_rank, ep_rank_loc, mpi_rank}; 37 int *my_triple_list = new int[3*(ep_size+remote_ep_size)]; 38 39 40 MPI_Allgather(my_triple, 3, MPI_INT, my_triple_list, 3, MPI_INT, *newintracomm); 62 41 63 42 64 int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank; 65 int intra_ep_size, intra_num_ep, intra_mpi_size; 43 #ifdef _showinfo 44 for(int i=0; i<ep_size+remote_ep_size; i++) 45 { 46 if(new_ep_rank == i) 47 { 48 for(int j=0; j<ep_size+remote_ep_size; j++) 49 { 50 printf("rank %d : my_triple_list[%d] = %d %d %d\n", i, j, my_triple_list[3*j], my_triple_list[3*j+1], my_triple_list[3*j+2]); 51 } 52 printf("\n"); 53 } 54 MPI_Barrier(*newintracomm); 55 } 56 #endif 66 57 67 intra_ep_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 68 intra_ep_rank_loc = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 69 intra_mpi_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 70 intra_ep_size = (*newintracomm)->ep_comm_ptr->size_rank_info[0].second; 71 intra_num_ep = (*newintracomm)->ep_comm_ptr->size_rank_info[1].second; 72 intra_mpi_size = (*newintracomm)->ep_comm_ptr->size_rank_info[2].second; 73 74 75 MPI_Barrier_local(*newintracomm); 76 77 78 int *reorder; 79 if(intra_ep_rank_loc == 0) 58 if((*newintracomm)->ep_comm_ptr->size_rank_info[1].first==0) 80 59 { 81 reorder = new int[intra_ep_size]; 82 } 83 84 85 MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm); 86 if(intra_ep_rank_loc == 0) 87 { 88 ::MPI_Bcast(reorder, intra_ep_size, to_mpi_type(MPI_INT), 0, to_mpi_comm((*newintracomm)->mpi_comm)); 89 90 vector< pair<int, int> > tmp_rank_map(intra_ep_size); 91 92 93 for(int i=0; i<intra_ep_size; i++) 60 for(int i=0; i<ep_size+remote_ep_size; i++) 94 61 { 95 tmp_rank_map[reorder[i]] = (*newintracomm)->ep_rank_map->at(i);62 (*newintracomm)->ep_comm_ptr->comm_list[0]->ep_rank_map->at(my_triple_list[3*i]) = std::make_pair(my_triple_list[3*i+1], my_triple_list[3*i+2]); 96 63 } 97 98 //(*newintracomm)->rank_map->swap(tmp_rank_map);99 (*newintracomm)->ep_rank_map->clear();100 for(int i=0; i<tmp_rank_map.size(); i++)101 {102 (*newintracomm)->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(i, tmp_rank_map[i].first, tmp_rank_map[i].second));103 }104 105 106 tmp_rank_map.clear();107 64 } 108 65 109 66 MPI_Barrier_local(*newintracomm); 110 67 111 (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 68 #ifdef _showinfo 112 69 113 if(intra_ep_rank_loc == 0)70 for(int i=0; i<ep_size+remote_ep_size; i++) 114 71 { 115 delete[] reorder; 116 } 117 118 return MPI_SUCCESS; 119 } 120 121 122 123 124 125 int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm) 126 { 127 128 129 assert(inter_comm->is_intercomm); 130 131 // determine if only one MPI proc 132 133 // to be completed ...... 134 135 // multiple MPI proc and high differs 136 137 int newcomm_ep_rank = inter_comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 138 int newcomm_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->size_rank_info[1].first; 139 int newcomm_num_ep = inter_comm->ep_comm_ptr->intercomm->size_rank_info[1].second; 140 141 int ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 142 int ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 143 int num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 144 145 if(newcomm_ep_rank_loc == 0) 146 { 147 ::MPI_Comm *mpi_intracomm = new ::MPI_Comm; 148 ::MPI_Intercomm_merge(to_mpi_comm(inter_comm->ep_comm_ptr->intercomm->mpi_inter_comm), high, mpi_intracomm); 149 150 MPI_Info info; 151 MPI_Comm *ep_comm; 152 MPI_Comm_create_endpoints(mpi_intracomm, newcomm_num_ep, info, ep_comm); 153 154 inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge = ep_comm; 155 } 156 157 MPI_Barrier_local(inter_comm); 158 159 int remote_num_ep = newcomm_num_ep - num_ep; 160 161 *newintracomm = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[high? remote_num_ep+ep_rank_loc : ep_rank_loc]; 162 163 int ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 164 int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->intercomm_rank_map->size(); 165 166 //printf("ep_size = %d, remote_ep_size = %d\n", ep_size, remote_ep_size); 167 168 (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = high? remote_ep_size+ep_rank : ep_rank; 169 170 int my_triple[3]; 171 my_triple[0] = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 172 my_triple[1] = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 173 my_triple[2] = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 174 175 int *my_triple_list = new int[3 * (*newintracomm)->ep_comm_ptr->size_rank_info[0].second]; 176 177 178 MPI_Allgather(my_triple, 3, MPI_INT, my_triple_list, 3, MPI_INT, *newintracomm); 179 180 if((*newintracomm)->ep_comm_ptr->size_rank_info[1].first == 0) 181 { 182 (*newintracomm)->ep_rank_map->clear(); 183 for(int i=0; i<(*newintracomm)->ep_comm_ptr->size_rank_info[0].second; i++) 72 if(new_ep_rank == i) 184 73 { 185 (*newintracomm)->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(my_triple_list[3*i], my_triple_list[3*i+1], my_triple_list[3*i+2])); 74 for(EP_RANK_MAP::iterator it = (*newintracomm)->ep_rank_map->begin(); it != (*newintracomm)->ep_rank_map->end(); it++) 75 { 76 printf("rank %d : ep_rank_map[%d] = %d %d\n", i, it->first, it->second.first, it->second.second); 77 } 78 printf("\n"); 186 79 } 187 } 188 189 #ifdef _showinfo 190 MPI_Barrier_local(inter_comm); 191 if((*newintracomm)->ep_comm_ptr->size_rank_info[0].first == 15) 192 { 193 for(std::map<int, std::pair<int, int> >::iterator it = (*newintracomm)->ep_rank_map->begin(); it != (*newintracomm)->ep_rank_map->end(); it++) 194 { 195 printf("(%d %d %d)\n", it->first, it->second.first, it->second.second); 196 } 80 MPI_Barrier(*newintracomm); 197 81 } 198 82 #endif 199 200 delete my_triple_list;83 84 delete[] my_triple_list; 201 85 } 202 86 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_message.cpp
r1522 r1527 32 32 std::list<MPI_Request* >::iterator it; 33 33 34 //show_EP_PendingRequests(EP_PendingRequests);34 show_EP_PendingRequests(EP_PendingRequests); 35 35 36 36 … … 46 46 } 47 47 48 if((*(*it))->state == 2)49 {50 EP_PendingRequests->erase(it);51 52 memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size());53 it = EP_PendingRequests->begin();54 continue;55 }56 57 48 Message_Check(((*(*it))->comm)); 58 49 } … … 72 63 } 73 64 74 if((*(*it))->state == 2) 75 { 76 EP_PendingRequests->erase(it); 77 78 memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size()); 79 it = EP_PendingRequests->begin(); 80 continue; 81 } 82 83 if((*(*it))->state == 0) 65 if((*(*it))->probed == false) 84 66 { 85 67 #pragma omp critical (_query0) … … 107 89 MPI_Imrecv((*(*it))->buf, recv_count, (*(*it))->ep_datatype, message, *it); 108 90 (*(*it))->type = 3; 109 (*(*it))-> state = 1;91 (*(*it))->probed = true; 110 92 111 93 memcheck("delete "<< status.mpi_status <<" : in ep_lib::Request_Check, delete status.mpi_status"); … … 133 115 int Message_Check(MPI_Comm comm) 134 116 { 135 if(!comm->is_ep) return MPI_SUCCESS; 136 137 if(comm->is_intercomm) 138 { 139 Message_Check_intercomm(comm); 140 } 141 142 return Message_Check_intracomm(comm); 143 144 } 145 146 147 int Message_Check_intracomm(MPI_Comm comm) 117 if(comm->is_ep) return Message_Check_endpoint(comm); 118 } 119 120 121 int Message_Check_endpoint(MPI_Comm comm) 148 122 { 149 123 … … 204 178 } 205 179 206 207 int Message_Check_intercomm(MPI_Comm comm)208 {209 if(!comm->ep_comm_ptr->intercomm->mpi_inter_comm) return 0;210 211 Debug("Message probing for intercomm\n");212 213 int flag = true;214 ::MPI_Message message;215 ::MPI_Status status;216 int current_ep_rank;217 MPI_Comm_rank(comm, ¤t_ep_rank);218 219 while(flag) // loop until the end of global queue "comm->ep_comm_ptr->intercomm->mpi_inter_comm"220 {221 Debug("Message probing for intracomm\n");222 223 #pragma omp critical (_mpi_call)224 {225 ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &flag, &status);226 if(flag)227 {228 Debug("find message in mpi comm \n");229 ::MPI_Mprobe(status.MPI_SOURCE, status.MPI_TAG, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &message, &status);230 }231 }232 233 234 if(flag)235 {236 237 MPI_Message msg = new ep_message;238 msg->mpi_message = new ::MPI_Message(message);239 240 memcheck("new "<< msg <<" : in ep_lib::Message_Check, msg = new ep_message");241 memcheck("new "<< msg->mpi_message <<" : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message");242 243 244 msg->ep_tag = bitset<15>(status.MPI_TAG >> 16).to_ulong();245 int src_loc = bitset<8> (status.MPI_TAG >> 8) .to_ulong();246 int dest_loc = bitset<8> (status.MPI_TAG) .to_ulong();247 int src_mpi = status.MPI_SOURCE;248 249 msg->ep_src = get_ep_rank(comm, src_loc, src_mpi);250 #ifdef _showinfo251 printf("status.MPI_TAG = %d, src_loc = %d, dest_loc = %d, ep_tag = %d\n", status.MPI_TAG, src_loc, dest_loc, msg->ep_tag);252 #endif253 254 msg->mpi_status = new ::MPI_Status(status);255 memcheck("new "<< msg->mpi_status <<" : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status");256 257 #pragma omp critical (_query)258 {259 #pragma omp flush260 comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->push_back(msg);261 memcheck("comm->ep_comm_ptr->comm_list["<<dest_loc<<"]->ep_comm_ptr->message_queue->size = "<<comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->size());262 #pragma omp flush263 }264 }265 }266 267 Message_Check_intracomm(comm);268 269 return MPI_SUCCESS;270 }271 272 273 180 274 181 void show_EP_PendingRequests(std::list< ep_lib::MPI_Request* > * EP_PendingRequest) -
XIOS/dev/branch_openmp/extern/ep_dev/ep_probe.cpp
r1522 r1527 8 8 int MPI_Iprobe_mpi(int src, int tag, MPI_Comm comm, int *flag, MPI_Status *status) 9 9 { 10 ::MPI_Status mpi_status; 11 12 ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_status); 13 14 status->mpi_status = new ::MPI_Status(mpi_status); 10 15 status->ep_src = src; 11 16 status->ep_tag = tag; 12 return ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, to_mpi_status_ptr(*status));13 17 } 18 19 20 int MPI_Improbe_mpi(int src, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status) 21 { 22 ::MPI_Status mpi_status; 23 ::MPI_Message mpi_message; 24 25 #ifdef _openmpi 26 #pragma omp critical (_mpi_call) 27 { 28 ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_status); 29 if(*flag) 30 { 31 ::MPI_Mprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_message, &mpi_status); 32 } 33 } 34 #elif _intelmpi 35 ::MPI_Improbe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_message, &mpi_status); 36 #endif 37 38 status->mpi_status = new ::MPI_Status(mpi_status); 39 status->ep_src = src; 40 status->ep_tag = tag; 41 42 (*message)->mpi_message = &message; 43 (*message)->ep_src = src; 44 (*message)->ep_tag = tag; 45 } 46 47 48 14 49 15 50 int MPI_Iprobe(int src, int tag, MPI_Comm comm, int *flag, MPI_Status *status) … … 24 59 { 25 60 Debug("MPI_Iprobe with EP\n"); 26 61 27 62 *flag = false; 28 63 … … 53 88 } 54 89 90 91 55 92 56 93 … … 63 100 { 64 101 Debug("calling MPI_Improbe MPI\n"); 65 66 ::MPI_Status mpi_status; 67 ::MPI_Message mpi_message; 68 69 #ifdef _openmpi 70 #pragma omp critical (_mpi_call) 71 { 72 ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_status); 73 if(*flag) 74 { 75 ::MPI_Mprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_message, &mpi_status); 76 } 77 } 78 #elif _intelmpi 79 ::MPI_Improbe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_message, &mpi_status); 80 #endif 81 82 status->mpi_status = &mpi_status; 83 status->ep_src = src; 84 status->ep_tag = tag; 85 86 (*message)->mpi_message = &message; 87 (*message)->ep_src = src; 88 (*message)->ep_tag = tag; 89 90 91 return 0; 92 } 93 94 102 return MPI_Improbe_mpi(src, tag, comm, flag, message, status); 103 } 95 104 96 105 #pragma omp flush -
XIOS/dev/branch_openmp/extern/ep_dev/ep_recv.cpp
r1522 r1527 40 40 { 41 41 if(!comm->is_ep) return MPI_Irecv_mpi(buf, count, datatype, src, tag, comm, request); 42 43 if(comm->is_intercomm) 44 { 45 src = comm->inter_rank_map->at(src); 46 printf("new src = %d\n", src); 47 } 42 48 43 49 Debug("MPI_Irecv with EP"); … … 54 60 (*request)->comm = comm; 55 61 (*request)->type = 2; 56 (*request)-> state = 0;62 (*request)->probed = false; 57 63 58 64 … … 118 124 (*request)->ep_src = (*message)->ep_src; 119 125 120 (*request)-> state = 1;126 (*request)->probed = true; 121 127 122 128 ::MPI_Imrecv(buf, count, to_mpi_type(datatype), to_mpi_message_ptr(*message), to_mpi_request_ptr(*request)); -
XIOS/dev/branch_openmp/extern/ep_dev/ep_reduce.cpp
r1503 r1527 288 288 { 289 289 290 if(!comm->is_ep && comm->mpi_comm) 291 { 292 return ::MPI_Reduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), root, to_mpi_comm(comm->mpi_comm)); 293 } 290 if(!comm->is_ep) return ::MPI_Reduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), root, to_mpi_comm(comm->mpi_comm)); 291 if(comm->is_intercomm) return MPI_Reduce_intercomm(sendbuf, recvbuf, count, datatype, op, root, comm); 294 292 295 293 … … 339 337 340 338 339 int MPI_Reduce_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm) 340 { 341 printf("MPI_Reduce_intercomm not yet implemented\n"); 342 MPI_Abort(comm, 0); 343 } 341 344 } 342 345 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_reduce_scatter.cpp
r1500 r1527 19 19 int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 20 20 { 21 if(!comm->is_ep) 22 { 23 return ::MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 24 } 21 if(!comm->is_ep) return ::MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 22 if(comm->is_intercomm) return MPI_Reduce_scatter_intercomm(sendbuf, recvbuf, recvcounts, datatype, op, comm); 25 23 26 24 … … 81 79 } 82 80 81 int MPI_Reduce_scatter_intercomm(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 82 { 83 printf("MPI_Reduce_scatter_intercomm not yet implemented\n"); 84 MPI_Abort(comm, 0); 85 } 86 83 87 84 88 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_request.hpp
r1517 r1527 18 18 // 3: Imrecv 19 19 20 int state; // 0: new20 //int state; // 0: new 21 21 // 1: imrecvd 22 22 // 2: tested or waited 23 bool probed; 23 24 24 25 void* buf; -
XIOS/dev/branch_openmp/extern/ep_dev/ep_scan.cpp
r1503 r1527 350 350 int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 351 351 { 352 if(!comm->is_ep) 353 { 354 return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 355 } 352 if(!comm->is_ep) return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 353 if(comm->is_intercomm) return MPI_Scan_intercomm(sendbuf, recvbuf, count, datatype, op, comm); 356 354 357 355 valid_type(datatype); … … 446 444 } 447 445 446 int MPI_Scan_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 447 { 448 printf("MPI_Scan_intercomm not yet implemented\n"); 449 MPI_Abort(comm, 0); 450 } 451 448 452 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_scatter.cpp
r1503 r1527 42 42 int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) 43 43 { 44 if(!comm->is_ep) 45 { 46 return ::MPI_Scatter(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 47 } 44 if(!comm->is_ep) return ::MPI_Scatter(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 45 if(comm->is_intercomm) return MPI_Scatter_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm); 48 46 49 47 assert(sendcount == recvcount); … … 97 95 98 96 99 100 // if(is_root) printf("\nranks = %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d\n", ranks[0], ranks[1], ranks[2], ranks[3], ranks[4], ranks[5], ranks[6], ranks[7],101 // ranks[8], ranks[9], ranks[10], ranks[11], ranks[12], ranks[13], ranks[14], ranks[15]);102 103 97 if(is_root) 104 98 for(int i=0; i<ep_size; i++) … … 106 100 memcpy(tmp_sendbuf + i*datasize*count, sendbuf + ranks[i]*datasize*count, count*datasize); 107 101 } 108 109 // if(is_root) printf("\ntmp_sendbuf = %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d\n", static_cast<int*>(tmp_sendbuf)[0], static_cast<int*>(tmp_sendbuf)[2], static_cast<int*>(tmp_sendbuf)[4], static_cast<int*>(tmp_sendbuf)[6],110 // static_cast<int*>(tmp_sendbuf)[8], static_cast<int*>(tmp_sendbuf)[10], static_cast<int*>(tmp_sendbuf)[12], static_cast<int*>(tmp_sendbuf)[14],111 // static_cast<int*>(tmp_sendbuf)[16], static_cast<int*>(tmp_sendbuf)[18], static_cast<int*>(tmp_sendbuf)[20], static_cast<int*>(tmp_sendbuf)[22],112 // static_cast<int*>(tmp_sendbuf)[24], static_cast<int*>(tmp_sendbuf)[26], static_cast<int*>(tmp_sendbuf)[28], static_cast<int*>(tmp_sendbuf)[30] );113 114 102 115 103 // MPI_Scatterv from root to masters … … 127 115 ::MPI_Scatterv(tmp_sendbuf, recvcounts.data(), displs.data(), to_mpi_type(sendtype), local_recvbuf, num_ep*count, to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 128 116 129 // printf("local_recvbuf = %d %d %d %d\n", static_cast<int*>(local_recvbuf)[0], static_cast<int*>(local_recvbuf)[1], static_cast<int*>(local_recvbuf)[2], static_cast<int*>(local_recvbuf)[3]);130 // static_cast<int*>(local_recvbuf)[4], static_cast<int*>(local_recvbuf)[5], static_cast<int*>(local_recvbuf)[6], static_cast<int*>(local_recvbuf)[7]);131 117 } 132 118 … … 138 124 } 139 125 126 127 int MPI_Scatter_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) 128 { 129 printf("MPI_Scatter_intercomm not yet implemented\n"); 130 MPI_Abort(comm, 0); 131 } 132 140 133 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_scatterv.cpp
r1503 r1527 45 45 MPI_Datatype recvtype, int root, MPI_Comm comm) 46 46 { 47 if(!comm->is_ep) 48 { 49 return ::MPI_Scatterv(sendbuf, sendcounts, displs, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 50 } 47 if(!comm->is_ep) return ::MPI_Scatterv(sendbuf, sendcounts, displs, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 48 if(comm->is_intercomm) return MPI_Scatterv_intercomm(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm); 51 49 52 50 assert(sendtype == recvtype); … … 105 103 for(int i=0; i<ep_size; i++) 106 104 { 107 //printf("i=%d : start from %d, src displs = %d, count = %d\n ", i, local_displs/datasize, displs[ranks[i]], sendcounts[ranks[i]]);108 105 memcpy(tmp_sendbuf+local_displs, sendbuf + displs[ranks[i]]*datasize, sendcounts[ranks[i]]*datasize); 109 106 local_displs += sendcounts[ranks[i]]*datasize; 110 107 } 111 112 //for(int i=0; i<ep_size*2; i++) printf("%d\t", static_cast<int*>(const_cast<void*>(tmp_sendbuf))[i]);113 108 } 114 109 … … 129 124 130 125 ::MPI_Scatterv(tmp_sendbuf, recvcounts.data(), my_displs.data(), to_mpi_type(sendtype), local_sendbuf, num_ep*count, to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 131 132 // printf("my_displs = %d %d %d %d\n", my_displs[0], my_displs[1], my_displs[2], my_displs[3] );133 134 // printf("%d %d %d %d %d %d %d %d\n", static_cast<int*>(local_sendbuf)[0], static_cast<int*>(local_sendbuf)[1], static_cast<int*>(local_sendbuf)[2], static_cast<int*>(local_sendbuf)[3],135 // static_cast<int*>(local_sendbuf)[4], static_cast<int*>(local_sendbuf)[5], static_cast<int*>(local_sendbuf)[6], static_cast<int*>(local_sendbuf)[7]);136 126 } 137 127 … … 154 144 155 145 146 int MPI_Scatterv_intercomm(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount, 147 MPI_Datatype recvtype, int root, MPI_Comm comm) 148 { 149 printf("MPI_Scatterv_intercomm not yet implemented\n"); 150 MPI_Abort(comm, 0); 151 } 152 153 156 154 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_send.cpp
r1518 r1527 18 18 { 19 19 if(!comm->is_ep) return MPI_Send_mpi(buf, count, datatype, dest, tag, comm); 20 if(comm->is_intercomm) return MPI_Send_intercomm(buf, count, datatype, dest, tag, comm);20 if(comm->is_intercomm) dest = comm->inter_rank_map->at(dest); 21 21 22 22 Debug("\nMPI_Send with EP\n"); … … 38 38 { 39 39 if(!comm->is_ep) return MPI_Ssend_mpi(buf, count, datatype, dest, tag, comm); 40 if(comm->is_intercomm) return MPI_Ssend_intercomm(buf, count, datatype, dest, tag, comm);40 if(comm->is_intercomm) dest = comm->inter_rank_map->at(dest); 41 41 42 42 Debug("\nMPI_Ssend with EP\n"); … … 61 61 { 62 62 if(!comm->is_ep) return MPI_Isend_mpi(buf, count, datatype, dest, tag, comm, request); 63 if(comm->is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request);63 if(comm->is_intercomm) dest = comm->inter_rank_map->at(dest); 64 64 65 65 Debug("\nMPI_Isend with EP\n"); … … 104 104 105 105 if(!comm->is_ep) return MPI_Issend_mpi(buf, count, datatype, dest, tag, comm, request); 106 if(comm->is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request);106 if(comm->is_intercomm) dest = comm->inter_rank_map->at(dest); 107 107 108 108 Debug("\nMPI_Issend with EP\n"); … … 144 144 return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); 145 145 } 146 147 int MPI_Send_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 148 { 149 Debug("\nMPI_Send_intercomm with EP\n"); 150 MPI_Request request; 151 MPI_Status status; 152 MPI_Isend(buf, count, datatype, dest, tag, comm, &request); 153 MPI_Wait(&request, &status); 154 return MPI_SUCCESS; 155 } 156 146 157 147 158 148 int MPI_Ssend_mpi(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) … … 163 153 164 154 165 int MPI_Ssend_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm)166 {167 Debug("\nMPI_Ssend_intercomm with EP\n");168 MPI_Request request;169 MPI_Status status;170 MPI_Issend(buf, count, datatype, dest, tag, comm, &request);171 MPI_Wait(&request, &status);172 return MPI_SUCCESS;173 }174 175 155 int MPI_Isend_mpi(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 176 156 { … … 195 175 } 196 176 197 int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 198 { 199 Debug("\nMPI_Isend_intercomm with EP\n"); 200 201 #ifdef _check_sum 202 check_sum_send(buf, count, datatype, dest, tag, comm); 203 #endif 204 205 int src_comm_label = comm->ep_comm_ptr->comm_label; 206 int dest_comm_label = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).second.second; 207 208 int src_rank; 209 MPI_Comm_rank(comm, &src_rank); 210 211 212 *request = new ep_request; 213 memcheck("new "<< *request <<" : in ep_lib::MPI_Isend_intercomm, *request = new ep_request"); 214 215 (*request)->mpi_request = new ::MPI_Request; 216 memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend_intercomm, (*request)->mpi_request = new ::MPI_Request"); 217 218 219 int ep_src_loc = comm->ep_rank_map->at(src_rank).first; 220 int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).first; 221 int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); 222 int mpi_dest = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).second.first; 223 224 #ifdef _showinfo 225 printf("Send : ep_src_loc = %d, ep_dest_loc = %d, mpi_src = %d, mpi_dest = %d, mpi_tag = %d\n", ep_src_loc, ep_dest_loc, comm->ep_comm_ptr->size_rank_info[2].first, mpi_dest, mpi_tag); 226 #endif 227 228 229 (*request)->ep_src = src_rank; 230 (*request)->ep_tag = tag; 231 (*request)->ep_datatype = datatype; 232 233 (*request)->type = 1; // used in wait 234 (*request)->comm = comm; 235 (*request)->buf = const_cast<void*>(buf); 236 237 238 if(src_comm_label == dest_comm_label) 239 { 240 Debug("\nMPI_Isend_intercomm with EP_intracomm\n"); 241 return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 242 } 243 244 else 245 { 246 Debug("\nMPI_Isend_intercomm with EP_intercomm\n"); 247 return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), to_mpi_request_ptr(*request)); 248 } 249 } 250 251 177 252 178 253 179 … … 275 201 276 202 277 int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request)278 {279 Debug("\nMPI_Issend_intercomm with EP\n");280 281 #ifdef _check_sum282 check_sum_send(buf, count, datatype, dest, tag, comm);283 #endif284 285 int src_comm_label = comm->ep_comm_ptr->comm_label;286 int dest_comm_label = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).second.second;287 288 int src_rank;289 MPI_Comm_rank(comm, &src_rank);290 291 292 *request = new ep_request;293 memcheck("new "<< *request <<" : in ep_lib::MPI_Issend_intercomm, *request = new ep_request");294 295 (*request)->mpi_request = new ::MPI_Request;296 memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend_intercomm, (*request)->mpi_request = new ::MPI_Request");297 298 299 int ep_src_loc = comm->ep_rank_map->at(src_rank).first;300 int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).first;301 int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc);302 int mpi_dest = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).second.first;303 304 #ifdef _showinfo305 printf("ep_src_loc = %d, ep_dest_loc = %d, mpi_src = %d, mpi_dest = %d, mpi_tag = %d\n", ep_src_loc, ep_dest_loc, comm->ep_comm_ptr->size_rank_info[2].first, mpi_dest, mpi_tag);306 #endif307 308 (*request)->ep_src = src_rank;309 (*request)->ep_tag = tag;310 (*request)->ep_datatype = datatype;311 312 (*request)->type = 1; // used in wait313 (*request)->comm = comm;314 (*request)->buf = const_cast<void*>(buf);315 316 317 if(src_comm_label == dest_comm_label)318 {319 Debug("\nMPI_Issend_intercomm with EP_intracomm\n");320 return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request));321 }322 323 else324 {325 Debug("\nMPI_Issend_intercomm with EP_intercomm\n");326 return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), to_mpi_request_ptr(*request));327 }328 }329 330 203 } 331 204 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_size.cpp
r1525 r1527 22 22 if(comm->is_ep) 23 23 { 24 if(comm->is_intercomm) 25 { 26 Debug("MPI_Comm_remote_size with EP_intercomm"); 27 return *size = comm->ep_comm_ptr->intercomm->intercomm_rank_map->size(); 28 } 29 else 30 { 31 Debug("MPI_Comm_remote_size with EP_intracomm"); 32 return *size=0; 33 } 34 } 35 return MPI_Comm_remote_size_mpi(comm, size); 36 } 37 38 int MPI_Comm_remote_size2(MPI_Comm comm, int *size) 39 { 40 if(comm->is_ep) 41 { 42 if(comm->is_intercomm) 43 { 44 Debug("MPI_Comm_remote_size with EP_intercomm"); 45 return *size = comm->inter_rank_map->size(); 46 } 47 return *size=0; 24 return *size = comm->is_intercomm? comm->inter_rank_map->size() : 0; 48 25 } 49 26 return MPI_Comm_remote_size_mpi(comm, size); -
XIOS/dev/branch_openmp/extern/ep_dev/ep_split.cpp
r1511 r1527 9 9 { 10 10 11 void vec_simplify(std::vector<int> *inout_vector)12 {13 std::vector<int> out_vec;14 int found=false;15 for(std::vector<int>::iterator it_in = inout_vector->begin() ; it_in != inout_vector->end(); ++it_in)16 {17 for(std::vector<int>::iterator it = out_vec.begin() ; it != out_vec.end(); ++it)18 {19 if(*it_in == *it)20 {21 found=true;22 break;23 }24 else found=false;25 }26 if(found == false)27 {28 out_vec.push_back(*it_in);29 }30 }31 inout_vector->swap(out_vec);32 }33 34 11 void vec_simplify(std::vector<int> *in_vector, std::vector<int> *out_vector) 35 12 { … … 274 251 *newcomm = comm->ep_comm_ptr->comm_list[0]->mem_bridge[new_ep_rank_loc]; 275 252 memcheck("in MPI_Split ep_rank="<< ep_rank <<" : *newcomm = "<< *newcomm); 276 277 (*newcomm)->ep_comm_ptr->comm_label = color;278 253 279 254 (*newcomm)->ep_comm_ptr->size_rank_info[0].first = new_ep_rank; -
XIOS/dev/branch_openmp/extern/ep_dev/ep_test.cpp
r1522 r1527 25 25 { 26 26 printf("MPI_Test : Error in request type\n"); 27 28 27 exit(1); 29 28 } … … 39 38 40 39 ::MPI_Status mpi_status; 41 42 43 40 ::MPI_Test(to_mpi_request_ptr(*request), flag, &mpi_status); 44 41 … … 51 48 status->ep_datatype = (*request)->ep_datatype; 52 49 53 (*request)->state = 2;50 //(*request)->state = 2; 54 51 55 52 memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Test, delete (*request)->mpi_request"); … … 61 58 62 59 return Request_Check(); 63 64 60 } 65 61 … … 101 97 array_of_statuses[i].ep_datatype = array_of_requests[i]->ep_datatype; 102 98 103 array_of_requests[i]->state = 2;99 //array_of_requests[i]->state = 2; 104 100 105 101 memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Testall, array_of_requests["<<i<<"]->mpi_request"); … … 109 105 110 106 } 111 112 107 } 113 108 114 109 return Request_Check(); 115 116 110 } 117 111 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_type.cpp
r1517 r1527 33 33 } 34 34 35 /*36 37 MPI_Aint::MPI_Aint(void* aint)38 {39 mpi_aint = new ::MPI_Aint;40 *(static_cast< ::MPI_Aint*>(mpi_aint)) = *(static_cast< ::MPI_Aint*>(aint));41 }42 43 MPI_Aint::MPI_Aint(int aint)44 {45 mpi_aint = new ::MPI_Aint;46 *(static_cast< ::MPI_Aint*>(mpi_aint)) = aint;47 }48 49 MPI_Aint MPI_Aint::operator=(int a)50 {51 mpi_aint = new ::MPI_Aint;52 *(static_cast< int*>(mpi_aint)) = a;53 }54 55 MPI_Fint::MPI_Fint(void* fint)56 {57 mpi_fint = new ::MPI_Fint;58 *(static_cast< ::MPI_Fint*>(mpi_fint)) = *(static_cast< ::MPI_Fint*>(fint));59 }60 61 62 */63 64 65 66 35 } 67 36 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_type.hpp
r1525 r1527 38 38 #include "ep_barrier.hpp" 39 39 #include "ep_comm.hpp" 40 #include "ep_intercomm.hpp"41 40 #include "ep_window.hpp" 42 41 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_wait.cpp
r1522 r1527 26 26 { 27 27 printf("MPI_Wait : Error in request type\n"); 28 29 28 exit(1); 30 29 } … … 41 40 status->ep_datatype = (*request)->ep_datatype; 42 41 43 (*request)->state = 2;42 //(*request)->state = 2; 44 43 45 44 memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Wait, delete (*request)->mpi_request"); … … 95 94 array_of_statuses[i].ep_datatype = array_of_requests[i]->ep_datatype; 96 95 97 array_of_requests[i]->state = 2;96 //array_of_requests[i]->state = 2; 98 97 99 98 memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]->mpi_request"); -
XIOS/dev/branch_openmp/extern/ep_dev/main.cpp
r1525 r1527 31 31 MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); 32 32 MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); 33 34 #pragma omp parallel default(shared) 35 { 36 MPI_Comm_rank(MPI_COMM_WORLD , &mpi_rank); 37 38 int num_ep = omp_get_num_threads(); 39 MPI_Info info; 40 41 //printf("mpi_rank = %d, thread_num = %d\n", mpi_rank, omp_get_thread_num()); 42 43 MPI_Comm *ep_comm; 44 #pragma omp master 45 { 46 MPI_Comm *ep_comm; 47 MPI_Comm_create_endpoints(MPI_COMM_WORLD->mpi_comm, num_ep, info, ep_comm); 48 passage = ep_comm; 49 } 50 51 #pragma omp barrier 52 53 MPI_Comm comm; // this should act as EP_COMM_WORLD 54 55 comm = passage[omp_get_thread_num()]; 56 57 // TEST OF COMM_SPLIT 58 { 59 60 MPI_Barrier(comm); 61 int rank, size; 62 MPI_Comm_rank(comm, &rank); 63 MPI_Comm_size(comm, &size); 64 65 int tab_color[16] = {2, 2, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1}; // used for config 1 66 int tab_key[16] = {2, 1, 4, 1, 0, 4, 3, 3, 4, 3, 4, 4, 4, 0, 0, 2}; // used for config 1 67 68 int color = tab_color[rank]; 69 int key = tab_key[rank]; 70 71 72 MPI_Comm split_comm; 73 MPI_Comm_split(comm, color, key, &split_comm); 74 75 76 77 int split_rank, split_size; 78 MPI_Comm_rank(split_comm, &split_rank); 79 MPI_Comm_size(split_comm, &split_size); 80 81 #ifdef _Memory_check 82 printf("rank = %d, color = %d, key = %d, split_rank = %d, local_rank=%d\n", rank, color, key, split_rank, split_comm->ep_comm_ptr->size_rank_info[1].first); 83 #endif 84 85 MPI_Barrier(comm); 86 MPI_Barrier(comm); 87 MPI_Barrier(comm); 88 89 if(rank == 0) printf(" \t MPI_Comm_split \t OK\n"); 90 91 92 int local_leader = 0; 93 int remote_leader = color==2? 13: 4; // used for config 1 94 95 MPI_Comm peer_comm = comm; 96 97 MPI_Comm inter_comm; 98 MPI_Intercomm_create(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm); 99 100 int inter_rank, inter_size, remote_size; 101 MPI_Comm_rank(inter_comm, &inter_rank); 102 MPI_Comm_size(inter_comm, &inter_size); 103 MPI_Comm_remote_size(inter_comm, &remote_size); 104 105 106 MPI_Barrier(comm); 107 MPI_Barrier(comm); 108 109 if(rank == 0) printf(" \t MPI_Intercomm_create \t OK\n"); 110 111 //printf("rank = %d, split_rank = %d, split_size = %d, inter_rank=%d, inter_size=%d, remote_size=%d\n", rank, split_rank, split_size, inter_rank, inter_size, remote_size); 112 113 114 MPI_Comm_free(&split_comm); 115 116 117 MPI_Barrier(comm); 118 119 120 if(color==2 && split_rank==0) 121 { 122 double sendbuf[9]={1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9}; 123 MPI_Request send_request; 124 MPI_Status send_status; 125 MPI_Isend(sendbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &send_request); 126 MPI_Wait(&send_request, &send_status); 127 } 128 129 if(color==1 && split_rank==0) 130 { 131 double recvbuf[9]; 132 MPI_Request recv_request; 133 MPI_Status recv_status; 134 MPI_Irecv(recvbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &recv_request); 135 MPI_Wait(&recv_request, &recv_status); 136 for(int i=0; i<9; i++) 137 { 138 printf("recvbuf[%d] = %lf\n", i, recvbuf[i]); 139 } 140 } 141 142 MPI_Barrier(comm); 143 MPI_Barrier(comm); 144 145 if(rank == 0) printf(" \t Test iP2P for intercomm \t OK\n"); 146 147 148 MPI_Comm inter_comm_dup; 149 MPI_Comm_dup(inter_comm, &inter_comm_dup); 150 int inter_dup_rank, inter_dup_size; 151 MPI_Comm_rank(inter_comm_dup, &inter_dup_rank); 152 MPI_Comm_size(inter_comm_dup, &inter_dup_size); 153 154 //printf("rank = %d, split_rank = %d, split_size = %d, inter_dup_rank=%d, inter_dup_size=%d\n", rank, split_rank, split_size, inter_dup_rank, inter_dup_size); 155 156 157 158 MPI_Comm inter_comm_dup_merged; 159 bool high; 160 if(color==2) high=true; 161 else high = false; 162 163 MPI_Intercomm_merge(inter_comm_dup, high, &inter_comm_dup_merged); 164 int merged_rank, merged_size; 165 MPI_Comm_rank(inter_comm_dup_merged, &merged_rank); 166 MPI_Comm_size(inter_comm_dup_merged, &merged_size); 167 168 //printf("merged_rank=%d, merged_rank_loc=%d, root_ep_loc = %d\n", merged_rank, inter_comm_dup_merged->ep_comm_ptr->size_rank_info[1].first, inter_comm_dup_merged->ep_rank_map->at(0).first); 169 170 MPI_Barrier(comm); 171 MPI_Barrier(comm); 172 173 int sendbuf=0; 174 if(merged_rank == 0) sendbuf = 99; 175 MPI_Bcast(&sendbuf, 1, MPI_INT, 0, inter_comm_dup_merged); 176 printf("merged_rank = %d, sendbuf = %d\n", merged_rank, sendbuf); 177 178 179 180 MPI_Barrier(comm); 181 182 MPI_Comm_free(&inter_comm_dup_merged); 183 MPI_Comm_free(&inter_comm_dup); 184 185 186 187 188 MPI_Comm_free(&inter_comm); 189 190 } 191 192 193 194 MPI_Barrier(comm); 195 MPI_Comm_free(&comm); 196 } 197 198 //MPI_Finalize(); 199 //return 0; 200 201 202 33 203 34 204 … … 899 1069 // TEST OF GATHER 900 1070 { 1071 int inter_comm_dup_merged_rank; 1072 MPI_Comm_rank(inter_comm_dup_merged, &inter_comm_dup_merged_rank); 1073 1074 int inter_comm_dup_merged_size; 1075 MPI_Comm_size(inter_comm_dup_merged, &inter_comm_dup_merged_size); 1076 901 1077 902 1078 … … 907 1083 gather_root = rand() % inter_comm_dup_merged_size; 908 1084 } 1085 1086 909 1087 910 1088 MPI_Bcast(&gather_root, 1, MPI_INT, 0, inter_comm_dup_merged); 1089 1090 printf("rank = %d, inter_comm_dup_merged_rank = %d, inter_comm_dup_merged_size = %d, gather_root = %d\n", rank, inter_comm_dup_merged_rank, inter_comm_dup_merged_size, gather_root); 1091 911 1092 912 1093 double sendbuf[2]; … … 1133 1314 } 1134 1315 1316 1135 1317 MPI_Barrier(comm); 1136 1318 MPI_Barrier(comm); … … 1202 1384 1203 1385 MPI_Comm inter_comm; 1204 MPI_Intercomm_create 2(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm);1386 MPI_Intercomm_create(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm); 1205 1387 1206 1388 int inter_rank; … … 1214 1396 MPI_Comm_free(&split_comm); 1215 1397 1216 MPI_Barrier 2(inter_comm);1217 MPI_Comm_free 2(&inter_comm);1398 MPI_Barrier(inter_comm); 1399 MPI_Comm_free(&inter_comm); 1218 1400 1219 1401 }
Note: See TracChangeset
for help on using the changeset viewer.