[1646] | 1 | #ifdef _usingEP |
---|
[1603] | 2 | #include "ep_lib.hpp" |
---|
| 3 | #include <mpi.h> |
---|
| 4 | #include "ep_mpi.hpp" |
---|
| 5 | |
---|
| 6 | |
---|
| 7 | namespace ep_lib |
---|
| 8 | { |
---|
| 9 | |
---|
| 10 | int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) |
---|
| 11 | { |
---|
| 12 | if(!comm->is_ep) return ::MPI_Alltoall(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); |
---|
| 13 | if(comm->is_intercomm) return MPI_Alltoall_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); |
---|
| 14 | |
---|
| 15 | |
---|
| 16 | assert(valid_type(sendtype) && valid_type(recvtype)); |
---|
| 17 | assert(sendcount == recvcount); |
---|
| 18 | |
---|
| 19 | ::MPI_Aint datasize, llb; |
---|
| 20 | ::MPI_Type_get_extent(to_mpi_type(sendtype), &llb, &datasize); |
---|
| 21 | |
---|
| 22 | int count = sendcount; |
---|
| 23 | |
---|
| 24 | int ep_rank = comm->ep_comm_ptr->size_rank_info[0].first; |
---|
| 25 | int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; |
---|
| 26 | int mpi_rank = comm->ep_comm_ptr->size_rank_info[2].first; |
---|
| 27 | int ep_size = comm->ep_comm_ptr->size_rank_info[0].second; |
---|
| 28 | int num_ep = comm->ep_comm_ptr->size_rank_info[1].second; |
---|
| 29 | int mpi_size = comm->ep_comm_ptr->size_rank_info[2].second; |
---|
| 30 | |
---|
| 31 | void* tmp_recvbuf; |
---|
| 32 | if(ep_rank == 0) tmp_recvbuf = new void*[count * ep_size * ep_size * datasize]; |
---|
| 33 | |
---|
| 34 | MPI_Gather(sendbuf, count*ep_size, sendtype, tmp_recvbuf, count*ep_size, recvtype, 0, comm); |
---|
| 35 | |
---|
| 36 | // reorder tmp_buf |
---|
| 37 | void* tmp_sendbuf; |
---|
| 38 | if(ep_rank == 0) tmp_sendbuf = new void*[count * ep_size * ep_size * datasize]; |
---|
| 39 | |
---|
| 40 | if(ep_rank == 0) |
---|
| 41 | for(int i=0; i<ep_size; i++) |
---|
| 42 | { |
---|
| 43 | for(int j=0; j<ep_size; j++) |
---|
| 44 | { |
---|
| 45 | memcpy(tmp_sendbuf + j*ep_size*count*datasize + i*count*datasize, tmp_recvbuf + i*ep_size*count*datasize + j*count*datasize, count*datasize); |
---|
| 46 | } |
---|
| 47 | } |
---|
| 48 | |
---|
| 49 | MPI_Scatter(tmp_sendbuf, ep_size*count, sendtype, recvbuf, ep_size*recvcount, recvtype, 0, comm); |
---|
| 50 | |
---|
| 51 | if(ep_rank == 0) |
---|
| 52 | { |
---|
| 53 | delete[] tmp_recvbuf; |
---|
| 54 | delete[] tmp_sendbuf; |
---|
| 55 | } |
---|
| 56 | } |
---|
| 57 | |
---|
| 58 | |
---|
| 59 | int MPI_Alltoall_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) |
---|
| 60 | { |
---|
| 61 | printf("MPI_Alltoall_intercomm not yet implemented\n"); |
---|
| 62 | MPI_Abort(comm, 0); |
---|
| 63 | } |
---|
| 64 | } |
---|
| 65 | |
---|
| 66 | |
---|
[1646] | 67 | #endif |
---|