/*! \file ep_send.hpp \since 2 may 2016 \brief Definitions of MPI send functions: MPI_Send, MPI_Ssend, MPI_Isend, MPI_Issend */ #include "ep_lib.hpp" #include #include "ep_declaration.hpp" #include "ep_mpi.hpp" namespace ep_lib { int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) { if(!comm->is_ep) return MPI_Send_mpi(buf, count, datatype, dest, tag, comm); if(comm->is_intercomm) return MPI_Send_intercomm(buf, count, datatype, dest, tag, comm); Debug("\nMPI_Send with EP\n"); int ep_src_loc = comm->ep_comm_ptr->size_rank_info[1].first; int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first; int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); int mpi_dest = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second; #ifdef _check_sum check_sum_send(buf, count, datatype, dest, tag, comm); #endif return ::MPI_Send(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm)); } int MPI_Ssend(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) { if(!comm->is_ep) return MPI_Ssend_mpi(buf, count, datatype, dest, tag, comm); if(comm->is_intercomm) return MPI_Ssend_intercomm(buf, count, datatype, dest, tag, comm); Debug("\nMPI_Ssend with EP\n"); int ep_src_loc = comm->ep_comm_ptr->size_rank_info[1].first; int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first; int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); int mpi_dest = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second; #ifdef _check_sum check_sum_send(buf, count, datatype, dest, tag, comm); #endif return ::MPI_Ssend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm)); } int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) { if(!comm->is_ep) return MPI_Isend_mpi(buf, count, datatype, dest, tag, comm, request); if(comm->is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request); Debug("\nMPI_Isend with EP\n"); int src_rank; MPI_Comm_rank(comm, &src_rank); #ifdef _check_sum check_sum_send(buf, count, datatype, dest, tag, comm); #endif *request = new ep_request; memcheck("new "<< *request <<" : in ep_lib::MPI_Isend, *request = new ep_request"); (*request)->mpi_request = new ::MPI_Request; memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend, (*request)->mpi_request = new ::MPI_Request"); int ep_src_loc = comm->ep_comm_ptr->size_rank_info[1].first; int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first; int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); int mpi_dest = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second; (*request)->ep_src = src_rank; (*request)->ep_tag = tag; (*request)->ep_datatype = datatype; (*request)->type = 1; // used in wait (*request)->comm = comm; (*request)->buf = const_cast(buf); return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); } int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) { if(!comm->is_ep) return MPI_Issend_mpi(buf, count, datatype, dest, tag, comm, request); if(comm->is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request); Debug("\nMPI_Issend with EP\n"); int src_rank; MPI_Comm_rank(comm, &src_rank); #ifdef _check_sum check_sum_send(buf, count, datatype, dest, tag, comm); #endif *request = new ep_request; memcheck("new "<< *request <<" : in ep_lib::MPI_Issend, *request = new ep_request"); (*request)->mpi_request = new ::MPI_Request; memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend, (*request)->mpi_request = new ::MPI_Request"); int ep_src_loc = comm->ep_comm_ptr->size_rank_info[1].first; int ep_dest_loc = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).first; int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); int mpi_dest = comm->ep_comm_ptr->comm_list[0]->ep_rank_map->at(dest).second; (*request)->ep_src = src_rank; (*request)->ep_tag = tag; (*request)->ep_datatype = datatype; (*request)->type = 1; // used in wait (*request)->comm = comm; (*request)->buf = const_cast(buf); return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); } int MPI_Send_mpi(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) { Debug("\nMPI_Send with MPI\n"); return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); } int MPI_Send_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) { Debug("\nMPI_Send with EP_Intercomm\n"); MPI_Request request; MPI_Status status; MPI_Isend(buf, count, datatype, dest, tag, comm, &request); MPI_Wait(&request, &status); return MPI_SUCCESS; } int MPI_Ssend_mpi(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) { Debug("\nMPI_Ssend with MPI\n"); return ::MPI_Ssend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); } int MPI_Ssend_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) { Debug("\nMPI_Ssend with EP_Intercomm\n"); MPI_Request request; MPI_Status status; MPI_Issend(buf, count, datatype, dest, tag, comm, &request); MPI_Wait(&request, &status); return MPI_SUCCESS; } int MPI_Isend_mpi(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) { Debug("\nMPI_Isend with MPI\n"); int src_rank; MPI_Comm_rank(comm, &src_rank); *request = new ep_request; memcheck("new "<< *request <<" : in ep_lib::MPI_Isend, *request = new ep_request"); (*request)->mpi_request = new ::MPI_Request; memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend, (*request)->mpi_request = new ::MPI_Request"); (*request)->ep_src = src_rank; (*request)->ep_tag = tag; (*request)->ep_datatype = datatype; (*request)->type = 1; (*request)->comm = comm; return ::MPI_Isend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); } int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) { Debug("MPI_Isend with EP_Intercomm\n"); #ifdef _check_sum check_sum_send(buf, count, datatype, dest, tag, comm); #endif int dest_remote_ep_rank = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; int dest_remote_comm_label = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).second; int src_ep_rank = comm->ep_comm_ptr->intercomm->size_rank_info[0].first; int src_comm_label; src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).second; //Message_Check(comm); if(dest_remote_comm_label == src_comm_label) // mpi_dest differs { int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; int ep_src_loc = comm->ep_rank_map->at(inter_src).first; int ep_dest_loc = comm->ep_rank_map->at(dest_remote_ep_rank).first; int mpi_dest = comm->ep_rank_map->at(dest_remote_ep_rank).second; int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); *request = new ep_request; memcheck("new "<< *request <<" : in ep_lib::MPI_Isend_intercomm, *request = new ep_request"); (*request)->mpi_request = new ::MPI_Request; memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend_intercomm, (*request)->mpi_request = new ::MPI_Request"); (*request)->type = 1; // used in wait (*request)->comm = comm; (*request)->ep_src = src_ep_rank; (*request)->ep_tag = tag; (*request)->ep_datatype = datatype; return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); } else // dest_remote_comm_label != src_comm_label { int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; int ep_src_loc = comm->ep_rank_map->at(inter_src).first; int ep_dest_loc = comm->ep_comm_ptr->intercomm->inter_rank_map->at(dest_remote_ep_rank).first; int mpi_dest = comm->ep_comm_ptr->intercomm->inter_rank_map->at(dest_remote_ep_rank).second; int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); *request = new ep_request; memcheck("new "<< *request <<" : in ep_lib::MPI_Isend_intercomm, *request = new ep_request"); (*request)->mpi_request = new ::MPI_Request; memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend_intercomm, (*request)->mpi_request = new ::MPI_Request"); (*request)->type = 1; // used in wait (*request)->comm = comm; (*request)->ep_src = src_ep_rank; (*request)->ep_tag = tag; (*request)->ep_datatype = datatype; return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), to_mpi_request_ptr(*request)); } } int MPI_Issend_mpi(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) { Debug("\nMPI_Issend with MPI\n"); int src_rank; MPI_Comm_rank(comm, &src_rank); *request = new ep_request; memcheck("new "<< *request <<" : in ep_lib::MPI_Issend, *request = new ep_request"); (*request)->mpi_request = new ::MPI_Request; memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend, (*request)->mpi_request = new ::MPI_Request"); (*request)->ep_src = src_rank; (*request)->ep_tag = tag; (*request)->ep_datatype = datatype; (*request)->type = 1; (*request)->comm = comm; return ::MPI_Issend(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); } int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) { Debug("MPI_Issend with EP_Intercomm\n"); #ifdef _check_sum check_sum_send(buf, count, datatype, dest, tag, comm); #endif int dest_remote_ep_rank = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).first; int dest_remote_comm_label = comm->ep_comm_ptr->intercomm->remote_rank_map->at(dest).second; int src_ep_rank = comm->ep_comm_ptr->intercomm->size_rank_info[0].first; int src_comm_label; src_comm_label = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).second; //Message_Check(comm); if(dest_remote_comm_label == src_comm_label) // mpi_dest differs { int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; int ep_src_loc = comm->ep_rank_map->at(inter_src).first; int ep_dest_loc = comm->ep_rank_map->at(dest_remote_ep_rank).first; int mpi_dest = comm->ep_rank_map->at(dest_remote_ep_rank).second; int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); *request = new ep_request; memcheck("new "<< *request <<" : in ep_lib::MPI_Issend_intercomm, *request = new ep_request"); (*request)->mpi_request = new ::MPI_Request; memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend_intercomm, (*request)->mpi_request = new ::MPI_Request"); (*request)->type = 1; // used in wait (*request)->comm = comm; (*request)->ep_src = src_ep_rank; (*request)->ep_tag = tag; (*request)->ep_datatype = datatype; return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); } else // dest_remote_comm_label != src_comm_label { int inter_src = comm->ep_comm_ptr->intercomm->local_rank_map->at(src_ep_rank).first; int ep_src_loc = comm->ep_rank_map->at(inter_src).first; int ep_dest_loc = comm->ep_comm_ptr->intercomm->inter_rank_map->at(dest_remote_ep_rank).first; int mpi_dest = comm->ep_comm_ptr->intercomm->inter_rank_map->at(dest_remote_ep_rank).second; int mpi_tag = tag_combine(tag, ep_src_loc, ep_dest_loc); *request = new ep_request; memcheck("new "<< *request <<" : in ep_lib::MPI_Issend_intercomm, *request = new ep_request"); (*request)->mpi_request = new ::MPI_Request; memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend_intercomm, (*request)->mpi_request = new ::MPI_Request"); (*request)->type = 1; // used in wait (*request)->comm = comm; (*request)->ep_src = src_ep_rank; (*request)->ep_tag = tag; (*request)->ep_datatype = datatype; return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), to_mpi_request_ptr(*request)); } } }