Changeset 1499 for XIOS/dev/branch_openmp/extern
- Timestamp:
- 05/16/18 15:56:55 (6 years ago)
- Location:
- XIOS/dev/branch_openmp/extern/ep_dev
- Files:
-
- 20 edited
Legend:
- Unmodified
- Added
- Removed
-
XIOS/dev/branch_openmp/extern/ep_dev/ep_barrier.cpp
r1381 r1499 61 61 int MPI_Barrier_local(MPI_Comm comm) 62 62 { 63 //Message_Check(comm); 63 64 comm.ep_barrier->wait(); 64 65 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_create.cpp
r1381 r1499 14 14 15 15 namespace ep_lib { 16 17 /*! 18 Dynamic creation of endpoints for each MPI process. 19 The output of this function is an array of communicator handles of length num_ep, where each handle 20 corresponds to a nez local randk in the output communicator. 21 Once created, endpoints behave as MPI processes. 22 \param [in] mpi_comm Parent MPI communicator. 23 \param [in] num_ep Number of endpoints per process. 24 \param [out] info Information of the EP creation. 25 \param [out] out_comm_hdls Handles of EP communicators. 26 */ 27 // #ifdef _intelmpi 28 // int MPI_Comm_create_endpoints(int base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls) 29 // { 30 // int base_rank; 31 // int base_size; 32 33 // ::MPI_Comm mpi_base_comm = static_cast< ::MPI_Comm> (base_comm_ptr); 34 35 // ::MPI_Comm_size(mpi_base_comm, &base_size); // ep_lib::mpi_comm_size 36 // ::MPI_Comm_rank(mpi_base_comm, &base_rank); // ep_lib::mpi_comm_rank 37 // // parent_comm can also be endpoints communicators 38 39 // std::vector<int> recv_num_ep(base_size); 40 41 // out_comm_hdls = new MPI_Comm[num_ep]; 42 43 // for (int idx = 0; idx < num_ep; ++idx) 44 // { 45 // out_comm_hdls[idx].is_ep = true; 46 // out_comm_hdls[idx].is_intercomm = false; 47 // out_comm_hdls[idx].ep_comm_ptr = new ep_communicator; 48 // out_comm_hdls[idx].mpi_comm = base_comm_ptr; 49 // out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls; 50 // out_comm_hdls[idx].ep_comm_ptr->comm_label = 0; 51 // } 52 53 // ::MPI_Allgather(&num_ep, 1, static_cast< ::MPI_Datatype>(MPI_INT), &recv_num_ep[0], 1, static_cast< ::MPI_Datatype>(MPI_INT), mpi_base_comm); 54 55 56 // int sum = 0; // representing total ep number of process with smaller rank 57 // for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; } 58 59 // int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0); 60 61 // out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep); 62 63 // out_comm_hdls[0].my_buffer = new BUFFER; 64 65 // out_comm_hdls[0].rank_map = new RANK_MAP; 66 // out_comm_hdls[0].rank_map->resize(ep_size); 67 68 69 // for (int i = 1; i < num_ep; i++) 70 // { 71 // out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier; 72 // out_comm_hdls[i].my_buffer = out_comm_hdls[0].my_buffer; 73 // out_comm_hdls[i].rank_map = out_comm_hdls[0].rank_map; 74 // } 75 76 77 // for (int i = 0; i < num_ep; i++) 78 // { 79 // out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size); 80 // out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep); 81 // out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size); 82 83 // out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list; 84 // } 85 86 87 // int ind = 0; 88 89 // for(int i=0; i<base_size; i++) 90 // { 91 // for(int j=0; j<recv_num_ep[i]; j++) 92 // { 93 // out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i); 94 // ind++; 95 // } 96 // } 97 98 99 100 // return 0; 101 102 // } //MPI_Comm_create_endpoints 103 104 // #elif _openmpi 105 // int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls) 106 // { 107 108 // int base_rank; 109 // int base_size; 110 111 // ::MPI_Comm mpi_base_comm = static_cast< ::MPI_Comm> (base_comm_ptr); 112 113 // ::MPI_Comm_size(mpi_base_comm, &base_size); // ep_lib::mpi_comm_size 114 // ::MPI_Comm_rank(mpi_base_comm, &base_rank); // ep_lib::mpi_comm_rank 115 // // parent_comm can also be endpoints communicators ? 116 // std::vector<int> recv_num_ep(base_size); 117 118 // out_comm_hdls = new MPI_Comm[num_ep]; 119 120 // for (int idx = 0; idx < num_ep; ++idx) 121 // { 122 // out_comm_hdls[idx].is_ep = true; 123 // out_comm_hdls[idx].is_intercomm = false; 124 // out_comm_hdls[idx].ep_comm_ptr = new ep_communicator; 125 // out_comm_hdls[idx].mpi_comm = base_comm_ptr; 126 // out_comm_hdls[idx].ep_comm_ptr->comm_list = out_comm_hdls; 127 // out_comm_hdls[idx].ep_comm_ptr->comm_label = 0; 128 // } 129 130 // ::MPI_Allgather(&num_ep, 1, static_cast< ::MPI_Datatype> (MPI_INT), 131 // &recv_num_ep[0], 1, static_cast< ::MPI_Datatype> (MPI_INT), mpi_base_comm); 132 133 // int sum = 0; // representing total ep number of process with smaller rank 134 // for (int i = 0; i < base_rank; ++i) {sum += recv_num_ep[i]; } 135 136 // int ep_size = std::accumulate(recv_num_ep.begin(), recv_num_ep.end(), 0); 137 138 // out_comm_hdls[0].ep_barrier = new OMPbarrier(num_ep); 139 // out_comm_hdls[0].my_buffer = new BUFFER; 140 141 // out_comm_hdls[0].rank_map = new RANK_MAP; 142 // out_comm_hdls[0].rank_map->resize(ep_size); 143 144 145 // for (int i = 1; i < num_ep; i++) 146 // { 147 // out_comm_hdls[i].ep_barrier = out_comm_hdls[0].ep_barrier; 148 // out_comm_hdls[i].my_buffer = out_comm_hdls[0].my_buffer; 149 // out_comm_hdls[i].rank_map = out_comm_hdls[0].rank_map; 150 // } 151 152 153 // for (int i = 0; i < num_ep; i++) 154 // { 155 // out_comm_hdls[i].ep_comm_ptr->size_rank_info[0] = std::make_pair(sum+i, ep_size); 156 // out_comm_hdls[i].ep_comm_ptr->size_rank_info[1] = std::make_pair(i, num_ep); 157 // out_comm_hdls[i].ep_comm_ptr->size_rank_info[2] = std::make_pair(base_rank, base_size); 158 159 // out_comm_hdls[i].ep_comm_ptr->message_queue = new Message_list; 160 // } 161 162 163 // int ind = 0; 164 165 // for(int i=0; i<base_size; i++) 166 // { 167 // for(int j=0; j<recv_num_ep[i]; j++) 168 // { 169 // out_comm_hdls[0].rank_map->at(ind) = make_pair(j, i); 170 // ind++; 171 // } 172 // } 173 174 // return 0; 175 176 // } //MPI_Comm_create_endpoints 177 178 // #endif 179 16 180 17 181 int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls) … … 26 190 ::MPI_Comm_rank(mpi_base_comm, &base_rank); // ep_lib::mpi_comm_rank 27 191 // parent_comm can also be endpoints communicators 28 192 29 193 std::vector<int> recv_num_ep(base_size); 30 31 32 194 33 195 out_comm_hdls = new MPI_Comm[num_ep]; -
XIOS/dev/branch_openmp/extern/ep_dev/ep_declaration.cpp
r1385 r1499 24 24 ::MPI_Datatype MPI_UNSIGNED_LONG_STD = MPI_UNSIGNED_LONG; 25 25 ::MPI_Datatype MPI_UNSIGNED_CHAR_STD = MPI_UNSIGNED_CHAR; 26 ::MPI_Datatype MPI_UINT64_T_STD = MPI_UINT64_T; 27 ::MPI_Datatype MPI_LONG_LONG_INT_STD = MPI_LONG_LONG_INT; 26 28 27 29 #undef MPI_INT … … 32 34 #undef MPI_UNSIGNED_LONG 33 35 #undef MPI_UNSIGNED_CHAR 36 #undef MPI_UINT64_T 37 #undef MPI_LONG_LONG_INT 34 38 35 39 … … 37 41 ::MPI_Op MPI_MAX_STD = MPI_MAX; 38 42 ::MPI_Op MPI_MIN_STD = MPI_MIN; 43 ::MPI_Op MPI_LOR_STD = MPI_LOR; 39 44 ::MPI_Op MPI_REPLACE_STD = MPI_REPLACE; 40 45 … … 42 47 #undef MPI_MAX 43 48 #undef MPI_MIN 49 #undef MPI_LOR 44 50 #undef MPI_REPLACE 45 46 51 47 52 … … 55 60 extern ::MPI_Datatype MPI_UNSIGNED_LONG_STD; 56 61 extern ::MPI_Datatype MPI_UNSIGNED_CHAR_STD; 62 extern ::MPI_Datatype MPI_UINT64_T_STD; 63 extern ::MPI_Datatype MPI_LONG_LONG_INT_STD; 64 57 65 58 66 extern ::MPI_Op MPI_SUM_STD; 59 67 extern ::MPI_Op MPI_MAX_STD; 60 68 extern ::MPI_Op MPI_MIN_STD; 69 extern ::MPI_Op MPI_LOR_STD; 61 70 extern ::MPI_Op MPI_REPLACE_STD; 62 71 … … 75 84 ep_lib::MPI_Datatype MPI_UNSIGNED_LONG = &MPI_UNSIGNED_LONG_STD; 76 85 ep_lib::MPI_Datatype MPI_UNSIGNED_CHAR = &MPI_UNSIGNED_CHAR_STD; 86 ep_lib::MPI_Datatype MPI_UINT64_T = &MPI_UINT64_T_STD; 87 ep_lib::MPI_Datatype MPI_LONG_LONG_INT = &MPI_LONG_LONG_INT_STD; 88 77 89 78 90 ep_lib::MPI_Op MPI_SUM = &MPI_SUM_STD; 79 91 ep_lib::MPI_Op MPI_MAX = &MPI_MAX_STD; 80 92 ep_lib::MPI_Op MPI_MIN = &MPI_MIN_STD; 93 ep_lib::MPI_Op MPI_LOR = &MPI_LOR_STD; 81 94 ep_lib::MPI_Op MPI_REPLACE = &MPI_REPLACE_STD; 82 95 … … 88 101 89 102 90 int MPI_MODE_NOPRECEDE_STD = MPI_MODE_NOPRECEDE;91 int MPI_MODE_NOCHECK_STD = MPI_MODE_NOCHECK;92 int MPI_MODE_NOPUT_STD = MPI_MODE_NOPUT;93 int MPI_MODE_NOSTORE_STD = MPI_MODE_NOSTORE;94 int MPI_MODE_NOSUCCEED_STD = MPI_MODE_NOSUCCEED;95 103 96 #undef MPI_MODE_NOPRECEDE97 #undef MPI_MODE_NOCHECK98 #undef MPI_MODE_NOPUT99 #undef MPI_MODE_NOSTORE100 #undef MPI_MODE_NOSUCCEED101 104 102 int MPI_MODE_NOPRECEDE = MPI_MODE_NOPRECEDE_STD;103 int MPI_MODE_NOCHECK = MPI_MODE_NOCHECK_STD;104 int MPI_MODE_NOPUT = MPI_MODE_NOPUT_STD;105 int MPI_MODE_NOSTORE = MPI_MODE_NOSTORE_STD;106 int MPI_MODE_NOSUCCEED = MPI_MODE_NOSUCCEED_STD; -
XIOS/dev/branch_openmp/extern/ep_dev/ep_declaration.hpp
r1385 r1499 9 9 #undef MPI_UNSIGNED_LONG 10 10 #undef MPI_UNSIGNED_CHAR 11 #undef MPI_UINT64_T 12 #undef MPI_LONG_LONG_INT 11 13 12 14 #undef MPI_SUM 13 15 #undef MPI_MAX 14 16 #undef MPI_MIN 17 #undef MPI_LOR 15 18 #undef MPI_REPLACE 16 19 … … 29 32 extern ep_lib::MPI_Datatype MPI_UNSIGNED_LONG; 30 33 extern ep_lib::MPI_Datatype MPI_UNSIGNED_CHAR; 34 extern ep_lib::MPI_Datatype MPI_UINT64_T; 35 extern ep_lib::MPI_Datatype MPI_LONG_LONG_INT; 31 36 32 37 extern ep_lib::MPI_Op MPI_SUM; 33 38 extern ep_lib::MPI_Op MPI_MAX; 34 39 extern ep_lib::MPI_Op MPI_MIN; 40 extern ep_lib::MPI_Op MPI_LOR; 35 41 extern ep_lib::MPI_Op MPI_REPLACE; 36 42 … … 43 49 44 50 45 46 47 51 #endif // EP_DECLARATION_HPP_INCLUDED 48 52 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_dup.cpp
r1381 r1499 94 94 out_comm[i].ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 95 95 96 out_comm[i].ep_comm_ptr->intercomm->intercomm_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_rank_map; 97 out_comm[i].ep_comm_ptr->intercomm->local_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_rank_map; 98 out_comm[i].ep_comm_ptr->intercomm->remote_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->remote_rank_map; 96 int map_size = 0; 97 map_size = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_rank_map->size(); 98 out_comm[i].ep_comm_ptr->intercomm->intercomm_rank_map->resize(map_size); 99 for(int ii=0; ii<map_size; ii++) 100 out_comm[i].ep_comm_ptr->intercomm->intercomm_rank_map->at(ii) = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_rank_map->at(ii); 101 102 map_size = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_rank_map->size(); 103 out_comm[i].ep_comm_ptr->intercomm->local_rank_map->resize(map_size); 104 for(int ii=0; ii<map_size; ii++) 105 out_comm[i].ep_comm_ptr->intercomm->local_rank_map->at(ii) = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_rank_map->at(ii); 106 107 map_size = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->remote_rank_map->size(); 108 out_comm[i].ep_comm_ptr->intercomm->remote_rank_map->resize(map_size); 109 for(int ii=0; ii<map_size; ii++) 110 out_comm[i].ep_comm_ptr->intercomm->remote_rank_map->at(ii) = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->remote_rank_map->at(ii); 111 112 113 //out_comm[i].ep_comm_ptr->intercomm->intercomm_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->intercomm_rank_map; 114 //out_comm[i].ep_comm_ptr->intercomm->local_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_rank_map; 115 //out_comm[i].ep_comm_ptr->intercomm->remote_rank_map = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->remote_rank_map; 99 116 100 117 out_comm[i].ep_comm_ptr->intercomm->local_comm = comm.ep_comm_ptr->comm_list[i].ep_comm_ptr->intercomm->local_comm; -
XIOS/dev/branch_openmp/extern/ep_dev/ep_free.cpp
r1381 r1499 9 9 int MPI_Comm_free(MPI_Comm *comm) 10 10 { 11 12 11 if(! comm->is_ep) 13 12 { … … 53 52 { 54 53 comm->rank_map->clear(); 54 delete comm->rank_map; 55 55 Debug("rank_map emptied\n"); 56 56 } … … 59 59 { 60 60 comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue->clear(); 61 delete comm->ep_comm_ptr->comm_list[i].ep_comm_ptr->message_queue; 61 62 Debug("message queue freed\n"); 62 63 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_gather.cpp
r1381 r1499 35 35 for(int i=0; i<num_ep; i++) 36 36 memcpy(recvbuf + datasize * i * count, comm.my_buffer->void_buffer[i], datasize * count); 37 38 //printf("local_recvbuf = %d %d \n", static_cast<int*>(recvbuf)[0], static_cast<int*>(recvbuf)[1] ); 37 39 } 38 40 … … 101 103 if(is_root) 102 104 { 105 // printf("tmp_recvbuf = %d %d %d %d %d %d %d %d\n", static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1], 106 // static_cast<int*>(tmp_recvbuf)[2], static_cast<int*>(tmp_recvbuf)[3], 107 // static_cast<int*>(tmp_recvbuf)[4], static_cast<int*>(tmp_recvbuf)[5], 108 // static_cast<int*>(tmp_recvbuf)[6], static_cast<int*>(tmp_recvbuf)[7] ); 109 103 110 int offset; 104 111 for(int i=0; i<ep_size; i++) … … 106 113 offset = displs[comm.rank_map->at(i).second] + comm.rank_map->at(i).first * sendcount; 107 114 memcpy(recvbuf + i*sendcount*datasize, tmp_recvbuf+offset*datasize, sendcount*datasize); 115 116 108 117 } 109 118 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_init.cpp
r1381 r1499 11 11 int MPI_Init_thread(int *argc, char*** argv, int required, int*provided) 12 12 { 13 //printf("MPI_Init_thread\n"); 14 13 15 int id = omp_get_thread_num(); 14 16 … … 22 24 int MPI_Init(int *argc, char ***argv) 23 25 { 26 //printf("MPI_init called\n"); 24 27 int id = omp_get_thread_num(); 25 28 … … 33 36 int MPI_Initialized(int *flag) 34 37 { 38 //printf("MPI_initialized called\n"); 39 35 40 ::MPI_Initialized(flag); 36 41 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.cpp
r1381 r1499 24 24 25 25 MPI_Barrier(local_comm); 26 27 26 28 27 29 int leader_ranks[6]; //! 0: rank in world, 1: mpi_size, 2: rank_in_peer. -
XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_kernel.cpp
r1381 r1499 116 116 117 117 } 118 119 118 120 119 121 send_buf[0] = size_info[1]; … … 366 368 #pragma omp critical (write_to_tag_list) 367 369 tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm)); 370 //printf("tag_list size = %lu\n", tag_list.size()); 368 371 } 369 372 } … … 568 571 // (*newintercomm).rank_map->at(i).first, (*newintercomm).rank_map->at(i).second); 569 572 573 // MPI_Comm *test_comm = newintercomm->ep_comm_ptr->intercomm->local_comm; 574 // int test_rank; 575 // MPI_Comm_rank(*test_comm, &test_rank); 576 // printf("=================test_rank = %d\n", test_rank); 577 578 570 579 571 580 return MPI_SUCCESS; … … 620 629 MPI_Status status; 621 630 631 632 622 633 MPI_Comm_rank(peer_comm, &leader_rank_in_peer[0]); 623 634 … … 714 725 *newintercomm = iter->second[my_position]; 715 726 found = true; 727 // tag_list.erase(iter); 716 728 break; 717 729 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_world.cpp
r1381 r1499 9 9 { 10 10 11 // #ifdef _openmpi 12 13 // int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm) 14 // { 15 16 // int ep_rank, ep_rank_loc, mpi_rank; 17 // int ep_size, num_ep, mpi_size; 18 19 // ep_rank = local_comm.ep_comm_ptr->size_rank_info[0].first; 20 // ep_rank_loc = local_comm.ep_comm_ptr->size_rank_info[1].first; 21 // mpi_rank = local_comm.ep_comm_ptr->size_rank_info[2].first; 22 // ep_size = local_comm.ep_comm_ptr->size_rank_info[0].second; 23 // num_ep = local_comm.ep_comm_ptr->size_rank_info[1].second; 24 // mpi_size = local_comm.ep_comm_ptr->size_rank_info[2].second; 25 26 27 // std::vector<int> rank_info[4]; //! 0->rank_in_world of local_comm, 1->rank_in_local_parent of local_comm 28 // //! 2->rank_in_world of remote_comm, 3->rank_in_local_parent of remote_comm 29 30 // int rank_in_world; 31 // int rank_in_local_parent; 32 33 // int local_ep_size = ep_size; 34 // int remote_ep_size; 35 36 // ::MPI_Comm peer_comm = to_mpi_comm(peer_comm_ptr); 37 // ::MPI_Comm local_mpi_comm = to_mpi_comm(local_comm.mpi_comm); 38 39 // ::MPI_Comm_rank(peer_comm, &rank_in_world); 40 41 // ::MPI_Comm_rank(local_mpi_comm, &rank_in_local_parent); 42 43 // bool is_proc_master = false; 44 // bool is_local_leader = false; 45 // bool is_final_master = false; 46 47 48 // if(ep_rank == local_leader) { is_proc_master = true; is_local_leader = true; is_final_master = true;} 49 // if(ep_rank_loc == 0 && mpi_rank != local_comm.rank_map->at(local_leader).second) is_proc_master = true; 50 51 52 // int size_info[4]; //! used for choose size of rank_info 0-> mpi_size of local_comm, 1-> mpi_size of remote_comm 53 54 // int leader_info[4]; //! 0->world rank of local_leader, 1->world rank of remote leader 55 56 57 // std::vector<int> ep_info[2]; //! 0-> num_ep in local_comm, 1->num_ep in remote_comm 58 59 // std::vector<int> new_rank_info[4]; 60 // std::vector<int> new_ep_info[2]; 61 62 // std::vector<int> offset; 63 64 // if(is_proc_master) 65 // { 66 67 // size_info[0] = mpi_size; 68 69 // rank_info[0].resize(size_info[0]); 70 // rank_info[1].resize(size_info[0]); 71 72 73 74 // ep_info[0].resize(size_info[0]); 75 76 // vector<int> send_buf(6); 77 // vector<int> recv_buf(3*size_info[0]); 78 79 // send_buf[0] = rank_in_world; 80 // send_buf[1] = rank_in_local_parent; 81 // send_buf[2] = num_ep; 82 83 // ::MPI_Allgather(send_buf.data(), 3, to_mpi_type(MPI_INT), recv_buf.data(), 3, to_mpi_type(MPI_INT), local_mpi_comm); 84 85 // for(int i=0; i<size_info[0]; i++) 86 // { 87 // rank_info[0][i] = recv_buf[3*i]; 88 // rank_info[1][i] = recv_buf[3*i+1]; 89 // ep_info[0][i] = recv_buf[3*i+2]; 90 // } 91 92 // if(is_local_leader) 93 // { 94 // leader_info[0] = rank_in_world; 95 // leader_info[1] = mpi_remote_leader; 96 97 // ::MPI_Status mpi_status; 98 99 // send_buf[0] = size_info[0]; 100 // send_buf[1] = local_ep_size; 101 102 // ::MPI_Send(send_buf.data(), 2, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 103 104 // ::MPI_Recv(recv_buf.data(), 2, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); 105 106 // recv_buf[2] = rank_in_world; 107 // recv_buf[3] = mpi_remote_leader; 108 109 // } 110 111 // ::MPI_Bcast(recv_buf.data(), 4, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 112 113 // size_info[1] = recv_buf[0]; 114 // remote_ep_size = recv_buf[1]; 115 // leader_info[0] = recv_buf[2]; 116 // leader_info[1] = recv_buf[3]; 117 118 // rank_info[2].resize(size_info[1]); 119 // rank_info[3].resize(size_info[1]); 120 121 // ep_info[1].resize(size_info[1]); 122 123 // send_buf.resize(3*size_info[0]); 124 // recv_buf.resize(3*size_info[1]); 125 126 // if(is_local_leader) 127 // { 128 // ::MPI_Status mpi_status; 129 130 131 // std::copy ( rank_info[0].data(), rank_info[0].data() + size_info[0], send_buf.begin() ); 132 // std::copy ( rank_info[1].data(), rank_info[1].data() + size_info[0], send_buf.begin() + size_info[0] ); 133 // std::copy ( ep_info[0].data(), ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[0] ); 134 135 // ::MPI_Send(send_buf.data(), 3*size_info[0], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 136 137 // ::MPI_Recv(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); 138 139 // } 140 141 // ::MPI_Bcast(recv_buf.data(), 3*size_info[1], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 142 143 // std::copy ( recv_buf.data(), recv_buf.data() + size_info[1], rank_info[2].begin() ); 144 // std::copy ( recv_buf.data() + size_info[1], recv_buf.data() + 2*size_info[1], rank_info[3].begin() ); 145 // std::copy ( recv_buf.data() + 2*size_info[1], recv_buf.data() + 3*size_info[1], ep_info[1].begin() ); 146 147 // offset.resize(size_info[0]); 148 149 // if(leader_info[0]<leader_info[1]) // erase all ranks doubled with remote_comm, except the local leader 150 // { 151 152 // bool found = false; 153 // int ep_tmp; 154 // int ep_local; 155 // int ep_remote; 156 // for(int i=0; i<size_info[0]; i++) 157 // { 158 // int target = rank_info[0][i]; 159 // found = false; 160 // for(int j=0; j<size_info[1]; j++) 161 // { 162 // if(target == rank_info[2][j]) 163 // { 164 // found = true; 165 // ep_tmp = ep_info[1][j]; 166 // ep_local = ep_info[0][j]; 167 // ep_remote = ep_info[1][j]; 168 // break; 169 // } 170 // } 171 // if(found) 172 // { 173 174 // if(target == leader_info[0]) // the leader is doubled in remote 175 // { 176 // new_rank_info[0].push_back(target); 177 // new_rank_info[1].push_back(rank_info[1][i]); 178 179 // new_ep_info[0].push_back(ep_local + ep_remote); 180 // offset[i] = 0; 181 // } 182 // else 183 // { 184 // offset[i] = ep_local; 185 // } 186 // } 187 // else 188 // { 189 // new_rank_info[0].push_back(target); 190 // new_rank_info[1].push_back(rank_info[1][i]); 191 192 // new_ep_info[0].push_back(ep_info[0][i]); 193 194 // offset[i] = 0; 195 // } 196 197 // } 198 // } 199 200 // else // erase rank doubled with remote leader 201 // { 202 203 // bool found = false; 204 // int ep_tmp; 205 // int ep_local; 206 // int ep_remote; 207 // for(int i=0; i<size_info[0]; i++) 208 // { 209 // int target = rank_info[0][i]; 210 // found = false; 211 // for(int j=0; j<size_info[1]; j++) 212 // { 213 214 // if(target == rank_info[2][j]) 215 // { 216 // found = true; 217 // ep_tmp = ep_info[1][j]; 218 // ep_local = ep_info[0][j]; 219 // ep_remote = ep_info[1][j]; 220 // break; 221 // } 222 // } 223 // if(found) 224 // { 225 // if(target != leader_info[1]) 226 // { 227 // new_rank_info[0].push_back(target); 228 // new_rank_info[1].push_back(rank_info[1][i]); 229 230 // new_ep_info[0].push_back(ep_local + ep_remote); 231 // offset[i] = 0; 232 // } 233 // else // found remote leader 234 // { 235 // offset[i] = ep_remote; 236 // } 237 // } 238 // else 239 // { 240 // new_rank_info[0].push_back(target); 241 // new_rank_info[1].push_back(rank_info[1][i]); 242 243 // new_ep_info[0].push_back(ep_info[0][i]); 244 // offset[i] = 0; 245 // } 246 // } 247 // } 248 249 // if(offset[mpi_rank] == 0) 250 // { 251 // is_final_master = true; 252 // } 253 254 255 // // size_info[4]: 2->size of new_ep_info for local, 3->size of new_ep_info for remote 256 257 // if(is_local_leader) 258 // { 259 // size_info[2] = new_ep_info[0].size(); 260 // ::MPI_Status mpi_status; 261 262 // ::MPI_Send(&size_info[2], 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 263 264 // ::MPI_Recv(&size_info[3], 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); 265 // } 266 267 // ::MPI_Bcast(&size_info[2], 2, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 268 269 // new_rank_info[2].resize(size_info[3]); 270 // new_rank_info[3].resize(size_info[3]); 271 // new_ep_info[1].resize(size_info[3]); 272 273 // send_buf.resize(size_info[2]); 274 // recv_buf.resize(size_info[3]); 275 276 // if(is_local_leader) 277 // { 278 // ::MPI_Status mpi_status; 279 280 // std::copy ( new_rank_info[0].data(), new_rank_info[0].data() + size_info[2], send_buf.begin() ); 281 // std::copy ( new_rank_info[1].data(), new_rank_info[1].data() + size_info[2], send_buf.begin() + size_info[2] ); 282 // std::copy ( new_ep_info[0].data(), new_ep_info[0].data() + size_info[0], send_buf.begin() + 2*size_info[2] ); 283 284 // ::MPI_Send(send_buf.data(), 3*size_info[2], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 285 // ::MPI_Recv(recv_buf.data(), 3*size_info[3], to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); 286 287 // } 288 289 // ::MPI_Bcast(recv_buf.data(), 3*size_info[3], to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 290 291 // std::copy ( recv_buf.data(), recv_buf.data() + size_info[3], new_rank_info[2].begin() ); 292 // std::copy ( recv_buf.data() + size_info[3], recv_buf.data() + 2*size_info[3], new_rank_info[3].begin() ); 293 // std::copy ( recv_buf.data() + 2*size_info[3], recv_buf.data() + 3*size_info[3], new_ep_info[1].begin() ); 294 295 // } 296 297 298 299 // if(is_proc_master) 300 // { 301 // // leader_info[4]: 2-> rank of local leader in new_group generated comm; 302 // // 3-> rank of remote leader in new_group generated comm; 303 // ::MPI_Group local_group; 304 // ::MPI_Group new_group; 305 // ::MPI_Comm *new_comm = new ::MPI_Comm; 306 // ::MPI_Comm *intercomm = new ::MPI_Comm; 307 308 // ::MPI_Comm_group(local_mpi_comm, &local_group); 309 310 // ::MPI_Group_incl(local_group, size_info[2], new_rank_info[1].data(), &new_group); 311 312 // ::MPI_Comm_create(local_mpi_comm, new_group, new_comm); 313 314 315 316 // if(is_local_leader) 317 // { 318 // ::MPI_Comm_rank(*new_comm, &leader_info[2]); 319 // } 320 321 // ::MPI_Bcast(&leader_info[2], 1, to_mpi_type(MPI_INT), local_comm.rank_map->at(local_leader).second, local_mpi_comm); 322 323 // if(new_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 324 // { 325 // ::MPI_Barrier(*new_comm); 326 327 // ::MPI_Intercomm_create(*new_comm, leader_info[2], peer_comm, leader_info[1], tag, intercomm); 328 329 // int id; 330 // ::MPI_Comm_rank(*new_comm, &id); 331 // int my_num_ep = new_ep_info[0][id]; 332 333 // MPI_Comm *ep_intercomm; 334 // MPI_Info info; 335 // MPI_Comm_create_endpoints(new_comm, my_num_ep, info, ep_intercomm); 336 337 338 339 // for(int i= 0; i<my_num_ep; i++) 340 // { 341 // ep_intercomm[i].is_intercomm = true; 342 343 // ep_intercomm[i].ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 344 // ep_intercomm[i].ep_comm_ptr->intercomm->mpi_inter_comm = intercomm; 345 // ep_intercomm[i].ep_comm_ptr->comm_label = leader_info[0]; 346 // } 347 348 // #pragma omp critical (write_to_tag_list) 349 // tag_list.push_back(make_pair( make_pair(tag, min(leader_info[0], leader_info[1])) , ep_intercomm)); 350 // } 351 352 353 // } 354 355 356 // MPI_Barrier_local(local_comm); 357 358 // vector<int> bcast_buf(8); 359 // if(is_local_leader) 360 // { 361 // std::copy(size_info, size_info+4, bcast_buf.begin()); 362 // std::copy(leader_info, leader_info+4, bcast_buf.begin()+4); 363 // } 364 365 // MPI_Bcast(bcast_buf.data(), 8, MPI_INT, local_leader, local_comm); 366 367 // if(!is_local_leader) 368 // { 369 // std::copy(bcast_buf.begin(), bcast_buf.begin()+4, size_info); 370 // std::copy(bcast_buf.begin()+4, bcast_buf.begin()+8, leader_info); 371 // } 372 373 // if(!is_local_leader) 374 // { 375 // new_rank_info[1].resize(size_info[2]); 376 // ep_info[1].resize(size_info[1]); 377 // offset.resize(size_info[0]); 378 // } 379 380 // bcast_buf.resize(size_info[2]+size_info[1]+size_info[0]+1); 381 382 // if(is_local_leader) 383 // { 384 // bcast_buf[0] = remote_ep_size; 385 // std::copy(new_rank_info[1].data(), new_rank_info[1].data()+size_info[2], bcast_buf.begin()+1); 386 // std::copy(ep_info[1].data(), ep_info[1].data()+size_info[1], bcast_buf.begin()+size_info[2]+1); 387 // std::copy(offset.data(), offset.data()+size_info[0], bcast_buf.begin()+size_info[2]+size_info[1]+1); 388 // } 389 390 // MPI_Bcast(bcast_buf.data(), size_info[2]+size_info[1]+size_info[0]+1, MPI_INT, local_leader, local_comm); 391 392 // if(!is_local_leader) 393 // { 394 // remote_ep_size = bcast_buf[0]; 395 // std::copy(bcast_buf.data()+1, bcast_buf.data()+1+size_info[2], new_rank_info[1].begin()); 396 // std::copy(bcast_buf.data()+1+size_info[2], bcast_buf.data()+1+size_info[2]+size_info[1], ep_info[1].begin()); 397 // std::copy(bcast_buf.data()+1+size_info[2]+size_info[1], bcast_buf.data()+1+size_info[2]+size_info[1]+size_info[0], offset.begin()); 398 // } 399 400 401 // int my_position = offset[rank_in_local_parent]+ep_rank_loc; 402 403 404 // MPI_Barrier_local(local_comm); 405 // #pragma omp flush 406 407 408 // #pragma omp critical (read_from_tag_list) 409 // { 410 // bool found = false; 411 // while(!found) 412 // { 413 // for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++) 414 // { 415 // if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1]))) 416 // { 417 // *newintercomm = iter->second[my_position]; 418 419 // found = true; 420 // //tag_list.erase(iter); 421 // break; 422 // } 423 // } 424 // } 425 // } 426 427 // MPI_Barrier(local_comm); 428 429 // if(is_local_leader) 430 // { 431 // int local_flag = true; 432 // int remote_flag = false; 433 // ::MPI_Status mpi_status; 434 435 // ::MPI_Send(&local_flag, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 436 437 // ::MPI_Recv(&remote_flag, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &mpi_status); 438 // } 439 440 // MPI_Barrier(local_comm); 441 442 // if(is_proc_master) 443 // { 444 // for(std::list<std::pair < std::pair<int,int>, MPI_Comm* > >::iterator iter = tag_list.begin(); iter!=tag_list.end(); iter++) 445 // { 446 // if((*iter).first == make_pair(tag, min(leader_info[0], leader_info[1]))) 447 // { 448 // tag_list.erase(iter); 449 // break; 450 // } 451 // } 452 // } 453 454 455 456 // int intercomm_ep_rank, intercomm_ep_rank_loc, intercomm_mpi_rank; 457 // int intercomm_ep_size, intercomm_num_ep, intercomm_mpi_size; 458 459 // intercomm_ep_rank = newintercomm->ep_comm_ptr->size_rank_info[0].first; 460 // intercomm_ep_rank_loc = newintercomm->ep_comm_ptr->size_rank_info[1].first; 461 // intercomm_mpi_rank = newintercomm->ep_comm_ptr->size_rank_info[2].first; 462 // intercomm_ep_size = newintercomm->ep_comm_ptr->size_rank_info[0].second; 463 // intercomm_num_ep = newintercomm->ep_comm_ptr->size_rank_info[1].second; 464 // intercomm_mpi_size = newintercomm->ep_comm_ptr->size_rank_info[2].second; 465 466 // MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm); 467 468 // int my_rank_map_elem[2]; 469 470 471 // my_rank_map_elem[0] = intercomm_ep_rank; 472 473 // my_rank_map_elem[1] = (*newintercomm).ep_comm_ptr->comm_label; 474 475 // vector<pair<int, int> > local_rank_map_array; 476 // vector<pair<int, int> > remote_rank_map_array; 477 478 479 // (*newintercomm).ep_comm_ptr->intercomm->local_rank_map = new RANK_MAP; 480 // (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->resize(local_ep_size); 481 482 // MPI_Allgather2(my_rank_map_elem, 2, MPI_INT, (*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2, MPI_INT, local_comm); 483 484 // (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 485 // (*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->resize(remote_ep_size); 486 487 // int local_intercomm_size = intercomm_ep_size; 488 // int remote_intercomm_size; 489 490 491 492 493 // if(is_local_leader) 494 // { 495 // ::MPI_Status status; 496 497 // ::MPI_Send((*newintercomm).ep_comm_ptr->intercomm->local_rank_map->data(), 2*local_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 498 499 // ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 500 501 // ::MPI_Send(&local_intercomm_size, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 502 503 // ::MPI_Recv(&remote_intercomm_size, 1, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 504 // } 505 506 // MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->remote_rank_map->data(), 2*remote_ep_size, MPI_INT, local_leader, local_comm); 507 // MPI_Bcast(&remote_intercomm_size, 1, MPI_INT, 0, *newintercomm); 508 509 510 // (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; 511 // (*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size); 512 513 // (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[0] = local_comm.ep_comm_ptr->size_rank_info[0]; 514 // (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[1] = local_comm.ep_comm_ptr->size_rank_info[1]; 515 // (*newintercomm).ep_comm_ptr->intercomm->size_rank_info[2] = local_comm.ep_comm_ptr->size_rank_info[2]; 516 517 518 // if(is_local_leader) 519 // { 520 // ::MPI_Status status; 521 522 // ::MPI_Send((*newintercomm).rank_map->data(), 2*local_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm); 523 524 // ::MPI_Recv((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), mpi_remote_leader, tag, peer_comm, &status); 525 // } 526 527 // MPI_Bcast((*newintercomm).ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, to_mpi_type(MPI_INT), 0, *newintercomm); 528 529 // (*newintercomm).ep_comm_ptr->intercomm->local_comm = &(local_comm.ep_comm_ptr->comm_list[ep_rank_loc]); 530 // (*newintercomm).ep_comm_ptr->intercomm->intercomm_tag = local_comm.ep_comm_ptr->comm_label; 531 532 533 // return MPI_SUCCESS; 534 535 // } 536 537 538 539 // #elif _intelmpi 11 540 int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm) 12 541 { … … 526 1055 } 527 1056 1057 // #endif 1058 528 1059 } 529 1060 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.cpp
r1388 r1499 62 62 if( comm.ep_comm_ptr->intercomm->remote_rank_map->at(i).first == inter_rank ) 63 63 { 64 //printf("get_ep_rank for intercomm, ep_rank_loc = %d, mpi_rank = %d => ep_src = %d\n", ep_rank_loc, mpi_rank, i); 64 65 return i; 65 66 } … … 210 211 || datatype == MPI_CHAR 211 212 || datatype == MPI_LONG 212 || datatype == MPI_UNSIGNED_LONG) 213 || datatype == MPI_UNSIGNED_LONG 214 || datatype == MPI_UINT64_T) 213 215 { 214 216 return true; … … 223 225 if( op == MPI_MAX 224 226 || op == MPI_MIN 225 || op == MPI_SUM) 227 || op == MPI_SUM 228 || op == MPI_LOR) 226 229 { 227 230 return true; … … 272 275 273 276 274 275 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.hpp
r1388 r1499 9 9 #include "ep_lib_fortran.hpp" 10 10 #include "ep_lib_win.hpp" 11 12 #include "ep_global.hpp"13 11 14 12 namespace ep_lib … … 37 35 int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count); 38 36 37 // #ifdef _openmpi 38 // int MPI_Comm_create_endpoints(void* mpi_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from MPI to create endpoints 39 // #elif _intelmpi 40 // int MPI_Comm_create_endpoints(int mpi_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from MPI to create endpoints 41 // #endif 42 39 43 int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from EP to create endpoints 40 44 … … 80 84 } 81 85 86 //MPI_Datatype to_mpi(ep_lib::MPI_Datatype type); 87 82 88 83 89 #endif // EP_LIB_HPP_INCLUDED -
XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_collective.hpp
r1381 r1499 26 26 27 27 int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm); 28 29 28 int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm); 30 29 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_intercomm.hpp
r1381 r1499 32 32 int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 33 33 34 // #ifdef _intelmpi 35 // int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, int peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm); 36 // #elif _openmpi 37 // int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm); 38 // #endif 39 34 40 int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm); 35 41 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_reduce.cpp
r1381 r1499 29 29 30 30 template<typename T> 31 T lor_op(T a, T b) 32 { 33 return a||b; 34 } 35 36 template<typename T> 31 37 void reduce_max(const T * buffer, T* recvbuf, int count) 32 38 { … … 44 50 { 45 51 transform(buffer, buffer+count, recvbuf, recvbuf, std::plus<T>()); 52 } 53 54 template<typename T> 55 void reduce_lor(const T * buffer, T* recvbuf, int count) 56 { 57 transform(buffer, buffer+count, recvbuf, recvbuf, lor_op<T>); 46 58 } 47 59 … … 112 124 } 113 125 126 else if(datatype == MPI_UINT64_T) 127 { 128 assert(datasize == sizeof(uint64_t)); 129 for(int i=1; i<num_ep; i++) 130 reduce_max<uint64_t>(static_cast<uint64_t*>(comm.my_buffer->void_buffer[i]), static_cast<uint64_t*>(recvbuf), count); 131 } 132 133 else if(datatype == MPI_LONG_LONG_INT) 134 { 135 assert(datasize == sizeof(long long)); 136 for(int i=1; i<num_ep; i++) 137 reduce_max<long long>(static_cast<long long*>(comm.my_buffer->void_buffer[i]), static_cast<long long*>(recvbuf), count); 138 } 139 114 140 else printf("datatype Error\n"); 115 141 … … 160 186 } 161 187 188 else if(datatype == MPI_UINT64_T) 189 { 190 assert(datasize == sizeof(uint64_t)); 191 for(int i=1; i<num_ep; i++) 192 reduce_min<uint64_t>(static_cast<uint64_t*>(comm.my_buffer->void_buffer[i]), static_cast<uint64_t*>(recvbuf), count); 193 } 194 195 else if(datatype == MPI_LONG_LONG_INT) 196 { 197 assert(datasize == sizeof(long long)); 198 for(int i=1; i<num_ep; i++) 199 reduce_min<long long>(static_cast<long long*>(comm.my_buffer->void_buffer[i]), static_cast<long long*>(recvbuf), count); 200 } 201 162 202 else printf("datatype Error\n"); 163 203 … … 209 249 } 210 250 251 else if(datatype ==MPI_UINT64_T) 252 { 253 assert(datasize == sizeof(uint64_t)); 254 for(int i=1; i<num_ep; i++) 255 reduce_sum<uint64_t>(static_cast<uint64_t*>(comm.my_buffer->void_buffer[i]), static_cast<uint64_t*>(recvbuf), count); 256 } 257 258 else if(datatype ==MPI_LONG_LONG_INT) 259 { 260 assert(datasize == sizeof(long long)); 261 for(int i=1; i<num_ep; i++) 262 reduce_sum<long long>(static_cast<long long*>(comm.my_buffer->void_buffer[i]), static_cast<long long*>(recvbuf), count); 263 } 264 211 265 else printf("datatype Error\n"); 212 266 267 } 268 269 if(op == MPI_LOR) 270 { 271 if(datatype != MPI_INT) 272 printf("datatype Error, must be MPI_INT\n"); 273 else 274 { 275 assert(datasize == sizeof(int)); 276 for(int i=1; i<num_ep; i++) 277 reduce_lor<int>(static_cast<int*>(comm.my_buffer->void_buffer[i]), static_cast<int*>(recvbuf), count); 278 } 213 279 } 214 280 } -
XIOS/dev/branch_openmp/extern/ep_dev/ep_size.cpp
r1381 r1499 32 32 Debug("Calling EP_Comm_size\n"); 33 33 34 if( to_mpi_comm(comm.mpi_comm) != *static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm))34 if(comm.mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 35 35 { 36 36 ::MPI_Comm mpi_comm = to_mpi_comm(comm.mpi_comm); … … 38 38 39 39 ::MPI_Comm_size(mpi_comm, &mpi_size); 40 Debug("============ ep comm size called for non ep comm\n");40 printf("============ ep comm size called for non ep comm, %d\n", mpi_size); 41 41 *size = mpi_size; 42 42 return 0; … … 46 46 { 47 47 *size = MPI_UNDEFINED; 48 Debug("============ ep comm size called for non ep comm not defined\n");48 printf("============ ep comm size called for non ep comm not defined\n"); 49 49 return 0; 50 50 } … … 55 55 if(!comm.is_ep) 56 56 { 57 if( to_mpi_comm(comm.mpi_comm) != *static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm))57 if(comm.mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL.mpi_comm)) 58 58 { 59 59 ::MPI_Comm mpi_comm = to_mpi_comm(comm.mpi_comm); -
XIOS/dev/branch_openmp/extern/ep_dev/ep_type.hpp
r1482 r1499 184 184 struct BUFFER 185 185 { 186 void * void_buffer[1 6];186 void * void_buffer[12]; 187 187 }; 188 188 … … 248 248 249 249 250 250 251 class MPI_Aint 251 252 { … … 264 265 { 265 266 public: 266 267 267 268 void* mpi_fint; 268 269 269 270 MPI_Fint() {} 270 271 MPI_Fint(void* fint); 271 272 //MPI_Fint(int f): mpi_fint(f) {} 272 273 }; 274 273 274 }; 275 275 276 class MPI_Win 276 277 { 277 278 public: 278 bool is_ep; 279 MPI_Comm comm; 280 281 void* server_win[16]; 282 void* client_win; 283 void* null_win[16]; 284 }; 279 bool is_ep; 280 MPI_Comm comm; 281 282 void* server_win[16]; 283 void* client_win; 284 void* null_win[16]; 285 }; 286 285 287 286 288 static MPI_Comm *passage; … … 295 297 // #pragma omp threadprivate(fc_comm_map_ptr) 296 298 // // <MPI_Fint,thread_num> EP_Comm 297 299 298 300 } 299 301 -
XIOS/dev/branch_openmp/extern/ep_dev/ep_wait.cpp
r1381 r1499 31 31 while(request->type == 2) Request_Check(); 32 32 33 34 //::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request); 33 35 ::MPI_Status mpi_status; 34 36 ::MPI_Wait(static_cast< ::MPI_Request*>(request->mpi_request), &mpi_status); 37 38 39 40 41 // request->mpi_request = mpi_request; 35 42 36 43 status->mpi_status = &mpi_status; … … 39 46 status->ep_datatype = request->ep_datatype; 40 47 48 49 41 50 return MPI_SUCCESS; 42 51 … … 71 80 { 72 81 finished.at(i) = 1; 82 //mpi_request[i] = static_cast< ::MPI_Request >(array_of_requests[i].mpi_request); 73 83 mpi_request[i] = *static_cast< ::MPI_Request*>(array_of_requests[i].mpi_request); 74 84 } -
XIOS/dev/branch_openmp/extern/ep_dev/makefile
r1382 r1499 1 1 appname := EP_test 2 2 3 CXX := mpiicc -D_intelmpi - openmp3 CXX := mpiicc -D_intelmpi -qopenmp 4 4 CXXFLAGS := -std=c++11 -g 5 5 #CXXFLAGS := -std=c++11 -g
Note: See TracChangeset
for help on using the changeset viewer.