Ignore:
Timestamp:
06/01/18 19:36:12 (4 years ago)
Author:
yushan
Message:

optimisation partially finished. To do : special case for intercomm_create and intercomm_merge

File:
1 edited

Legend:

Unmodified
Added
Removed
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp

    r1513 r1518  
    124124  int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm) 
    125125  { 
     126     
    126127 
    127128    assert(inter_comm->is_intercomm); 
    128129 
    129     //if(inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->comm_label == -99) 
    130     //{ 
    131     //    return MPI_Intercomm_merge_unique_leader(inter_comm, high, newintracomm); 
    132     //} 
     130    // determine if only one MPI proc 
     131 
     132        // to be completed ...... 
     133 
     134    // multiple MPI proc and high differs 
     135 
     136    int newcomm_ep_rank = inter_comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 
     137    int newcomm_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->size_rank_info[1].first; 
     138    int newcomm_num_ep = inter_comm->ep_comm_ptr->intercomm->size_rank_info[1].second; 
     139 
     140    int ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 
     141    int ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
     142    int num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
     143 
     144    if(newcomm_ep_rank_loc == 0) 
     145    { 
     146      ::MPI_Comm *mpi_intracomm = new ::MPI_Comm; 
     147      ::MPI_Intercomm_merge(to_mpi_comm(inter_comm->ep_comm_ptr->intercomm->mpi_inter_comm), high, mpi_intracomm); 
     148       
     149      MPI_Info info; 
     150      MPI_Comm *ep_comm; 
     151      MPI_Comm_create_endpoints(mpi_intracomm, newcomm_num_ep, info, ep_comm); 
     152 
     153      inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge = ep_comm; 
     154    } 
     155 
     156    MPI_Barrier_local(inter_comm); 
     157 
     158    int remote_num_ep = newcomm_num_ep - num_ep; 
     159 
     160    *newintracomm = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[high? remote_num_ep+ep_rank_loc : ep_rank_loc]; 
     161 
     162    int ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 
     163    int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->intercomm_rank_map->size(); 
     164 
     165    //printf("ep_size = %d, remote_ep_size = %d\n", ep_size, remote_ep_size); 
     166 
     167    (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = high? remote_ep_size+ep_rank : ep_rank; 
     168 
     169    int my_triple[3]; 
     170    my_triple[0] = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 
     171    my_triple[1] = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 
     172    my_triple[2] = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
     173 
     174    int *my_triple_list = new int[3 * (*newintracomm)->ep_comm_ptr->size_rank_info[0].second]; 
     175 
     176 
     177    MPI_Allgather(my_triple, 3, MPI_INT, my_triple_list, 3, MPI_INT, *newintracomm); 
     178 
     179    if((*newintracomm)->ep_comm_ptr->size_rank_info[1].first == 0) 
     180    { 
     181      (*newintracomm)->ep_rank_map->clear(); 
     182      for(int i=0; i<(*newintracomm)->ep_comm_ptr->size_rank_info[0].second; i++) 
     183      { 
     184        (*newintracomm)->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(my_triple_list[3*i], my_triple_list[3*i+1], my_triple_list[3*i+2])); 
     185      } 
     186    } 
     187 
     188#ifdef _showinfo 
     189    MPI_Barrier_local(inter_comm); 
     190    if((*newintracomm)->ep_comm_ptr->size_rank_info[0].first == 15) 
     191    { 
     192      for(std::map<int, std::pair<int, int> >::iterator it = (*newintracomm)->ep_rank_map->begin(); it != (*newintracomm)->ep_rank_map->end(); it++) 
     193      { 
     194        printf("(%d  %d  %d)\n", it->first, it->second.first, it->second.second); 
     195      } 
     196    } 
     197#endif 
     198     
     199    delete my_triple_list; 
     200  
     201 
     202    return 0; 
     203 
    133204 
    134205 
    135206    Debug("intercomm_merge kernel\n"); 
    136207 
    137     int ep_rank_loc; 
    138     int num_ep; 
     208    //int ep_rank_loc; 
     209    //int num_ep; 
    139210 
    140211    ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
     
    143214 
    144215 
    145     int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size(); 
     216    //int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size(); 
    146217 
    147218 
Note: See TracChangeset for help on using the changeset viewer.