Ignore:
Timestamp:
05/28/18 09:54:32 (6 years ago)
Author:
yushan
Message:

save dev

File:
1 edited

Legend:

Unmodified
Added
Removed

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp

    r1381 r1500  
    1818    int ep_size, num_ep, mpi_size; 
    1919 
    20     ep_rank = inter_comm.ep_comm_ptr->size_rank_info[0].first; 
    21     ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first; 
    22     mpi_rank = inter_comm.ep_comm_ptr->size_rank_info[2].first; 
    23     ep_size = inter_comm.ep_comm_ptr->size_rank_info[0].second; 
    24     num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second; 
    25     mpi_size = inter_comm.ep_comm_ptr->size_rank_info[2].second; 
     20    ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 
     21    ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
     22    mpi_rank = inter_comm->ep_comm_ptr->size_rank_info[2].first; 
     23    ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 
     24    num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
     25    mpi_size = inter_comm->ep_comm_ptr->size_rank_info[2].second; 
    2626 
    2727    int local_high = high; 
    2828    int remote_high; 
    2929 
    30     int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size(); 
     30    int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size(); 
    3131 
    3232    int local_ep_rank, local_ep_rank_loc, local_mpi_rank; 
    3333    int local_ep_size, local_num_ep, local_mpi_size; 
    3434 
    35     local_ep_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first; 
    36     local_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 
    37     local_mpi_rank = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first; 
    38     local_ep_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second; 
    39     local_num_ep = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 
    40     local_mpi_size = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second; 
     35    local_ep_rank = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first; 
     36    local_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 
     37    local_mpi_rank = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first; 
     38    local_ep_size = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second; 
     39    local_num_ep = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 
     40    local_mpi_size = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second; 
    4141 
    4242 
    … …  
    4545      MPI_Status status[2]; 
    4646      MPI_Request request[2]; 
    47       MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[0]); 
    48       MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm.ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[1]); 
     47      MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[0]); 
     48      MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[1]); 
    4949 
    5050      MPI_Waitall(2, request, status); 
    … …  
    5252 
    5353 
    54     MPI_Bcast(&remote_high, 1, MPI_INT, 0, *(inter_comm.ep_comm_ptr->intercomm->local_comm)); 
     54    MPI_Bcast(&remote_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->local_comm); 
    5555 
    5656 
    … …  
    6464    int intra_ep_size, intra_num_ep, intra_mpi_size; 
    6565 
    66     intra_ep_rank = newintracomm->ep_comm_ptr->size_rank_info[0].first; 
    67     intra_ep_rank_loc = newintracomm->ep_comm_ptr->size_rank_info[1].first; 
    68     intra_mpi_rank = newintracomm->ep_comm_ptr->size_rank_info[2].first; 
    69     intra_ep_size = newintracomm->ep_comm_ptr->size_rank_info[0].second; 
    70     intra_num_ep = newintracomm->ep_comm_ptr->size_rank_info[1].second; 
    71     intra_mpi_size = newintracomm->ep_comm_ptr->size_rank_info[2].second; 
     66    intra_ep_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 
     67    intra_ep_rank_loc = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 
     68    intra_mpi_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
     69    intra_ep_size = (*newintracomm)->ep_comm_ptr->size_rank_info[0].second; 
     70    intra_num_ep = (*newintracomm)->ep_comm_ptr->size_rank_info[1].second; 
     71    intra_mpi_size = (*newintracomm)->ep_comm_ptr->size_rank_info[2].second; 
    7272 
    7373 
    … …  
    8585    if(intra_ep_rank_loc == 0) 
    8686    { 
    87       ::MPI_Bcast(reorder, intra_ep_size, to_mpi_type(MPI_INT), 0, to_mpi_comm(newintracomm->mpi_comm)); 
     87      ::MPI_Bcast(reorder, intra_ep_size, to_mpi_type(MPI_INT), 0, to_mpi_comm((*newintracomm)->mpi_comm)); 
    8888 
    8989      vector< pair<int, int> > tmp_rank_map(intra_ep_size); 
    … …  
    9292      for(int i=0; i<intra_ep_size; i++) 
    9393      { 
    94         tmp_rank_map[reorder[i]] = newintracomm->rank_map->at(i) ; 
     94        tmp_rank_map[reorder[i]] = (*newintracomm)->rank_map->at(i) ; 
    9595      } 
    9696 
    97       newintracomm->rank_map->swap(tmp_rank_map); 
     97      (*newintracomm)->rank_map->swap(tmp_rank_map); 
    9898 
    9999      tmp_rank_map.clear(); 
    … …  
    102102    MPI_Barrier_local(*newintracomm); 
    103103 
    104     (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 
     104    (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 
    105105 
    106106    if(intra_ep_rank_loc == 0) 
    … …  
    119119  { 
    120120 
    121     assert(inter_comm.is_intercomm); 
    122  
    123     if(inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->comm_label == -99) 
     121    assert(inter_comm->is_intercomm); 
     122 
     123    if(inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->comm_label == -99) 
    124124    { 
    125125        return MPI_Intercomm_merge_unique_leader(inter_comm, high, newintracomm); 
    … …  
    132132    int num_ep; 
    133133 
    134     ep_rank_loc = inter_comm.ep_comm_ptr->size_rank_info[1].first; 
    135     num_ep = inter_comm.ep_comm_ptr->size_rank_info[1].second; 
    136  
    137  
    138  
    139     int remote_ep_size = inter_comm.ep_comm_ptr->intercomm->remote_rank_map->size(); 
     134    ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
     135    num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
     136 
     137 
     138 
     139    int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size(); 
    140140 
    141141 
    … …  
    149149    { 
    150150 
    151       ::MPI_Comm mpi_comm = to_mpi_comm(inter_comm.ep_comm_ptr->intercomm->mpi_inter_comm); 
     151      ::MPI_Comm mpi_comm = to_mpi_comm(inter_comm->ep_comm_ptr->intercomm->mpi_inter_comm); 
    152152 
    153153      ::MPI_Intercomm_merge(mpi_comm, high, mpi_intracomm); 
    … …  
    155155      MPI_Comm_create_endpoints(mpi_intracomm, num_ep, info, ep_intracomm); 
    156156 
    157       inter_comm.ep_comm_ptr->comm_list->mem_bridge = ep_intracomm; 
     157      inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge = ep_intracomm; 
    158158 
    159159    } 
    … …  
    165165     
    166166    int my_ep_rank = high? inter_rank+remote_ep_size : inter_rank; 
    167     int my_ep_rank_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 
    168     int my_num_ep_loc = inter_comm.ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 
    169     int my_num_ep_total = inter_comm.ep_comm_ptr->comm_list->mem_bridge[0].ep_comm_ptr->size_rank_info[1].second; 
    170     int my_ep_size = inter_comm.ep_comm_ptr->comm_list->mem_bridge[0].ep_comm_ptr->size_rank_info[0].second; 
     167    int my_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 
     168    int my_num_ep_loc = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 
     169    int my_num_ep_total = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[0]->ep_comm_ptr->size_rank_info[1].second; 
     170    int my_ep_size = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[0]->ep_comm_ptr->size_rank_info[0].second; 
    171171 
    172172    int tmp_intra_ep_rank_loc = high?my_ep_rank_loc+my_num_ep_total-my_num_ep_loc: my_ep_rank_loc; 
    173173 
    174174 
    175     *newintracomm = inter_comm.ep_comm_ptr->comm_list->mem_bridge[tmp_intra_ep_rank_loc]; 
    176  
    177     int newintracomm_ep_rank = (*newintracomm).ep_comm_ptr->size_rank_info[0].first; 
    178     int newintracomm_ep_rank_loc = (*newintracomm).ep_comm_ptr->size_rank_info[1].first; 
    179     int newintracomm_mpi_rank = (*newintracomm).ep_comm_ptr->size_rank_info[2].first; 
    180     int newintracomm_ep_size = (*newintracomm).ep_comm_ptr->size_rank_info[0].second; 
    181     int newintracomm_num_ep = (*newintracomm).ep_comm_ptr->size_rank_info[1].second; 
    182     int newintracomm_mpi_size = (*newintracomm).ep_comm_ptr->size_rank_info[2].second; 
     175    *newintracomm = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[tmp_intra_ep_rank_loc]; 
     176 
     177    int newintracomm_ep_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 
     178    int newintracomm_ep_rank_loc = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 
     179    int newintracomm_mpi_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
     180    int newintracomm_ep_size = (*newintracomm)->ep_comm_ptr->size_rank_info[0].second; 
     181    int newintracomm_num_ep = (*newintracomm)->ep_comm_ptr->size_rank_info[1].second; 
     182    int newintracomm_mpi_size = (*newintracomm)->ep_comm_ptr->size_rank_info[2].second; 
    183183 
    184184 
    … …  
    186186    buf[0] = my_ep_rank; 
    187187    buf[1] = tmp_intra_ep_rank_loc; 
    188     buf[2] = newintracomm->ep_comm_ptr->size_rank_info[2].first; 
     188    buf[2] = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
    189189 
    190190    // printf("my_ep_rank = %d, tmp_intra_ep_rank_loc = %d, mpi_rank = %d\n", my_ep_rank, tmp_intra_ep_rank_loc, newintracomm->ep_comm_ptr->size_rank_info[2].first); 
    … …  
    206206    for(int i=0; i<newintracomm_ep_size; i++) 
    207207    { 
    208       (*newintracomm).rank_map->at(rankmap_buf[3*i]).first  = rankmap_buf[3*i+1]; 
    209       (*newintracomm).rank_map->at(rankmap_buf[3*i]).second = rankmap_buf[3*i+2]; 
    210     } 
    211  
    212  
    213     (*newintracomm).ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 
    214     (*newintracomm).ep_comm_ptr->size_rank_info[1].first = tmp_intra_ep_rank_loc; 
     208      (*newintracomm)->rank_map->at(rankmap_buf[3*i]).first  = rankmap_buf[3*i+1]; 
     209      (*newintracomm)->rank_map->at(rankmap_buf[3*i]).second = rankmap_buf[3*i+2]; 
     210    } 
     211 
     212 
     213    (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 
     214    (*newintracomm)->ep_comm_ptr->size_rank_info[1].first = tmp_intra_ep_rank_loc; 
    215215     
    216216 
Note: See TracChangeset for help on using the changeset viewer.