Ignore:
Timestamp:
06/12/18 11:54:13 (6 years ago)
Author:
yushan
Message:

update intercomm_merge and remove redundant files

File:
1 edited

Legend:

Unmodified
Added
Removed

  • XIOS/dev/branch_openmp/extern/src_ep_dev/ep_merge.cpp

    r1520 r1539  
    1010{ 
    1111 
    12   int MPI_Intercomm_merge_unique_leader(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm) 
     12  int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm) 
    1313  { 
    14     Debug("intercomm_merge with unique leader\n"); 
     14     
     15    int ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 
     16    int ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 
     17    int ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
     18    int num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
     19    int mpi_rank = inter_comm->ep_comm_ptr->size_rank_info[2].first; 
    1520 
     21    int remote_ep_size = inter_comm->inter_rank_map->size(); 
    1622 
    17  
    18     int ep_rank, ep_rank_loc, mpi_rank; 
    19     int ep_size, num_ep, mpi_size; 
    20  
    21     ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 
    22     ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
    23     mpi_rank = inter_comm->ep_comm_ptr->size_rank_info[2].first; 
    24     ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 
    25     num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
    26     mpi_size = inter_comm->ep_comm_ptr->size_rank_info[2].second; 
    27  
    28     int local_high = high; 
    29     int remote_high; 
    30  
    31     int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size(); 
    32  
    33     int local_ep_rank, local_ep_rank_loc, local_mpi_rank; 
    34     int local_ep_size, local_num_ep, local_mpi_size; 
    35  
    36     //local_ep_rank = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first; 
    37     //local_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 
    38     //local_mpi_rank = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first; 
    39     //local_ep_size = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second; 
    40     //local_num_ep = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 
    41     //local_mpi_size = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second; 
    42  
    43  
    44     if(local_ep_rank == 0) 
    45     { 
    46       MPI_Status status[2]; 
    47       MPI_Request request[2]; 
    48       MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[0]); 
    49       MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[1]); 
    50  
    51       MPI_Waitall(2, request, status); 
    52     } 
    53  
    54  
    55     //MPI_Bcast(&remote_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->local_comm); 
    56  
     23    int new_ep_rank = high? remote_ep_size + ep_rank : ep_rank; 
    5724 
    5825 
    5926    MPI_Comm_dup(inter_comm, newintracomm); 
    6027 
    61     int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size; 
     28    (*newintracomm)->is_intercomm = false; 
     29    (*newintracomm)->inter_rank_map->clear(); 
     30    delete (*newintracomm)->inter_rank_map; 
     31 
     32    (*newintracomm)->ep_comm_ptr->size_rank_info[0].second = ep_size + remote_ep_size; 
     33     
     34    /////////////////////////////////// 
     35    int int_high = high? 1 : 0; 
     36    int sum_high; 
     37     
     38    MPI_Allreduce(&int_high, &sum_high, 1, MPI_INT, MPI_SUM, *newintracomm); 
     39 
     40    if(sum_high==0 || sum_high==ep_size+remote_ep_size) 
     41    { 
     42      printf("MPI_Intercomm_merge error: please define high with different value...\n"); 
     43      MPI_Abort(inter_comm, 0); 
     44    } 
     45     
     46     
     47     
     48    /////////////////////////////////// 
     49     
     50    (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = new_ep_rank; 
     51 
     52    // modif ep_rank_map 
     53    int my_triple[3] = {new_ep_rank, ep_rank_loc, mpi_rank}; 
     54    int *my_triple_list = new int[3*(ep_size+remote_ep_size)]; 
     55 
     56     
     57    MPI_Allgather(my_triple, 3, MPI_INT, my_triple_list, 3, MPI_INT, *newintracomm);  
    6258 
    6359 
    64     int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank; 
    65     int intra_ep_size, intra_num_ep, intra_mpi_size; 
     60#ifdef _showinfo 
     61    for(int i=0; i<ep_size+remote_ep_size; i++) 
     62    { 
     63      if(new_ep_rank == i) 
     64      { 
     65        for(int j=0; j<ep_size+remote_ep_size; j++) 
     66        { 
     67          printf("rank %d : my_triple_list[%d] = %d %d %d\n", i, j, my_triple_list[3*j], my_triple_list[3*j+1], my_triple_list[3*j+2]); 
     68        } 
     69        printf("\n"); 
     70      } 
     71      MPI_Barrier(*newintracomm); 
     72    } 
     73#endif 
    6674 
    67     intra_ep_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 
    68     intra_ep_rank_loc = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 
    69     intra_mpi_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
    70     intra_ep_size = (*newintracomm)->ep_comm_ptr->size_rank_info[0].second; 
    71     intra_num_ep = (*newintracomm)->ep_comm_ptr->size_rank_info[1].second; 
    72     intra_mpi_size = (*newintracomm)->ep_comm_ptr->size_rank_info[2].second; 
    73  
    74  
    75     MPI_Barrier_local(*newintracomm); 
    76  
    77  
    78     int *reorder; 
    79     if(intra_ep_rank_loc == 0) 
     75    if((*newintracomm)->ep_comm_ptr->size_rank_info[1].first==0) 
    8076    { 
    81       reorder = new int[intra_ep_size]; 
    82     } 
    83  
    84  
    85     MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm); 
    86     if(intra_ep_rank_loc == 0) 
    87     { 
    88       ::MPI_Bcast(reorder, intra_ep_size, to_mpi_type(MPI_INT), 0, to_mpi_comm((*newintracomm)->mpi_comm)); 
    89  
    90       vector< pair<int, int> > tmp_rank_map(intra_ep_size); 
    91  
    92  
    93       for(int i=0; i<intra_ep_size; i++) 
     77      for(int i=0; i<ep_size+remote_ep_size; i++) 
    9478      { 
    95         tmp_rank_map[reorder[i]] = (*newintracomm)->ep_rank_map->at(i) ; 
     79        (*newintracomm)->ep_comm_ptr->comm_list[0]->ep_rank_map->at(my_triple_list[3*i]) = std::make_pair(my_triple_list[3*i+1], my_triple_list[3*i+2]); 
    9680      } 
    97  
    98       //(*newintracomm)->rank_map->swap(tmp_rank_map); 
    99       (*newintracomm)->ep_rank_map->clear(); 
    100       for(int i=0; i<tmp_rank_map.size(); i++) 
    101       { 
    102         (*newintracomm)->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(i, tmp_rank_map[i].first, tmp_rank_map[i].second)); 
    103       } 
    104        
    105  
    106       tmp_rank_map.clear(); 
    10781    } 
    10882 
    10983    MPI_Barrier_local(*newintracomm); 
    11084 
    111     (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 
     85#ifdef _showinfo 
    11286 
    113     if(intra_ep_rank_loc == 0) 
     87    for(int i=0; i<ep_size+remote_ep_size; i++) 
    11488    { 
    115       delete[] reorder; 
    116     } 
    117  
    118     return MPI_SUCCESS; 
    119   } 
    120  
    121  
    122  
    123  
    124  
    125   int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm) 
    126   { 
    127      
    128  
    129     assert(inter_comm->is_intercomm); 
    130  
    131     // determine if only one MPI proc 
    132  
    133         // to be completed ...... 
    134  
    135     // multiple MPI proc and high differs 
    136  
    137     int newcomm_ep_rank = inter_comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 
    138     int newcomm_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->size_rank_info[1].first; 
    139     int newcomm_num_ep = inter_comm->ep_comm_ptr->intercomm->size_rank_info[1].second; 
    140  
    141     int ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 
    142     int ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
    143     int num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
    144  
    145     if(newcomm_ep_rank_loc == 0) 
    146     { 
    147       ::MPI_Comm *mpi_intracomm = new ::MPI_Comm; 
    148       ::MPI_Intercomm_merge(to_mpi_comm(inter_comm->ep_comm_ptr->intercomm->mpi_inter_comm), high, mpi_intracomm); 
    149        
    150       MPI_Info info; 
    151       MPI_Comm *ep_comm; 
    152       MPI_Comm_create_endpoints(mpi_intracomm, newcomm_num_ep, info, ep_comm); 
    153  
    154       inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge = ep_comm; 
    155     } 
    156  
    157     MPI_Barrier_local(inter_comm); 
    158  
    159     int remote_num_ep = newcomm_num_ep - num_ep; 
    160  
    161     *newintracomm = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[high? remote_num_ep+ep_rank_loc : ep_rank_loc]; 
    162  
    163     int ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 
    164     int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->intercomm_rank_map->size(); 
    165  
    166     //printf("ep_size = %d, remote_ep_size = %d\n", ep_size, remote_ep_size); 
    167  
    168     (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = high? remote_ep_size+ep_rank : ep_rank; 
    169  
    170     int my_triple[3]; 
    171     my_triple[0] = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 
    172     my_triple[1] = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 
    173     my_triple[2] = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
    174  
    175     int *my_triple_list = new int[3 * (*newintracomm)->ep_comm_ptr->size_rank_info[0].second]; 
    176  
    177  
    178     MPI_Allgather(my_triple, 3, MPI_INT, my_triple_list, 3, MPI_INT, *newintracomm); 
    179  
    180     if((*newintracomm)->ep_comm_ptr->size_rank_info[1].first == 0) 
    181     { 
    182       (*newintracomm)->ep_rank_map->clear(); 
    183       for(int i=0; i<(*newintracomm)->ep_comm_ptr->size_rank_info[0].second; i++) 
     89      if(new_ep_rank == i) 
    18490      { 
    185         (*newintracomm)->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(my_triple_list[3*i], my_triple_list[3*i+1], my_triple_list[3*i+2])); 
     91        for(EP_RANK_MAP::iterator it = (*newintracomm)->ep_rank_map->begin(); it != (*newintracomm)->ep_rank_map->end(); it++) 
     92        { 
     93          printf("rank %d : ep_rank_map[%d] = %d %d\n", i, it->first, it->second.first, it->second.second); 
     94        } 
     95        printf("\n"); 
    18696      } 
    187     } 
    188  
    189 #ifdef _showinfo 
    190     MPI_Barrier_local(inter_comm); 
    191     if((*newintracomm)->ep_comm_ptr->size_rank_info[0].first == 15) 
    192     { 
    193       for(std::map<int, std::pair<int, int> >::iterator it = (*newintracomm)->ep_rank_map->begin(); it != (*newintracomm)->ep_rank_map->end(); it++) 
    194       { 
    195         printf("(%d  %d  %d)\n", it->first, it->second.first, it->second.second); 
    196       } 
     97      MPI_Barrier(*newintracomm); 
    19798    } 
    19899#endif 
    199      
    200     delete my_triple_list; 
     100 
     101    delete[] my_triple_list; 
     102 
    201103  } 
    202104   
Note: See TracChangeset for help on using the changeset viewer.