Ignore:
Timestamp:
05/30/18 20:08:15 (3 years ago)
Author:
yushan
Message:

save dev

File:
1 edited

Legend:

Unmodified
Added
Removed
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_kernel.cpp

    r1503 r1511  
    2020    num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second; 
    2121    mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second; 
     22 
     23 
     24    // step 1 : local leaders exchange ep_size then bcast to all ep in local_comm 
     25 
     26    int remote_ep_size; 
     27    bool is_local_leader = ep_rank==local_leader? true: false; 
     28 
     29    if(is_local_leader) 
     30    { 
     31      MPI_Request request[2]; 
     32      MPI_Status status[2]; 
     33      MPI_Isend(&ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request[0]); 
     34      MPI_Irecv(&remote_ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request[1]); 
     35      MPI_Waitall(2, request, status); 
     36    } 
     37 
     38    MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm); 
     39 
     40#ifdef _showinfo 
     41    MPI_Barrier(peer_comm); 
     42    MPI_Barrier(peer_comm); 
     43    #pragma omp critical(stdoutput) 
     44    printf("peer_rank = %d, ep_size = %d, remote_ep_size = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_size, remote_ep_size); 
     45    MPI_Barrier(peer_comm); 
     46    MPI_Barrier(peer_comm);  
     47#endif 
     48 
     49    // step 2 : gather ranks in world for both local and remote comm 
     50 
     51    int rank_in_world; 
     52    ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &rank_in_world); 
     53 
     54    int *ranks_in_world_local  = new int[ep_size]; 
     55    int *ranks_in_world_remote = new int[remote_ep_size]; 
     56 
     57    MPI_Allgather(&rank_in_world, 1, MPI_INT, ranks_in_world_local, 1, MPI_INT, local_comm); 
     58 
     59    if(is_local_leader) 
     60    { 
     61      MPI_Request request[2]; 
     62      MPI_Status status[2]; 
     63      MPI_Isend(ranks_in_world_local,  ep_size,        MPI_INT, remote_leader, tag, peer_comm, &request[0]); 
     64      MPI_Irecv(ranks_in_world_remote, remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request[1]); 
     65      MPI_Waitall(2, request, status); 
     66    } 
     67 
     68    MPI_Bcast(ranks_in_world_remote, remote_ep_size, MPI_INT, local_leader, local_comm); 
     69 
     70#ifdef _showinfo 
     71 
     72    MPI_Barrier(peer_comm); 
     73    MPI_Barrier(peer_comm); 
     74 
     75    if(remote_leader == 4) 
     76    { 
     77      for(int i=0; i<ep_size; i++) 
     78      { 
     79        if(ep_rank == i) 
     80        { 
     81          printf("peer_rank = %d, ranks_in_world_local = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first); 
     82          for(int i=0; i<ep_size; i++) 
     83          { 
     84            printf("%d\t", ranks_in_world_local[i]); 
     85          } 
     86     
     87          printf("\npeer_rank = %d, ranks_in_world_remote = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first); 
     88          for(int i=0; i<remote_ep_size; i++) 
     89          { 
     90            printf("%d\t", ranks_in_world_remote[i]); 
     91          } 
     92          printf("\n"); 
     93          
     94        } 
     95         
     96        MPI_Barrier(local_comm); 
     97        MPI_Barrier(local_comm); 
     98        MPI_Barrier(local_comm); 
     99      } 
     100    } 
     101 
     102    MPI_Barrier(peer_comm); 
     103    MPI_Barrier(peer_comm); 
     104    MPI_Barrier(peer_comm); 
     105     
     106    if(remote_leader == 13) 
     107    { 
     108      for(int i=0; i<ep_size; i++) 
     109      { 
     110        if(ep_rank == i) 
     111        { 
     112          printf("peer_rank = %d, ranks_in_world_local = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first); 
     113          for(int i=0; i<ep_size; i++) 
     114          { 
     115            printf("%d\t", ranks_in_world_local[i]); 
     116          } 
     117     
     118          printf("\npeer_rank = %d, ranks_in_world_remote = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first); 
     119          for(int i=0; i<remote_ep_size; i++) 
     120          { 
     121            printf("%d\t", ranks_in_world_remote[i]); 
     122          } 
     123          printf("\n"); 
     124          
     125        } 
     126         
     127        MPI_Barrier(local_comm); 
     128        MPI_Barrier(local_comm); 
     129        MPI_Barrier(local_comm); 
     130      } 
     131    } 
     132 
     133    MPI_Barrier(peer_comm); 
     134    MPI_Barrier(peer_comm); 
     135 
     136#endif 
     137 
     138 
     139    // step 3 : determine the priority and ownership of each ep 
     140 
     141    int local_leader_rank_in_peer; 
     142    int remote_leader_rank_in_peer; 
     143 
     144    if(is_local_leader) MPI_Comm_rank(peer_comm, &local_leader_rank_in_peer); 
     145 
     146    if(is_local_leader) 
     147    { 
     148      MPI_Request request[2]; 
     149      MPI_Status status[2]; 
     150      MPI_Isend(&local_leader_rank_in_peer,  1, MPI_INT, remote_leader, tag, peer_comm, &request[0]); 
     151      MPI_Irecv(&remote_leader_rank_in_peer, 1, MPI_INT, remote_leader, tag, peer_comm, &request[1]); 
     152      MPI_Waitall(2, request, status); 
     153    } 
     154 
     155    MPI_Bcast(&local_leader_rank_in_peer,  1, MPI_INT, local_leader, local_comm); 
     156    MPI_Bcast(&remote_leader_rank_in_peer, 1, MPI_INT, local_leader, local_comm); 
     157 
     158    bool priority = local_leader_rank_in_peer > remote_leader_rank_in_peer? true : false; 
     159 
     160#ifdef _showinfo 
     161    MPI_Barrier(peer_comm); 
     162    MPI_Barrier(peer_comm); 
     163    printf("peer_rank = %d, priority = %d, local_leader_rank_in_peer = %d, remote_leader_rank_in_peer = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, priority, local_leader_rank_in_peer, remote_leader_rank_in_peer); 
     164    MPI_Barrier(peer_comm); 
     165    MPI_Barrier(peer_comm); 
     166#endif 
     167 
     168    int local_leader_rank_in_world = ranks_in_world_local[local_leader]; 
     169    int remote_leader_rank_in_world; 
     170 
     171    if(is_local_leader) 
     172    { 
     173      MPI_Request request[2]; 
     174      MPI_Status status[2]; 
     175      MPI_Isend(&local_leader_rank_in_world,  1, MPI_INT, remote_leader, tag, peer_comm, &request[0]); 
     176      MPI_Irecv(&remote_leader_rank_in_world, 1, MPI_INT, remote_leader, tag, peer_comm, &request[1]); 
     177      MPI_Waitall(2, request, status); 
     178    } 
     179 
     180    MPI_Bcast(&remote_leader_rank_in_world, 1, MPI_INT, local_leader, local_comm); 
     181 
     182    int ownership; 
     183 
     184    if(rank_in_world == ranks_in_world_local[local_leader]) ownership = 1; 
     185    else if(rank_in_world == remote_leader_rank_in_world)   ownership = 0; 
     186    else 
     187    { 
     188      ownership = 1; 
     189      for(int i=0; i<remote_ep_size; i++) 
     190      { 
     191        if(rank_in_world == ranks_in_world_remote[i]) 
     192        { 
     193          ownership = priority? 1 : 0; 
     194          break; 
     195        }   
     196      } 
     197    } 
     198     
     199#ifdef _showinfo 
     200    MPI_Barrier(peer_comm); 
     201    MPI_Barrier(peer_comm); 
     202    printf("peer_rank = %d, priority = %d, ownership = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, priority, ownership); 
     203    MPI_Barrier(peer_comm); 
     204    MPI_Barrier(peer_comm); 
     205#endif 
     206 
     207 
     208    // step 4 : extract local_comm and create intercomm 
     209 
     210    bool is_involved = is_local_leader || (!is_local_leader && ep_rank_loc == 0 && rank_in_world != local_leader_rank_in_world); 
     211 
     212#ifdef _showinfo 
     213 
     214    MPI_Barrier(peer_comm); 
     215    MPI_Barrier(peer_comm); 
     216    printf("peer_rank = %d, is_involved = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, is_involved); 
     217    MPI_Barrier(peer_comm); 
     218    MPI_Barrier(peer_comm); 
     219 
     220#endif 
     221 
     222    if(is_involved) 
     223    { 
     224      ::MPI_Group local_group; 
     225      ::MPI_Group extracted_group; 
     226      ::MPI_Comm *extracted_comm = new ::MPI_Comm; 
     227 
     228 
     229      ::MPI_Comm_group(to_mpi_comm(local_comm->mpi_comm), &local_group); 
     230 
     231      int *ownership_list = new int[mpi_size]; 
     232      int *mpi_rank_list = new int[mpi_size]; 
     233 
     234      ::MPI_Allgather(&ownership, 1, to_mpi_type(MPI_INT), ownership_list, 1, to_mpi_type(MPI_INT), to_mpi_comm(local_comm->mpi_comm)); 
     235      ::MPI_Allgather(&mpi_rank,  1, to_mpi_type(MPI_INT), mpi_rank_list,  1, to_mpi_type(MPI_INT), to_mpi_comm(local_comm->mpi_comm)); 
     236 
     237       
     238      int n=0; 
     239      for(int i=0; i<mpi_size; i++) 
     240      {  
     241        n+=ownership_list[i]; 
     242      } 
     243 
     244      int *new_mpi_rank_list = new int[n]; 
     245      int j=0; 
     246      for(int i=0; i<mpi_size; i++) 
     247      {  
     248        if(ownership_list[i] !=0) 
     249        { 
     250          new_mpi_rank_list[j++] = mpi_rank_list[i]; 
     251        } 
     252      } 
     253 
     254 
     255      ::MPI_Group_incl(local_group, n, new_mpi_rank_list, &extracted_group); 
     256 
     257      ::MPI_Comm_create(to_mpi_comm(local_comm->mpi_comm), extracted_group, extracted_comm); 
     258 
     259      ::MPI_Comm *mpi_inter_comm = new ::MPI_Comm; 
     260 
     261      int local_leader_rank_in_extracted_comm; 
     262 
     263      if(is_local_leader) 
     264      { 
     265        ::MPI_Comm_rank(*extracted_comm, &local_leader_rank_in_extracted_comm); 
     266      } 
     267 
     268      ::MPI_Bcast(&local_leader_rank_in_extracted_comm, 1, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, to_mpi_comm(local_comm->mpi_comm)); 
     269 
     270      int local_leader_rank_in_peer_mpi; 
     271      int remote_leader_rank_in_peer_mpi; 
     272 
     273      ::MPI_Comm_rank(to_mpi_comm(peer_comm->mpi_comm), &local_leader_rank_in_peer_mpi); 
     274 
     275      if(is_local_leader) 
     276      { 
     277        MPI_Request request[2]; 
     278        MPI_Status status[2]; 
     279        MPI_Isend(&local_leader_rank_in_peer_mpi,  1, MPI_INT, remote_leader, tag, peer_comm, &request[0]); 
     280        MPI_Irecv(&remote_leader_rank_in_peer_mpi, 1, MPI_INT, remote_leader, tag, peer_comm, &request[1]); 
     281        MPI_Waitall(2, request, status); 
     282      } 
     283 
     284      ::MPI_Bcast(&remote_leader_rank_in_peer_mpi, 1, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, to_mpi_comm(local_comm->mpi_comm));       
     285 
     286 
     287      if(ownership) 
     288        ::MPI_Intercomm_create(*extracted_comm, local_leader_rank_in_extracted_comm, to_mpi_comm(peer_comm->mpi_comm), remote_leader_rank_in_peer_mpi, tag, mpi_inter_comm); 
     289 
     290 
     291      // step 5 :: determine new num_ep 
     292 
     293      int num_ep_count=0; 
     294 
     295      for(int i=0; i<ep_size; i++) 
     296      { 
     297        if(rank_in_world == ranks_in_world_local[i]) 
     298          num_ep_count++; 
     299      } 
     300       
     301      for(int i=0; i<remote_ep_size; i++) 
     302      { 
     303        if(rank_in_world == ranks_in_world_remote[i]) 
     304          num_ep_count++; 
     305      } 
     306 
     307      //printf("peer_rank = %d, num_ep_count = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, num_ep_count); 
     308 
     309 
     310      // step 6 : create endpoints from extracted_comm  
     311 
     312      if(ownership) 
     313      { 
     314        MPI_Comm *ep_comm; 
     315        MPI_Info info; 
     316        MPI_Comm_create_endpoints(extracted_comm, num_ep_count, info, ep_comm); 
     317 
     318        for(int i=0; i<num_ep_count; i++) 
     319        { 
     320          ep_comm[i]->is_intercomm = true; 
     321          ep_comm[i]->ep_comm_ptr->comm_label = ranks_in_world_local[local_leader]; 
     322          ep_comm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
     323          ep_comm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter_comm; 
     324        } 
     325 
     326        //delete ep_comm[0]->ep_rank_map; 
     327 
     328 
     329        #pragma omp critical (write_to_tag_list)       
     330        intercomm_list.push_back(make_pair( make_pair(tag, min(local_leader_rank_in_world, remote_leader_rank_in_world)) , make_pair(ep_comm , make_pair(num_ep_count, 0)))); 
     331        #pragma omp flush 
     332#ifdef _showinfo 
     333        for(int i=0; i<num_ep_count; i++) 
     334          printf("peer_rank = %d, ep_comm = %p, ep_comm[%d] -> new_ep_rank = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_comm, i, ep_comm[i]->ep_comm_ptr->size_rank_info[0].first); 
     335#endif 
     336      } 
     337 
     338 
     339      delete ownership_list; 
     340      delete mpi_rank_list; 
     341      delete new_mpi_rank_list; 
     342 
     343    } 
     344 
     345    int repeated=0; 
     346    for(int i=0; i<remote_ep_size; i++) 
     347    { 
     348      if(rank_in_world == ranks_in_world_remote[i]) 
     349        repeated++; 
     350    } 
     351 
     352    int my_turn = ownership? ep_rank_loc : ep_rank_loc+repeated; 
     353 
     354#ifdef _showinfo 
     355 
     356    MPI_Barrier(peer_comm); 
     357    MPI_Barrier(peer_comm); 
     358    printf("peer_rank = %d, ep_rank_loc = %d, ownership = %d, repeated = %d, my_turn = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_rank_loc, ownership, repeated, my_turn); 
     359    MPI_Barrier(peer_comm); 
     360    MPI_Barrier(peer_comm); 
     361 
     362#endif 
     363 
     364 
     365    #pragma omp flush 
     366    #pragma omp critical (read_from_intercomm_list) 
     367    { 
     368      bool flag=true; 
     369      while(flag) 
     370      { 
     371        for(std::list<std::pair<std::pair<int, int> , std::pair<MPI_Comm * , std::pair<int, int> > > >::iterator iter = intercomm_list.begin(); iter!=intercomm_list.end(); iter++) 
     372        { 
     373          if(iter->first == make_pair(tag, min(local_leader_rank_in_world, remote_leader_rank_in_world))) 
     374          { 
     375            *newintercomm = iter->second.first[my_turn]; 
     376             
     377            iter->second.second.second++; 
     378             
     379            if(iter->second.second.first == iter->second.second.second) 
     380              intercomm_list.erase(iter); 
     381 
     382            flag = false; 
     383            break;  
     384          } 
     385        } 
     386      } 
     387    } 
     388 
     389#ifdef _showinfo 
     390 
     391    MPI_Barrier(peer_comm); 
     392    MPI_Barrier(peer_comm); 
     393    printf("peer_rank = %d, test_rank = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, (*newintercomm)->ep_comm_ptr->size_rank_info[0].first); 
     394    MPI_Barrier(peer_comm); 
     395    MPI_Barrier(peer_comm); 
     396 
     397#endif 
     398 
     399 
     400    // step 7 : create intercomm_rank_map for local leaders 
     401     
     402    int my_quadruple[4]; 
     403 
     404    my_quadruple[0] = ep_rank; 
     405    my_quadruple[1] = (*newintercomm)->ep_comm_ptr->size_rank_info[1].first; 
     406    my_quadruple[2] = (*newintercomm)->ep_comm_ptr->size_rank_info[2].first; 
     407    my_quadruple[3] = (*newintercomm)->ep_comm_ptr->comm_label; 
     408 
     409 
     410#ifdef _showinfo 
     411 
     412    MPI_Barrier(peer_comm); 
     413    MPI_Barrier(peer_comm); 
     414    printf("peer_rank = %d, my_quadruple = %d %d %d %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, my_quadruple[0], my_quadruple[1], my_quadruple[2], my_quadruple[3]); 
     415    MPI_Barrier(peer_comm); 
     416    MPI_Barrier(peer_comm); 
     417#endif 
     418 
     419    int *local_quadruple_list; 
     420    int *remote_quadruple_list; 
     421    if(is_involved) 
     422    { 
     423      local_quadruple_list = new int[4*ep_size]; 
     424      remote_quadruple_list = new int[4*remote_ep_size]; 
     425 
     426    } 
     427 
     428    MPI_Gather(my_quadruple, 4, MPI_INT, local_quadruple_list, 4, MPI_INT, local_leader, local_comm); 
     429 
     430 
     431    if(is_local_leader) 
     432    { 
     433      MPI_Request request[2]; 
     434      MPI_Status status[2]; 
     435      MPI_Isend(local_quadruple_list,  4*ep_size,        MPI_INT, remote_leader, tag, peer_comm, &request[0]); 
     436      MPI_Irecv(remote_quadruple_list, 4*remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request[1]); 
     437      MPI_Waitall(2, request, status); 
     438    } 
     439 
     440    if(is_involved) 
     441    { 
     442      ::MPI_Bcast(remote_quadruple_list, 4*remote_ep_size, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, to_mpi_comm(local_comm->mpi_comm)); 
     443      (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new INTERCOMM_RANK_MAP; 
     444      for(int i=0; i<remote_ep_size; i++) 
     445      { 
     446        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->insert(std::pair<int, std::pair< int, std::pair<int, int> > >(remote_quadruple_list[4*i], remote_quadruple_list[4*i+1], remote_quadruple_list[4*i+2], remote_quadruple_list[4*i+3])); 
     447      } 
     448    } 
     449 
     450 
     451    // step 8 : associate intercomm_rank_map to endpoints 
     452 
     453    int *leader_rank_in_world_local_gathered = new int[(*newintercomm)->ep_comm_ptr->size_rank_info[1].second]; 
     454 
     455    MPI_Allgather_local(&local_leader_rank_in_world, 1, MPI_INT, leader_rank_in_world_local_gathered, *newintercomm); 
     456 
     457 
     458    int new_rank_loc = (*newintercomm)->ep_comm_ptr->size_rank_info[1].first; 
     459    int *new_rank_loc_local_gathered = new int[(*newintercomm)->ep_comm_ptr->size_rank_info[1].second]; 
     460 
     461    MPI_Allgather_local(&new_rank_loc, 1, MPI_INT, new_rank_loc_local_gathered, *newintercomm); 
     462 
     463    //printf("peer_rank = %d, leader_rank_in_world_local_gathered = %d %d %d %d, new_rank_loc_local_gathered = %d %d %d %d\n",  
     464    //  peer_comm->ep_comm_ptr->size_rank_info[0].first, leader_rank_in_world_local_gathered[0], leader_rank_in_world_local_gathered[1], leader_rank_in_world_local_gathered[2], leader_rank_in_world_local_gathered[3], 
     465    //  new_rank_loc_local_gathered[0], new_rank_loc_local_gathered[1], new_rank_loc_local_gathered[2], new_rank_loc_local_gathered[3]); 
     466 
     467    if(is_involved) 
     468    { 
     469      if((*newintercomm)->ep_comm_ptr->size_rank_info[1].first == 0) 
     470      { 
     471        (*newintercomm)->ep_rank_map->clear(); 
     472        delete (*newintercomm)->ep_rank_map; 
     473      } 
     474      (*newintercomm)->ep_rank_map = new EP_RANK_MAP[ep_size]; 
     475      *((*newintercomm)->ep_rank_map) = *(local_comm->ep_rank_map); 
     476    } 
     477 
     478    MPI_Barrier_local(*newintercomm); 
     479 
     480     
     481    if(!is_involved) 
     482    { 
     483      int target; 
     484      for(int i=0; i<(*newintercomm)->ep_comm_ptr->size_rank_info[1].second; i++) 
     485      { 
     486        if(local_leader_rank_in_world == leader_rank_in_world_local_gathered[i]) 
     487        { 
     488          target = i; 
     489          break; 
     490        } 
     491      } 
     492      (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = (*newintercomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->intercomm_rank_map; 
     493      (*newintercomm)->ep_rank_map = (*newintercomm)->ep_comm_ptr->comm_list[target]->ep_rank_map; 
     494    } 
     495 
     496 
     497    //printf("peer_rank = %d, intercomm_rank_map add = %p\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map); 
     498     
     499 
     500    if(peer_comm->ep_comm_ptr->size_rank_info[0].first == 5) 
     501    { 
     502      int receiver = rand()%10; 
     503      printf("receiver = %d, intercomm_local_rank = %d, intercomm_mpi_rank = %d, comm_label = %d\n", receiver, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).first,  
     504        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.first, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.second); 
     505    } 
     506 
     507    if(peer_comm->ep_comm_ptr->size_rank_info[0].first == 9) 
     508    { 
     509      int receiver = rand()%6; 
     510      printf("receiver = %d, intercomm_local_rank = %d, intercomm_mpi_rank = %d, comm_label = %d\n", receiver, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).first,  
     511        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.first, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.second); 
     512    } 
     513 
     514    //printf("peer_rank = %d, rank_map add = %p\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, (*newintercomm)->ep_rank_map); 
     515 
     516 
     517    if(peer_comm->ep_comm_ptr->size_rank_info[0].first == 5) 
     518    { 
     519      for(int i=0; i<ep_size; i++) 
     520      { 
     521        printf("rank_map->at(%d) = %d, %d\n", i, (*newintercomm)->ep_rank_map->at(i).first, (*newintercomm)->ep_rank_map->at(i).second); 
     522      } 
     523    } 
     524 
     525 
     526    // clean up 
     527 
     528    delete ranks_in_world_local; 
     529    delete ranks_in_world_remote; 
     530 
     531    if(is_involved) 
     532    { 
     533      delete local_quadruple_list; 
     534      delete remote_quadruple_list; 
     535    } 
     536 
     537    delete leader_rank_in_world_local_gathered; 
     538    delete new_rank_loc_local_gathered; 
     539 
     540 
     541  } 
     542 
     543  int MPI_Intercomm_create_kernel_bkp(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 
     544  { 
     545    int ep_rank, ep_rank_loc, mpi_rank; 
     546    int ep_size, num_ep, mpi_size; 
     547 
     548    ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first; 
     549    ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first; 
     550    mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first; 
     551    ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second; 
     552    num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second; 
     553    mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second; 
     554 
    22555 
    23556    std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm 
     
    5311064 
    5321065 
    533     (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; 
    534     (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size); 
     1066    //(*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP; 
     1067    //(*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size); 
    5351068 
    5361069 
     
    5581091      for(ii=0; ii<vec2map.size(); ii++) 
    5591092      { 
    560         (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(vec2map[ii].first) = make_pair(vec2map[ii].second.first, vec2map[ii].second.second); 
     1093        //(*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(vec2map[ii].first) = make_pair(vec2map[ii].second.first, vec2map[ii].second.second); 
    5611094      } 
    5621095       
     
    5641097    } 
    5651098 
    566     MPI_Bcast((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, new_bcast_root, *newintercomm); 
     1099    //MPI_Bcast((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, new_bcast_root, *newintercomm); 
    5671100 
    5681101    (*newintercomm)->ep_comm_ptr->intercomm->local_comm = (local_comm->ep_comm_ptr->comm_list[ep_rank_loc]); 
Note: See TracChangeset for help on using the changeset viewer.