Changeset 1517


Ignore:
Timestamp:
06/01/18 15:24:53 (6 years ago)
Author:
yushan
Message:

save dev

Location:
XIOS/dev/branch_openmp/extern/ep_dev
Files:
1 added
21 edited

Legend:

Unmodified
Added
Removed

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_barrier.cpp

    r1515 r1517  
    1111    if(comm->is_intercomm) return MPI_Barrier_intercomm(comm); 
    1212 
    13     if(comm->is_ep) 
     13    if(comm->is_ep)  
    1414    { 
    15       int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
    16  
    17       MPI_Barrier_local(comm); 
    18  
    19       if(ep_rank_loc == 0) 
    20       { 
    21         ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
    22  
    23         ::MPI_Barrier(mpi_comm); 
    24       } 
    25  
    26       MPI_Barrier_local(comm); 
    27  
    28       return 0; 
    29     } 
    30     else if(comm->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
    31     { 
    32       ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
    33       ::MPI_Barrier(mpi_comm); 
    34       return 0; 
     15      return MPI_Barrier_intracomm(comm); 
    3516    } 
    3617 
    37     else return 0; 
     18     
     19    return MPI_Barrier_mpi(comm); 
     20 
    3821  } 
    3922 
    40  
    41   int MPI_Barrier_intercomm(MPI_Comm comm) 
     23  int MPI_Barrier_intracomm(MPI_Comm comm) 
    4224  { 
    43  
    44     int ep_rank; 
    45     MPI_Comm_rank(comm, &ep_rank); 
    46  
    47     int ep_rank_loc = comm->ep_rank_map->at(ep_rank).first; 
     25    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
    4826 
    4927    MPI_Barrier_local(comm); 
    … …  
    5129    if(ep_rank_loc == 0) 
    5230    { 
    53       //::MPI_Comm mpi_comm = to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm); 
    54       ::MPI_Barrier(to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm)); 
     31      ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
     32 
     33      ::MPI_Barrier(mpi_comm); 
    5534    } 
    5635 
    … …  
    5837 
    5938    return 0; 
     39  } 
    6040 
     41  int MPI_Barrier_intercomm(MPI_Comm comm) 
     42  { 
     43    MPI_Barrier_local(comm); 
     44 
     45    if(comm->ep_comm_ptr->intercomm->size_rank_info[1].first == 0) 
     46      ::MPI_Barrier(to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm)); 
     47 
     48    MPI_Barrier_local(comm); 
     49  } 
     50 
     51  int MPI_Barrier_mpi(MPI_Comm comm) 
     52  { 
     53    return ::MPI_Barrier(to_mpi_comm(comm->mpi_comm)); 
    6154  } 
    6255 
    … …  
    6457  int MPI_Barrier_local(MPI_Comm comm) 
    6558  { 
    66     //Message_Check(comm); 
    6759    comm->ep_barrier->wait(); 
    6860  } 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_comm.hpp

    r1515 r1517  
    3737 
    3838    ep_intercomm *intercomm; 
    39  
    40     ep_communicator(); 
    41     bool operator == (ep_communicator right); 
    42     bool operator != (ep_communicator right); 
    4339     
    4440  }; 
    … …  
    5349 
    5450    BUFFER     *my_buffer; 
     51     
    5552    ep_barrier *ep_barrier; 
     53 
    5654    EP_RANK_MAP   *ep_rank_map;      // for intercomm : = ep_rank_map of newcomm 
     55     
     56 
     57 
    5758    void* mpi_comm; 
    5859    ep_communicator *ep_comm_ptr; 
    … …  
    6061    void* mpi_bridge; 
    6162 
    62     ep_comm();   
     63    ep_comm(){};   
    6364    ep_comm(void* comm); 
    64  
    65  
    66     bool operator == (ep_comm right); 
    67     bool operator != (ep_comm right); 
    68     bool is_null(); 
    6965 
    7066  }; 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_create.cpp

    r1511 r1517  
    4141 
    4242    out_comm_hdls = new MPI_Comm[num_ep]; 
     43#ifdef _showinfo 
     44    printf("new out_comm_hdls = %p\n", out_comm_hdls); 
     45#endif 
    4346 
     47    ::MPI_Comm *parent_comm = new ::MPI_Comm; 
     48    ::MPI_Comm_dup(to_mpi_comm(base_comm_ptr), parent_comm); 
     49 
     50#ifdef _showinfo 
     51    printf("new out_comm_hdls->mpi_comm = %p\n", parent_comm); 
     52#endif 
     53     
    4454    for (int idx = 0; idx < num_ep; ++idx) 
    4555    { 
     56 
    4657      out_comm_hdls[idx] = new ep_comm; 
     58#ifdef _showinfo 
     59      printf("new out_comm_hdls[%d] = %p\n", idx, out_comm_hdls[idx]); 
     60#endif 
     61 
    4762      out_comm_hdls[idx]->is_ep = true; 
    4863      out_comm_hdls[idx]->is_intercomm = false; 
     64       
    4965      out_comm_hdls[idx]->ep_comm_ptr = new ep_communicator;      
    50       *(static_cast< ::MPI_Comm*>(out_comm_hdls[idx]->mpi_comm)) = *(static_cast< ::MPI_Comm*>(base_comm_ptr)); 
     66#ifdef _showinfo 
     67      printf("new out_comm_hdls[%d]->ep_comm_ptr = %p\n", idx, out_comm_hdls[idx]->ep_comm_ptr); 
     68#endif 
     69 
     70 
     71      out_comm_hdls[idx]->mpi_comm = parent_comm; 
    5172      out_comm_hdls[idx]->ep_comm_ptr->comm_list = out_comm_hdls; 
    5273      out_comm_hdls[idx]->ep_comm_ptr->comm_label = 0; 
    … …  
    6283 
    6384    out_comm_hdls[0]->ep_barrier = new ep_barrier(num_ep); 
     85#ifdef _showinfo 
     86    printf("new out_comm_hdls[0]->ep_barrier = %p\n", out_comm_hdls[0]->ep_barrier); 
     87#endif 
     88 
    6489    out_comm_hdls[0]->my_buffer = new BUFFER; 
     90#ifdef _showinfo 
     91    printf("new out_comm_hdls[0]->my_buffer = %p\n", out_comm_hdls[0]->my_buffer); 
     92#endif 
    6593 
    6694    out_comm_hdls[0]->ep_rank_map = new EP_RANK_MAP; 
    67  
     95#ifdef _showinfo 
     96    printf("new out_comm_hdls[0]->ep_rank_map = %p\n", out_comm_hdls[0]->ep_rank_map); 
     97#endif 
    6898 
    6999    for (int i = 1; i < num_ep; i++) 
    … …  
    82112 
    83113      out_comm_hdls[i]->ep_comm_ptr->message_queue = new Message_list; 
     114#ifdef _showinfo 
     115      printf("new out_comm_hdls[%d]->ep_comm_ptr->message_queue = %p\n", i, out_comm_hdls[i]->ep_comm_ptr->message_queue); 
     116#endif 
     117 
    84118    } 
    85119 
    … …  
    91125      for(int j=0; j<recv_num_ep[i]; j++) 
    92126      { 
    93         //out_comm_hdls[0]->rank_map->at(ind) = make_pair(j, i); 
    94127        out_comm_hdls[0]->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(ind, j, i)); 
    95128        ind++; 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_dup.cpp

    r1515 r1517  
    77{ 
    88 
     9  int MPI_Comm_dup_mpi(MPI_Comm comm, MPI_Comm *newcomm) 
     10  { 
     11    newcomm = new MPI_Comm; 
     12    (*newcomm)->is_ep = false; 
     13 
     14    ::MPI_Comm *output = new ::MPI_Comm; 
     15       
     16    ::MPI_Comm_dup(to_mpi_comm(comm->mpi_comm), output); 
     17 
     18    (*newcomm)->mpi_comm = output; 
     19  } 
     20 
     21  int MPI_Comm_dup_intracomm(MPI_Comm comm, MPI_Comm *newcomm) 
     22  { 
     23    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     24    int num_ep      = comm->ep_comm_ptr->size_rank_info[1].second; 
     25 
     26 
     27    if(0 == ep_rank_loc) 
     28    { 
     29      MPI_Info info; 
     30      MPI_Comm *out_comm; 
     31 
     32      MPI_Comm_create_endpoints(comm->mpi_comm, num_ep, info, out_comm); 
     33      comm->ep_comm_ptr->comm_list[0]->mem_bridge = out_comm; 
     34    } 
     35 
     36    MPI_Barrier_local(comm); 
     37 
     38    *newcomm = (comm->ep_comm_ptr->comm_list[0]->mem_bridge[ep_rank_loc]); 
     39     
     40  } 
     41 
    942  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm) 
    1043  { 
     44     
    1145    if(!comm->is_ep) 
    1246    { 
    13       Debug("Comm_dup MPI\n"); 
    14       newcomm = new MPI_Comm; 
    15       (*newcomm)->is_ep = comm->is_ep; 
    16  
    17       ::MPI_Comm input = to_mpi_comm(comm->mpi_comm); 
    18       ::MPI_Comm *output = new ::MPI_Comm; 
    19  
    20        
    21       ::MPI_Comm_dup(input, output); 
    22  
    23       (*newcomm)->mpi_comm = output; 
    24  
    25       return 0; 
     47      Debug("MPI_Comm_dup with MPI\n"); 
     48      return MPI_Comm_dup_mpi(comm, newcomm);  
    2649    } 
    2750 
    2851    if(comm->is_intercomm) return MPI_Comm_dup_intercomm(comm, newcomm); 
    2952 
    30     // for intracomm 
    31     if(comm->mpi_comm == static_cast< ::MPI_Comm* >(MPI_COMM_NULL->mpi_comm)) return 0; 
     53     
     54    return MPI_Comm_dup_intracomm(comm, newcomm);  
    3255 
    33  
    34     int my_rank = comm->ep_comm_ptr->size_rank_info[1].first; 
    35     int num_ep  = comm->ep_comm_ptr->size_rank_info[1].second; 
    36  
    37  
    38     if(0 == my_rank) 
    39     { 
    40       MPI_Info info; 
    41       MPI_Comm *out_comm; 
    42       ::MPI_Comm *mpi_dup = new ::MPI_Comm; 
    43  
    44       ::MPI_Comm in_comm = to_mpi_comm(comm->mpi_comm); 
    45  
    46       ::MPI_Comm_dup(in_comm, mpi_dup); 
    47  
    48       MPI_Comm_create_endpoints(mpi_dup, num_ep, info, out_comm); 
    49       comm->ep_comm_ptr->comm_list[0]->mem_bridge = out_comm; 
    50     } 
    51  
    52     MPI_Barrier(comm); 
    53  
    54     *newcomm = (comm->ep_comm_ptr->comm_list[0]->mem_bridge[my_rank]); 
    55  
    56     return MPI_SUCCESS; 
     56    
    5757  } 
    5858 
    … …  
    6060  { 
    6161     
    62     if(comm->mpi_comm == static_cast< ::MPI_Comm* >(MPI_COMM_NULL->mpi_comm)) return 0; 
    63  
    64     int my_rank = comm->ep_comm_ptr->size_rank_info[1].first; 
    65     int num_ep  = comm->ep_comm_ptr->size_rank_info[1].second; 
     62    int newcomm_ep_rank =comm->ep_comm_ptr->intercomm->size_rank_info[0].first;  
     63    int newcomm_ep_rank_loc = comm->ep_comm_ptr->intercomm->size_rank_info[1].first; 
     64    int newcomm_num_ep = comm->ep_comm_ptr->intercomm->size_rank_info[1].second; 
    6665 
    6766 
    68     if(0 == my_rank) 
     67    if(0 == newcomm_ep_rank_loc) 
    6968    { 
     69      //printf("in dup , newcomm_ep_rank_loc = 0 :  ep %d\n", comm->ep_comm_ptr->intercomm->size_rank_info[0].first); 
     70 
    7071      MPI_Info info; 
    7172      MPI_Comm *out_comm; 
    72       ::MPI_Comm *mpi_dup = new ::MPI_Comm; 
    7373 
    74       ::MPI_Comm in_comm = to_mpi_comm(comm->mpi_comm); 
     74      MPI_Comm_create_endpoints(comm->mpi_comm, newcomm_num_ep, info, out_comm); 
    7575 
    76       ::MPI_Comm_dup(in_comm, mpi_dup); 
     76      ::MPI_Comm *mpi_inter_comm = new ::MPI_Comm; 
     77      ::MPI_Comm_dup(to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), mpi_inter_comm); 
    7778 
    78       MPI_Comm_create_endpoints(mpi_dup, num_ep, info, out_comm); 
     79      for(int i=0; i<newcomm_num_ep; i++) 
     80      { 
     81        out_comm[i]->is_intercomm = true; 
     82        out_comm[i]->ep_comm_ptr->comm_label = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->comm_label; 
     83        out_comm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
     84#ifdef _showinfo 
     85        printf("new out_comm[%d]->ep_comm_ptr->intercomm = %p\n", i, out_comm[i]->ep_comm_ptr->intercomm); 
     86#endif 
     87        out_comm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter_comm;       
     88      } 
    7989 
    80       ::MPI_Comm *mpi_inter = new ::MPI_Comm; 
    81  
    82       ::MPI_Comm_dup(to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), mpi_inter); 
    83        
    84       for(int i=0; i<num_ep; i++) 
    85       { 
    86         out_comm[i]->ep_comm_ptr->comm_label = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->comm_label; 
    87         out_comm[i]->ep_comm_ptr->intercomm = new ep_intercomm; 
    88  
    89         out_comm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter; 
    90         out_comm[i]->is_intercomm = true; 
    91  
    92         out_comm[i]->ep_comm_ptr->intercomm->inter_rank_map = new RANK_MAP; 
    93         out_comm[i]->ep_comm_ptr->intercomm->loc_rank_map = new RANK_MAP; 
    94         out_comm[i]->ep_comm_ptr->intercomm->remote_rank_map = new RANK_MAP; 
    95  
    96         int map_size = 0; 
    97         map_size = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->intercomm_rank_map->size(); 
    98         out_comm[i]->ep_comm_ptr->intercomm->inter_rank_map->resize(map_size); 
    99         for(int ii=0; ii<map_size; ii++) 
    100           out_comm[i]->ep_comm_ptr->intercomm->intercomm_rank_map->at(ii) = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->intercomm_rank_map->at(ii); 
    101  
    102         map_size = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->local_rank_map->size(); 
    103         out_comm[i]->ep_comm_ptr->intercomm->loc_rank_map->resize(map_size); 
    104         for(int ii=0; ii<map_size; ii++) 
    105           out_comm[i]->ep_comm_ptr->intercomm->local_rank_map->at(ii) = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->local_rank_map->at(ii); 
    106  
    107         map_size = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->remote_rank_map->size(); 
    108         out_comm[i]->ep_comm_ptr->intercomm->remote_rank_map->resize(map_size); 
    109         for(int ii=0; ii<map_size; ii++) 
    110           out_comm[i]->ep_comm_ptr->intercomm->remote_rank_map->at(ii) = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->remote_rank_map->at(ii); 
    111  
    112  
    113         //out_comm[i]->ep_comm_ptr->intercomm->intercomm_rank_map = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->intercomm_rank_map; 
    114         //out_comm[i]->ep_comm_ptr->intercomm->local_rank_map = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->local_rank_map; 
    115         //out_comm[i]->ep_comm_ptr->intercomm->remote_rank_map = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->remote_rank_map; 
    116  
    117         //out_comm[i]->ep_comm_ptr->intercomm->local_comm = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->local_comm;         
    118         out_comm[i]->ep_comm_ptr->intercomm->intercomm_tag = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->intercomm_tag; 
    119  
    120         for(int j =0; j<3; j++) 
    121         { 
    122           out_comm[i]->ep_comm_ptr->intercomm->size_rank_info[j] = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->size_rank_info[j]; 
    123         } 
    124  
    125       } 
    12690 
    12791      comm->ep_comm_ptr->comm_list[0]->mem_bridge = out_comm; 
    12892    } 
    12993 
    130     MPI_Barrier(comm); 
     94    MPI_Barrier_local(comm); 
    13195 
    132     *newcomm = comm->ep_comm_ptr->comm_list[0]->mem_bridge[my_rank]; 
     96    *newcomm = (comm->ep_comm_ptr->comm_list[0]->mem_bridge[newcomm_ep_rank_loc]); 
     97 
     98    (*newcomm)->ep_comm_ptr->size_rank_info[0] = comm->ep_comm_ptr->size_rank_info[0]; 
     99    (*newcomm)->ep_comm_ptr->size_rank_info[1] = comm->ep_comm_ptr->size_rank_info[1]; 
     100    (*newcomm)->ep_comm_ptr->size_rank_info[2] = comm->ep_comm_ptr->size_rank_info[2]; 
     101 
     102    (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[0] = comm->ep_comm_ptr->intercomm->size_rank_info[0]; 
     103    (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[1] = comm->ep_comm_ptr->intercomm->size_rank_info[1]; 
     104    (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[2] = comm->ep_comm_ptr->intercomm->size_rank_info[2]; 
     105 
     106 
     107    int ep_rank_loc = (*newcomm)->ep_comm_ptr->size_rank_info[1].first; 
     108     
     109    if(ep_rank_loc == 0) 
     110    { 
     111      //printf("in dup , ep_rank_loc = 0 :  ep %d\n", (*newcomm)->ep_comm_ptr->size_rank_info[0].first); 
     112      (*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new INTERCOMM_RANK_MAP; 
     113      (*newcomm)->ep_comm_ptr->intercomm->local_rank_map = new EP_RANK_MAP; 
     114 
     115      *(*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = *comm->ep_comm_ptr->intercomm->intercomm_rank_map; 
     116      *(*newcomm)->ep_comm_ptr->intercomm->local_rank_map = *comm->ep_comm_ptr->intercomm->local_rank_map; 
     117    } 
     118 
     119    MPI_Barrier_local(comm); 
     120 
     121    if(ep_rank_loc !=0 ) 
     122    { 
     123      (*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = (*newcomm)->ep_comm_ptr->comm_list[0]->ep_comm_ptr->intercomm->intercomm_rank_map; 
     124      (*newcomm)->ep_comm_ptr->intercomm->local_rank_map = (*newcomm)->ep_comm_ptr->comm_list[0]->ep_comm_ptr->intercomm->local_rank_map; 
     125    } 
     126 
    133127     
    134128 
    135     return MPI_SUCCESS; 
     129 
     130 
     131     
    136132  } 
    137133 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_finalize.cpp

    r1500 r1517  
    2222  int MPI_Abort(MPI_Comm comm, int errorcode) 
    2323  { 
    24     int id = omp_get_thread_num(); 
     24    if(!comm->is_ep) 
     25      return MPI_Abort_mpi(comm, errorcode); 
    2526 
    26     if(id == 0) 
     27    else 
    2728    { 
    28       ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
    29       ::MPI_Abort(mpi_comm, errorcode); 
     29      if(comm->ep_comm_ptr->size_rank_info[1].first == 0) 
     30      { 
     31        ::MPI_Abort(to_mpi_comm(comm->mpi_comm), errorcode); 
     32      } 
    3033    } 
    31     return 0; 
     34  } 
     35 
     36  int MPI_Abort_mpi(MPI_Comm comm, int errorcode) 
     37  { 
     38    return ::MPI_Abort(to_mpi_comm(comm->mpi_comm), errorcode); 
    3239  } 
    3340 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_free.cpp

    r1515 r1517  
    1111    if(! (*comm)->is_ep) 
    1212    { 
    13       if((*comm)->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
    14       { 
    15         ::MPI_Comm mpi_comm = to_mpi_comm((*comm)->mpi_comm); 
    16  
    17         ::MPI_Comm_free(&mpi_comm); 
    18         Debug("comm is MPI, freed\n"); 
    19       } 
    20       return 0; 
    21     } 
    22  
    23     else if((*comm)->is_intercomm) 
    24     { 
    25       return MPI_Comm_free_intercomm(comm); 
    26     } 
    27  
    28     else 
    29     { 
    30       int ep_rank_loc, num_ep; 
    31  
    32       ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first; 
    33       num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second; 
    34  
    35       MPI_Barrier(*comm); 
    36  
    37       if(ep_rank_loc == 0) 
    38       { 
    39         Debug("comm is EP, mpi_comm_ptr != NULL\n"); 
    40  
    41         delete (*comm)->my_buffer; 
    42  
    43  
    44         if((*comm)->ep_barrier != NULL) 
    45         { 
    46           (*comm)->ep_barrier->~ep_barrier(); 
    47           Debug("ep_barrier freed\n"); 
    48         } 
    49  
    50  
    51         if( ! (*comm)->ep_rank_map->empty() ) 
    52         { 
    53           (*comm)->ep_rank_map->clear(); 
    54           delete (*comm)->ep_rank_map; 
    55           Debug("ep_rank_map emptied and freed\n"); 
    56         } 
    57  
    58         for(int i=0; i<num_ep; i++) 
    59         { 
    60           (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue->clear(); 
    61           delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue; 
    62           Debug("message queue freed\n"); 
    63            
    64  
    65           if((*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr != NULL) 
    66           { 
    67             delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr; 
    68             Debug("ep_comm_ptr freed\n"); 
    69           } 
    70         } 
    71  
    72         if( to_mpi_comm((*comm)->mpi_comm) != *static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)  
    73           && to_mpi_comm((*comm)->mpi_comm) != *static_cast< ::MPI_Comm*>(MPI_COMM_WORLD->mpi_comm)) 
    74         { 
    75           ::MPI_Comm mpi_comm = to_mpi_comm((*comm)->mpi_comm); 
    76           ::MPI_Comm_free(&mpi_comm); 
    77           Debug("mpi_comm freed\n"); 
    78         } 
    79  
    80        //if(comm != NULL) {delete[] comm->ep_comm_ptr->comm_list; Debug("comm freed\n");} 
    81  
    82       } 
    83  
    84       return 0; 
    85     } 
    86  
    87  
    88  
    89   } 
    90  
    91   int MPI_Comm_free_intercomm(MPI_Comm *comm) 
    92   { 
     13      return MPI_Comm_free_mpi(comm); 
     14    } 
     15 
     16    else  
     17    { 
     18      if((*comm)->is_intercomm) 
     19        return MPI_Comm_free_intercomm(comm); 
     20      else 
     21        return MPI_Comm_free_intracomm(comm); 
     22    } 
     23  } 
     24 
     25 
     26 
     27  int MPI_Comm_free_mpi(MPI_Comm *comm) 
     28  { 
     29    Debug("MPI_Comm_free with MPI\n"); 
     30 
     31    return ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->mpi_comm)); 
     32     
     33  } 
     34 
     35  int MPI_Comm_free_intracomm(MPI_Comm *comm) 
     36  { 
     37    Debug("MPI_Comm_free with EP_intracomm\n"); 
     38 
    9339    int ep_rank_loc, num_ep; 
    94     //MPI_Barrier(*comm); 
    95  
    96     return 0; 
    9740 
    9841    ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first; 
    9942    num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second; 
    10043 
    101  
    102  
    103     //MPI_Barrier(*comm); 
     44    MPI_Barrier(*comm); 
    10445 
    10546    if(ep_rank_loc == 0) 
    10647    { 
    107       Debug("comm is EP, mpi_comm_ptr != NULL\n"); 
    108  
    109  
    110       if((*comm)->ep_barrier != NULL) 
    111       { 
    112         (*comm)->ep_barrier->~ep_barrier(); 
    113         Debug("ep_barrier freed\n"); 
    114       } 
    115  
    116  
    117       if( ! (*comm)->ep_rank_map->empty() ) 
    118       { 
    119         (*comm)->ep_rank_map->clear(); 
    120         delete (*comm)->ep_rank_map; 
    121         Debug("ep_rank_map emptied and freed\n"); 
    122       } 
     48 
     49#ifdef _showinfo 
     50      printf("delete (*comm)->my_buffer = %p\n", (*comm)->my_buffer); 
     51#endif 
     52      delete (*comm)->my_buffer; 
     53 
     54 
     55       
     56#ifdef _showinfo         
     57      printf("delete (*comm)->ep_barrier = %p\n", (*comm)->ep_barrier); 
     58#endif 
     59      delete (*comm)->ep_barrier; 
     60       
     61 
     62 
     63      (*comm)->ep_rank_map->clear(); 
     64#ifdef _showinfo 
     65      printf("delete (*comm)->ep_rank_map = %p\n", (*comm)->ep_rank_map); 
     66#endif 
     67      delete (*comm)->ep_rank_map; 
     68       
    12369 
    12470      for(int i=0; i<num_ep; i++) 
    12571      { 
    12672        (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue->clear(); 
    127         Debug("message queue freed\n"); 
    128  
    129         #pragma omp critical (memory_free) 
    130         if((*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm != NULL) 
    131         { 
    132           (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->local_rank_map->clear(); 
    133           (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->remote_rank_map->clear(); 
    134           (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm->intercomm_rank_map->clear(); 
    135           Debug("intercomm local/remote/intercomm_rank_map emptied\n"); 
    136         }   
    137  
    138         if((*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr != NULL) 
    139         { 
    140           delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr; 
    141           Debug("ep_comm_ptr freed\n"); 
    142         } 
     73#ifdef _showinfo 
     74        printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr->message_queue = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue); 
     75#endif 
     76        delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue; 
     77           
     78 
     79#ifdef _showinfo 
     80        printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr); 
     81#endif 
     82        delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr; 
     83 
     84 
     85#ifdef _showinfo 
     86        printf("delete (*comm)->ep_comm_ptr->comm_list[%d] = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]); 
     87#endif 
     88        delete (*comm)->ep_comm_ptr->comm_list[i]; 
    14389      } 
    14490 
    145       if((*comm)->mpi_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
     91#ifdef _showinfo 
     92      printf("delete (*comm)->mpi_comm = %p\n", (*comm)->mpi_comm); 
     93#endif 
     94      ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->mpi_comm)); 
     95       
     96#ifdef _showinfo 
     97      printf("delete (*comm)->ep_comm_ptr->comm_list = %p\n", (*comm)->ep_comm_ptr->comm_list); 
     98#endif 
     99      delete[] (*comm)->ep_comm_ptr->comm_list; 
     100    } 
     101  } 
     102 
     103 
     104 
     105 
     106 
     107 
     108 
     109  int MPI_Comm_free_intercomm(MPI_Comm *comm) 
     110  { 
     111    int ep_rank; 
     112    MPI_Comm_rank(*comm, &ep_rank); 
     113    int ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first; 
     114    int num_ep =      (*comm)->ep_comm_ptr->size_rank_info[1].second; 
     115     
     116    int newcomm_ep_rank =(*comm)->ep_comm_ptr->intercomm->size_rank_info[0].first;  
     117    int newcomm_ep_rank_loc = (*comm)->ep_comm_ptr->intercomm->size_rank_info[1].first; 
     118    int newcomm_num_ep = (*comm)->ep_comm_ptr->intercomm->size_rank_info[1].second; 
     119 
     120    MPI_Barrier(*comm); 
     121 
     122    if(ep_rank_loc == 0) 
     123    { 
     124      (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map->clear(); 
     125#ifdef _showinfo 
     126      printf("delete (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map = %p\n", (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map); 
     127#endif 
     128      delete (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map; 
     129 
     130      (*comm)->ep_comm_ptr->intercomm->local_rank_map->clear(); 
     131#ifdef _showinfo 
     132      printf("delete (*comm)->ep_comm_ptr->intercomm->local_rank_map = %p\n", (*comm)->ep_comm_ptr->intercomm->local_rank_map); 
     133#endif 
     134      delete (*comm)->ep_comm_ptr->intercomm->local_rank_map; 
     135    } 
     136 
     137    if(newcomm_ep_rank_loc == 0) 
     138    { 
     139 
     140#ifdef _showinfo 
     141      printf("delete (*comm)->my_buffer = %p\n", (*comm)->my_buffer); 
     142#endif 
     143      delete (*comm)->my_buffer; 
     144 
     145 
     146       
     147#ifdef _showinfo         
     148      printf("delete (*comm)->ep_barrier = %p\n", (*comm)->ep_barrier); 
     149#endif 
     150      delete (*comm)->ep_barrier; 
     151       
     152 
     153      (*comm)->ep_rank_map->clear(); 
     154#ifdef _showinfo 
     155      printf("delete (*comm)->ep_rank_map = %p\n", (*comm)->ep_rank_map); 
     156#endif 
     157      delete (*comm)->ep_rank_map; 
     158       
     159#ifdef _showinfo 
     160      printf("delete (*comm)->ep_comm_ptr->intercomm->mpi_inter_comm = %p\n", (*comm)->ep_comm_ptr->intercomm->mpi_inter_comm); 
     161#endif 
     162      ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->ep_comm_ptr->intercomm->mpi_inter_comm)); 
     163 
     164      for(int i=0; i<newcomm_num_ep; i++) 
    146165      { 
    147         ::MPI_Comm mpi_comm = to_mpi_comm((*comm)->mpi_comm); 
    148         ::MPI_Comm_free(&mpi_comm); 
    149         Debug("mpi_comm freed\n"); 
     166        (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue->clear(); 
     167#ifdef _showinfo 
     168        printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr->message_queue = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue); 
     169#endif 
     170        delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue; 
     171 
     172#ifdef _showinfo 
     173        printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr->intercomm = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm); 
     174#endif 
     175        delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm;         
     176           
     177 
     178#ifdef _showinfo 
     179        printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr); 
     180#endif 
     181        delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr; 
     182 
     183 
     184#ifdef _showinfo 
     185        printf("delete (*comm)->ep_comm_ptr->comm_list[%d] = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]); 
     186#endif 
     187        delete (*comm)->ep_comm_ptr->comm_list[i]; 
     188 
    150189      } 
    151190 
    152       if((*comm)->ep_comm_ptr->intercomm->mpi_inter_comm != static_cast< ::MPI_Comm*>(MPI_COMM_NULL->mpi_comm)) 
    153       { 
    154         ::MPI_Comm mpi_comm = to_mpi_comm((*comm)->ep_comm_ptr->comm_list[0]->ep_comm_ptr->intercomm->mpi_inter_comm); 
    155         ::MPI_Comm_free(&mpi_comm); 
    156         Debug("mpi_intercomm freed\n"); 
    157       } 
    158  
    159      if(comm != NULL) {delete[] (*comm)->ep_comm_ptr->comm_list; Debug("comm freed\n");} 
    160  
    161     } 
    162  
    163      
    164      
    165     return 0; 
    166   } 
    167  
     191#ifdef _showinfo 
     192      printf("delete (*comm)->mpi_comm = %p\n", (*comm)->mpi_comm); 
     193#endif 
     194      ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->mpi_comm)); 
     195       
     196#ifdef _showinfo 
     197      printf("delete (*comm)->ep_comm_ptr->comm_list = %p\n", (*comm)->ep_comm_ptr->comm_list); 
     198#endif 
     199      delete[] (*comm)->ep_comm_ptr->comm_list; 
     200    } 
     201  } 
    168202 
    169203 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_info.hpp

    r1502 r1517  
    1212      void* mpi_info; 
    1313 
    14       ep_info(); 
     14      ep_info(){}; 
    1515      ep_info(void* info); 
    1616  }; 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_init.cpp

    r1499 r1517  
    1111  int MPI_Init_thread(int *argc, char*** argv, int required, int*provided) 
    1212  { 
    13     //printf("MPI_Init_thread\n"); 
     13    Debug("MPI_Init_thread with EP/MPI\n"); 
    1414 
    15     int id = omp_get_thread_num(); 
    16  
    17     if(id == 0) 
     15    if(omp_get_thread_num() == 0) 
    1816    { 
    1917      ::MPI_Init_thread(argc, argv, required, provided); 
    2018    } 
    21     return 0; 
    2219  } 
    2320 
    2421  int MPI_Init(int *argc, char ***argv) 
    2522  { 
    26     //printf("MPI_init called\n"); 
    27     int id = omp_get_thread_num(); 
     23    Debug("MPI_Init with EP/MPI\n"); 
    2824 
    29     if(id == 0) 
     25    if(omp_get_thread_num() == 0) 
    3026    { 
    3127      ::MPI_Init(argc, argv); 
    3228    } 
    33         return 0; 
    3429  } 
    3530 
    3631  int MPI_Initialized(int *flag) 
    3732  { 
    38     //printf("MPI_initialized called\n"); 
     33    Debug("MPI_Initialized with EP/MPI\n"); 
    3934 
    40     ::MPI_Initialized(flag); 
    41      
    42     return 0; 
     35    return ::MPI_Initialized(flag); 
    4336  } 
    4437 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.hpp

    r1515 r1517  
    2424 
    2525 
    26     RANK_MAP *inter_rank_map; 
     26    RANK_MAP *inter_rank_map;   
    2727    RANK_MAP *loc_rank_map; 
    2828    RANK_MAP *remote_rank_map; 
    2929 
    3030 
    31     SIZE_RANK_INFO size_rank_info[3]; 
     31    SIZE_RANK_INFO size_rank_info[3];        // size_rank_info of newcomm 
    3232 
    3333 
    34     //ep_comm *local_comm; 
    3534    int intercomm_tag; 
    36  
    37     ep_intercomm(); 
    38     //~ep_intercomm(){delete mpi_inter_comm;} 
    39     bool operator == (ep_intercomm right); 
    40     bool operator != (ep_intercomm right); 
    4135     
    4236  }; 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_kernel.cpp

    r1515 r1517  
    353353        MPI_Info info; 
    354354        MPI_Comm_create_endpoints(extracted_comm, num_ep_count, info, ep_comm); 
     355       
     356#ifdef _showinfo 
     357        printf("new ep_comm->ep_comm_ptr->intercomm->mpi_inter_comm = %p\n", mpi_inter_comm); 
     358#endif 
    355359 
    356360        for(int i=0; i<num_ep_count; i++) 
    … …  
    359363          ep_comm[i]->ep_comm_ptr->comm_label = ranks_in_world_local[local_leader]; 
    360364          ep_comm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
     365#ifdef _showinfo 
     366          printf("new ep_comm[%d]->ep_comm_ptr->intercomm = %p\n", i, ep_comm[i]->ep_comm_ptr->intercomm); 
     367#endif 
    361368          ep_comm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter_comm; 
     369           
    362370        } 
    363371 
    … …  
    496504      ::MPI_Bcast(remote_quadruple_list, 4*remote_ep_size, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, to_mpi_comm(local_comm->mpi_comm)); 
    497505      (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new INTERCOMM_RANK_MAP; 
     506#ifdef _showinfo 
     507      printf("new intercomm_rank_map = %p\n", (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map); 
     508#endif 
    498509      for(int i=0; i<remote_ep_size; i++) 
    499510      { 
    … …  
    522533    if(is_involved) 
    523534    { 
    524       (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map = new EP_RANK_MAP[ep_size]; 
    525       *((*newintercomm)->ep_comm_ptr->intercomm->local_rank_map) = *(local_comm->ep_rank_map); 
     535 
     536      (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map = new EP_RANK_MAP; 
     537      printf("new local_rank_map = %p\n", (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map); 
     538 
     539      for(std::map<int, std::pair<int, int> >::iterator it = local_comm->ep_rank_map->begin(); it != local_comm->ep_rank_map->end(); it++) 
     540      { 
     541        (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map->insert(std::pair<int, std::pair< int, int > >(it->first, it->second.first, it->second.second)); 
     542      } 
    526543    } 
    527544 
    … …  
    541558      } 
    542559      (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = (*newintercomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->intercomm_rank_map; 
    543       (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map = (*newintercomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->local_rank_map; 
    544     } 
    545  
    546     (*newintercomm)->ep_comm_ptr->size_rank_info[0]=local_comm->ep_comm_ptr->size_rank_info[0]; 
    547     (*newintercomm)->ep_comm_ptr->size_rank_info[1]=local_comm->ep_comm_ptr->size_rank_info[1]; 
    548     (*newintercomm)->ep_comm_ptr->size_rank_info[2]=local_comm->ep_comm_ptr->size_rank_info[2]; 
    549      
     560      (*newintercomm)->ep_comm_ptr->intercomm->local_rank_map     = (*newintercomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->local_rank_map; 
     561    } 
     562 
     563    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[0] = (*newintercomm)->ep_comm_ptr->size_rank_info[0]; 
     564    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[1] = (*newintercomm)->ep_comm_ptr->size_rank_info[1]; 
     565    (*newintercomm)->ep_comm_ptr->intercomm->size_rank_info[2] = (*newintercomm)->ep_comm_ptr->size_rank_info[2]; 
     566 
     567 
     568    (*newintercomm)->ep_comm_ptr->size_rank_info[0] = local_comm->ep_comm_ptr->size_rank_info[0]; 
     569    (*newintercomm)->ep_comm_ptr->size_rank_info[1] = local_comm->ep_comm_ptr->size_rank_info[1]; 
     570    (*newintercomm)->ep_comm_ptr->size_rank_info[2] = local_comm->ep_comm_ptr->size_rank_info[2]; 
     571 
     572 
     573#ifdef _showinfo 
     574    MPI_Barrier(peer_comm); 
     575    MPI_Barrier(peer_comm); 
     576 
     577    printf("peer_rank = %d, size_rank_info = %d %d %d %d %d %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first,  
     578      (*newintercomm)->ep_comm_ptr->size_rank_info[0].first, (*newintercomm)->ep_comm_ptr->size_rank_info[0].second, 
     579      (*newintercomm)->ep_comm_ptr->size_rank_info[1].first, (*newintercomm)->ep_comm_ptr->size_rank_info[1].second, 
     580      (*newintercomm)->ep_comm_ptr->size_rank_info[2].first, (*newintercomm)->ep_comm_ptr->size_rank_info[2].second); 
     581     
     582    MPI_Barrier(peer_comm); 
     583    MPI_Barrier(peer_comm); 
     584 
     585#endif 
     586 
    550587/* 
    551588    if(peer_comm->ep_comm_ptr->size_rank_info[0].first == 5) 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.cpp

    r1515 r1517  
    44#include <iostream> 
    55#include <fstream> 
     6#include "ep_mpi.hpp" 
    67 
    78using namespace std; 
    … …  
    1415namespace ep_lib 
    1516{  
    16   bool ep_comm::is_null() 
    17   { 
    18     if(!this->is_intercomm) 
    19       return this->mpi_comm == MPI_COMM_NULL->mpi_comm; 
    20     else 
    21       return this->ep_comm_ptr->intercomm->mpi_inter_comm == MPI_COMM_NULL->mpi_comm; 
    22   } 
    2317 
    2418  int tag_combine(int real_tag, int src, int dest) 
    … …  
    4741  int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count) 
    4842  { 
    49  
    50     ::MPI_Status *mpi_status = static_cast< ::MPI_Status* >(status->mpi_status); 
    51     ::MPI_Datatype *mpi_datatype = static_cast< ::MPI_Datatype*>(datatype); 
    52  
    53     ::MPI_Get_count(mpi_status, *mpi_datatype, count); 
     43    return ::MPI_Get_count(to_mpi_status_ptr(*status), to_mpi_type(datatype), count); 
    5444  } 
    5545 
    … …  
    227217{ 
    228218  return *(static_cast< MPI_Comm* >(comm)); 
     219}  
     220 
     221MPI_Comm* to_mpi_comm_ptr(void* comm) 
     222{ 
     223  return static_cast< MPI_Comm* >(comm); 
    229224}   
    230225 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.hpp

    r1515 r1517  
    44#include "ep_type.hpp" 
    55#include "ep_lib_intercomm.hpp" 
     6#include "ep_lib_intracomm.hpp" 
    67#include "ep_lib_local.hpp" 
    78#include "ep_lib_collective.hpp" 
    … …  
    1011#include "ep_lib_win.hpp" 
    1112#include "ep_lib_mpi.hpp" 
     13//#include "ep_mpi.hpp" 
    1214 
    1315 
    … …  
    2830 
    2931  int MPI_Comm_free(MPI_Comm* comm); 
    30   int MPI_Comm_free_intracomm(MPI_Comm* comm); 
    3132 
    3233  int MPI_Finalize(); 
    … …  
    3738 
    3839  int MPI_Get_count(const MPI_Status *status, MPI_Datatype datatype, int *count); 
    39  
    40   // #ifdef _openmpi 
    41   // int MPI_Comm_create_endpoints(void* mpi_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from MPI to create endpoints 
    42   // #elif _intelmpi 
    43   // int MPI_Comm_create_endpoints(int mpi_comm, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from MPI to create endpoints 
    44   // #endif 
    4540   
    46   int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from EP to create endpoints 
     41  int MPI_Comm_remote_size(MPI_Comm comm, int *size); 
    4742 
    4843  int MPI_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 
    … …  
    5146  int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
    5247 
    53   int tag_combine(int real_tag, int src, int dest); 
    54   int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank); 
    55  
    56   int Message_Check(MPI_Comm comm); 
    57   int Message_Check_intracomm(MPI_Comm comm); 
    58   int Request_Check(); 
    5948 
    6049  int MPI_Recv  (void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status); 
    … …  
    7867  int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr); 
    7968 
     69  int MPI_Comm_test_inter(MPI_Comm comm, int *flag); 
     70 
    8071 
    8172  void check_sum_send(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, int type); 
    … …  
    8576  bool valid_op(MPI_Op op); 
    8677 
     78  int tag_combine(int real_tag, int src, int dest); 
     79  int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank); 
     80 
     81  int Message_Check(MPI_Comm comm); 
     82 
     83  int Request_Check(); 
     84 
     85  int test_sendrecv(MPI_Comm comm); 
     86 
     87 
    8788} 
    8889 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_collective.hpp

    r1499 r1517  
    1010 
    1111  int MPI_Barrier(MPI_Comm comm); 
     12   
    1213 
    1314  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm); 
    … …  
    2627 
    2728  int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm); 
     29   
    2830  int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm); 
    2931 
    … …  
    4143  int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm); 
    4244 
     45  int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm); 
     46 
     47  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
     48 
     49  int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
     50   
     51   
     52  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm); 
     53   
     54  int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
     55 
     56  int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from EP to create endpoints 
     57 
     58 
     59 
     60  int MPI_Intercomm_merge(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm); 
     61 
     62  int MPI_Intercomm_merge_unique_leader(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm); 
     63 
    4364} 
    4465 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_intercomm.hpp

    r1499 r1517  
    88  typedef void* MPI_Op; 
    99 
    10    
     10  int MPI_Send_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 
     11  int MPI_Ssend_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 
     12  int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
     13 
     14  int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
     15 
     16 
    1117  int MPI_Comm_dup_intercomm(MPI_Comm comm, MPI_Comm *newcomm); 
    1218 
    … …  
    2228 
    2329  int MPI_Iprobe_intercomm(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status); 
    24   int MPI_Iprobe_any_source(int tag, MPI_Comm comm, int *flag, MPI_Status *status); 
    2530  int MPI_Improbe_intercomm(int source, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status); 
    26   int MPI_Improbe_any_source(int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status); 
    2731 
    28   int MPI_Comm_split(MPI_Comm comm, int color, int key, MPI_Comm *newcomm); 
    2932 
    30   int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
    3133 
    32   int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
    33    
    34   // #ifdef _intelmpi 
    35   // int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, int peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm); 
    36   // #elif _openmpi 
    37   // int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm); 
    38   // #endif 
    39  
    40   int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm); 
    41    
    42   int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
    43  
    44   int MPI_Intercomm_merge(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm); 
    45  
    46   int MPI_Intercomm_merge_unique_leader(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm); 
    47  
    48   int MPI_Comm_remote_size(MPI_Comm comm, int *size); 
    49  
    50   int MPI_Comm_test_inter(MPI_Comm comm, int *flag); 
    51  
    52   int test_sendrecv(MPI_Comm comm); 
    5334 
    5435} 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_mpi.hpp

    r1515 r1517  
    1010 
    1111  int MPI_Send_mpi      (void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 
    12   int MPI_Send_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 
    13    
    1412  int MPI_Ssend_mpi      (void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 
    15   int MPI_Ssend_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 
    16    
    1713  int MPI_Isend_mpi      (const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
    18   int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
    19    
    2014  int MPI_Issend_mpi      (const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
    21   int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
    2215   
    2316   
    2417  int MPI_Recv_mpi      (void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Status *status); 
    25    
    2618  int MPI_Irecv_mpi      (void *buf, int count, MPI_Datatype datatype, int src, int tag, MPI_Comm comm, MPI_Request *request); 
     19 
    2720 
    2821  int MPI_Comm_rank_mpi(MPI_Comm comm, int* rank); 
    … …  
    3023  int MPI_Comm_remote_size_mpi(MPI_Comm comm, int* size); 
    3124 
     25  int MPI_Comm_dup_mpi(MPI_Comm comm, MPI_Comm *newcomm); 
     26  int MPI_Comm_free_mpi(MPI_Comm *comm); 
     27 
     28  int MPI_Barrier_mpi(MPI_Comm comm); 
     29   
     30  int MPI_Abort_mpi(MPI_Comm comm, int errorcode); 
    3231} 
    3332 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_message.cpp

    r1515 r1517  
    6161        if(probed) 
    6262        { 
    63           ::MPI_Get_count(to_mpi_status_ptr(status), to_mpi_type((*(*it))->ep_datatype), &recv_count); 
     63          MPI_Get_count(&status, (*(*it))->ep_datatype, &recv_count); 
    6464           
    6565          MPI_Imrecv((*(*it))->buf, recv_count, (*(*it))->ep_datatype, message, *it); 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_message.hpp

    r1502 r1517  
    1515      void* mpi_status; 
    1616 
    17       ep_message() {} 
    18       ep_message(void* message); 
    1917  }; 
    2018   

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_mpi.hpp

    r1500 r1517  
    77MPI_Op       to_mpi_op(ep_lib::MPI_Op op); 
    88MPI_Comm     to_mpi_comm(void* comm); 
     9MPI_Comm*    to_mpi_comm_ptr(void* comm); 
    910MPI_Message  to_mpi_message(void* message); 
    1011MPI_Info     to_mpi_info(ep_lib::MPI_Info info); 

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_request.hpp

    r1503 r1517  
    3434      ep_request() {} 
    3535      ep_request(void* request); 
    36       bool operator == (ep_request right); 
    3736  }; 
    3837   

  • XIOS/dev/branch_openmp/extern/ep_dev/ep_type.cpp

    r1503 r1517  
    1111namespace ep_lib 
    1212{  
    13    
    14   ep_intercomm::ep_intercomm() 
     13  ep_comm::ep_comm(void* comm) 
    1514  { 
    16     intercomm_rank_map = NULL; 
    17     local_rank_map = NULL; 
    18     remote_rank_map = NULL; 
    19     mpi_inter_comm = new ::MPI_Comm; 
    20   } 
    21  
    22   bool ep_intercomm::operator == (ep_intercomm right) 
    23   { 
    24     bool a = intercomm_rank_map == right.intercomm_rank_map; 
    25     bool b = local_rank_map == right.local_rank_map; 
    26     bool c = remote_rank_map == right.remote_rank_map; 
    27     bool d = *(static_cast< ::MPI_Comm*>(mpi_inter_comm)) == *(static_cast< ::MPI_Comm*>(right.mpi_inter_comm)); 
    28     bool e = size_rank_info == right.size_rank_info; 
    29     bool f = intercomm_tag == right.intercomm_tag; 
    30     return a&&b&&c&&d&&e&&f; 
    31   } 
    32  
    33   bool ep_intercomm::operator != (ep_intercomm right) 
    34   { 
    35     bool a = intercomm_rank_map != right.intercomm_rank_map; 
    36     bool b = local_rank_map != right.local_rank_map; 
    37     bool c = remote_rank_map != right.remote_rank_map; 
    38     bool d = *(static_cast< ::MPI_Comm*>(mpi_inter_comm)) != *(static_cast< ::MPI_Comm*>(right.mpi_inter_comm)); 
    39     bool e = size_rank_info != right.size_rank_info; 
    40     bool f = intercomm_tag != right.intercomm_tag; 
    41     return a||b||c||d||e||f; 
    42   } 
    43  
    44   ep_communicator::ep_communicator() 
    45   { 
    46     comm_list = NULL; 
    47     message_queue = NULL; 
    48     intercomm = NULL; 
    49   } 
    50  
    51   ep_message::ep_message(void* message) 
    52   { 
    53     mpi_message = new ::MPI_Message; 
    54     *(static_cast< ::MPI_Message*>(mpi_message)) = *(static_cast< ::MPI_Message*>(message)); 
     15    is_ep = false; 
     16    is_intercomm = false; 
     17    mpi_comm = static_cast< ::MPI_Comm*>(comm); 
    5518  } 
    5619 
    5720  ep_info::ep_info(void* info) 
    5821  { 
    59     mpi_info = new ::MPI_Info; 
    60     *(static_cast< ::MPI_Info*>(mpi_info)) = *(static_cast< ::MPI_Info*>(info)); 
     22    mpi_info = static_cast< ::MPI_Info*>(info); 
    6123  } 
    62    
    63   ep_info::ep_info() 
    64   { 
    65     mpi_info = new ::MPI_Info; 
    66     *(static_cast< ::MPI_Info*>(mpi_info)) = *(static_cast< ::MPI_Info*>(MPI_INFO_NULL->mpi_info)); 
    67   } 
    68    
     24 
    6925  ep_request::ep_request(void* request) 
    7026  { 
    71     mpi_request = new ::MPI_Request; 
    72     *(static_cast< ::MPI_Request*>(mpi_request)) = *static_cast< ::MPI_Request*>(request); 
     27    mpi_request = static_cast< ::MPI_Request*>(request); 
    7328  } 
    7429   
    7530  ep_status::ep_status(void* status) 
    7631  { 
    77     mpi_status = new ::MPI_Status; 
    78     *(static_cast< ::MPI_Status*>(mpi_status)) = *static_cast< ::MPI_Status*>(status); 
     32    mpi_status = static_cast< ::MPI_Status*>(status); 
    7933  } 
     34 
     35  /* 
    8036   
    8137  MPI_Aint::MPI_Aint(void* aint) 
    … …  
    10359  } 
    10460 
    105   bool ep_communicator::operator == (ep_communicator right) 
    106   { 
    107     bool a = size_rank_info == right.size_rank_info; 
    108     bool b = comm_label == right.comm_label; 
    109     bool c = intercomm == right.intercomm; 
    110     return a&&b&&c; 
    111   } 
    112  
    113   bool ep_communicator::operator != (ep_communicator right) 
    114   { 
    115     bool a = size_rank_info != right.size_rank_info; 
    116     bool b = comm_label != right.comm_label; 
    117     bool c = intercomm != right.intercomm; 
    118     return a||b||c; 
    119   } 
    120  
    121   ep_comm::ep_comm() 
    122   { 
    123     is_ep = true; 
    124     is_intercomm = false; 
    125     my_buffer = NULL; 
    126     ep_barrier = NULL; 
    127     ep_rank_map = NULL; 
    128     ep_comm_ptr = NULL; 
    129     mem_bridge = NULL; 
    130     mpi_bridge = NULL; 
    131     mpi_comm = new ::MPI_Comm; 
    132   } 
    133  
    13461   
    135  
    136   ep_comm::ep_comm(void* comm) 
    137   { 
    138     is_ep = false; 
    139     is_intercomm = false; 
    140     my_buffer = NULL; 
    141     ep_barrier = NULL; 
    142     ep_rank_map = NULL; 
    143     ep_comm_ptr = NULL; 
    144     mem_bridge = NULL; 
    145     mpi_bridge = NULL; 
    146     mpi_comm = new ::MPI_Comm; 
    147     *(static_cast< ::MPI_Comm*>(mpi_comm)) = *(static_cast< ::MPI_Comm*>(comm)); 
    148   } 
    149  
     62*/ 
    15063   
    151  
    152   bool ep_comm::operator == (ep_comm right) 
    153   { 
    154     bool a = is_ep == right.is_ep; 
    155     bool b = is_intercomm == right.is_intercomm; 
    156     bool c = *(static_cast< ::MPI_Comm*>(mpi_comm)) == *(static_cast< ::MPI_Comm*>(right.mpi_comm)); 
    157     bool d = is_ep ? ep_comm_ptr == right.ep_comm_ptr : true; 
    158     return a&&b&&c&&d; 
    159   } 
    160  
    161   bool ep_comm::operator != (ep_comm right) 
    162   { 
    163     bool a = is_ep != right.is_ep; 
    164     bool b = is_intercomm != right.is_intercomm; 
    165     bool c = *(static_cast< ::MPI_Comm*>(mpi_comm)) != *(static_cast< ::MPI_Comm*>(right.mpi_comm)); 
    166     bool d = is_ep ? ep_comm_ptr != right.ep_comm_ptr : true; 
    167     return a||b||c||d; 
    168   } 
    169  
    170   bool ep_request::operator == (ep_request right) 
    171   { 
    172     bool b = type == right.type; 
    173     bool c = buf == right.buf; 
    174     bool d = ep_src == right.ep_src; 
    175     bool e = ep_tag == right.ep_tag; 
    176     bool f = ep_datatype == right.ep_datatype; 
    177     return b&&c&&d&&e&&f; 
    178   } 
    17964 
    18065 

  • XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

    r1515 r1517  
    1919  srand (time(NULL)); 
    2020 
    21   printf("Testing ep_lib\n"); 
     21  //printf("Testing ep_lib\n"); 
    2222  int required=3, provided; 
    2323 
    … …  
    3232  MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); 
    3333  
     34   
     35 
    3436  #pragma omp parallel default(shared) 
    3537  { 
    … …  
    858860      MPI_Comm_free(&split_comm); 
    859861 
    860       MPI_Barrier(comm); 
    861       MPI_Barrier(comm); 
    862  
     862 
     863      MPI_Barrier(comm); 
     864      MPI_Barrier(comm); 
     865 
     866      MPI_Comm inter_comm_dup; 
     867      MPI_Comm_dup(inter_comm, &inter_comm_dup); 
     868 
     869      MPI_Barrier(comm); 
     870      MPI_Barrier(comm); 
     871 
     872 
     873      MPI_Comm_free(&inter_comm_dup); 
     874 
     875 
     876      MPI_Barrier(comm); 
     877      MPI_Barrier(comm); 
     878 
     879      MPI_Barrier(inter_comm); 
    863880      MPI_Comm_free(&inter_comm); 
    864881       
Note: See TracChangeset for help on using the changeset viewer.