Changeset 1527


Ignore:
Timestamp:
06/08/18 15:59:53 (2 years ago)
Author:
yushan
Message:

save dev

Location:
XIOS/dev/branch_openmp/extern/ep_dev
Files:
1 added
2 deleted
43 edited

Legend:

Unmodified
Added
Removed
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_allgather.cpp

    r1511 r1527  
    4343  { 
    4444 
    45     if(!comm->is_ep && comm->mpi_comm) 
    46     { 
    47       return ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 
    48     } 
     45    if(!comm->is_ep) return ::MPI_Allgather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 
     46    if(comm->is_intercomm) return MPI_Allgather_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); 
     47 
    4948 
    5049    assert(sendcount == recvcount); 
     
    119118    MPI_Bcast_local(recvbuf, count*ep_size, datatype, 0, comm); 
    120119 
    121     MPI_Barrier(comm); 
    122  
    123  
    124120    if(is_master) 
    125121    { 
     
    131127  } 
    132128 
     129  int MPI_Allgather_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) 
     130  { 
     131    printf("MPI_Allgather_intercomm not yet implemented\n"); 
     132    MPI_Abort(comm, 0); 
     133  } 
    133134 
    134135 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_allgatherv.cpp

    r1503 r1527  
    2020  { 
    2121 
    22     if(!comm->is_ep && comm->mpi_comm) 
    23     { 
    24       return ::MPI_Allgatherv(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcounts, displs, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 
    25     } 
    26  
    27     if(!comm->mpi_comm) return 0; 
     22    if(!comm->is_ep) return ::MPI_Allgatherv(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcounts, displs, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 
     23    if(comm->is_intercomm) return MPI_Allgatherv_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm); 
    2824 
    2925 
     
    114110      delete[] tmp_recvbuf; 
    115111    } 
    116  
    117  
    118112  } 
    119113 
    120114 
     115  int MPI_Allgatherv_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, MPI_Comm comm) 
     116  { 
     117    printf("MPI_Allgatherv_intercomm not yet implemented\n"); 
     118    MPI_Abort(comm, 0); 
     119  } 
     120 
    121121 
    122122} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_allocate.cpp

    r1500 r1527  
    1212  { 
    1313    ::MPI_Alloc_mem(to_mpi_aint(size), to_mpi_info(info), baseptr); 
    14     //::MPI_Alloc_mem(size.mpi_aint, MPI_INFO_NULL_STD, baseptr); 
    1514    return 0; 
    1615   } 
     
    1918  { 
    2019    ::MPI_Alloc_mem(size, *(static_cast< ::MPI_Info*>(info->mpi_info)), baseptr); 
    21     //::MPI_Alloc_mem(size, MPI_INFO_NULL_STD, baseptr); 
    2220    return 0; 
    2321  } 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_allreduce.cpp

    r1500 r1527  
    1717 { 
    1818 
    19  
    20  
    2119  int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
    2220  { 
    23     if(!comm->is_ep && comm->mpi_comm) 
    24     { 
    25       return ::MPI_Allreduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    26     } 
    27  
     21    if(!comm->is_ep) return ::MPI_Allreduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
     22    if(comm->is_intercomm) return MPI_Allreduce_intercomm(sendbuf, recvbuf, count, datatype, op, comm); 
    2823 
    2924 
     
    6661    } 
    6762 
    68     MPI_Barrier_local(comm); 
    6963  } 
    7064 
    71    
     65 
     66  int MPI_Allreduce_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
     67  { 
     68    printf("MPI_Allreduce_intercomm not yet implemented\n"); 
     69    MPI_Abort(comm, 0); 
     70  } 
    7271 
    7372 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_alltoall.cpp

    r1500 r1527  
    99  int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) 
    1010  { 
    11     if(!comm->is_ep) 
    12     { 
    13       return ::MPI_Alltoall(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 
    14     } 
     11    if(!comm->is_ep) return ::MPI_Alltoall(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), to_mpi_comm(comm->mpi_comm)); 
     12    if(comm->is_intercomm) return MPI_Alltoall_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); 
    1513 
    1614 
     
    5553      delete[] tmp_sendbuf; 
    5654    } 
    57      
    58     MPI_Barrier(comm); 
    59  
    60     return 0; 
    6155  } 
    6256 
     57 
     58  int MPI_Alltoall_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) 
     59  { 
     60    printf("MPI_Alltoall_intercomm not yet implemented\n"); 
     61    MPI_Abort(comm, 0); 
     62  } 
    6363} 
    6464 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_barrier.cpp

    r1525 r1527  
    99  int MPI_Barrier(MPI_Comm comm) 
    1010  { 
    11     if(comm->is_intercomm) return MPI_Barrier_intercomm(comm); 
    1211 
    1312    if(comm->is_ep)  
    1413    { 
    15       return MPI_Barrier_intracomm(comm); 
     14      return MPI_Barrier_endpoint(comm); 
    1615    } 
    1716 
     
    2120  } 
    2221 
    23   int MPI_Barrier_intracomm(MPI_Comm comm) 
     22  int MPI_Barrier_endpoint(MPI_Comm comm) 
    2423  { 
    2524    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     
    3534 
    3635    MPI_Barrier_local(comm); 
    37  
    38     return 0; 
    3936  } 
    4037 
    41   int MPI_Barrier2(MPI_Comm comm) 
    42   { 
    43     int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
    44  
    45     MPI_Barrier_local(comm); 
    46  
    47     if(ep_rank_loc == 0) 
    48     { 
    49       ::MPI_Comm mpi_comm = to_mpi_comm(comm->mpi_comm); 
    50  
    51       ::MPI_Barrier(mpi_comm); 
    52     } 
    53  
    54     MPI_Barrier_local(comm); 
    55  
    56     return 0; 
    57   } 
    58  
    59   int MPI_Barrier_intercomm(MPI_Comm comm) 
    60   { 
    61     MPI_Barrier_local(comm); 
    62  
    63     if(comm->ep_comm_ptr->intercomm->size_rank_info[1].first == 0) 
    64       ::MPI_Barrier(to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm)); 
    65  
    66     MPI_Barrier_local(comm); 
    67   } 
    6838 
    6939  int MPI_Barrier_mpi(MPI_Comm comm) 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_bcast.cpp

    r1503 r1527  
    4545  { 
    4646 
    47     if(!comm->is_ep) 
    48     { 
    49       #pragma omp single nowait 
    50       ::MPI_Bcast(buffer, count, to_mpi_type(datatype), root, to_mpi_comm(comm->mpi_comm)); 
    51       return 0; 
    52     } 
     47    if(!comm->is_ep) return ::MPI_Bcast(buffer, count, to_mpi_type(datatype), root, to_mpi_comm(comm->mpi_comm)); 
     48    if(comm->is_intercomm) return MPI_Bcast_intercomm(buffer, count, datatype, root, comm); 
    5349 
    5450 
     
    5955    int root_mpi_rank = comm->ep_rank_map->at(root).second; 
    6056    int root_ep_rank_loc = comm->ep_rank_map->at(root).first; 
     57 
     58    //printf("ep_rank = %d, root_mpi_rank = %d, root_ep_rank_loc = %d\n", ep_rank, root_mpi_rank, root_ep_rank_loc); 
    6159 
    6260 
     
    6967    else                          MPI_Bcast_local(buffer, count, datatype, 0, comm); 
    7068 
    71     return 0; 
    7269  } 
    7370 
    7471 
    75  
     72  int MPI_Bcast_intercomm(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm) 
     73  { 
     74    printf("MPI_Bcast_intercomm not yet implemented\n"); 
     75    MPI_Abort(comm, 0); 
     76  } 
    7677 
    7778 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_comm.hpp

    r1525 r1527  
    33 
    44#include "ep_message.hpp" 
    5 #include "ep_intercomm.hpp" 
    65#include "ep_barrier.hpp" 
    76#include "ep_buffer.hpp" 
     
    2625                                      // 2: mpi_rank,    mpi_size 
    2726 
    28                                       // for intercomm : = size_rank_info of local_comm 
    29  
    30  
    3127    ep_comm **comm_list; 
    3228 
    3329    Message_list *message_queue; 
    3430 
    35  
    36     int comm_label; 
    37  
    38     ep_intercomm *intercomm; 
    39      
    4031  }; 
    4132   
     
    5243    ep_barrier *ep_barrier; 
    5344 
    54     EP_RANK_MAP   *ep_rank_map;      // for intercomm : = ep_rank_map of newcomm 
     45    EP_RANK_MAP   *ep_rank_map; 
    5546     
    5647    INTER_RANK_MAP *inter_rank_map; 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_create.cpp

    r1517 r1527  
    7171      out_comm_hdls[idx]->mpi_comm = parent_comm; 
    7272      out_comm_hdls[idx]->ep_comm_ptr->comm_list = out_comm_hdls; 
    73       out_comm_hdls[idx]->ep_comm_ptr->comm_label = 0; 
    7473    } 
    7574 
     
    130129    } 
    131130 
    132     return 0; 
    133  
    134131  } //MPI_Comm_create_endpoints 
    135132 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_dup.cpp

    r1518 r1527  
    77{ 
    88 
    9   int MPI_Comm_dup_mpi(MPI_Comm comm, MPI_Comm *newcomm) 
     9   
     10  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm) 
    1011  { 
    11     newcomm = new MPI_Comm; 
    12     (*newcomm)->is_ep = false; 
     12     
     13    if(!comm->is_ep) 
     14    { 
     15      Debug("MPI_Comm_dup with MPI\n"); 
     16      return MPI_Comm_dup_mpi(comm, newcomm);  
     17    } 
    1318 
    14     ::MPI_Comm *output = new ::MPI_Comm; 
    15        
    16     ::MPI_Comm_dup(to_mpi_comm(comm->mpi_comm), output); 
    17  
    18     (*newcomm)->mpi_comm = output; 
     19    return MPI_Comm_dup_endpoint(comm, newcomm);  
     20    
    1921  } 
    2022 
    21   int MPI_Comm_dup_intracomm(MPI_Comm comm, MPI_Comm *newcomm) 
     23 
     24   
     25  int MPI_Comm_dup_endpoint(MPI_Comm comm, MPI_Comm *newcomm) 
    2226  { 
    2327    int ep_rank_loc = comm->ep_comm_ptr->size_rank_info[1].first; 
     
    3741 
    3842    *newcomm = (comm->ep_comm_ptr->comm_list[0]->mem_bridge[ep_rank_loc]); 
     43 
     44    if(comm->is_intercomm) 
     45    { 
     46      (*newcomm)->is_intercomm = true; 
     47      (*newcomm)->ep_comm_ptr->size_rank_info[0] = comm->ep_comm_ptr->size_rank_info[0]; 
     48      (*newcomm)->inter_rank_map = new INTER_RANK_MAP; 
     49 
     50      for(INTER_RANK_MAP::iterator it = comm->inter_rank_map->begin(); it !=comm->inter_rank_map->end(); it++) 
     51      { 
     52        (*newcomm)->inter_rank_map->insert(std::make_pair(it->first, it->second)); 
     53      } 
     54 
     55    } 
    3956     
    4057  } 
    4158 
    42   int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm) 
     59  int MPI_Comm_dup_mpi(MPI_Comm comm, MPI_Comm *newcomm) 
    4360  { 
    44      
    45     if(!comm->is_ep) 
    46     { 
    47       Debug("MPI_Comm_dup with MPI\n"); 
    48       return MPI_Comm_dup_mpi(comm, newcomm);  
    49     } 
     61    newcomm = new MPI_Comm; 
     62    (*newcomm)->is_ep = false; 
    5063 
    51     if(comm->is_intercomm) return MPI_Comm_dup_intercomm(comm, newcomm); 
     64    ::MPI_Comm *output = new ::MPI_Comm; 
     65       
     66    ::MPI_Comm_dup(to_mpi_comm(comm->mpi_comm), output); 
    5267 
    53      
    54     return MPI_Comm_dup_intracomm(comm, newcomm);  
    55  
    56     
    57   } 
    58  
    59   int MPI_Comm_dup_intercomm(MPI_Comm comm, MPI_Comm *newcomm) 
    60   { 
    61      
    62     int newcomm_ep_rank =comm->ep_comm_ptr->intercomm->size_rank_info[0].first;  
    63     int newcomm_ep_rank_loc = comm->ep_comm_ptr->intercomm->size_rank_info[1].first; 
    64     int newcomm_num_ep = comm->ep_comm_ptr->intercomm->size_rank_info[1].second; 
    65  
    66  
    67     if(0 == newcomm_ep_rank_loc) 
    68     { 
    69       //printf("in dup , newcomm_ep_rank_loc = 0 :  ep %d\n", comm->ep_comm_ptr->intercomm->size_rank_info[0].first); 
    70  
    71       MPI_Info info; 
    72       MPI_Comm *out_comm; 
    73  
    74       MPI_Comm_create_endpoints(comm->mpi_comm, newcomm_num_ep, info, out_comm); 
    75  
    76       ::MPI_Comm *mpi_inter_comm = new ::MPI_Comm; 
    77       ::MPI_Comm_dup(to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), mpi_inter_comm); 
    78  
    79       for(int i=0; i<newcomm_num_ep; i++) 
    80       { 
    81         out_comm[i]->is_intercomm = true; 
    82         out_comm[i]->ep_comm_ptr->comm_label = comm->ep_comm_ptr->comm_list[i]->ep_comm_ptr->comm_label; 
    83         out_comm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm; 
    84 #ifdef _showinfo 
    85         printf("new out_comm[%d]->ep_comm_ptr->intercomm = %p\n", i, out_comm[i]->ep_comm_ptr->intercomm); 
    86 #endif 
    87         out_comm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter_comm; 
    88       } 
    89  
    90  
    91       comm->ep_comm_ptr->comm_list[0]->mem_bridge = out_comm; 
    92     } 
    93  
    94     MPI_Barrier_local(comm); 
    95  
    96     *newcomm = (comm->ep_comm_ptr->comm_list[0]->mem_bridge[newcomm_ep_rank_loc]); 
    97  
    98     (*newcomm)->ep_comm_ptr->size_rank_info[0] = comm->ep_comm_ptr->size_rank_info[0]; 
    99     (*newcomm)->ep_comm_ptr->size_rank_info[1] = comm->ep_comm_ptr->size_rank_info[1]; 
    100     (*newcomm)->ep_comm_ptr->size_rank_info[2] = comm->ep_comm_ptr->size_rank_info[2]; 
    101  
    102     (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[0] = comm->ep_comm_ptr->intercomm->size_rank_info[0]; 
    103     (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[1] = comm->ep_comm_ptr->intercomm->size_rank_info[1]; 
    104     (*newcomm)->ep_comm_ptr->intercomm->size_rank_info[2] = comm->ep_comm_ptr->intercomm->size_rank_info[2]; 
    105  
    106     (*newcomm)->ep_comm_ptr->intercomm->intercomm_tag = comm->ep_comm_ptr->intercomm->intercomm_tag; 
    107  
    108  
    109     int ep_rank_loc = (*newcomm)->ep_comm_ptr->size_rank_info[1].first; 
    110      
    111     if(ep_rank_loc == 0) 
    112     { 
    113       int world_rank; 
    114       MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); 
    115  
    116       (*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new INTERCOMM_RANK_MAP; 
    117       (*newcomm)->ep_comm_ptr->intercomm->local_rank_map = new EP_RANK_MAP; 
    118  
    119       *(*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = *comm->ep_comm_ptr->intercomm->intercomm_rank_map; 
    120       *(*newcomm)->ep_comm_ptr->intercomm->local_rank_map     = *comm->ep_comm_ptr->intercomm->local_rank_map; 
    121     } 
    122  
    123     MPI_Barrier_local(comm); 
    124  
    125     if(ep_rank_loc !=0 ) 
    126     { 
    127       int target = (*newcomm)->ep_comm_ptr->intercomm->intercomm_tag; 
    128       (*newcomm)->ep_comm_ptr->intercomm->intercomm_rank_map = (*newcomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->intercomm_rank_map;  
    129       (*newcomm)->ep_comm_ptr->intercomm->local_rank_map     = (*newcomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->local_rank_map; 
    130     } 
    131  
    132      
    133  
    134  
    135  
    136      
     68    (*newcomm)->mpi_comm = output; 
    13769  } 
    13870 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_exscan.cpp

    r1503 r1527  
    228228  int MPI_Exscan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
    229229  { 
    230     if(!comm->is_ep) 
    231     { 
    232       return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    233     } 
     230    if(!comm->is_ep) return ::MPI_Exscan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
     231    if(comm->is_intercomm) return MPI_Exscan_intercomm(sendbuf, recvbuf, count, datatype, op, comm); 
    234232     
    235233    valid_type(datatype); 
     
    291289 
    292290    if(ep_rank_loc == 0) 
     291    { 
    293292      ::MPI_Exscan(MPI_IN_PLACE, tmp_recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    294  
    295     // printf(" ID=%d : %d  %d \n", ep_rank, static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]); 
     293    } 
    296294     
    297295    MPI_Exscan_local(tmp_sendbuf, tmp_recvbuf, count, datatype, op, comm); 
    298  
    299      // printf(" ID=%d : after local tmp_sendbuf = %d %d ; tmp_recvbuf = %d  %d \n", ep_rank, static_cast<int*>(tmp_sendbuf)[0], static_cast<int*>(tmp_sendbuf)[1], static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1]); 
    300  
    301296 
    302297 
     
    314309 
    315310    else memcpy(recvbuf, tmp_recvbuf, datasize*count); 
    316      
    317  
    318  
    319311 
    320312    delete[] tmp_sendbuf; 
     
    323315  } 
    324316 
     317 
     318  int MPI_Exscan_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
     319  { 
     320    printf("MPI_Exscan_intercomm not yet implemented\n"); 
     321    MPI_Abort(comm, 0); 
     322  } 
     323 
    325324} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_finalize.cpp

    r1517 r1527  
    99  int MPI_Finalize() 
    1010  { 
    11     printf("calling EP Finalize\n"); 
    12  
    13     int id = omp_get_thread_num(); 
    14  
    15     if(id == 0) 
     11    #pragma omp master 
    1612    { 
     13      printf("calling EP Finalize\n"); 
    1714      ::MPI_Finalize(); 
    1815    } 
    19     return 0; 
    2016  } 
    2117 
    2218  int MPI_Abort(MPI_Comm comm, int errorcode) 
    23   { 
    24     if(!comm->is_ep) 
    25       return MPI_Abort_mpi(comm, errorcode); 
    26  
    27     else 
    28     { 
    29       if(comm->ep_comm_ptr->size_rank_info[1].first == 0) 
    30       { 
    31         ::MPI_Abort(to_mpi_comm(comm->mpi_comm), errorcode); 
    32       } 
    33     } 
    34   } 
    35  
    36   int MPI_Abort_mpi(MPI_Comm comm, int errorcode) 
    3719  { 
    3820    return ::MPI_Abort(to_mpi_comm(comm->mpi_comm), errorcode); 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_free.cpp

    r1525 r1527  
    99  int MPI_Comm_free(MPI_Comm *comm) 
    1010  { 
    11     if(! (*comm)->is_ep) 
    12     { 
    13       return MPI_Comm_free_mpi(comm); 
    14     } 
    15  
    16     else  
    17     { 
    18       if((*comm)->is_intercomm) 
    19         return MPI_Comm_free_intercomm(comm); 
    20       else 
    21         return MPI_Comm_free_intracomm(comm); 
    22     } 
     11    if(! (*comm)->is_ep) return MPI_Comm_free_mpi(comm); 
     12    else return MPI_Comm_free_endpoint(comm); 
    2313  } 
    2414 
     
    3323  } 
    3424 
    35   int MPI_Comm_free_intracomm(MPI_Comm *comm) 
     25  int MPI_Comm_free_endpoint(MPI_Comm *comm) 
    3626  { 
    3727    Debug("MPI_Comm_free with EP_intracomm\n"); 
     
    4030 
    4131    ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first; 
     32    int ep_rank = (*comm)->ep_comm_ptr->size_rank_info[0].first; 
    4233    num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second; 
    4334 
     
    4637    if(ep_rank_loc == 0) 
    4738    { 
     39 
     40      if((*comm)->is_intercomm) 
     41      { 
     42        for(int i=0; i<num_ep; i++) 
     43        { 
     44          (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map->clear(); 
     45#ifdef _showinfo 
     46          printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->inter_rank_map = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map); 
     47#endif 
     48          delete (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map; 
     49        } 
     50      } 
     51 
     52 
    4853 
    4954#ifdef _showinfo 
     
    103108 
    104109 
    105   int MPI_Comm_free2(MPI_Comm *comm) 
    106   { 
    107     Debug("MPI_Comm_free with EP_intracomm\n"); 
    108  
    109     int ep_rank_loc, num_ep; 
    110  
    111     ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first; 
    112     num_ep = (*comm)->ep_comm_ptr->size_rank_info[1].second; 
    113  
    114     MPI_Barrier2(*comm); 
    115  
    116     if(ep_rank_loc == 0) 
    117     { 
    118  
    119 #ifdef _showinfo 
    120       printf("delete (*comm)->my_buffer = %p\n", (*comm)->my_buffer); 
    121 #endif 
    122       delete (*comm)->my_buffer; 
    123  
    124  
    125        
    126 #ifdef _showinfo         
    127       printf("delete (*comm)->ep_barrier = %p\n", (*comm)->ep_barrier); 
    128 #endif 
    129       delete (*comm)->ep_barrier; 
    130        
    131  
    132  
    133       (*comm)->ep_rank_map->clear(); 
    134 #ifdef _showinfo 
    135       printf("delete (*comm)->ep_rank_map = %p\n", (*comm)->ep_rank_map); 
    136 #endif 
    137       delete (*comm)->ep_rank_map; 
    138        
    139  
    140       for(int i=0; i<num_ep; i++) 
    141       { 
    142         if((*comm)->is_intercomm) 
    143         { 
    144           (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map->clear(); 
    145 #ifdef _showinfo 
    146           printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->inter_rank_map = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map); 
    147 #endif 
    148           delete (*comm)->ep_comm_ptr->comm_list[i]->inter_rank_map; 
    149         } 
    150  
    151         (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue->clear(); 
    152 #ifdef _showinfo 
    153         printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr->message_queue = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue); 
    154 #endif 
    155         delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue; 
    156            
    157  
    158 #ifdef _showinfo 
    159         printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr); 
    160 #endif 
    161         delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr; 
    162  
    163  
    164 #ifdef _showinfo 
    165         printf("delete (*comm)->ep_comm_ptr->comm_list[%d] = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]); 
    166 #endif 
    167         delete (*comm)->ep_comm_ptr->comm_list[i]; 
    168       } 
    169  
    170 #ifdef _showinfo 
    171       printf("delete (*comm)->mpi_comm = %p\n", (*comm)->mpi_comm); 
    172 #endif 
    173       ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->mpi_comm)); 
    174        
    175 #ifdef _showinfo 
    176       printf("delete (*comm)->ep_comm_ptr->comm_list = %p\n", (*comm)->ep_comm_ptr->comm_list); 
    177 #endif 
    178       delete[] (*comm)->ep_comm_ptr->comm_list; 
    179     } 
    180   } 
    181  
    182  
    183  
    184   int MPI_Comm_free_intercomm(MPI_Comm *comm) 
    185   { 
    186     int ep_rank; 
    187     MPI_Comm_rank(*comm, &ep_rank); 
    188     int ep_rank_loc = (*comm)->ep_comm_ptr->size_rank_info[1].first; 
    189     int num_ep =      (*comm)->ep_comm_ptr->size_rank_info[1].second; 
    190      
    191     int newcomm_ep_rank =(*comm)->ep_comm_ptr->intercomm->size_rank_info[0].first;  
    192     int newcomm_ep_rank_loc = (*comm)->ep_comm_ptr->intercomm->size_rank_info[1].first; 
    193     int newcomm_num_ep = (*comm)->ep_comm_ptr->intercomm->size_rank_info[1].second; 
    194  
    195     MPI_Barrier(*comm); 
    196  
    197     if(ep_rank_loc == 0) 
    198     { 
    199       (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map->clear(); 
    200 #ifdef _showinfo 
    201       printf("delete (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map = %p\n", (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map); 
    202 #endif 
    203       delete (*comm)->ep_comm_ptr->intercomm->intercomm_rank_map; 
    204  
    205       (*comm)->ep_comm_ptr->intercomm->local_rank_map->clear(); 
    206 #ifdef _showinfo 
    207       printf("delete (*comm)->ep_comm_ptr->intercomm->local_rank_map = %p\n", (*comm)->ep_comm_ptr->intercomm->local_rank_map); 
    208 #endif 
    209       delete (*comm)->ep_comm_ptr->intercomm->local_rank_map; 
    210     } 
    211  
    212     if(newcomm_ep_rank_loc == 0) 
    213     { 
    214  
    215 #ifdef _showinfo 
    216       printf("delete (*comm)->my_buffer = %p\n", (*comm)->my_buffer); 
    217 #endif 
    218       delete (*comm)->my_buffer; 
    219  
    220  
    221        
    222 #ifdef _showinfo         
    223       printf("delete (*comm)->ep_barrier = %p\n", (*comm)->ep_barrier); 
    224 #endif 
    225       delete (*comm)->ep_barrier; 
    226        
    227  
    228       (*comm)->ep_rank_map->clear(); 
    229 #ifdef _showinfo 
    230       printf("delete (*comm)->ep_rank_map = %p\n", (*comm)->ep_rank_map); 
    231 #endif 
    232       delete (*comm)->ep_rank_map; 
    233        
    234 #ifdef _showinfo 
    235       printf("delete (*comm)->ep_comm_ptr->intercomm->mpi_inter_comm = %p\n", (*comm)->ep_comm_ptr->intercomm->mpi_inter_comm); 
    236 #endif 
    237       ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->ep_comm_ptr->intercomm->mpi_inter_comm)); 
    238  
    239       for(int i=0; i<newcomm_num_ep; i++) 
    240       { 
    241         (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue->clear(); 
    242 #ifdef _showinfo 
    243         printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr->message_queue = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue); 
    244 #endif 
    245         delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->message_queue; 
    246  
    247 #ifdef _showinfo 
    248         printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr->intercomm = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm); 
    249 #endif 
    250         delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr->intercomm;         
    251            
    252  
    253 #ifdef _showinfo 
    254         printf("delete (*comm)->ep_comm_ptr->comm_list[%d]->ep_comm_ptr = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr); 
    255 #endif 
    256         delete (*comm)->ep_comm_ptr->comm_list[i]->ep_comm_ptr; 
    257  
    258  
    259 #ifdef _showinfo 
    260         printf("delete (*comm)->ep_comm_ptr->comm_list[%d] = %p\n", i, (*comm)->ep_comm_ptr->comm_list[i]); 
    261 #endif 
    262         delete (*comm)->ep_comm_ptr->comm_list[i]; 
    263  
    264       } 
    265  
    266 #ifdef _showinfo 
    267       printf("delete (*comm)->mpi_comm = %p\n", (*comm)->mpi_comm); 
    268 #endif 
    269       ::MPI_Comm_free(to_mpi_comm_ptr((*comm)->mpi_comm)); 
    270        
    271 #ifdef _showinfo 
    272       printf("delete (*comm)->ep_comm_ptr->comm_list = %p\n", (*comm)->ep_comm_ptr->comm_list); 
    273 #endif 
    274       delete[] (*comm)->ep_comm_ptr->comm_list; 
    275     } 
    276   } 
    277  
    278  
    279110} 
    280111 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_gather.cpp

    r1503 r1527  
    3535      for(int i=0; i<num_ep; i++) 
    3636        memcpy(recvbuf + datasize * i * count, comm->my_buffer->void_buffer[i], datasize * count); 
    37  
    38       //printf("local_recvbuf = %d %d \n", static_cast<int*>(recvbuf)[0], static_cast<int*>(recvbuf)[1] ); 
    3937    } 
    4038 
     
    4442  int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) 
    4543  { 
    46     if(!comm->is_ep) 
    47     { 
    48       return ::MPI_Gather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), 
    49                    root, to_mpi_comm(comm->mpi_comm)); 
    50     } 
     44    if(!comm->is_ep) return ::MPI_Gather(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), 
     45                                         root, to_mpi_comm(comm->mpi_comm)); 
     46    if(comm->is_intercomm) return MPI_Gather_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm); 
    5147 
    5248    assert(sendcount == recvcount && sendtype == recvtype); 
     
    10399    if(is_root) 
    104100    { 
    105       // printf("tmp_recvbuf = %d %d %d %d %d %d %d %d\n", static_cast<int*>(tmp_recvbuf)[0], static_cast<int*>(tmp_recvbuf)[1], 
    106       //                                                   static_cast<int*>(tmp_recvbuf)[2], static_cast<int*>(tmp_recvbuf)[3], 
    107       //                                                   static_cast<int*>(tmp_recvbuf)[4], static_cast<int*>(tmp_recvbuf)[5], 
    108       //                                                   static_cast<int*>(tmp_recvbuf)[6], static_cast<int*>(tmp_recvbuf)[7] ); 
    109  
    110101      int offset; 
    111102      for(int i=0; i<ep_size; i++) 
     
    128119  } 
    129120 
     121 
     122  int MPI_Gather_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) 
     123  { 
     124    printf("MPI_Gather_intercomm not yet implemented\n"); 
     125    MPI_Abort(comm, 0); 
     126  } 
    130127} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_gatherv.cpp

    r1503 r1527  
    4646  { 
    4747   
    48     if(!comm->is_ep) 
    49     { 
    50       return ::MPI_Gatherv(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, const_cast<int*>(input_recvcounts), const_cast<int*>(input_displs), 
    51                     to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 
    52     } 
    53  
     48    if(!comm->is_ep) return ::MPI_Gatherv(const_cast<void*>(sendbuf), sendcount, to_mpi_type(sendtype), recvbuf, const_cast<int*>(input_recvcounts), const_cast<int*>(input_displs), 
     49                                          to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 
     50    if(comm->is_intercomm) return MPI_Gatherv_intercomm(sendbuf, sendcount, sendtype, recvbuf, input_recvcounts, input_displs, recvtype, root, comm); 
    5451 
    5552    assert(sendtype == recvtype); 
     
    169166  } 
    170167 
     168  int MPI_Gatherv_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int input_recvcounts[], const int input_displs[], 
     169                            MPI_Datatype recvtype, int root, MPI_Comm comm) 
     170  { 
     171    printf("MPI_Gatherv_intercomm not yet implemented\n"); 
     172    MPI_Abort(comm, 0); 
     173  } 
     174 
    171175} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_init.cpp

    r1525 r1527  
    1515    Debug("MPI_Init_thread with EP/MPI\n"); 
    1616 
    17     if(omp_get_thread_num() == 0) 
     17    #pragma omp master 
    1818    { 
    1919      ::MPI_Init_thread(argc, argv, required, provided); 
     
    2626    Debug("MPI_Init with EP/MPI\n"); 
    2727 
    28     if(omp_get_thread_num() == 0) 
     28    #pragma omp master 
    2929    { 
    3030      ::MPI_Init(argc, argv); 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.cpp

    r1525 r1527  
    1414namespace ep_lib 
    1515{ 
    16   int MPI_Intercomm_create2(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 
     16  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 
    1717  { 
    1818    assert(local_comm->is_ep); 
     
    371371 
    372372    (*newintercomm)->inter_rank_map = new INTER_RANK_MAP; 
     373   
    373374 
    374375    int rank_info[2]; 
     
    449450  } 
    450451 
    451  
    452   int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 
    453   { 
    454     assert(local_comm->is_ep); 
    455  
    456     int ep_rank, ep_rank_loc, mpi_rank; 
    457     int ep_size, num_ep, mpi_size; 
    458  
    459     ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first; 
    460     ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first; 
    461     mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first; 
    462     ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second; 
    463     num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second; 
    464     mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second; 
    465  
    466  
    467     MPI_Barrier(local_comm); 
    468  
    469     int leader_ranks_in_peer[3];  // local_leader_rank_in_peer 
    470                                   // remote_leader_rank_in_peer 
    471                                   // size of peer 
    472  
    473     if(ep_rank == local_leader) 
    474     { 
    475       MPI_Comm_rank(peer_comm, &leader_ranks_in_peer[0]); 
    476       leader_ranks_in_peer[1] = remote_leader; 
    477       MPI_Comm_size(peer_comm, &leader_ranks_in_peer[2]); 
    478     } 
    479  
    480     MPI_Bcast(leader_ranks_in_peer, 3, MPI_INT, local_leader, local_comm); 
    481  
    482     if(leader_ranks_in_peer[0] != leader_ranks_in_peer[2]) 
    483     { 
    484       Debug("calling MPI_Intercomm_create_kernel\n"); 
    485       return MPI_Intercomm_create_kernel(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm); 
    486     } 
    487  
    488     else 
    489     { 
    490       if(leader_ranks_in_peer[2] == 1) 
    491       { 
    492         Debug("calling MPI_Intercomm_create_unique\n"); 
    493         return MPI_Intercomm_create_unique_leader(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);  
    494          
    495       } 
    496       else 
    497       { 
    498         Debug("calling MPI_Intercomm_create_world\n"); 
    499         return MPI_Intercomm_create_from_world(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);    
    500       } 
    501        
    502     } 
    503  
    504   } 
    505  
    506   int MPI_Comm_test_inter(MPI_Comm comm, int *flag) 
    507   { 
    508     if(comm->is_ep) 
    509     { 
    510       return *flag = comm->is_intercomm; 
    511     }  
    512     else 
    513     { 
    514       return ::MPI_Comm_test_inter(to_mpi_comm(comm->mpi_comm), flag); 
    515     } 
    516   } 
    517  
    518  
    519452} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_kernel.cpp

    r1522 r1527  
    99namespace ep_lib 
    1010{ 
    11   int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 
     11  /*int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 
    1212  { 
    1313    int ep_rank, ep_rank_loc, mpi_rank; 
     
    619619    // clean up // 
    620620    ////////////// 
    621  
     621/* 
    622622    delete ranks_in_world_local; 
    623623    delete ranks_in_world_remote; 
     
    635635  } 
    636636 
    637  
     637*/ 
    638638 
    639639   
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_unique.cpp

    r1515 r1527  
    1010{ 
    1111 
    12  
     12/* 
    1313  int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm) 
    1414  { 
     
    225225    return MPI_SUCCESS; 
    226226  } 
    227  
     227*/ 
    228228} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_world.cpp

    r1512 r1527  
    99{ 
    1010 
    11  
     11/* 
    1212  int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm) 
    1313  { 
     
    541541 
    542542  } 
    543  
     543*/ 
    544544  // #endif 
    545545 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.cpp

    r1525 r1527  
    3232 
    3333  int get_ep_rank(MPI_Comm comm, int ep_rank_loc, int mpi_rank) 
    34   { 
    35     /*if(comm->is_intercomm) 
    36     { 
    37       for(std::map<int, std::pair< int, std::pair<int, int> > >::iterator it = comm->ep_comm_ptr->intercomm->intercomm_rank_map->begin(); it != comm->ep_comm_ptr->intercomm->intercomm_rank_map->end(); it++) 
    38       { 
    39         if(   ( it->second.first  == ep_rank_loc ) 
    40            && ( it->second.second.first == mpi_rank ) ) 
    41         { 
    42           return it->first; 
    43         } 
    44       } 
    45       printf("rank not find for EP_intercomm\n"); 
    46       int err; 
    47       return MPI_Abort(comm, err); 
    48     }*/ 
    49      
     34  {     
    5035    for(std::map<int, std::pair<int, int> >::iterator it = comm->ep_rank_map->begin(); it != comm->ep_rank_map->end(); it++) 
    5136    { 
     
    5742    } 
    5843    printf("rank not find for EP_intracomm\n"); 
    59     int err; 
    60     return MPI_Abort(comm, err); 
     44    return MPI_Abort(comm, 0); 
    6145  } 
    6246   
     
    7054  { 
    7155    return ::MPI_Wtime(); 
    72  
     56  } 
     57 
     58  int MPI_Comm_test_inter(MPI_Comm comm, int *flag) 
     59  { 
     60    if(comm->is_ep) return *flag = comm->is_intercomm; 
     61    else return ::MPI_Comm_test_inter(to_mpi_comm(comm->mpi_comm), flag); 
    7362  } 
    7463 
     
    139128  } 
    140129 
    141   int test_sendrecv(MPI_Comm comm) 
    142   { 
    143     int myRank; 
    144     MPI_Comm_rank(comm, &myRank); 
    145     bool amClient = false; 
    146     bool amServer = false; 
    147     if(myRank<=3) amClient = true; 
    148     else amServer = true; 
    149  
    150     if(amServer) 
    151     { 
    152       int send_buf[4]; 
    153       MPI_Request send_request[8]; 
    154       MPI_Status send_status[8]; 
    155  
    156        
    157        
    158       for(int j=0; j<4; j++)  // 4 buffers 
    159       { 
    160         for(int i=0; i<2; i++) 
    161         { 
    162           send_buf[j] = (myRank+1)*100 + j; 
    163           MPI_Isend(&send_buf[j], 1, MPI_INT, i*2, 9999, comm, &send_request[i*4+j]); 
    164         } 
    165       } 
    166        
    167  
    168       MPI_Waitall(8, send_request, send_status); 
    169     } 
    170  
    171  
    172     if(amClient&&myRank%2==0) // Clients leaders 
    173     { 
    174       int recv_buf[8]; 
    175       MPI_Request recv_request[8]; 
    176       MPI_Status recv_status[8]; 
    177  
    178       for(int i=0; i<2; i++)  // 2 servers 
    179       { 
    180         for(int j=0; j<4; j++) 
    181         { 
    182           MPI_Irecv(&recv_buf[i*4+j], 1, MPI_INT, i+4, 9999, comm, &recv_request[i*4+j]); 
    183         } 
    184       } 
    185  
    186       MPI_Waitall(8, recv_request, recv_status); 
    187       printf("============ client %d, recv_buf = %d, %d, %d, %d, %d, %d, %d, %d ================\n",  
    188               myRank, recv_buf[0], recv_buf[1], recv_buf[2], recv_buf[3], recv_buf[4], recv_buf[5], recv_buf[6], recv_buf[7]); 
    189     } 
    190  
    191     MPI_Barrier(comm); 
    192  
    193   } 
    194130 
    195131  bool valid_type(MPI_Datatype datatype) 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib.hpp

    r1525 r1527  
    44#include "ep_type.hpp" 
    55#include "ep_lib_intercomm.hpp" 
    6 #include "ep_lib_intracomm.hpp" 
     6#include "ep_lib_endpoint.hpp" 
    77#include "ep_lib_local.hpp" 
    88#include "ep_lib_collective.hpp" 
    9 #include "ep_tag.hpp" 
    109#include "ep_lib_fortran.hpp" 
    1110#include "ep_lib_win.hpp" 
    1211#include "ep_lib_mpi.hpp" 
    13 //#include "ep_mpi.hpp" 
    1412 
    1513 
     
    3028 
    3129  int MPI_Comm_free(MPI_Comm* comm); 
    32   int MPI_Comm_free2(MPI_Comm* comm); 
    3330 
    3431  int MPI_Finalize(); 
     
    4138   
    4239  int MPI_Comm_remote_size(MPI_Comm comm, int *size); 
    43   int MPI_Comm_remote_size2(MPI_Comm comm, int *size); 
    4440 
    4541 
     
    6763   
    6864  int MPI_Alloc_mem(MPI_Aint size, MPI_Info info, void *baseptr); 
    69    
    7065  int MPI_Alloc_mem(unsigned long size, MPI_Info info, void *baseptr); 
    7166 
     
    9388} 
    9489 
    95 //MPI_Datatype to_mpi(ep_lib::MPI_Datatype type); 
    96  
    97  
    9890#endif // EP_LIB_HPP_INCLUDED 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_collective.hpp

    r1525 r1527  
    1010 
    1111  int MPI_Barrier(MPI_Comm comm); 
    12   int MPI_Barrier2(MPI_Comm comm); 
    13    
     12  
    1413 
    1514  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm); 
     
    4746 
    4847  int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
    49   int MPI_Intercomm_create2(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
    5048 
    51   int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
     49  //int MPI_Intercomm_create_kernel(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
    5250   
    5351   
    54   int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm); 
     52  //int MPI_Intercomm_create_from_world(MPI_Comm local_comm, int local_leader, void* peer_comm_ptr, int mpi_remote_leader, int tag, MPI_Comm *newintercomm); 
    5553   
    56   int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
     54  //int MPI_Intercomm_create_unique_leader(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); 
    5755 
    5856  int MPI_Comm_create_endpoints(void* base_comm_ptr, int num_ep, MPI_Info info, MPI_Comm *& out_comm_hdls); // from EP to create endpoints 
     
    6260  int MPI_Intercomm_merge(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm); 
    6361 
    64   int MPI_Intercomm_merge_unique_leader(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm); 
     62  //int MPI_Intercomm_merge_unique_leader(MPI_Comm intercomm, bool high, MPI_Comm *newintracomm); 
    6563 
    6664} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_intercomm.hpp

    r1517 r1527  
    88  typedef void* MPI_Op; 
    99 
    10   int MPI_Send_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 
    11   int MPI_Ssend_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); 
    12   int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
     10  int MPI_Allgather_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm); 
     11  int MPI_Allgatherv_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int recvcounts[], const int displs[], MPI_Datatype recvtype, MPI_Comm comm); 
    1312 
    14   int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
     13  int MPI_Allreduce_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm); 
    1514 
     15  int MPI_Alltoall_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm); 
    1616 
    17   int MPI_Comm_dup_intercomm(MPI_Comm comm, MPI_Comm *newcomm); 
     17  int MPI_Bcast_intercomm(void *buffer, int count, MPI_Datatype datatype, int root, MPI_Comm comm); 
    1818 
    19   int MPI_Comm_free_intercomm(MPI_Comm* comm); 
     19  int MPI_Exscan_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm); 
     20  int MPI_Scan_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm); 
    2021 
    21   int MPI_Barrier_intercomm(MPI_Comm comm); 
     22  int MPI_Gather_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm); 
    2223 
    23   int Message_Check_intercomm(MPI_Comm comm); 
     24  int MPI_Gatherv_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, const int input_recvcounts[], const int input_displs[], 
     25                            MPI_Datatype recvtype, int root, MPI_Comm comm); 
    2426 
    25   int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
    26   int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request); 
     27  int MPI_Reduce_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm); 
    2728 
     29  int MPI_Reduce_scatter_intercomm(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm); 
    2830 
    29   int MPI_Iprobe_intercomm(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status); 
    30   int MPI_Improbe_intercomm(int source, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status); 
    31  
    32  
    33  
    34  
     31  int MPI_Scatter_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm); 
     32  int MPI_Scatterv_intercomm(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount, 
     33                   MPI_Datatype recvtype, int root, MPI_Comm comm); 
    3534} 
    3635 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_lib_mpi.hpp

    r1522 r1527  
    1919 
    2020  int MPI_Iprobe_mpi(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status); 
     21  int MPI_Improbe_mpi(int src, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status); 
    2122   
    2223  int MPI_Comm_rank_mpi(MPI_Comm comm, int* rank); 
     
    2829 
    2930  int MPI_Barrier_mpi(MPI_Comm comm); 
    30    
    31   int MPI_Abort_mpi(MPI_Comm comm, int errorcode); 
     31 
    3232} 
    3333 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_merge.cpp

    r1522 r1527  
    1010{ 
    1111 
    12   int MPI_Intercomm_merge_unique_leader(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm) 
     12  int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm) 
    1313  { 
    14     Debug("intercomm_merge with unique leader\n"); 
     14     
     15    int ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 
     16    int ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 
     17    int ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
     18    int num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
     19    int mpi_rank = inter_comm->ep_comm_ptr->size_rank_info[2].first; 
    1520 
     21    int remote_ep_size = inter_comm->inter_rank_map->size(); 
    1622 
    17  
    18     int ep_rank, ep_rank_loc, mpi_rank; 
    19     int ep_size, num_ep, mpi_size; 
    20  
    21     ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 
    22     ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
    23     mpi_rank = inter_comm->ep_comm_ptr->size_rank_info[2].first; 
    24     ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 
    25     num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
    26     mpi_size = inter_comm->ep_comm_ptr->size_rank_info[2].second; 
    27  
    28     int local_high = high; 
    29     int remote_high; 
    30  
    31     int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->remote_rank_map->size(); 
    32  
    33     int local_ep_rank, local_ep_rank_loc, local_mpi_rank; 
    34     int local_ep_size, local_num_ep, local_mpi_size; 
    35  
    36     //local_ep_rank = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].first; 
    37     //local_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].first; 
    38     //local_mpi_rank = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].first; 
    39     //local_ep_size = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[0].second; 
    40     //local_num_ep = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[1].second; 
    41     //local_mpi_size = inter_comm->ep_comm_ptr->intercomm->local_comm->ep_comm_ptr->size_rank_info[2].second; 
    42  
    43  
    44     if(local_ep_rank == 0) 
    45     { 
    46       MPI_Status status[2]; 
    47       MPI_Request request[2]; 
    48       MPI_Isend(&local_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[0]); 
    49       MPI_Irecv(&remote_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->intercomm_tag, inter_comm, &request[1]); 
    50  
    51       MPI_Waitall(2, request, status); 
    52     } 
    53  
    54  
    55     //MPI_Bcast(&remote_high, 1, MPI_INT, 0, inter_comm->ep_comm_ptr->intercomm->local_comm); 
    56  
     23    int new_ep_rank = high? remote_ep_size + ep_rank : ep_rank; 
    5724 
    5825 
    5926    MPI_Comm_dup(inter_comm, newintracomm); 
    6027 
    61     int my_ep_rank = local_high<remote_high? local_ep_rank: local_ep_rank+remote_ep_size; 
     28    (*newintracomm)->is_intercomm = false; 
     29    (*newintracomm)->inter_rank_map->clear(); 
     30    delete (*newintracomm)->inter_rank_map; 
     31 
     32    (*newintracomm)->ep_comm_ptr->size_rank_info[0].second = ep_size + remote_ep_size; 
     33    (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = new_ep_rank; 
     34 
     35    // modif ep_rank_map 
     36    int my_triple[3] = {new_ep_rank, ep_rank_loc, mpi_rank}; 
     37    int *my_triple_list = new int[3*(ep_size+remote_ep_size)]; 
     38 
     39     
     40    MPI_Allgather(my_triple, 3, MPI_INT, my_triple_list, 3, MPI_INT, *newintracomm);  
    6241 
    6342 
    64     int intra_ep_rank, intra_ep_rank_loc, intra_mpi_rank; 
    65     int intra_ep_size, intra_num_ep, intra_mpi_size; 
     43#ifdef _showinfo 
     44    for(int i=0; i<ep_size+remote_ep_size; i++) 
     45    { 
     46      if(new_ep_rank == i) 
     47      { 
     48        for(int j=0; j<ep_size+remote_ep_size; j++) 
     49        { 
     50          printf("rank %d : my_triple_list[%d] = %d %d %d\n", i, j, my_triple_list[3*j], my_triple_list[3*j+1], my_triple_list[3*j+2]); 
     51        } 
     52        printf("\n"); 
     53      } 
     54      MPI_Barrier(*newintracomm); 
     55    } 
     56#endif 
    6657 
    67     intra_ep_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 
    68     intra_ep_rank_loc = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 
    69     intra_mpi_rank = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
    70     intra_ep_size = (*newintracomm)->ep_comm_ptr->size_rank_info[0].second; 
    71     intra_num_ep = (*newintracomm)->ep_comm_ptr->size_rank_info[1].second; 
    72     intra_mpi_size = (*newintracomm)->ep_comm_ptr->size_rank_info[2].second; 
    73  
    74  
    75     MPI_Barrier_local(*newintracomm); 
    76  
    77  
    78     int *reorder; 
    79     if(intra_ep_rank_loc == 0) 
     58    if((*newintracomm)->ep_comm_ptr->size_rank_info[1].first==0) 
    8059    { 
    81       reorder = new int[intra_ep_size]; 
    82     } 
    83  
    84  
    85     MPI_Gather(&my_ep_rank, 1, MPI_INT, reorder, 1, MPI_INT, 0, *newintracomm); 
    86     if(intra_ep_rank_loc == 0) 
    87     { 
    88       ::MPI_Bcast(reorder, intra_ep_size, to_mpi_type(MPI_INT), 0, to_mpi_comm((*newintracomm)->mpi_comm)); 
    89  
    90       vector< pair<int, int> > tmp_rank_map(intra_ep_size); 
    91  
    92  
    93       for(int i=0; i<intra_ep_size; i++) 
     60      for(int i=0; i<ep_size+remote_ep_size; i++) 
    9461      { 
    95         tmp_rank_map[reorder[i]] = (*newintracomm)->ep_rank_map->at(i) ; 
     62        (*newintracomm)->ep_comm_ptr->comm_list[0]->ep_rank_map->at(my_triple_list[3*i]) = std::make_pair(my_triple_list[3*i+1], my_triple_list[3*i+2]); 
    9663      } 
    97  
    98       //(*newintracomm)->rank_map->swap(tmp_rank_map); 
    99       (*newintracomm)->ep_rank_map->clear(); 
    100       for(int i=0; i<tmp_rank_map.size(); i++) 
    101       { 
    102         (*newintracomm)->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(i, tmp_rank_map[i].first, tmp_rank_map[i].second)); 
    103       } 
    104        
    105  
    106       tmp_rank_map.clear(); 
    10764    } 
    10865 
    10966    MPI_Barrier_local(*newintracomm); 
    11067 
    111     (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = my_ep_rank; 
     68#ifdef _showinfo 
    11269 
    113     if(intra_ep_rank_loc == 0) 
     70    for(int i=0; i<ep_size+remote_ep_size; i++) 
    11471    { 
    115       delete[] reorder; 
    116     } 
    117  
    118     return MPI_SUCCESS; 
    119   } 
    120  
    121  
    122  
    123  
    124  
    125   int MPI_Intercomm_merge(MPI_Comm inter_comm, bool high, MPI_Comm *newintracomm) 
    126   { 
    127      
    128  
    129     assert(inter_comm->is_intercomm); 
    130  
    131     // determine if only one MPI proc 
    132  
    133         // to be completed ...... 
    134  
    135     // multiple MPI proc and high differs 
    136  
    137     int newcomm_ep_rank = inter_comm->ep_comm_ptr->intercomm->size_rank_info[0].first; 
    138     int newcomm_ep_rank_loc = inter_comm->ep_comm_ptr->intercomm->size_rank_info[1].first; 
    139     int newcomm_num_ep = inter_comm->ep_comm_ptr->intercomm->size_rank_info[1].second; 
    140  
    141     int ep_rank = inter_comm->ep_comm_ptr->size_rank_info[0].first; 
    142     int ep_rank_loc = inter_comm->ep_comm_ptr->size_rank_info[1].first; 
    143     int num_ep = inter_comm->ep_comm_ptr->size_rank_info[1].second; 
    144  
    145     if(newcomm_ep_rank_loc == 0) 
    146     { 
    147       ::MPI_Comm *mpi_intracomm = new ::MPI_Comm; 
    148       ::MPI_Intercomm_merge(to_mpi_comm(inter_comm->ep_comm_ptr->intercomm->mpi_inter_comm), high, mpi_intracomm); 
    149        
    150       MPI_Info info; 
    151       MPI_Comm *ep_comm; 
    152       MPI_Comm_create_endpoints(mpi_intracomm, newcomm_num_ep, info, ep_comm); 
    153  
    154       inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge = ep_comm; 
    155     } 
    156  
    157     MPI_Barrier_local(inter_comm); 
    158  
    159     int remote_num_ep = newcomm_num_ep - num_ep; 
    160  
    161     *newintracomm = inter_comm->ep_comm_ptr->comm_list[0]->mem_bridge[high? remote_num_ep+ep_rank_loc : ep_rank_loc]; 
    162  
    163     int ep_size = inter_comm->ep_comm_ptr->size_rank_info[0].second; 
    164     int remote_ep_size = inter_comm->ep_comm_ptr->intercomm->intercomm_rank_map->size(); 
    165  
    166     //printf("ep_size = %d, remote_ep_size = %d\n", ep_size, remote_ep_size); 
    167  
    168     (*newintracomm)->ep_comm_ptr->size_rank_info[0].first = high? remote_ep_size+ep_rank : ep_rank; 
    169  
    170     int my_triple[3]; 
    171     my_triple[0] = (*newintracomm)->ep_comm_ptr->size_rank_info[0].first; 
    172     my_triple[1] = (*newintracomm)->ep_comm_ptr->size_rank_info[1].first; 
    173     my_triple[2] = (*newintracomm)->ep_comm_ptr->size_rank_info[2].first; 
    174  
    175     int *my_triple_list = new int[3 * (*newintracomm)->ep_comm_ptr->size_rank_info[0].second]; 
    176  
    177  
    178     MPI_Allgather(my_triple, 3, MPI_INT, my_triple_list, 3, MPI_INT, *newintracomm); 
    179  
    180     if((*newintracomm)->ep_comm_ptr->size_rank_info[1].first == 0) 
    181     { 
    182       (*newintracomm)->ep_rank_map->clear(); 
    183       for(int i=0; i<(*newintracomm)->ep_comm_ptr->size_rank_info[0].second; i++) 
     72      if(new_ep_rank == i) 
    18473      { 
    185         (*newintracomm)->ep_rank_map->insert(std::pair< int, std::pair<int,int> >(my_triple_list[3*i], my_triple_list[3*i+1], my_triple_list[3*i+2])); 
     74        for(EP_RANK_MAP::iterator it = (*newintracomm)->ep_rank_map->begin(); it != (*newintracomm)->ep_rank_map->end(); it++) 
     75        { 
     76          printf("rank %d : ep_rank_map[%d] = %d %d\n", i, it->first, it->second.first, it->second.second); 
     77        } 
     78        printf("\n"); 
    18679      } 
    187     } 
    188  
    189 #ifdef _showinfo 
    190     MPI_Barrier_local(inter_comm); 
    191     if((*newintracomm)->ep_comm_ptr->size_rank_info[0].first == 15) 
    192     { 
    193       for(std::map<int, std::pair<int, int> >::iterator it = (*newintracomm)->ep_rank_map->begin(); it != (*newintracomm)->ep_rank_map->end(); it++) 
    194       { 
    195         printf("(%d  %d  %d)\n", it->first, it->second.first, it->second.second); 
    196       } 
     80      MPI_Barrier(*newintracomm); 
    19781    } 
    19882#endif 
    199      
    200     delete my_triple_list; 
     83 
     84    delete[] my_triple_list; 
    20185  } 
    20286   
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_message.cpp

    r1522 r1527  
    3232    std::list<MPI_Request* >::iterator it; 
    3333     
    34 //    show_EP_PendingRequests(EP_PendingRequests); 
     34    show_EP_PendingRequests(EP_PendingRequests); 
    3535     
    3636     
     
    4646      } 
    4747       
    48       if((*(*it))->state == 2) 
    49       { 
    50         EP_PendingRequests->erase(it); 
    51          
    52         memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size()); 
    53         it = EP_PendingRequests->begin(); 
    54         continue; 
    55       } 
    56        
    5748      Message_Check(((*(*it))->comm)); 
    5849    } 
     
    7263      } 
    7364       
    74       if((*(*it))->state == 2) 
    75       { 
    76         EP_PendingRequests->erase(it); 
    77          
    78         memcheck("EP_PendingRequests["<<ep_rank<<"]->size() = " << EP_PendingRequests->size()); 
    79         it = EP_PendingRequests->begin(); 
    80         continue; 
    81       } 
    82        
    83       if((*(*it))->state == 0) 
     65      if((*(*it))->probed == false) 
    8466      { 
    8567        #pragma omp critical (_query0) 
     
    10789          MPI_Imrecv((*(*it))->buf, recv_count, (*(*it))->ep_datatype, message, *it); 
    10890          (*(*it))->type = 3; 
    109           (*(*it))->state = 1; 
     91          (*(*it))->probed = true; 
    11092 
    11193          memcheck("delete "<< status.mpi_status <<" : in ep_lib::Request_Check, delete status.mpi_status"); 
     
    133115  int Message_Check(MPI_Comm comm) 
    134116  { 
    135     if(!comm->is_ep) return MPI_SUCCESS; 
    136  
    137     if(comm->is_intercomm) 
    138     { 
    139       Message_Check_intercomm(comm); 
    140     } 
    141      
    142     return Message_Check_intracomm(comm); 
    143  
    144   } 
    145    
    146    
    147   int Message_Check_intracomm(MPI_Comm comm) 
     117    if(comm->is_ep) return Message_Check_endpoint(comm); 
     118  } 
     119   
     120   
     121  int Message_Check_endpoint(MPI_Comm comm) 
    148122  { 
    149123     
     
    204178  } 
    205179 
    206  
    207   int Message_Check_intercomm(MPI_Comm comm) 
    208   { 
    209     if(!comm->ep_comm_ptr->intercomm->mpi_inter_comm) return 0; 
    210  
    211     Debug("Message probing for intercomm\n"); 
    212  
    213     int flag = true; 
    214     ::MPI_Message message; 
    215     ::MPI_Status status; 
    216     int current_ep_rank; 
    217     MPI_Comm_rank(comm, &current_ep_rank); 
    218  
    219     while(flag) // loop until the end of global queue "comm->ep_comm_ptr->intercomm->mpi_inter_comm" 
    220     { 
    221       Debug("Message probing for intracomm\n"); 
    222  
    223       #pragma omp critical (_mpi_call) 
    224       { 
    225         ::MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &flag, &status); 
    226         if(flag) 
    227         { 
    228           Debug("find message in mpi comm \n"); 
    229           ::MPI_Mprobe(status.MPI_SOURCE, status.MPI_TAG, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), &message, &status); 
    230         } 
    231       } 
    232        
    233  
    234       if(flag) 
    235       { 
    236  
    237         MPI_Message msg = new ep_message;  
    238         msg->mpi_message = new ::MPI_Message(message); 
    239  
    240         memcheck("new "<< msg <<" : in ep_lib::Message_Check, msg = new ep_message"); 
    241         memcheck("new "<< msg->mpi_message <<" : in ep_lib::Message_Check, msg->mpi_message = new ::MPI_Message"); 
    242                
    243  
    244         msg->ep_tag  = bitset<15>(status.MPI_TAG >> 16).to_ulong();  
    245         int src_loc  = bitset<8> (status.MPI_TAG >> 8) .to_ulong();  
    246         int dest_loc = bitset<8> (status.MPI_TAG)           .to_ulong(); 
    247         int src_mpi  = status.MPI_SOURCE; 
    248               
    249         msg->ep_src  = get_ep_rank(comm, src_loc,  src_mpi);    
    250 #ifdef _showinfo 
    251         printf("status.MPI_TAG = %d, src_loc = %d, dest_loc = %d, ep_tag = %d\n", status.MPI_TAG, src_loc, dest_loc, msg->ep_tag); 
    252 #endif 
    253  
    254         msg->mpi_status = new ::MPI_Status(status);  
    255         memcheck("new "<< msg->mpi_status <<" : in ep_lib::Message_Check, msg->mpi_status = new ::MPI_Status"); 
    256  
    257         #pragma omp critical (_query) 
    258         { 
    259           #pragma omp flush 
    260           comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->push_back(msg); 
    261           memcheck("comm->ep_comm_ptr->comm_list["<<dest_loc<<"]->ep_comm_ptr->message_queue->size = "<<comm->ep_comm_ptr->comm_list[dest_loc]->ep_comm_ptr->message_queue->size()); 
    262           #pragma omp flush 
    263         } 
    264       } 
    265     } 
    266  
    267     Message_Check_intracomm(comm); 
    268  
    269     return MPI_SUCCESS; 
    270   } 
    271    
    272    
    273180   
    274181  void show_EP_PendingRequests(std::list< ep_lib::MPI_Request* > * EP_PendingRequest) 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_probe.cpp

    r1522 r1527  
    88  int MPI_Iprobe_mpi(int src, int tag, MPI_Comm comm, int *flag, MPI_Status *status) 
    99  { 
     10    ::MPI_Status mpi_status; 
     11 
     12    ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_status); 
     13 
     14    status->mpi_status = new ::MPI_Status(mpi_status); 
    1015    status->ep_src = src; 
    1116    status->ep_tag = tag; 
    12     return ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, to_mpi_status_ptr(*status)); 
    1317  } 
     18 
     19 
     20  int MPI_Improbe_mpi(int src, int tag, MPI_Comm comm, int *flag, MPI_Message *message, MPI_Status *status) 
     21  { 
     22    ::MPI_Status mpi_status; 
     23    ::MPI_Message mpi_message; 
     24 
     25    #ifdef _openmpi 
     26    #pragma omp critical (_mpi_call) 
     27    { 
     28      ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_status); 
     29      if(*flag) 
     30      { 
     31        ::MPI_Mprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_message, &mpi_status); 
     32      } 
     33    } 
     34    #elif _intelmpi 
     35    ::MPI_Improbe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_message, &mpi_status); 
     36    #endif 
     37       
     38    status->mpi_status = new ::MPI_Status(mpi_status); 
     39    status->ep_src = src; 
     40    status->ep_tag = tag; 
     41 
     42    (*message)->mpi_message = &message; 
     43    (*message)->ep_src = src; 
     44    (*message)->ep_tag = tag; 
     45  } 
     46 
     47 
     48 
    1449 
    1550  int MPI_Iprobe(int src, int tag, MPI_Comm comm, int *flag, MPI_Status *status) 
     
    2459    { 
    2560      Debug("MPI_Iprobe with EP\n"); 
    26        
     61 
    2762      *flag = false; 
    2863     
     
    5388  } 
    5489 
     90   
     91 
    5592 
    5693 
     
    63100    { 
    64101      Debug("calling MPI_Improbe MPI\n"); 
    65  
    66       ::MPI_Status mpi_status; 
    67       ::MPI_Message mpi_message; 
    68  
    69       #ifdef _openmpi 
    70       #pragma omp critical (_mpi_call) 
    71       { 
    72         ::MPI_Iprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_status); 
    73         if(*flag) 
    74         { 
    75           ::MPI_Mprobe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), &mpi_message, &mpi_status); 
    76         } 
    77       } 
    78       #elif _intelmpi 
    79         ::MPI_Improbe(src<0? MPI_ANY_SOURCE : src, tag<0? MPI_ANY_TAG: tag, to_mpi_comm(comm->mpi_comm), flag, &mpi_message, &mpi_status); 
    80       #endif 
    81          
    82       status->mpi_status = &mpi_status; 
    83       status->ep_src = src; 
    84       status->ep_tag = tag; 
    85  
    86       (*message)->mpi_message = &message; 
    87       (*message)->ep_src = src; 
    88       (*message)->ep_tag = tag; 
    89        
    90        
    91       return 0; 
    92     } 
    93  
    94     
     102      return MPI_Improbe_mpi(src, tag, comm, flag, message, status); 
     103    }  
    95104 
    96105    #pragma omp flush 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_recv.cpp

    r1522 r1527  
    4040  { 
    4141    if(!comm->is_ep) return MPI_Irecv_mpi(buf, count, datatype, src, tag, comm, request); 
     42 
     43    if(comm->is_intercomm) 
     44    { 
     45      src = comm->inter_rank_map->at(src); 
     46      printf("new src = %d\n", src); 
     47    } 
    4248     
    4349    Debug("MPI_Irecv with EP"); 
     
    5460    (*request)->comm = comm; 
    5561    (*request)->type = 2; 
    56     (*request)->state = 0; 
     62    (*request)->probed = false; 
    5763     
    5864 
     
    118124    (*request)->ep_src = (*message)->ep_src; 
    119125     
    120     (*request)->state = 1; 
     126    (*request)->probed = true; 
    121127         
    122128    ::MPI_Imrecv(buf, count, to_mpi_type(datatype), to_mpi_message_ptr(*message), to_mpi_request_ptr(*request));                 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_reduce.cpp

    r1503 r1527  
    288288  { 
    289289 
    290     if(!comm->is_ep && comm->mpi_comm) 
    291     { 
    292       return ::MPI_Reduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), root, to_mpi_comm(comm->mpi_comm)); 
    293     } 
     290    if(!comm->is_ep) return ::MPI_Reduce(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), root, to_mpi_comm(comm->mpi_comm)); 
     291    if(comm->is_intercomm) return MPI_Reduce_intercomm(sendbuf, recvbuf, count, datatype, op, root, comm); 
    294292 
    295293 
     
    339337 
    340338 
     339  int MPI_Reduce_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root, MPI_Comm comm) 
     340  { 
     341    printf("MPI_Reduce_intercomm not yet implemented\n"); 
     342    MPI_Abort(comm, 0); 
     343  } 
    341344} 
    342345 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_reduce_scatter.cpp

    r1500 r1527  
    1919  int MPI_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
    2020  { 
    21     if(!comm->is_ep) 
    22     { 
    23       return ::MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    24     } 
     21    if(!comm->is_ep) return ::MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
     22    if(comm->is_intercomm) return MPI_Reduce_scatter_intercomm(sendbuf, recvbuf, recvcounts, datatype, op, comm); 
    2523 
    2624 
     
    8179  } 
    8280 
     81  int MPI_Reduce_scatter_intercomm(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
     82  { 
     83    printf("MPI_Reduce_scatter_intercomm not yet implemented\n"); 
     84    MPI_Abort(comm, 0); 
     85  } 
     86 
    8387   
    8488} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_request.hpp

    r1517 r1527  
    1818                // 3: Imrecv  
    1919                 
    20       int state; // 0: new  
     20      //int state; // 0: new  
    2121                 // 1: imrecvd 
    2222                 // 2: tested or waited 
     23      bool probed; 
    2324                    
    2425      void* buf; 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_scan.cpp

    r1503 r1527  
    350350  int MPI_Scan(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
    351351  { 
    352     if(!comm->is_ep) 
    353     { 
    354       return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
    355     } 
     352    if(!comm->is_ep) return ::MPI_Scan(sendbuf, recvbuf, count, to_mpi_type(datatype), to_mpi_op(op), to_mpi_comm(comm->mpi_comm)); 
     353    if(comm->is_intercomm) return MPI_Scan_intercomm(sendbuf, recvbuf, count, datatype, op, comm); 
    356354     
    357355    valid_type(datatype); 
     
    446444  } 
    447445 
     446  int MPI_Scan_intercomm(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) 
     447  { 
     448    printf("MPI_Scan_intercomm not yet implemented\n"); 
     449    MPI_Abort(comm, 0); 
     450  } 
     451 
    448452} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_scatter.cpp

    r1503 r1527  
    4242  int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) 
    4343  { 
    44     if(!comm->is_ep) 
    45     { 
    46       return ::MPI_Scatter(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 
    47     } 
     44    if(!comm->is_ep) return ::MPI_Scatter(sendbuf, sendcount, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 
     45    if(comm->is_intercomm) return MPI_Scatter_intercomm(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm); 
    4846    
    4947    assert(sendcount == recvcount); 
     
    9795 
    9896 
    99  
    100     // if(is_root) printf("\nranks = %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d\n", ranks[0], ranks[1], ranks[2], ranks[3], ranks[4], ranks[5], ranks[6], ranks[7],  
    101     //                                                                                   ranks[8], ranks[9], ranks[10], ranks[11], ranks[12], ranks[13], ranks[14], ranks[15]); 
    102  
    10397    if(is_root) 
    10498    for(int i=0; i<ep_size; i++) 
     
    106100      memcpy(tmp_sendbuf + i*datasize*count, sendbuf + ranks[i]*datasize*count, count*datasize); 
    107101    } 
    108  
    109     // if(is_root) printf("\ntmp_sendbuf = %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d\n", static_cast<int*>(tmp_sendbuf)[0], static_cast<int*>(tmp_sendbuf)[2], static_cast<int*>(tmp_sendbuf)[4], static_cast<int*>(tmp_sendbuf)[6],  
    110     //                                                                           static_cast<int*>(tmp_sendbuf)[8], static_cast<int*>(tmp_sendbuf)[10], static_cast<int*>(tmp_sendbuf)[12], static_cast<int*>(tmp_sendbuf)[14],  
    111     //                                                                           static_cast<int*>(tmp_sendbuf)[16], static_cast<int*>(tmp_sendbuf)[18], static_cast<int*>(tmp_sendbuf)[20], static_cast<int*>(tmp_sendbuf)[22],  
    112     //                                                                           static_cast<int*>(tmp_sendbuf)[24], static_cast<int*>(tmp_sendbuf)[26], static_cast<int*>(tmp_sendbuf)[28], static_cast<int*>(tmp_sendbuf)[30] ); 
    113  
    114102 
    115103    // MPI_Scatterv from root to masters 
     
    127115      ::MPI_Scatterv(tmp_sendbuf, recvcounts.data(), displs.data(), to_mpi_type(sendtype), local_recvbuf, num_ep*count, to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    128116 
    129       // printf("local_recvbuf = %d %d %d %d\n", static_cast<int*>(local_recvbuf)[0], static_cast<int*>(local_recvbuf)[1], static_cast<int*>(local_recvbuf)[2], static_cast<int*>(local_recvbuf)[3]); 
    130                                                           // static_cast<int*>(local_recvbuf)[4], static_cast<int*>(local_recvbuf)[5], static_cast<int*>(local_recvbuf)[6], static_cast<int*>(local_recvbuf)[7]); 
    131117    } 
    132118 
     
    138124  } 
    139125 
     126 
     127  int MPI_Scatter_intercomm(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, int root, MPI_Comm comm) 
     128  { 
     129    printf("MPI_Scatter_intercomm not yet implemented\n"); 
     130    MPI_Abort(comm, 0); 
     131  } 
     132 
    140133} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_scatterv.cpp

    r1503 r1527  
    4545                   MPI_Datatype recvtype, int root, MPI_Comm comm) 
    4646  { 
    47     if(!comm->is_ep) 
    48     { 
    49       return ::MPI_Scatterv(sendbuf, sendcounts, displs, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 
    50     } 
     47    if(!comm->is_ep) return ::MPI_Scatterv(sendbuf, sendcounts, displs, to_mpi_type(sendtype), recvbuf, recvcount, to_mpi_type(recvtype), root, to_mpi_comm(comm->mpi_comm)); 
     48    if(comm->is_intercomm) return MPI_Scatterv_intercomm(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm); 
    5149    
    5250    assert(sendtype == recvtype); 
     
    105103      for(int i=0; i<ep_size; i++) 
    106104      { 
    107         //printf("i=%d : start from %d, src displs = %d, count = %d\n ", i, local_displs/datasize, displs[ranks[i]], sendcounts[ranks[i]]); 
    108105        memcpy(tmp_sendbuf+local_displs, sendbuf + displs[ranks[i]]*datasize, sendcounts[ranks[i]]*datasize); 
    109106        local_displs += sendcounts[ranks[i]]*datasize; 
    110107      } 
    111        
    112       //for(int i=0; i<ep_size*2; i++) printf("%d\t", static_cast<int*>(const_cast<void*>(tmp_sendbuf))[i]); 
    113108    } 
    114109 
     
    129124 
    130125      ::MPI_Scatterv(tmp_sendbuf, recvcounts.data(), my_displs.data(), to_mpi_type(sendtype), local_sendbuf, num_ep*count, to_mpi_type(recvtype), root_mpi_rank, to_mpi_comm(comm->mpi_comm)); 
    131  
    132       // printf("my_displs = %d %d %d %d\n", my_displs[0], my_displs[1], my_displs[2], my_displs[3] ); 
    133  
    134       // printf("%d %d %d %d %d %d %d %d\n", static_cast<int*>(local_sendbuf)[0], static_cast<int*>(local_sendbuf)[1], static_cast<int*>(local_sendbuf)[2], static_cast<int*>(local_sendbuf)[3], 
    135       //                                     static_cast<int*>(local_sendbuf)[4], static_cast<int*>(local_sendbuf)[5], static_cast<int*>(local_sendbuf)[6], static_cast<int*>(local_sendbuf)[7]); 
    136126    } 
    137127 
     
    154144 
    155145 
     146  int MPI_Scatterv_intercomm(const void *sendbuf, const int sendcounts[], const int displs[], MPI_Datatype sendtype, void *recvbuf, int recvcount, 
     147                   MPI_Datatype recvtype, int root, MPI_Comm comm) 
     148  { 
     149    printf("MPI_Scatterv_intercomm not yet implemented\n"); 
     150    MPI_Abort(comm, 0); 
     151  } 
     152 
     153 
    156154} 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_send.cpp

    r1518 r1527  
    1818  { 
    1919    if(!comm->is_ep)       return MPI_Send_mpi(buf, count, datatype, dest, tag, comm); 
    20     if(comm->is_intercomm) return MPI_Send_intercomm(buf, count, datatype, dest, tag, comm); 
     20    if(comm->is_intercomm) dest = comm->inter_rank_map->at(dest); 
    2121     
    2222    Debug("\nMPI_Send with EP\n"); 
     
    3838  { 
    3939    if(!comm->is_ep)       return MPI_Ssend_mpi(buf, count, datatype, dest, tag, comm); 
    40     if(comm->is_intercomm) return MPI_Ssend_intercomm(buf, count, datatype, dest, tag, comm); 
     40    if(comm->is_intercomm) dest = comm->inter_rank_map->at(dest); 
    4141     
    4242    Debug("\nMPI_Ssend with EP\n"); 
     
    6161  { 
    6262    if(!comm->is_ep)       return MPI_Isend_mpi(buf, count, datatype, dest, tag, comm, request); 
    63     if(comm->is_intercomm) return MPI_Isend_intercomm(buf, count, datatype, dest, tag, comm, request); 
     63    if(comm->is_intercomm) dest = comm->inter_rank_map->at(dest); 
    6464 
    6565    Debug("\nMPI_Isend with EP\n"); 
     
    104104  
    105105    if(!comm->is_ep) return MPI_Issend_mpi(buf, count, datatype, dest, tag, comm, request); 
    106     if(comm->is_intercomm) return MPI_Issend_intercomm(buf, count, datatype, dest, tag, comm, request); 
     106    if(comm->is_intercomm) dest = comm->inter_rank_map->at(dest); 
    107107 
    108108    Debug("\nMPI_Issend with EP\n"); 
     
    144144    return ::MPI_Send(buf, count, to_mpi_type(datatype), dest, tag, to_mpi_comm(comm->mpi_comm)); 
    145145  } 
    146    
    147   int MPI_Send_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
    148   { 
    149     Debug("\nMPI_Send_intercomm with EP\n"); 
    150     MPI_Request request; 
    151     MPI_Status status; 
    152     MPI_Isend(buf, count, datatype, dest, tag, comm, &request); 
    153     MPI_Wait(&request, &status); 
    154     return MPI_SUCCESS; 
    155   } 
    156    
     146  
    157147   
    158148  int MPI_Ssend_mpi(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
     
    163153   
    164154 
    165   int MPI_Ssend_intercomm(void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm) 
    166   { 
    167     Debug("\nMPI_Ssend_intercomm with EP\n"); 
    168     MPI_Request request; 
    169     MPI_Status status; 
    170     MPI_Issend(buf, count, datatype, dest, tag, comm, &request); 
    171     MPI_Wait(&request, &status); 
    172     return MPI_SUCCESS; 
    173   } 
    174    
    175155  int MPI_Isend_mpi(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
    176156  { 
     
    195175  } 
    196176   
    197   int MPI_Isend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
    198   { 
    199     Debug("\nMPI_Isend_intercomm with EP\n"); 
    200  
    201 #ifdef _check_sum     
    202     check_sum_send(buf, count, datatype, dest, tag, comm); 
    203 #endif 
    204  
    205     int src_comm_label  = comm->ep_comm_ptr->comm_label; 
    206     int dest_comm_label = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).second.second; 
    207      
    208     int src_rank; 
    209     MPI_Comm_rank(comm, &src_rank); 
    210  
    211      
    212     *request = new ep_request; 
    213     memcheck("new "<< *request <<" : in ep_lib::MPI_Isend_intercomm, *request = new ep_request"); 
    214  
    215     (*request)->mpi_request = new ::MPI_Request; 
    216     memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Isend_intercomm, (*request)->mpi_request = new ::MPI_Request"); 
    217      
    218  
    219     int ep_src_loc  = comm->ep_rank_map->at(src_rank).first; 
    220     int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).first; 
    221     int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    222     int mpi_dest    = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).second.first; 
    223  
    224 #ifdef _showinfo 
    225     printf("Send : ep_src_loc = %d, ep_dest_loc = %d, mpi_src = %d, mpi_dest = %d, mpi_tag = %d\n", ep_src_loc, ep_dest_loc, comm->ep_comm_ptr->size_rank_info[2].first, mpi_dest, mpi_tag); 
    226 #endif 
    227      
    228  
    229     (*request)->ep_src  = src_rank; 
    230     (*request)->ep_tag  = tag; 
    231     (*request)->ep_datatype = datatype; 
    232  
    233     (*request)->type = 1;    // used in wait 
    234     (*request)->comm = comm; 
    235     (*request)->buf = const_cast<void*>(buf); 
    236  
    237  
    238     if(src_comm_label == dest_comm_label) 
    239     { 
    240       Debug("\nMPI_Isend_intercomm with EP_intracomm\n"); 
    241       return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
    242     } 
    243  
    244     else 
    245     { 
    246       Debug("\nMPI_Isend_intercomm with EP_intercomm\n"); 
    247       return ::MPI_Isend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), to_mpi_request_ptr(*request)); 
    248     } 
    249   } 
    250  
    251  
     177   
    252178 
    253179 
     
    275201 
    276202 
    277   int MPI_Issend_intercomm(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request) 
    278   { 
    279     Debug("\nMPI_Issend_intercomm with EP\n"); 
    280  
    281 #ifdef _check_sum     
    282     check_sum_send(buf, count, datatype, dest, tag, comm); 
    283 #endif 
    284  
    285     int src_comm_label  = comm->ep_comm_ptr->comm_label; 
    286     int dest_comm_label = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).second.second; 
    287      
    288     int src_rank; 
    289     MPI_Comm_rank(comm, &src_rank); 
    290  
    291      
    292     *request = new ep_request; 
    293     memcheck("new "<< *request <<" : in ep_lib::MPI_Issend_intercomm, *request = new ep_request"); 
    294  
    295     (*request)->mpi_request = new ::MPI_Request; 
    296     memcheck("new "<< (*request)->mpi_request <<" : in ep_lib::MPI_Issend_intercomm, (*request)->mpi_request = new ::MPI_Request"); 
    297      
    298  
    299     int ep_src_loc  = comm->ep_rank_map->at(src_rank).first; 
    300     int ep_dest_loc = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).first; 
    301     int mpi_tag     = tag_combine(tag, ep_src_loc, ep_dest_loc); 
    302     int mpi_dest    = comm->ep_comm_ptr->intercomm->intercomm_rank_map->at(dest).second.first; 
    303  
    304 #ifdef _showinfo 
    305     printf("ep_src_loc = %d, ep_dest_loc = %d, mpi_src = %d, mpi_dest = %d, mpi_tag = %d\n", ep_src_loc, ep_dest_loc, comm->ep_comm_ptr->size_rank_info[2].first, mpi_dest, mpi_tag); 
    306 #endif 
    307      
    308     (*request)->ep_src  = src_rank; 
    309     (*request)->ep_tag  = tag; 
    310     (*request)->ep_datatype = datatype; 
    311  
    312     (*request)->type = 1;    // used in wait 
    313     (*request)->comm = comm; 
    314     (*request)->buf = const_cast<void*>(buf); 
    315  
    316  
    317     if(src_comm_label == dest_comm_label) 
    318     { 
    319       Debug("\nMPI_Issend_intercomm with EP_intracomm\n"); 
    320       return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->mpi_comm), to_mpi_request_ptr(*request)); 
    321     } 
    322  
    323     else 
    324     { 
    325       Debug("\nMPI_Issend_intercomm with EP_intercomm\n"); 
    326       return ::MPI_Issend(buf, count, to_mpi_type(datatype), mpi_dest, mpi_tag, to_mpi_comm(comm->ep_comm_ptr->intercomm->mpi_inter_comm), to_mpi_request_ptr(*request)); 
    327     } 
    328   } 
    329    
    330203} 
    331204 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_size.cpp

    r1525 r1527  
    2222    if(comm->is_ep) 
    2323    { 
    24       if(comm->is_intercomm) 
    25       { 
    26         Debug("MPI_Comm_remote_size with EP_intercomm"); 
    27         return *size = comm->ep_comm_ptr->intercomm->intercomm_rank_map->size(); 
    28       } 
    29       else 
    30       { 
    31         Debug("MPI_Comm_remote_size with EP_intracomm"); 
    32         return *size=0; 
    33       } 
    34     } 
    35     return MPI_Comm_remote_size_mpi(comm, size); 
    36   } 
    37  
    38   int MPI_Comm_remote_size2(MPI_Comm comm, int *size) 
    39   { 
    40     if(comm->is_ep) 
    41     { 
    42       if(comm->is_intercomm) 
    43       { 
    44         Debug("MPI_Comm_remote_size with EP_intercomm"); 
    45         return *size = comm->inter_rank_map->size(); 
    46       } 
    47       return *size=0; 
     24      return *size = comm->is_intercomm? comm->inter_rank_map->size() : 0; 
    4825    } 
    4926    return MPI_Comm_remote_size_mpi(comm, size); 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_split.cpp

    r1511 r1527  
    99{ 
    1010 
    11   void vec_simplify(std::vector<int> *inout_vector) 
    12   { 
    13     std::vector<int> out_vec; 
    14     int found=false; 
    15     for(std::vector<int>::iterator it_in = inout_vector->begin() ; it_in != inout_vector->end(); ++it_in) 
    16     { 
    17       for(std::vector<int>::iterator it = out_vec.begin() ; it != out_vec.end(); ++it) 
    18       { 
    19         if(*it_in == *it) 
    20         { 
    21           found=true; 
    22           break; 
    23         } 
    24         else found=false; 
    25       } 
    26       if(found == false) 
    27       { 
    28         out_vec.push_back(*it_in); 
    29       } 
    30     } 
    31     inout_vector->swap(out_vec); 
    32   } 
    33    
    3411  void vec_simplify(std::vector<int> *in_vector, std::vector<int> *out_vector) 
    3512  { 
     
    274251        *newcomm = comm->ep_comm_ptr->comm_list[0]->mem_bridge[new_ep_rank_loc]; 
    275252        memcheck("in MPI_Split ep_rank="<< ep_rank <<" : *newcomm = "<< *newcomm); 
    276  
    277         (*newcomm)->ep_comm_ptr->comm_label = color; 
    278253         
    279254        (*newcomm)->ep_comm_ptr->size_rank_info[0].first = new_ep_rank; 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_test.cpp

    r1522 r1527  
    2525    { 
    2626      printf("MPI_Test : Error in request type\n"); 
    27   
    2827      exit(1); 
    2928    } 
     
    3938       
    4039      ::MPI_Status mpi_status; 
    41        
    42        
    4340      ::MPI_Test(to_mpi_request_ptr(*request), flag, &mpi_status); 
    4441       
     
    5148        status->ep_datatype = (*request)->ep_datatype; 
    5249         
    53         (*request)->state = 2; 
     50        //(*request)->state = 2; 
    5451                 
    5552        memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Test, delete (*request)->mpi_request"); 
     
    6158     
    6259    return Request_Check(); 
    63  
    6460  } 
    6561 
     
    10197        array_of_statuses[i].ep_datatype = array_of_requests[i]->ep_datatype; 
    10298         
    103         array_of_requests[i]->state = 2; 
     99        //array_of_requests[i]->state = 2; 
    104100       
    105101        memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Testall, array_of_requests["<<i<<"]->mpi_request"); 
     
    109105         
    110106      } 
    111        
    112107    } 
    113      
     108 
    114109    return Request_Check(); 
    115  
    116110  } 
    117111 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_type.cpp

    r1517 r1527  
    3333  } 
    3434 
    35   /* 
    36    
    37   MPI_Aint::MPI_Aint(void* aint) 
    38   { 
    39     mpi_aint = new ::MPI_Aint; 
    40     *(static_cast< ::MPI_Aint*>(mpi_aint)) = *(static_cast< ::MPI_Aint*>(aint)); 
    41   } 
    42  
    43   MPI_Aint::MPI_Aint(int aint) 
    44   { 
    45     mpi_aint = new ::MPI_Aint; 
    46     *(static_cast< ::MPI_Aint*>(mpi_aint)) = aint; 
    47   } 
    48  
    49   MPI_Aint MPI_Aint::operator=(int a) 
    50   { 
    51     mpi_aint = new ::MPI_Aint; 
    52     *(static_cast< int*>(mpi_aint)) = a;  
    53   } 
    54    
    55   MPI_Fint::MPI_Fint(void* fint) 
    56   { 
    57     mpi_fint = new ::MPI_Fint; 
    58     *(static_cast< ::MPI_Fint*>(mpi_fint)) = *(static_cast< ::MPI_Fint*>(fint)); 
    59   } 
    60  
    61    
    62 */ 
    63    
    64  
    65  
    6635} 
    6736 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_type.hpp

    r1525 r1527  
    3838#include "ep_barrier.hpp" 
    3939#include "ep_comm.hpp" 
    40 #include "ep_intercomm.hpp" 
    4140#include "ep_window.hpp" 
    4241 
  • XIOS/dev/branch_openmp/extern/ep_dev/ep_wait.cpp

    r1522 r1527  
    2626    { 
    2727      printf("MPI_Wait : Error in request type\n"); 
    28   
    2928      exit(1); 
    3029    } 
     
    4140    status->ep_datatype = (*request)->ep_datatype; 
    4241     
    43     (*request)->state = 2; 
     42    //(*request)->state = 2; 
    4443 
    4544    memcheck("delete "<< (*request)->mpi_request << " : in ep_lib::MPI_Wait, delete (*request)->mpi_request"); 
     
    9594      array_of_statuses[i].ep_datatype = array_of_requests[i]->ep_datatype; 
    9695       
    97       array_of_requests[i]->state = 2; 
     96      //array_of_requests[i]->state = 2; 
    9897       
    9998      memcheck("delete "<< array_of_requests[i]->mpi_request <<" : in ep_lib::MPI_Waitall, array_of_requests["<<i<<"]->mpi_request"); 
  • XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

    r1525 r1527  
    3131  MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); 
    3232  MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); 
     33 
     34  #pragma omp parallel default(shared) 
     35  { 
     36    MPI_Comm_rank(MPI_COMM_WORLD , &mpi_rank); 
     37 
     38    int num_ep = omp_get_num_threads(); 
     39    MPI_Info info; 
     40 
     41    //printf("mpi_rank = %d, thread_num = %d\n", mpi_rank, omp_get_thread_num()); 
     42 
     43    MPI_Comm *ep_comm; 
     44    #pragma omp master 
     45    { 
     46      MPI_Comm *ep_comm; 
     47      MPI_Comm_create_endpoints(MPI_COMM_WORLD->mpi_comm, num_ep, info, ep_comm); 
     48      passage = ep_comm;   
     49    } 
     50 
     51    #pragma omp barrier 
     52 
     53    MPI_Comm comm; // this should act as EP_COMM_WORLD 
     54         
     55    comm = passage[omp_get_thread_num()]; 
     56 
     57    // TEST OF COMM_SPLIT 
     58    { 
     59 
     60      MPI_Barrier(comm); 
     61      int rank, size; 
     62      MPI_Comm_rank(comm, &rank); 
     63      MPI_Comm_size(comm, &size); 
     64     
     65      int tab_color[16] = {2, 2, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1}; // used for config 1 
     66      int tab_key[16]   = {2, 1, 4, 1, 0, 4, 3, 3, 4, 3, 4, 4, 4, 0, 0, 2}; // used for config 1 
     67 
     68      int color = tab_color[rank]; 
     69      int key = tab_key[rank]; 
     70       
     71   
     72      MPI_Comm split_comm; 
     73      MPI_Comm_split(comm, color, key, &split_comm); 
     74 
     75 
     76     
     77      int split_rank, split_size; 
     78      MPI_Comm_rank(split_comm, &split_rank); 
     79      MPI_Comm_size(split_comm, &split_size); 
     80     
     81#ifdef _Memory_check     
     82      printf("rank = %d, color = %d, key = %d, split_rank = %d, local_rank=%d\n", rank, color, key, split_rank, split_comm->ep_comm_ptr->size_rank_info[1].first); 
     83#endif 
     84 
     85      MPI_Barrier(comm); 
     86      MPI_Barrier(comm); 
     87      MPI_Barrier(comm); 
     88       
     89      if(rank == 0) printf("            \t MPI_Comm_split \t OK\n"); 
     90 
     91 
     92      int local_leader = 0; 
     93      int remote_leader = color==2? 13: 4; // used for config 1 
     94 
     95      MPI_Comm peer_comm = comm; 
     96 
     97      MPI_Comm inter_comm; 
     98      MPI_Intercomm_create(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm); 
     99 
     100      int inter_rank, inter_size, remote_size; 
     101      MPI_Comm_rank(inter_comm, &inter_rank); 
     102      MPI_Comm_size(inter_comm, &inter_size); 
     103      MPI_Comm_remote_size(inter_comm, &remote_size); 
     104       
     105       
     106      MPI_Barrier(comm); 
     107      MPI_Barrier(comm); 
     108 
     109      if(rank == 0) printf("            \t MPI_Intercomm_create \t OK\n"); 
     110 
     111      //printf("rank = %d, split_rank = %d, split_size = %d, inter_rank=%d, inter_size=%d, remote_size=%d\n", rank, split_rank, split_size, inter_rank, inter_size, remote_size); 
     112 
     113 
     114      MPI_Comm_free(&split_comm); 
     115 
     116       
     117      MPI_Barrier(comm); 
     118 
     119 
     120      if(color==2 && split_rank==0) 
     121      { 
     122        double sendbuf[9]={1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9}; 
     123        MPI_Request send_request; 
     124        MPI_Status send_status; 
     125        MPI_Isend(sendbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &send_request); 
     126        MPI_Wait(&send_request, &send_status); 
     127      } 
     128 
     129      if(color==1 && split_rank==0) 
     130      { 
     131        double recvbuf[9]; 
     132        MPI_Request recv_request; 
     133        MPI_Status recv_status; 
     134        MPI_Irecv(recvbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &recv_request); 
     135        MPI_Wait(&recv_request, &recv_status); 
     136        for(int i=0; i<9; i++) 
     137        { 
     138          printf("recvbuf[%d] = %lf\n", i, recvbuf[i]); 
     139        } 
     140      } 
     141 
     142      MPI_Barrier(comm); 
     143      MPI_Barrier(comm); 
     144 
     145      if(rank == 0) printf("            \t Test iP2P for intercomm \t OK\n"); 
     146 
     147 
     148      MPI_Comm inter_comm_dup; 
     149      MPI_Comm_dup(inter_comm, &inter_comm_dup); 
     150      int inter_dup_rank, inter_dup_size; 
     151      MPI_Comm_rank(inter_comm_dup, &inter_dup_rank); 
     152      MPI_Comm_size(inter_comm_dup, &inter_dup_size); 
     153 
     154      //printf("rank = %d, split_rank = %d, split_size = %d, inter_dup_rank=%d, inter_dup_size=%d\n", rank, split_rank, split_size, inter_dup_rank, inter_dup_size); 
     155 
     156 
     157 
     158      MPI_Comm inter_comm_dup_merged; 
     159      bool high; 
     160      if(color==2) high=true; 
     161      else high = false; 
     162 
     163      MPI_Intercomm_merge(inter_comm_dup, high, &inter_comm_dup_merged); 
     164      int merged_rank, merged_size; 
     165      MPI_Comm_rank(inter_comm_dup_merged, &merged_rank); 
     166      MPI_Comm_size(inter_comm_dup_merged, &merged_size); 
     167 
     168      //printf("merged_rank=%d, merged_rank_loc=%d, root_ep_loc = %d\n", merged_rank, inter_comm_dup_merged->ep_comm_ptr->size_rank_info[1].first, inter_comm_dup_merged->ep_rank_map->at(0).first); 
     169 
     170      MPI_Barrier(comm); 
     171      MPI_Barrier(comm); 
     172             
     173      int sendbuf=0; 
     174      if(merged_rank == 0) sendbuf = 99; 
     175      MPI_Bcast(&sendbuf, 1, MPI_INT, 0, inter_comm_dup_merged); 
     176      printf("merged_rank = %d, sendbuf = %d\n", merged_rank, sendbuf); 
     177 
     178 
     179 
     180      MPI_Barrier(comm); 
     181 
     182      MPI_Comm_free(&inter_comm_dup_merged); 
     183      MPI_Comm_free(&inter_comm_dup); 
     184 
     185 
     186 
     187 
     188      MPI_Comm_free(&inter_comm); 
     189            
     190    } 
     191     
     192     
     193 
     194    MPI_Barrier(comm); 
     195    MPI_Comm_free(&comm); 
     196  } 
     197 
     198  //MPI_Finalize(); 
     199  //return 0; 
     200 
     201 
     202 
    33203  
    34204   
     
    8991069      // TEST OF GATHER 
    9001070      { 
     1071        int inter_comm_dup_merged_rank; 
     1072        MPI_Comm_rank(inter_comm_dup_merged, &inter_comm_dup_merged_rank); 
     1073 
     1074        int inter_comm_dup_merged_size; 
     1075        MPI_Comm_size(inter_comm_dup_merged, &inter_comm_dup_merged_size); 
     1076         
    9011077         
    9021078 
     
    9071083          gather_root = rand() % inter_comm_dup_merged_size; 
    9081084        } 
     1085 
     1086 
    9091087   
    9101088        MPI_Bcast(&gather_root, 1, MPI_INT, 0, inter_comm_dup_merged); 
     1089 
     1090        printf("rank = %d, inter_comm_dup_merged_rank = %d, inter_comm_dup_merged_size = %d, gather_root = %d\n", rank, inter_comm_dup_merged_rank, inter_comm_dup_merged_size, gather_root); 
     1091 
    9111092 
    9121093        double sendbuf[2]; 
     
    11331314      } 
    11341315 
     1316 
    11351317      MPI_Barrier(comm); 
    11361318      MPI_Barrier(comm); 
     
    12021384 
    12031385      MPI_Comm inter_comm; 
    1204       MPI_Intercomm_create2(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm); 
     1386      MPI_Intercomm_create(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm); 
    12051387 
    12061388      int inter_rank; 
     
    12141396      MPI_Comm_free(&split_comm); 
    12151397 
    1216       MPI_Barrier2(inter_comm); 
    1217       MPI_Comm_free2(&inter_comm); 
     1398      MPI_Barrier(inter_comm); 
     1399      MPI_Comm_free(&inter_comm); 
    12181400       
    12191401    } 
Note: See TracChangeset for help on using the changeset viewer.