source: XIOS/dev/branch_openmp/extern/ep_dev/ep_type.hpp @ 1503

Last change on this file since 1503 was 1503, checked in by yushan, 3 years ago

rank_map is passed from vector to map, in order to have more flexibility in comm_split

File size: 5.7 KB
Line 
1#ifndef EP_TYPE_HPP_INCLUDED
2#define EP_TYPE_HPP_INCLUDED
3
4
5#include <iostream>
6#include <stdlib.h>
7#include <stdio.h>
8#include <list>
9#include <map>
10#include <omp.h>
11#include <vector>
12#include <numeric>
13#include <bitset>
14#include <algorithm>
15#include <assert.h>
16#include <math.h>
17#include <string.h>
18
19#ifdef _Debug
20#define Debug(x) std::cout << x << std::endl
21#else
22#define Debug(x)
23#endif
24
25#ifdef _Memory_check
26#define memcheck(x) std::cout << x << std::endl
27#else
28#define memcheck(x)
29#endif
30
31
32#define BUFFER_SIZE 10000
33
34#include "ep_status.hpp"
35#include "ep_request.hpp"
36#include "ep_info.hpp"
37#include "ep_message.hpp"
38#include "ep_barrier.hpp"
39#include "ep_comm.hpp"
40#include "ep_intercomm.hpp"
41#include "ep_window.hpp"
42
43
44//typedef std::pair< int, int > SIZE_RANK_INFO; // < rank, size>
45
46//typedef std::vector< std::pair<int, int> > RANK_MAP;  // at(ep_rank) = <ep_rank_local, mpi_rank>
47//typedef std::map<int, std::pair<int, int> > EP_RANK_MAP;  // key(ep_rank) = <ep_rank_local, mpi_rank>
48
49//typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP;
50
51
52namespace ep_lib
53{
54
55  class ep_communicator;
56  class ep_intercomm;
57  class OMPbarrier;
58  typedef ep_communicator* EP_Comm;
59  //class MPI_Comm;
60
61/*
62  class MPI_Status
63  {
64    public:
65
66      void* ep_datatype;
67      int ep_src;
68      int ep_tag;
69      void* mpi_status;
70  };
71
72  class MPI_Message
73  {
74    public:
75
76      void* mpi_message;
77      int ep_src;
78      int ep_tag;
79      void* mpi_status;
80
81      MPI_Message() {}
82      MPI_Message(void* message);
83  };
84
85  typedef std::list<MPI_Message > Message_list;
86*/
87
88  class OMPbarrier
89  {
90    private:
91      int nbThreads;          //<The number of threads for this barrier
92      int currentNbThread;    //<The current number of threads waiting
93      bool sense;             //<Direct barrier feedback protection
94      omp_lock_t mutex;       //<To have an atomic int
95
96      OMPbarrier(OMPbarrier&){}
97      OMPbarrier& operator=(OMPbarrier&){return *this;}
98
99    public:
100      /** Constructor with the number of threads */
101      explicit OMPbarrier(const int inNbThreads)
102          : nbThreads(inNbThreads), currentNbThread(0), sense(false) {
103          omp_init_lock( &mutex );
104      }
105
106      /** Destructor, release the omp lock */
107      ~OMPbarrier(){
108          omp_destroy_lock( &mutex );
109      }
110
111      /** Perform a barrier */
112      void wait(){
113          const bool mySense = sense;
114          omp_set_lock( &mutex );
115          const int nbThreadsArrived = (++currentNbThread);
116          omp_unset_lock( &mutex );
117
118          if(nbThreadsArrived == nbThreads) {
119              currentNbThread = 0;
120              sense = !sense;
121              #pragma omp flush
122          }
123          else {
124              volatile const bool* const ptSense = &sense;
125              while( (*ptSense) == mySense){
126              }
127          }
128      }
129
130
131      /** Change the number of threads */
132      void setNbThreads(const int inNbThread){
133          omp_set_lock( &mutex );
134          nbThreads = inNbThread;
135          omp_unset_lock( &mutex );
136      }
137  };
138
139  /*
140  class ep_intercomm
141  {
142    public:
143
144    void *mpi_inter_comm;
145
146    RANK_MAP *intercomm_rank_map;
147    RANK_MAP *local_rank_map;
148    RANK_MAP *remote_rank_map;
149
150
151    SIZE_RANK_INFO size_rank_info[3];
152
153
154    MPI_Comm *local_comm;
155    int intercomm_tag;
156
157    ep_intercomm();
158    //~ep_intercomm(){delete mpi_inter_comm;}
159    bool operator == (ep_intercomm right);
160    bool operator != (ep_intercomm right);
161   
162  };
163
164  class ep_communicator
165  {
166    public:
167
168    SIZE_RANK_INFO size_rank_info[3]; // 0: ep_rank,     ep_size
169                                      // 1: ep_rank_loc, num_ep
170                                      // 2: mpi_rank,    mpi_size
171
172
173    MPI_Comm *comm_list;
174
175    Message_list *message_queue;
176
177
178    int comm_label;
179
180    ep_intercomm *intercomm;
181
182    ep_communicator();
183    bool operator == (ep_communicator right);
184    bool operator != (ep_communicator right);
185   
186  };
187
188
189  struct BUFFER
190  {
191    void * void_buffer[12];
192  };
193
194
195  class MPI_Comm
196  {
197    public:
198
199    bool is_ep;
200    bool is_intercomm;
201
202    BUFFER     *my_buffer;
203    OMPbarrier *ep_barrier;
204    RANK_MAP   *rank_map;
205    void* mpi_comm;
206    EP_Comm ep_comm_ptr;
207    MPI_Comm *mem_bridge;
208    void* mpi_bridge;
209
210    MPI_Comm(); 
211    MPI_Comm(void* comm);
212    //~MPI_Comm(){delete mpi_comm;}
213
214
215    bool operator == (MPI_Comm right);
216    bool operator != (MPI_Comm right);
217    bool is_null();
218
219  };
220
221
222  class MPI_Info
223  {
224    public:
225
226      void* mpi_info;
227
228      MPI_Info();
229      MPI_Info(void* info);
230  };
231
232
233  class MPI_Request
234  {
235    public:
236
237      void* mpi_request;
238
239      int type; //! type of the non-blocking communication. 1: Isend; 2:Irecv; 3:Imrecv; 4:Issend
240      void* buf;
241
242      int ep_src;
243      int ep_tag;
244      void* ep_datatype;
245
246      MPI_Comm comm;    //! EP communicator related to the communication
247
248      MPI_Request() {}
249      MPI_Request(void* request);
250      bool operator == (MPI_Request right);
251
252  };
253
254*/ 
255
256  class MPI_Aint
257  {
258    public:
259
260    void* mpi_aint;
261
262    MPI_Aint() {}
263    MPI_Aint(void* aint);
264    MPI_Aint(int aint);
265    MPI_Aint operator=(int a);
266    //MPI_Aint(int a): mpi_aint(a) {}
267  };
268
269  class MPI_Fint
270  {
271    public:
272 
273    void* mpi_fint;
274   
275    MPI_Fint() {}
276    MPI_Fint(void* fint);
277    //MPI_Fint(int f): mpi_fint(f) {}
278                                 
279  };
280 
281 
282
283
284  static MPI_Comm *passage;
285
286  static int TAG = 40000;
287
288  static std::list<std::pair<std::pair<int, int>, MPI_Comm * > > tag_list;
289
290  static std::map<std::pair<int, int>, MPI_Comm >  fc_comm_map;
291
292//  static std::map<std::pair<int, int>, MPI_Comm >  *fc_comm_map_ptr;
293//  #pragma omp threadprivate(fc_comm_map_ptr)
294//            //    <MPI_Fint,thread_num>   EP_Comm
295
296}
297
298
299
300#endif // EP_TYPE_HPP_INCLUDED
301
Note: See TracBrowser for help on using the repository browser.