1 | !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
2 | ! Math and Computer Science Division, Argonne National Laboratory ! |
---|
3 | !----------------------------------------------------------------------- |
---|
4 | ! CVS m_MCTWorld.F90,v 1.26 2007/06/01 19:56:25 rloy Exp |
---|
5 | ! CVS MCT_2_4_0 |
---|
6 | !BOP ------------------------------------------------------------------- |
---|
7 | ! |
---|
8 | ! !MODULE: m_MCTWorld -- MCTWorld Class |
---|
9 | ! |
---|
10 | ! !DESCRIPTION: |
---|
11 | ! MCTWorld is a datatype which acts as a component model registry. |
---|
12 | ! All models communicating through MCT must participate in initialization |
---|
13 | ! of MCTWorld. The single instance of MCTWorld, {\tt ThisMCTWorld} stores |
---|
14 | ! the component id and local and global processor rank of each component. |
---|
15 | ! This module contains methods for creating and destroying {\tt ThisMCTWorld} |
---|
16 | ! as well as inquiry functions. |
---|
17 | ! |
---|
18 | ! !INTERFACE: |
---|
19 | |
---|
20 | module m_MCTWorld |
---|
21 | ! |
---|
22 | ! !USES: |
---|
23 | use m_List, only : List ! Support for List components. |
---|
24 | |
---|
25 | implicit none |
---|
26 | |
---|
27 | private ! except |
---|
28 | |
---|
29 | ! !PUBLIC TYPES: |
---|
30 | |
---|
31 | public :: MCTWorld ! The MCTWorld class data structure |
---|
32 | |
---|
33 | type MCTWorld |
---|
34 | integer :: MCT_comm ! MCT communicator |
---|
35 | integer :: ncomps ! Total number of components |
---|
36 | integer :: mygrank ! Rank of this processor in |
---|
37 | ! global communicator. |
---|
38 | integer,dimension(:),pointer :: nprocspid => null() ! Number of processes |
---|
39 | ! each component is on (e.g. rank of its |
---|
40 | ! local communicator. |
---|
41 | integer,dimension(:,:),pointer :: idGprocid => null() ! Translate between local component rank |
---|
42 | ! rank in global communicator. |
---|
43 | ! idGprocid(modelid,localrank)=globalrank |
---|
44 | end type MCTWorld |
---|
45 | |
---|
46 | ! !PUBLIC DATA MEMBERS: |
---|
47 | |
---|
48 | type(MCTWorld) :: ThisMCTWorld ! declare the MCTWorld |
---|
49 | |
---|
50 | ! !PUBLIC MEMBER FUNCTIONS: |
---|
51 | public :: initialized ! Determine if MCT is initialized |
---|
52 | public :: init ! Create a MCTWorld |
---|
53 | public :: clean ! Destroy a MCTWorld |
---|
54 | public :: printnp ! Print contents of a MCTWorld |
---|
55 | public :: NumComponents ! Number of Components in the MCTWorld |
---|
56 | public :: ComponentNumProcs ! Number of processes owned by a given |
---|
57 | ! component |
---|
58 | public :: ComponentToWorldRank ! Given the rank of a process on a |
---|
59 | ! component, return its rank on the |
---|
60 | ! world communicator |
---|
61 | public :: ComponentRootRank ! Return the rank on the world |
---|
62 | ! communicator of the root process of |
---|
63 | ! a component |
---|
64 | public :: ThisMCTWorld ! Instantiation of the MCTWorld |
---|
65 | |
---|
66 | ! |
---|
67 | |
---|
68 | interface initialized ; module procedure & |
---|
69 | initialized_ |
---|
70 | end interface |
---|
71 | interface init ; module procedure & |
---|
72 | initd_, & |
---|
73 | initm_, & |
---|
74 | initr_ |
---|
75 | end interface |
---|
76 | interface clean ; module procedure clean_ ; end interface |
---|
77 | interface printnp ; module procedure printnp_ ; end interface |
---|
78 | interface NumComponents ; module procedure & |
---|
79 | NumComponents_ |
---|
80 | end interface |
---|
81 | interface ComponentNumProcs ; module procedure & |
---|
82 | ComponentNumProcs_ |
---|
83 | end interface |
---|
84 | interface ComponentToWorldRank ; module procedure & |
---|
85 | ComponentToWorldRank_ |
---|
86 | end interface |
---|
87 | interface ComponentRootRank ; module procedure & |
---|
88 | ComponentRootRank_ |
---|
89 | end interface |
---|
90 | |
---|
91 | |
---|
92 | |
---|
93 | ! !REVISION HISTORY: |
---|
94 | ! 19Jan01 - R. Jacob <jacob@mcs.anl.gov> - initial prototype |
---|
95 | ! 05Feb01 - J. Larson <larson@mcs.anl.gov> - added query and |
---|
96 | ! local-to-global mapping services NumComponents, |
---|
97 | ! ComponentNumProcs, ComponentToWorldRank, and ComponentRootRank |
---|
98 | ! 08Feb01 - R. Jacob <jacob@mcs.anl.gov> - add mylrank and mygrank |
---|
99 | ! to datatype |
---|
100 | ! 20Apr01 - R. Jacob <jacob@mcs.anl.gov> - remove allids from |
---|
101 | ! MCTWorld datatype. Not needed because component |
---|
102 | ! ids are always from 1 to number-of-components. |
---|
103 | ! 07Jun01 - R. Jacob <jacob@mcs.anl.gov> - remove myid, mynprocs |
---|
104 | ! and mylrank from MCTWorld datatype because they are not |
---|
105 | ! clearly defined in PCM mode. Add MCT_comm for future use. |
---|
106 | ! 03Aug01 - E. Ong <eong@mcs.anl.gov> - explicity specify starting |
---|
107 | ! address in mpi_irecv |
---|
108 | ! 27Nov01 - E. Ong <eong@mcs.anl.gov> - added R. Jacob's version of initd_ |
---|
109 | ! to support PCM mode. |
---|
110 | ! 15Feb02 - R. Jacob - elminate use of MP_COMM_WORLD. Use |
---|
111 | ! argument globalcomm instead. Create MCT_comm from |
---|
112 | ! globalcomm |
---|
113 | !EOP __________________________________________________________________ |
---|
114 | |
---|
115 | character(len=*),parameter :: myname='MCT::m_MCTWorld' |
---|
116 | |
---|
117 | contains |
---|
118 | |
---|
119 | |
---|
120 | |
---|
121 | !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
122 | ! Math and Computer Science Division, Argonne National Laboratory ! |
---|
123 | !BOP ------------------------------------------------------------------- |
---|
124 | ! |
---|
125 | ! !IROUTINE: initialized_ - determine if MCTWorld is initialized |
---|
126 | ! |
---|
127 | ! !DESCRIPTION: |
---|
128 | ! This routine may be used to determine whether {\tt MCTWorld::init} |
---|
129 | ! has been called. If not, the user must call {\tt init} before |
---|
130 | ! performing any other MCT library calls. |
---|
131 | ! |
---|
132 | ! !INTERFACE: |
---|
133 | |
---|
134 | logical function initialized_() |
---|
135 | |
---|
136 | ! |
---|
137 | ! !USES: |
---|
138 | ! |
---|
139 | |
---|
140 | ! !INPUT PARAMETERS: |
---|
141 | |
---|
142 | |
---|
143 | ! !REVISION HISTORY: |
---|
144 | ! 01June07 - R. Loy <rloy@mcs.anl.gov> - initial version |
---|
145 | !EOP ___________________________________________________________________ |
---|
146 | ! |
---|
147 | |
---|
148 | initialized_ = associated(ThisMCTWorld%nprocspid) |
---|
149 | |
---|
150 | end function initialized_ |
---|
151 | |
---|
152 | |
---|
153 | |
---|
154 | |
---|
155 | !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
156 | ! Math and Computer Science Division, Argonne National Laboratory ! |
---|
157 | !BOP ------------------------------------------------------------------- |
---|
158 | ! |
---|
159 | ! !IROUTINE: initm_ - initialize MCTWorld |
---|
160 | ! |
---|
161 | ! !DESCRIPTION: |
---|
162 | ! Do a distributed init of MCTWorld for the case where a set of processors |
---|
163 | ! contains more then one model and the models may not span the set of processors. |
---|
164 | ! {\tt ncomps} is the total number of components in the entire coupled system. |
---|
165 | ! {\tt globalcomm} encompasses all the models (typically this can be MPI\_COMM\_WORLD). |
---|
166 | ! {\tt mycomms} is an array of MPI communicators, each sized for the appropriate model |
---|
167 | ! and {\tt myids} is a corresponding array of integers containing the model ids for |
---|
168 | ! the models on this particular set of processors. |
---|
169 | ! |
---|
170 | ! This routine is called once for the models covered by the set of processors. |
---|
171 | ! |
---|
172 | ! !INTERFACE: |
---|
173 | |
---|
174 | subroutine initm_(ncomps,globalcomm,mycomms,myids) |
---|
175 | ! |
---|
176 | ! !USES: |
---|
177 | ! |
---|
178 | use m_mpif90 |
---|
179 | use m_die |
---|
180 | use m_stdio |
---|
181 | |
---|
182 | implicit none |
---|
183 | |
---|
184 | ! !INPUT PARAMETERS: |
---|
185 | |
---|
186 | integer, intent(in) :: ncomps ! number of components |
---|
187 | integer, intent(in) :: globalcomm ! global communicator |
---|
188 | integer, dimension(:),pointer :: mycomms ! my communicators |
---|
189 | integer, dimension(:),pointer :: myids ! component ids |
---|
190 | |
---|
191 | ! !REVISION HISTORY: |
---|
192 | ! 20Sep07 - T. Craig migrated code from initd routine |
---|
193 | ! 20Sep07 - T. Craig - made mycomms an array |
---|
194 | ! 03Nov19 - J. Edwards - Add barrier to improve peformance on high proc counts |
---|
195 | !EOP ___________________________________________________________________ |
---|
196 | ! |
---|
197 | character(len=*),parameter :: myname_=myname//'::initm_' |
---|
198 | integer :: ier,myGid,myLid,i,mysize,Gsize,j |
---|
199 | |
---|
200 | ! arrays allocated on the root to coordinate gathring of data |
---|
201 | ! and non-blocking receives by the root |
---|
202 | integer, dimension(:), allocatable :: compids,reqs,nprocs,Gprocids |
---|
203 | integer, dimension(:), allocatable :: root_nprocs |
---|
204 | integer, dimension(:,:),allocatable :: status,root_idGprocid |
---|
205 | integer, dimension(:,:),pointer :: tmparray |
---|
206 | integer,dimension(:),pointer :: apoint |
---|
207 | ! ------------------------------------------------------------------ |
---|
208 | |
---|
209 | ! Check that ncomps is a legal value |
---|
210 | if(ncomps < 1) then |
---|
211 | call die(myname_, "argument ncomps can't less than one!",ncomps) |
---|
212 | endif |
---|
213 | |
---|
214 | if (size(myids) /= size(mycomms)) then |
---|
215 | call die(myname_, "size of myids and mycomms inconsistent") |
---|
216 | endif |
---|
217 | |
---|
218 | ! make sure this has not been called already |
---|
219 | if(associated(ThisMCTWorld%nprocspid) ) then |
---|
220 | write(stderr,'(2a)') myname_, & |
---|
221 | 'MCTERROR: MCTWorld has already been initialized...Continuing' |
---|
222 | RETURN |
---|
223 | endif |
---|
224 | |
---|
225 | ! determine overall size |
---|
226 | call MP_comm_size(globalcomm,Gsize,ier) |
---|
227 | if(ier /= 0) call MP_perr_die(myname_,'MP_comm_size()',ier) |
---|
228 | |
---|
229 | ! determine my rank in comm_world |
---|
230 | call MP_comm_rank(globalcomm,myGid,ier) |
---|
231 | if(ier /= 0) call MP_perr_die(myname_,'MP_comm_rank()',ier) |
---|
232 | |
---|
233 | ! allocate space on global root to receive info about |
---|
234 | ! the other components |
---|
235 | if(myGid == 0) then |
---|
236 | allocate(nprocs(ncomps),compids(ncomps),& |
---|
237 | reqs(ncomps),status(MP_STATUS_SIZE,ncomps),& |
---|
238 | root_nprocs(ncomps),stat=ier) |
---|
239 | if (ier /= 0) then |
---|
240 | call die(myname_, 'allocate(nprocs,...)',ier) |
---|
241 | endif |
---|
242 | endif |
---|
243 | |
---|
244 | |
---|
245 | !!!!!!!!!!!!!!!!!! |
---|
246 | ! Gather the number of procs from the root of each component |
---|
247 | !!!!!!!!!!!!!!!!!! |
---|
248 | ! |
---|
249 | ! First on the global root, post a receive for each component |
---|
250 | if(myGid == 0) then |
---|
251 | do i=1,ncomps |
---|
252 | call MPI_IRECV(root_nprocs(i), 1, MP_INTEGER, MP_ANY_SOURCE,i, & |
---|
253 | globalcomm, reqs(i), ier) |
---|
254 | if(ier /= 0) call MP_perr_die(myname_,'MPI_IRECV(root_nprocs)',ier) |
---|
255 | enddo |
---|
256 | endif |
---|
257 | |
---|
258 | ! The local root on each component sends |
---|
259 | do i=1,size(myids) |
---|
260 | if(mycomms(i)/=MP_COMM_NULL) then |
---|
261 | call MP_comm_size(mycomms(i),mysize,ier) |
---|
262 | if(ier /= 0) call MP_perr_die(myname_,'MP_comm_size()',ier) |
---|
263 | call MP_comm_rank(mycomms(i),myLid,ier) |
---|
264 | if(ier /= 0) call MP_perr_die(myname_,'MP_comm_rank()',ier) |
---|
265 | if(myLid == 0) then |
---|
266 | call MPI_SEND(mysize,1,MP_INTEGER,0,myids(i),globalcomm,ier) |
---|
267 | if(ier /= 0) call MP_perr_die(myname_,'MPI_SEND(mysize)',ier) |
---|
268 | endif |
---|
269 | endif |
---|
270 | enddo |
---|
271 | |
---|
272 | ! Global root waits for all sends |
---|
273 | if(myGid == 0) then |
---|
274 | call MPI_WAITALL(size(reqs), reqs, status, ier) |
---|
275 | if(ier /= 0) call MP_perr_die(myname_,'MPI_WAITALL()',ier) |
---|
276 | endif |
---|
277 | ! Global root now knows how many processors each component is using |
---|
278 | |
---|
279 | !!!!!!!!!!!!!!!!!! |
---|
280 | ! end of nprocs |
---|
281 | !!!!!!!!!!!!!!!!!! |
---|
282 | |
---|
283 | |
---|
284 | ! allocate a tmp array for the receive on root. |
---|
285 | if(myGid == 0) then |
---|
286 | allocate(tmparray(0:Gsize-1,ncomps),stat=ier) |
---|
287 | if(ier/=0) call die(myname_,'allocate(tmparray)',ier) |
---|
288 | |
---|
289 | ! fill tmparray with a bad rank value for later error checking |
---|
290 | tmparray = -1 |
---|
291 | endif |
---|
292 | |
---|
293 | !!!!!!!!!!!!!!!!!! |
---|
294 | ! Gather the Gprocids from each local root |
---|
295 | !!!!!!!!!!!!!!!!!! |
---|
296 | ! |
---|
297 | ! First on the global root, post a receive for each component |
---|
298 | if(myGid == 0) then |
---|
299 | do i=1,ncomps |
---|
300 | apoint => tmparray(0:root_nprocs(i)-1,i) |
---|
301 | call MPI_IRECV(apoint, root_nprocs(i),MP_INTEGER, & |
---|
302 | MP_ANY_SOURCE,i,globalcomm, reqs(i), ier) |
---|
303 | if(ier /= 0) call MP_perr_die(myname_,'MPI_IRECV()',ier) |
---|
304 | enddo |
---|
305 | endif |
---|
306 | |
---|
307 | ! The root on each component sends |
---|
308 | do i=1,size(myids) |
---|
309 | if(mycomms(i)/=MP_COMM_NULL) then |
---|
310 | call MP_comm_size(mycomms(i),mysize,ier) |
---|
311 | if(ier /= 0) call MP_perr_die(myname_,'MP_comm_size()',ier) |
---|
312 | call MP_comm_rank(mycomms(i),myLid,ier) |
---|
313 | if(ier /= 0) call MP_perr_die(myname_,'MP_comm_rank()',ier) |
---|
314 | |
---|
315 | ! make the master list of global proc ids |
---|
316 | ! |
---|
317 | ! allocate space to hold global ids |
---|
318 | ! only needed on root, but allocate everywhere to avoid complaints. |
---|
319 | ! Don't allocate large size on non-root. |
---|
320 | if (myLid==0) then |
---|
321 | allocate(Gprocids(mysize),stat=ier) |
---|
322 | else |
---|
323 | allocate(Gprocids(1),stat=ier) |
---|
324 | endif |
---|
325 | if(ier/=0) call die(myname_,'allocate(Gprocids)',ier) |
---|
326 | ! gather over the LOCAL comm |
---|
327 | call MPI_GATHER(myGid,1,MP_INTEGER,Gprocids,1,MP_INTEGER,0,mycomms(i),ier) |
---|
328 | if(ier/=0) call die(myname_,'MPI_GATHER Gprocids',ier) |
---|
329 | |
---|
330 | ! This barrier needed for good performance on high-processor counts. |
---|
331 | call MPI_Barrier(mycomms(i), ier) |
---|
332 | if(ier/=0) call die(myname_,'MPI_Barrier Gprocids',ier) |
---|
333 | |
---|
334 | if(myLid == 0) then |
---|
335 | call MPI_SEND(Gprocids,mysize,MP_INTEGER,0,myids(i),globalcomm,ier) |
---|
336 | if(ier /= 0) call MP_perr_die(myname_,'MPI_SEND(Gprocids)',ier) |
---|
337 | endif |
---|
338 | |
---|
339 | deallocate(Gprocids,stat=ier) |
---|
340 | if(ier/=0) call die(myname_,'deallocate(Gprocids)',ier) |
---|
341 | endif |
---|
342 | enddo |
---|
343 | |
---|
344 | ! Global root waits for all sends |
---|
345 | if(myGid == 0) then |
---|
346 | call MPI_WAITALL(size(reqs), reqs, status, ier) |
---|
347 | if(ier /= 0) call MP_perr_die(myname_,'MPI_WAITALL(Gprocids)',ier) |
---|
348 | endif |
---|
349 | |
---|
350 | ! Now store the Gprocids in the World description and Broadcast |
---|
351 | |
---|
352 | if(myGid == 0) then |
---|
353 | allocate(root_idGprocid(ncomps,0:Gsize-1),stat=ier) |
---|
354 | if(ier/=0) call die(myname_,'allocate(root_idGprocid)',ier) |
---|
355 | |
---|
356 | root_idGprocid = transpose(tmparray) |
---|
357 | endif |
---|
358 | |
---|
359 | if(myGid /= 0) then |
---|
360 | allocate(root_nprocs(1),root_idGprocid(1,1),stat=ier) |
---|
361 | if(ier/=0) call die(myname_,'non-root allocate(root_idGprocid)',ier) |
---|
362 | endif |
---|
363 | |
---|
364 | !!!!!!!!!!!!!!!!!! |
---|
365 | ! end of Gprocids |
---|
366 | !!!!!!!!!!!!!!!!!! |
---|
367 | |
---|
368 | ! now call the init from root. |
---|
369 | call initr_(ncomps,globalcomm,root_nprocs,root_idGprocid) |
---|
370 | |
---|
371 | ! if(myGid==0 .or. myGid==17) then |
---|
372 | ! write(*,*)'MCTA',myGid,ThisMCTWorld%ncomps,ThisMCTWorld%MCT_comm,ThisMCTWorld%nprocspid |
---|
373 | ! do i=1,ThisMCTWorld%ncomps |
---|
374 | ! write(*,*)'MCTK',myGid,i,ThisMCTWorld%idGprocid(i,0:ThisMCTWorld%nprocspid(i)-1) |
---|
375 | ! enddo |
---|
376 | ! endif |
---|
377 | |
---|
378 | ! deallocate temporary arrays |
---|
379 | deallocate(root_nprocs,root_idGprocid,stat=ier) |
---|
380 | if(ier/=0) call die(myname_,'deallocate(root_nprocs,..)',ier) |
---|
381 | if(myGid == 0) then |
---|
382 | deallocate(compids,reqs,status,nprocs,tmparray,stat=ier) |
---|
383 | if(ier/=0) call die(myname_,'deallocate(compids,..)',ier) |
---|
384 | endif |
---|
385 | end subroutine initm_ |
---|
386 | |
---|
387 | !BOP ------------------------------------------------------------------- |
---|
388 | ! |
---|
389 | ! !IROUTINE: initd_ - initialize MCTWorld |
---|
390 | ! |
---|
391 | ! !DESCRIPTION: |
---|
392 | ! Do a distributed init of MCTWorld using the total number of components |
---|
393 | ! {\tt ncomps} and either a unique integer component id {\tt myid} or, |
---|
394 | ! if more than one model is placed on a processor, an array of integer ids |
---|
395 | ! specifying the models {\tt myids}. Also required is |
---|
396 | ! the local communicator {\tt mycomm} and global communicator {\tt globalcomm} |
---|
397 | ! which encompasses all the models (typically this can be MPI\_COMM\_WORLD). |
---|
398 | ! This routine must be called once by each component (using {\em myid}) or |
---|
399 | ! component group (using {\em myids}). |
---|
400 | ! |
---|
401 | ! !INTERFACE: |
---|
402 | |
---|
403 | subroutine initd_(ncomps,globalcomm,mycomm,myid,myids) |
---|
404 | ! |
---|
405 | ! !USES: |
---|
406 | ! |
---|
407 | use m_mpif90 |
---|
408 | use m_die |
---|
409 | use m_stdio |
---|
410 | |
---|
411 | implicit none |
---|
412 | |
---|
413 | ! !INPUT PARAMETERS: |
---|
414 | |
---|
415 | integer, intent(in) :: ncomps ! number of components |
---|
416 | integer, intent(in) :: globalcomm ! global communicator |
---|
417 | integer, intent(in) :: mycomm ! my communicator |
---|
418 | integer, intent(in),optional :: myid ! my component id |
---|
419 | integer, dimension(:),pointer,optional :: myids ! component ids |
---|
420 | |
---|
421 | ! !REVISION HISTORY: |
---|
422 | ! 19Jan01 - R. Jacob <jacob@mcs.anl.gov> - initial prototype |
---|
423 | ! 07Feb01 - R. Jacob <jacob@mcs.anl.gov> - non fatal error |
---|
424 | ! if init is called a second time. |
---|
425 | ! 08Feb01 - R. Jacob <jacob@mcs.anl.gov> - initialize the new |
---|
426 | ! mygrank and mylrank |
---|
427 | ! 20Apr01 - R. Jacob <jacob@mcs.anl.gov> - remove allids from |
---|
428 | ! MCTWorld datatype. Not needed because component |
---|
429 | ! ids are always from 1 to number-of-components. |
---|
430 | ! 22Jun01 - R. Jacob <jacob@mcs.anl.gov> - move Bcast and init |
---|
431 | ! of MCTWorld to initr_ |
---|
432 | ! 20Sep07 - T. Craig migrated code to new initm routine |
---|
433 | !EOP ___________________________________________________________________ |
---|
434 | ! |
---|
435 | character(len=*),parameter :: myname_=myname//'::initd_' |
---|
436 | integer :: msize,ier |
---|
437 | integer, dimension(:), pointer :: mycomm1d,myids1d |
---|
438 | |
---|
439 | ! ------------------------------------------------------------------ |
---|
440 | |
---|
441 | |
---|
442 | ! only one of myid and myids should be present |
---|
443 | if(present(myid) .and. present(myids)) then |
---|
444 | write(stderr,'(2a)') myname_, & |
---|
445 | 'MCTERROR: Must define myid or myids in MCTWord init' |
---|
446 | call die(myname_) |
---|
447 | endif |
---|
448 | |
---|
449 | if(.not.present(myid) .and. .not.present(myids)) then |
---|
450 | write(stderr,'(2a)') myname_, & |
---|
451 | 'MCTERROR: Must define one of myid or myids in MCTWord init' |
---|
452 | call die(myname_) |
---|
453 | endif |
---|
454 | |
---|
455 | if (present(myids)) then |
---|
456 | msize = size(myids) |
---|
457 | else |
---|
458 | msize = 1 |
---|
459 | endif |
---|
460 | |
---|
461 | allocate(mycomm1d(msize),myids1d(msize),stat=ier) |
---|
462 | if(ier/=0) call die(myname_,'non-root allocate(root_idGprocid)',ier) |
---|
463 | mycomm1d(:) = mycomm |
---|
464 | |
---|
465 | if (present(myids)) then |
---|
466 | myids1d(:) = myids(:) |
---|
467 | else |
---|
468 | myids1d(:) = myid |
---|
469 | endif |
---|
470 | |
---|
471 | call initm_(ncomps,globalcomm,mycomm1d,myids1d) |
---|
472 | |
---|
473 | deallocate(mycomm1d,myids1d) |
---|
474 | |
---|
475 | end subroutine initd_ |
---|
476 | |
---|
477 | !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
478 | ! Math and Computer Science Division, Argonne National Laboratory ! |
---|
479 | !BOP ------------------------------------------------------------------- |
---|
480 | ! |
---|
481 | ! !IROUTINE: initr_ - initialize MCTWorld from global root |
---|
482 | ! |
---|
483 | ! !DESCRIPTION: |
---|
484 | ! Initialize MCTWorld using information valid only on the global root. |
---|
485 | ! This is called by initm\_ but could also be called by the user |
---|
486 | ! for very complex model--processor geometries. |
---|
487 | ! |
---|
488 | ! !INTERFACE: |
---|
489 | |
---|
490 | subroutine initr_(ncomps,globalcomm,rnprocspid,ridGprocid) |
---|
491 | ! |
---|
492 | ! !USES: |
---|
493 | ! |
---|
494 | use m_mpif90 |
---|
495 | use m_die |
---|
496 | use m_stdio |
---|
497 | |
---|
498 | implicit none |
---|
499 | |
---|
500 | ! !INPUT PARAMETERS: |
---|
501 | |
---|
502 | integer, intent(in) :: ncomps ! total number of components |
---|
503 | integer, intent(in) :: globalcomm ! the global communicator |
---|
504 | integer, dimension(:),intent(in) :: rnprocspid ! number of processors for each component |
---|
505 | integer, dimension(:,:),intent(in) :: ridGprocid ! an array of size (1:ncomps) x (0:Gsize-1) |
---|
506 | ! which maps local ranks to global ranks |
---|
507 | ! it's actually 1:Gsize here |
---|
508 | |
---|
509 | ! !REVISION HISTORY: |
---|
510 | ! 22Jun01 - R. Jacob <jacob@mcs.anl.gov> - initial prototype |
---|
511 | !EOP ___________________________________________________________________ |
---|
512 | ! |
---|
513 | character(len=*),parameter :: myname_=myname//'::initr_' |
---|
514 | integer :: ier,Gsize,myGid,MCTcomm,i,j |
---|
515 | |
---|
516 | ! Check that ncomps is a legal value |
---|
517 | if(ncomps < 1) then |
---|
518 | call die(myname_, "argument ncomps can't less than one!",ncomps) |
---|
519 | endif |
---|
520 | |
---|
521 | ! determine overall size |
---|
522 | call MP_comm_size(globalcomm,Gsize,ier) |
---|
523 | if(ier /= 0) call MP_perr_die(myname_,'MP_comm_size()',ier) |
---|
524 | |
---|
525 | ! determine my rank in comm_world |
---|
526 | call MP_comm_rank(globalcomm,myGid,ier) |
---|
527 | if(ier /= 0) call MP_perr_die(myname_,'MP_comm_rank()',ier) |
---|
528 | |
---|
529 | ! create the MCT comm world |
---|
530 | call MP_comm_dup(globalcomm,MCTcomm,ier) |
---|
531 | if(ier /= 0) call MP_perr_die(myname_,'MP_comm_dup()',ier) |
---|
532 | |
---|
533 | allocate(ThisMCTWorld%nprocspid(ncomps),stat=ier) |
---|
534 | if(ier/=0) call die(myname_,'allocate(MCTWorld%nprocspid(:),...',ier) |
---|
535 | allocate(ThisMCTWorld%idGprocid(ncomps,0:Gsize-1),stat=ier) |
---|
536 | if(ier/=0) call die(myname_,'allocate(MCTWorld%nprocspid(:),...',ier) |
---|
537 | |
---|
538 | ! set the MCTWorld |
---|
539 | ThisMCTWorld%ncomps = ncomps |
---|
540 | ThisMCTWorld%MCT_comm = MCTcomm |
---|
541 | ThisMCTWorld%mygrank = myGid |
---|
542 | |
---|
543 | ! Now store the component ids in the World description and Broadcast |
---|
544 | if(myGid == 0) then |
---|
545 | ThisMCTWorld%nprocspid(1:ncomps) = rnprocspid(1:ncomps) |
---|
546 | ThisMCTWorld%idGprocid = ridGprocid |
---|
547 | endif |
---|
548 | |
---|
549 | call MPI_BCAST(ThisMCTWorld%nprocspid, ncomps, MP_INTEGER, 0, MCTcomm, ier) |
---|
550 | if(ier/=0) call MP_perr_die(myname_,'MPI_BCast nprocspid',ier) |
---|
551 | |
---|
552 | call MPI_BCAST(ThisMCTWorld%idGprocid, ncomps*Gsize,MP_INTEGER, 0,MCTcomm, ier) |
---|
553 | if(ier/=0) call MP_perr_die(myname_,'MPI_BCast Gprocids',ier) |
---|
554 | |
---|
555 | ! if(myGid==17) then |
---|
556 | ! do i=1,ThisMCTWorld%ncomps |
---|
557 | ! do j=1,ThisMCTWorld%nprocspid(i) |
---|
558 | ! write(*,*)'MCTK',myGid,i,j-1,ThisMCTWorld%idGprocid(i,j-1) |
---|
559 | ! enddo |
---|
560 | ! enddo |
---|
561 | ! endif |
---|
562 | |
---|
563 | end subroutine initr_ |
---|
564 | |
---|
565 | !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
566 | ! Math and Computer Science Division, Argonne National Laboratory ! |
---|
567 | !BOP ------------------------------------------------------------------- |
---|
568 | ! |
---|
569 | ! !IROUTINE: clean_ - Destroy a MCTWorld |
---|
570 | ! |
---|
571 | ! !DESCRIPTION: |
---|
572 | ! This routine deallocates the arrays of {\tt ThisMCTWorld} |
---|
573 | ! It also zeros out the integer components. |
---|
574 | ! |
---|
575 | ! !INTERFACE: |
---|
576 | |
---|
577 | subroutine clean_() |
---|
578 | ! |
---|
579 | ! !USES: |
---|
580 | ! |
---|
581 | use m_mpif90 |
---|
582 | use m_die |
---|
583 | |
---|
584 | implicit none |
---|
585 | |
---|
586 | ! !REVISION HISTORY: |
---|
587 | ! 19Jan01 - R. Jacob <jacob@mcs.anl.gov> - initial prototype |
---|
588 | ! 08Feb01 - R. Jacob <jacob@mcs.anl.gov> - clean the new |
---|
589 | ! mygrank and mylrank |
---|
590 | ! 20Apr01 - R. Jacob <jacob@mcs.anl.gov> - remove allids from |
---|
591 | ! MCTWorld datatype. Not needed because component |
---|
592 | ! ids are always from 1 to number-of-components. |
---|
593 | ! 07Jun01 - R. Jacob <jacob@mcs.anl.gov> - remove myid,mynprocs |
---|
594 | ! and mylrank. |
---|
595 | !EOP ___________________________________________________________________ |
---|
596 | |
---|
597 | character(len=*),parameter :: myname_=myname//'::clean_' |
---|
598 | integer :: ier |
---|
599 | |
---|
600 | deallocate(ThisMCTWorld%nprocspid,ThisMCTWorld%idGprocid,stat=ier) |
---|
601 | if(ier /= 0) call warn(myname_,'deallocate(MCTW,...)',ier) |
---|
602 | |
---|
603 | call MP_comm_free(ThisMCTWorld%MCT_comm, ier) |
---|
604 | if(ier /= 0) call MP_perr_die(myname_,'MP_comm_free()',ier) |
---|
605 | |
---|
606 | ThisMCTWorld%ncomps = 0 |
---|
607 | ThisMCTWorld%mygrank = 0 |
---|
608 | |
---|
609 | end subroutine clean_ |
---|
610 | |
---|
611 | !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
612 | ! Math and Computer Science Division, Argonne National Laboratory ! |
---|
613 | !BOP ------------------------------------------------------------------- |
---|
614 | ! |
---|
615 | ! !IROUTINE: NumComponents_ - Determine number of components in World. |
---|
616 | ! |
---|
617 | ! !DESCRIPTION: |
---|
618 | ! The function {\tt NumComponents\_} takes an input {\tt MCTWorld} |
---|
619 | ! argument {\tt World}, and returns the number of component models |
---|
620 | ! present. |
---|
621 | ! |
---|
622 | ! !INTERFACE: |
---|
623 | |
---|
624 | integer function NumComponents_(World) |
---|
625 | ! |
---|
626 | ! !USES: |
---|
627 | ! |
---|
628 | use m_die |
---|
629 | use m_stdio |
---|
630 | |
---|
631 | implicit none |
---|
632 | |
---|
633 | ! !INPUT PARAMETERS: |
---|
634 | |
---|
635 | type(MCTWorld), intent(in) :: World |
---|
636 | |
---|
637 | ! !REVISION HISTORY: |
---|
638 | ! 05Feb01 - J. Larson <larson@mcs.anl.gov> - initial version |
---|
639 | !EOP ___________________________________________________________________ |
---|
640 | ! |
---|
641 | character(len=*),parameter :: myname_=myname//'::NumComponents_' |
---|
642 | |
---|
643 | integer :: ncomps |
---|
644 | |
---|
645 | ncomps = World%ncomps |
---|
646 | |
---|
647 | if(ncomps <= 0) then |
---|
648 | write(stderr,'(2a,1i3)') myname,":: invalid no. of components = ",ncomps |
---|
649 | call die(myname_,'ncomps = ',ncomps) |
---|
650 | endif |
---|
651 | |
---|
652 | NumComponents_ = ncomps |
---|
653 | |
---|
654 | end function NumComponents_ |
---|
655 | |
---|
656 | !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
657 | ! Math and Computer Science Division, Argonne National Laboratory ! |
---|
658 | !BOP ------------------------------------------------------------------- |
---|
659 | ! |
---|
660 | ! !IROUTINE: ComponentNumProcs_ - Number of processes a component owns. |
---|
661 | ! |
---|
662 | ! !DESCRIPTION: |
---|
663 | ! The function {\tt ComponentNumProcs\_} takes an input {\tt MCTWorld} |
---|
664 | ! argument {\tt World}, and a component ID {\tt comp\_id}, and returns |
---|
665 | ! the number of processes owned by that component. |
---|
666 | ! |
---|
667 | ! !INTERFACE: |
---|
668 | |
---|
669 | integer function ComponentNumProcs_(World, comp_id) |
---|
670 | ! |
---|
671 | ! !USES: |
---|
672 | ! |
---|
673 | use m_die |
---|
674 | use m_stdio |
---|
675 | |
---|
676 | implicit none |
---|
677 | |
---|
678 | ! !INPUT PARAMETERS: |
---|
679 | type(MCTWorld), intent(in) :: World |
---|
680 | integer, intent(in) :: comp_id |
---|
681 | |
---|
682 | ! !REVISION HISTORY: |
---|
683 | ! 05Feb01 - J. Larson <larson@mcs.anl.gov> - initial version |
---|
684 | ! 07Jun01 - R. Jacob <jacob@mcs.anl.gov> - modify to use |
---|
685 | ! nprocspid and comp_id instead of World%mynprocs |
---|
686 | !EOP ___________________________________________________________________ |
---|
687 | ! |
---|
688 | character(len=*),parameter :: myname_=myname//'::ComponentNumPros_' |
---|
689 | |
---|
690 | integer :: mynprocs |
---|
691 | |
---|
692 | mynprocs = World%nprocspid(comp_id) |
---|
693 | |
---|
694 | if(mynprocs <= 0) then |
---|
695 | write(stderr,'(2a,1i6)') myname,":: invalid no. of processes = ",mynprocs |
---|
696 | call die(myname_,'Number of processes = ',mynprocs) |
---|
697 | endif |
---|
698 | |
---|
699 | ComponentNumProcs_ = mynprocs |
---|
700 | |
---|
701 | end function ComponentNumProcs_ |
---|
702 | |
---|
703 | !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
704 | ! Math and Computer Science Division, Argonne National Laboratory ! |
---|
705 | !BOP ------------------------------------------------------------------- |
---|
706 | ! |
---|
707 | ! !IROUTINE: ComponentToWorldRank_ - Determine rank on COMM_WORLD. |
---|
708 | ! |
---|
709 | ! !DESCRIPTION: |
---|
710 | ! The function {\tt ComponentToWorldRank\_} takes an input component ID |
---|
711 | ! {\tt comp\_id} and input rank on that component communicator |
---|
712 | ! {\tt comp\_rank}, and returns the rank of that process on the world |
---|
713 | ! communicator of {\tt MCTWorld}. |
---|
714 | ! |
---|
715 | ! !INTERFACE: |
---|
716 | |
---|
717 | integer function ComponentToWorldRank_(comp_rank, comp_id, World) |
---|
718 | ! |
---|
719 | ! !USES: |
---|
720 | ! |
---|
721 | use m_die |
---|
722 | use m_stdio |
---|
723 | |
---|
724 | implicit none |
---|
725 | |
---|
726 | ! !INPUT PARAMETERS: |
---|
727 | integer, intent(in) :: comp_rank ! process rank on the communicator |
---|
728 | ! associated with comp_id |
---|
729 | integer, intent(in) :: comp_id ! component id |
---|
730 | type(MCTWorld), intent(in) :: World ! World |
---|
731 | |
---|
732 | |
---|
733 | ! !REVISION HISTORY: |
---|
734 | ! 05Feb01 - J. Larson <larson@mcs.anl.gov> - initial version |
---|
735 | ! 14Jul02 - E. Ong <eong@mcs.anl.gov> - made argument checking required |
---|
736 | !EOP ___________________________________________________________________ |
---|
737 | ! |
---|
738 | character(len=*),parameter :: myname_=myname//'::ComponentToWorldRank_' |
---|
739 | |
---|
740 | logical :: valid |
---|
741 | integer :: n, world_rank |
---|
742 | |
---|
743 | |
---|
744 | ! Do we want the potentially time-consuming argument checks? |
---|
745 | ! The first time we use this function during execution on a |
---|
746 | ! given set of components and component ranks, we will. In |
---|
747 | ! later invocations, these argument checks are probably not |
---|
748 | ! necessary (unless one alters MCTWorld), and impose a cost |
---|
749 | ! one may wish to avoid. |
---|
750 | |
---|
751 | ! These checks are just conditional statements and are |
---|
752 | ! not particularly time-consuming. It's better to be safe |
---|
753 | ! than sorry. -EONG |
---|
754 | |
---|
755 | |
---|
756 | ! Check argument comp_id for validity--assume initially it is not... |
---|
757 | |
---|
758 | valid = .false. |
---|
759 | n = 0 |
---|
760 | |
---|
761 | if((comp_id <= World%ncomps) .and. & |
---|
762 | (comp_id > 0)) then |
---|
763 | valid = .true. |
---|
764 | endif |
---|
765 | |
---|
766 | if(.not. valid) then |
---|
767 | write(stderr,'(2a,1i7)') myname,":: invalid component id no. = ",& |
---|
768 | comp_id |
---|
769 | call die(myname_,'invalid comp_id = ',comp_id) |
---|
770 | endif |
---|
771 | |
---|
772 | ! Check argument comp_rank for validity on the communicator associated |
---|
773 | ! with comp_id. Assume initialy it is invalid. |
---|
774 | |
---|
775 | valid = .false. |
---|
776 | |
---|
777 | if((0 <= comp_rank) .or. & |
---|
778 | (comp_rank < ComponentNumProcs_(World, comp_id))) then |
---|
779 | valid = .true. |
---|
780 | endif |
---|
781 | |
---|
782 | if(.not. valid) then |
---|
783 | write(stderr,'(2a,1i5,1a,1i2)') myname, & |
---|
784 | ":: invalid process ID. = ", & |
---|
785 | comp_rank, "on component ",comp_id |
---|
786 | call die(myname_,'invalid comp_rank = ',comp_rank) |
---|
787 | endif |
---|
788 | |
---|
789 | |
---|
790 | ! If we have reached this point, the input data are valid. |
---|
791 | ! Return the global rank for comp_rank on component comp_id |
---|
792 | |
---|
793 | world_rank = World%idGprocid(comp_id, comp_rank) |
---|
794 | |
---|
795 | if(world_rank < 0) then |
---|
796 | write(stderr,'(2a,1i6)') myname,":: negative world rank = ",world_rank |
---|
797 | call die(myname_,'negative world rank = ',world_rank) |
---|
798 | endif |
---|
799 | |
---|
800 | ComponentToWorldRank_ = world_rank |
---|
801 | |
---|
802 | end function ComponentToWorldRank_ |
---|
803 | |
---|
804 | !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
805 | ! Math and Computer Science Division, Argonne National Laboratory ! |
---|
806 | !BOP ------------------------------------------------------------------- |
---|
807 | ! |
---|
808 | ! !IROUTINE: ComponentRootRank_ - Rank of component root on COMM_WORLD. |
---|
809 | ! |
---|
810 | ! !DESCRIPTION: |
---|
811 | ! The function {\tt ComponentRootRank\_} takes an input component ID |
---|
812 | ! {\tt comp\_id} and input {\tt MCTWorld} variable {\tt World}, and |
---|
813 | ! returns the global rank of the root of this component. |
---|
814 | ! |
---|
815 | ! !INTERFACE: |
---|
816 | |
---|
817 | integer function ComponentRootRank_(comp_id, World) |
---|
818 | ! |
---|
819 | ! !USES: |
---|
820 | ! |
---|
821 | use m_die |
---|
822 | use m_stdio |
---|
823 | |
---|
824 | implicit none |
---|
825 | |
---|
826 | ! !INPUT PARAMETERS: |
---|
827 | integer, intent(in) :: comp_id ! component id |
---|
828 | type(MCTWorld), intent(in) :: World ! World |
---|
829 | |
---|
830 | ! !REVISION HISTORY: |
---|
831 | ! 05Feb01 - J. Larson <larson@mcs.anl.gov> - initial version |
---|
832 | ! 14Jul02 - E. Ong <eong@mcs.anl.gov> - made argument checking required |
---|
833 | !EOP ___________________________________________________________________ |
---|
834 | ! |
---|
835 | character(len=*),parameter :: myname_=myname//'::ComponentRootRank_' |
---|
836 | |
---|
837 | integer :: world_comp_root |
---|
838 | |
---|
839 | ! Call ComponentToWorldRank_ assuming the root on a remote component |
---|
840 | ! has rank zero on the communicator associated with that component. |
---|
841 | |
---|
842 | world_comp_root = ComponentToWorldRank_(0, comp_id, World) |
---|
843 | |
---|
844 | if(world_comp_root < 0) then |
---|
845 | write(stderr,'(2a,1i6)') myname,":: negative world rank = ",& |
---|
846 | world_comp_root |
---|
847 | call die(myname_,'invalid root id = ',world_comp_root) |
---|
848 | endif |
---|
849 | |
---|
850 | ComponentRootRank_ = world_comp_root |
---|
851 | |
---|
852 | end function ComponentRootRank_ |
---|
853 | |
---|
854 | !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
855 | ! Math and Computer Science Division, Argonne National Laboratory ! |
---|
856 | !BOP ------------------------------------------------------------------- |
---|
857 | ! |
---|
858 | ! !IROUTINE: printnp_ - Print number of procs for a component id. |
---|
859 | ! |
---|
860 | ! !DESCRIPTION: |
---|
861 | ! Print out number of MPI processes for the givin component id. |
---|
862 | ! |
---|
863 | ! !INTERFACE: |
---|
864 | |
---|
865 | subroutine printnp_(compid,lun) |
---|
866 | ! |
---|
867 | ! !USES: |
---|
868 | ! |
---|
869 | use m_die |
---|
870 | use m_mpif90 |
---|
871 | |
---|
872 | implicit none |
---|
873 | |
---|
874 | !INPUT/OUTPUT PARAMETERS: |
---|
875 | integer, intent(in) :: compid |
---|
876 | integer, intent(in) :: lun |
---|
877 | |
---|
878 | ! !REVISION HISTORY: |
---|
879 | ! 06Jul12 - R. Jacob <jacob@mcs.anl.gov> - initial version |
---|
880 | !EOP ___________________________________________________________________ |
---|
881 | |
---|
882 | |
---|
883 | integer ier |
---|
884 | character(len=*),parameter :: myname_=myname//'::printnp_' |
---|
885 | |
---|
886 | write(lun,*) ThisMCTWorld%nprocspid(compid) |
---|
887 | |
---|
888 | end subroutine printnp_ |
---|
889 | |
---|
890 | |
---|
891 | end module m_MCTWorld |
---|