| 1 | |
|---|
| 2 | SUBROUTINE lbc_lnk_neicoll_/**/PRECISION( cdname, ptab, cd_nat, psgn, kfld, kfillmode, pfillval, khls, lsend, lrecv, ld4only ) |
|---|
| 3 | CHARACTER(len=*) , INTENT(in ) :: cdname ! name of the calling subroutine |
|---|
| 4 | TYPE(PTR_4d_/**/PRECISION), DIMENSION(:), INTENT(inout) :: ptab ! pointer of arrays on which apply the b.c. |
|---|
| 5 | CHARACTER(len=1), DIMENSION(:), INTENT(in ) :: cd_nat ! nature of array grid-points |
|---|
| 6 | REAL(PRECISION), DIMENSION(:), INTENT(in ) :: psgn ! sign used across the north fold boundary |
|---|
| 7 | INTEGER , INTENT(in ) :: kfld ! number of pt3d arrays |
|---|
| 8 | INTEGER , OPTIONAL, INTENT(in ) :: kfillmode ! filling method for halo over land (default = constant) |
|---|
| 9 | REAL(PRECISION), OPTIONAL, INTENT(in ) :: pfillval ! background value (used at closed boundaries) |
|---|
| 10 | INTEGER , OPTIONAL, INTENT(in ) :: khls ! halo size, default = nn_hls |
|---|
| 11 | LOGICAL, DIMENSION(4),OPTIONAL, INTENT(in ) :: lsend, lrecv ! communication with other 4 proc |
|---|
| 12 | LOGICAL, OPTIONAL, INTENT(in ) :: ld4only ! if .T., do only 4-neighbour comm (ignore corners) |
|---|
| 13 | ! |
|---|
| 14 | INTEGER :: ji, jj, jk , jl, jf, jn ! dummy loop indices |
|---|
| 15 | INTEGER :: ipi, ipj, ipk, ipl, ipf ! dimension of the input array |
|---|
| 16 | INTEGER :: ip0i, ip1i, im0i, im1i |
|---|
| 17 | INTEGER :: ip0j, ip1j, im0j, im1j |
|---|
| 18 | INTEGER :: ishti, ishtj, ishti2, ishtj2 |
|---|
| 19 | INTEGER :: iszS, iszR |
|---|
| 20 | INTEGER :: ierr |
|---|
| 21 | INTEGER :: ihls, idx |
|---|
| 22 | INTEGER :: impi_nc |
|---|
| 23 | INTEGER :: ifill_nfd |
|---|
| 24 | INTEGER, DIMENSION(4) :: iwewe, issnn |
|---|
| 25 | INTEGER, DIMENSION(8) :: isizei, ishtSi, ishtRi, ishtPi |
|---|
| 26 | INTEGER, DIMENSION(8) :: isizej, ishtSj, ishtRj, ishtPj |
|---|
| 27 | INTEGER, DIMENSION(8) :: ifill, iszall |
|---|
| 28 | INTEGER, DIMENSION(8) :: jnf |
|---|
| 29 | INTEGER, DIMENSION(:), ALLOCATABLE :: iScnt, iRcnt ! number of elements to be sent/received |
|---|
| 30 | INTEGER, DIMENSION(:), ALLOCATABLE :: iSdpl, iRdpl ! displacement in halos arrays |
|---|
| 31 | LOGICAL, DIMENSION(8) :: llsend, llrecv |
|---|
| 32 | REAL(PRECISION) :: zland |
|---|
| 33 | LOGICAL :: ll4only ! default: 8 neighbourgs |
|---|
| 34 | !!---------------------------------------------------------------------- |
|---|
| 35 | ! |
|---|
| 36 | ! ----------------------------------------- ! |
|---|
| 37 | ! 1. local variables initialization ! |
|---|
| 38 | ! ----------------------------------------- ! |
|---|
| 39 | ! |
|---|
| 40 | ipi = SIZE(ptab(1)%pt4d,1) |
|---|
| 41 | ipj = SIZE(ptab(1)%pt4d,2) |
|---|
| 42 | ipk = SIZE(ptab(1)%pt4d,3) |
|---|
| 43 | ipl = SIZE(ptab(1)%pt4d,4) |
|---|
| 44 | ipf = kfld |
|---|
| 45 | ! |
|---|
| 46 | IF( narea == 1 .AND. numcom == -1 ) CALL mpp_report( cdname, ipk, ipl, ipf, ld_lbc = .TRUE. ) |
|---|
| 47 | ! |
|---|
| 48 | ! take care of optional parameters |
|---|
| 49 | ! |
|---|
| 50 | ihls = nn_hls ! default definition |
|---|
| 51 | IF( PRESENT( khls ) ) ihls = khls |
|---|
| 52 | IF( ihls > n_hlsmax ) THEN |
|---|
| 53 | WRITE(ctmp1,*) TRIM(cdname), ' is calling lbc_lnk with khls > n_hlsmax : ', khls, '>', n_hlsmax |
|---|
| 54 | CALL ctl_stop( 'STOP', ctmp1 ) |
|---|
| 55 | ENDIF |
|---|
| 56 | IF( ipi /= Ni_0+2*ihls ) THEN |
|---|
| 57 | WRITE(ctmp1,*) TRIM(cdname), ' is calling lbc_lnk with an input array which does not match ihls along i: ', ipi, ihls, Ni_0 |
|---|
| 58 | CALL ctl_stop( 'STOP', ctmp1 ) |
|---|
| 59 | ENDIF |
|---|
| 60 | IF( ipj /= Nj_0+2*ihls ) THEN |
|---|
| 61 | WRITE(ctmp1,*) TRIM(cdname), ' is calling lbc_lnk with an input array which does not match ihls along j:', ipj, ihls , Nj_0 |
|---|
| 62 | CALL ctl_stop( 'STOP', ctmp1 ) |
|---|
| 63 | ENDIF |
|---|
| 64 | ! |
|---|
| 65 | ll4only = .FALSE. ! default definition |
|---|
| 66 | IF( PRESENT(ld4only) ) ll4only = ld4only |
|---|
| 67 | ! |
|---|
| 68 | impi_nc = mpi_nc_com8(ihls) ! default |
|---|
| 69 | IF( ll4only ) impi_nc = mpi_nc_com4(ihls) |
|---|
| 70 | ! |
|---|
| 71 | zland = 0._wp ! land filling value: zero by default |
|---|
| 72 | IF( PRESENT( pfillval ) ) zland = pfillval ! set land value |
|---|
| 73 | ! |
|---|
| 74 | ! define llsend and llrecv: logicals which say if mpi-neibourgs for send or receive exist or not. |
|---|
| 75 | IF ( PRESENT(lsend) .AND. PRESENT(lrecv) ) THEN ! localy defined neighbourgs |
|---|
| 76 | CALL ctl_stop( 'STOP', 'mpp_nc_generic+BDY not yet implemented') |
|---|
| 77 | !!$ ---> llsend(:) = lsend(:) ; llrecv(:) = lrecv(:) ??? |
|---|
| 78 | ELSE IF( PRESENT(lsend) .OR. PRESENT(lrecv) ) THEN |
|---|
| 79 | WRITE(ctmp1,*) TRIM(cdname), ' is calling lbc_lnk with only one of the two arguments lsend or lrecv' |
|---|
| 80 | CALL ctl_stop( 'STOP', ctmp1 ) |
|---|
| 81 | ELSE ! default neighbours |
|---|
| 82 | llsend(:) = mpiSnei(ihls,:) >= 0 |
|---|
| 83 | IF( ll4only ) llsend(5:8) = .FALSE. ! exclude corners |
|---|
| 84 | llrecv(:) = mpiRnei(ihls,:) >= 0 |
|---|
| 85 | IF( ll4only ) llrecv(5:8) = .FALSE. ! exclude corners |
|---|
| 86 | ENDIF |
|---|
| 87 | ! |
|---|
| 88 | ! define ifill: which method should be used to fill each parts (sides+corners) of the halos |
|---|
| 89 | ! default definition |
|---|
| 90 | DO jn = 1, 8 |
|---|
| 91 | IF( llrecv(jn) ) THEN ; ifill(jn) = jpfillmpi ! with an mpi communication |
|---|
| 92 | ELSEIF( l_SelfPerio(jn) ) THEN ; ifill(jn) = jpfillperio ! with self-periodicity |
|---|
| 93 | ELSEIF( PRESENT(kfillmode) ) THEN ; ifill(jn) = kfillmode ! localy defined |
|---|
| 94 | ELSE ; ifill(jn) = jpfillcst ! constant value (zland) |
|---|
| 95 | ENDIF |
|---|
| 96 | END DO |
|---|
| 97 | ! take care of "indirect self-periodicity" for the corners |
|---|
| 98 | DO jn = 5, 8 |
|---|
| 99 | IF(.NOT.l_SelfPerio(jn) .AND. l_SelfPerio(jpwe)) ifill(jn) = jpfillnothing ! no bi-perio but ew-perio: do corners later |
|---|
| 100 | IF(.NOT.l_SelfPerio(jn) .AND. l_SelfPerio(jpso)) ifill(jn) = jpfillnothing ! no bi-perio but ns-perio: do corners later |
|---|
| 101 | END DO |
|---|
| 102 | ! north fold treatment |
|---|
| 103 | IF( l_IdoNFold ) THEN |
|---|
| 104 | ifill_nfd = ifill(jpno) ! if we are here, this means llrecv(jpno) = .false. and l_SelfPerio(jpno) = .false. |
|---|
| 105 | ifill( (/jpno/) ) = jpfillnothing ! we do north fold -> do nothing for northern halo |
|---|
| 106 | ENDIF |
|---|
| 107 | |
|---|
| 108 | ! We first define the localization and size of the parts of the array that will be sent (s), received (r) |
|---|
| 109 | ! or used for periodocity (p). The localization is defined as "the bottom left corner - 1" in i and j directions. |
|---|
| 110 | ! This is a shift that will be applied later in the do loops to pick-up the appropriate part of the array |
|---|
| 111 | ! |
|---|
| 112 | ! all definitions bellow do not refer to N[ij][se]0 so we can use it with any local value of ihls |
|---|
| 113 | ! ! ________________________ |
|---|
| 114 | ip0i = 0 ! im0j = inner |__|________________|__| |
|---|
| 115 | ip1i = ihls ! im1j = inner - halo | |__|__________|__| | |
|---|
| 116 | im1i = ipi-2*ihls ! | | | | | | |
|---|
| 117 | im0i = ipi - ihls ! | | | | | | |
|---|
| 118 | ip0j = 0 ! | | | | | | |
|---|
| 119 | ip1j = ihls ! | |__|__________|__| | |
|---|
| 120 | im1j = ipj-2*ihls ! ip1j = halo |__|__|__________|__|__| |
|---|
| 121 | im0j = ipj - ihls ! ip0j = 0 |__|________________|__| |
|---|
| 122 | ! ! ip0i ip1i im1i im0i |
|---|
| 123 | ! |
|---|
| 124 | iwewe(:) = (/ jpwe,jpea,jpwe,jpea /) ; issnn(:) = (/ jpso,jpso,jpno,jpno /) |
|---|
| 125 | ! sides: west east south north ; corners: so-we, so-ea, no-we, no-ea |
|---|
| 126 | isizei(1:4) = (/ ihls, ihls, Ni_0, Ni_0 /) ; isizei(5:8) = ihls ! i- count |
|---|
| 127 | isizej(1:4) = (/ Nj_0, Nj_0, ihls, ihls /) ; isizej(5:8) = ihls ! j- count |
|---|
| 128 | ishtSi(1:4) = (/ ip1i, im1i, ip1i, ip1i /) ; ishtSi(5:8) = ishtSi( iwewe ) ! i- shift send data |
|---|
| 129 | ishtSj(1:4) = (/ ip1j, ip1j, ip1j, im1j /) ; ishtSj(5:8) = ishtSj( issnn ) ! j- shift send data |
|---|
| 130 | ishtRi(1:4) = (/ ip0i, im0i, ip1i, ip1i /) ; ishtRi(5:8) = ishtRi( iwewe ) ! i- shift received data location |
|---|
| 131 | ishtRj(1:4) = (/ ip1j, ip1j, ip0j, im0j /) ; ishtRj(5:8) = ishtRj( issnn ) ! j- shift received data location |
|---|
| 132 | ishtPi(1:4) = (/ im1i, ip1i, ip1i, ip1i /) ; ishtPi(5:8) = ishtPi( iwewe ) ! i- shift data used for periodicity |
|---|
| 133 | ishtPj(1:4) = (/ ip1j, ip1j, im1j, ip1j /) ; ishtPj(5:8) = ishtPj( issnn ) ! j- shift data used for periodicity |
|---|
| 134 | ! |
|---|
| 135 | ! -------------------------------- ! |
|---|
| 136 | ! 2. Prepare MPI exchanges ! |
|---|
| 137 | ! -------------------------------- ! |
|---|
| 138 | ! |
|---|
| 139 | ! Allocate local temporary arrays to be sent/received. |
|---|
| 140 | iszS = COUNT( llsend ) |
|---|
| 141 | iszR = COUNT( llrecv ) |
|---|
| 142 | ALLOCATE( iScnt(iszS), iRcnt(iszR), iSdpl(iszS), iRdpl(iszR) ) ! ok if iszS = 0 or iszR = 0 |
|---|
| 143 | iszall(:) = isizei(:) * isizej(:) * ipk * ipl * ipf |
|---|
| 144 | iScnt(:) = PACK( iszall, mask = llsend ) ! ok if mask = .false. |
|---|
| 145 | iRcnt(:) = PACK( iszall, mask = llrecv ) |
|---|
| 146 | iSdpl(1) = 0 |
|---|
| 147 | DO jn = 2,iszS |
|---|
| 148 | iSdpl(jn) = iSdpl(jn-1) + iScnt(jn-1) ! with _alltoallv: in units of sendtype |
|---|
| 149 | END DO |
|---|
| 150 | iRdpl(1) = 0 |
|---|
| 151 | DO jn = 2,iszR |
|---|
| 152 | iRdpl(jn) = iRdpl(jn-1) + iRcnt(jn-1) ! with _alltoallv: in units of sendtype |
|---|
| 153 | END DO |
|---|
| 154 | |
|---|
| 155 | ! Allocate buffer arrays to be sent/received if needed |
|---|
| 156 | iszS = SUM(iszall, mask = llsend) ! send buffer size |
|---|
| 157 | IF( ALLOCATED(BUFFSND) ) THEN |
|---|
| 158 | IF( SIZE(BUFFSND) < iszS ) DEALLOCATE(BUFFSND) ! send buffer is too small |
|---|
| 159 | ENDIF |
|---|
| 160 | IF( .NOT. ALLOCATED(BUFFSND) ) ALLOCATE( BUFFSND(iszS) ) |
|---|
| 161 | iszR = SUM(iszall, mask = llrecv) ! recv buffer size |
|---|
| 162 | IF( ALLOCATED(BUFFRCV) ) THEN |
|---|
| 163 | IF( SIZE(BUFFRCV) < iszR ) DEALLOCATE(BUFFRCV) ! recv buffer is too small |
|---|
| 164 | ENDIF |
|---|
| 165 | IF( .NOT. ALLOCATED(BUFFRCV) ) ALLOCATE( BUFFRCV(iszR) ) |
|---|
| 166 | |
|---|
| 167 | ! fill sending buffer with ptab(jf)%pt4d |
|---|
| 168 | idx = 1 |
|---|
| 169 | DO jn = 1, 8 |
|---|
| 170 | IF( llsend(jn) ) THEN |
|---|
| 171 | ishti = ishtSi(jn) |
|---|
| 172 | ishtj = ishtSj(jn) |
|---|
| 173 | DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1,isizej(jn) ; DO ji = 1,isizei(jn) |
|---|
| 174 | BUFFSND(idx) = ptab(jf)%pt4d(ishti+ji,ishtj+jj,jk,jl) |
|---|
| 175 | idx = idx + 1 |
|---|
| 176 | END DO ; END DO ; END DO ; END DO ; END DO |
|---|
| 177 | ENDIF |
|---|
| 178 | END DO |
|---|
| 179 | ! |
|---|
| 180 | ! ------------------------------------------------ ! |
|---|
| 181 | ! 3. Do all MPI exchanges in 1 unique call ! |
|---|
| 182 | ! ------------------------------------------------ ! |
|---|
| 183 | ! |
|---|
| 184 | IF( ln_timing ) CALL tic_tac(.TRUE.) |
|---|
| 185 | CALL mpi_neighbor_alltoallv (BUFFSND, iScnt, iSdpl, MPI_TYPE, BUFFRCV, iRcnt, iRdpl, MPI_TYPE, impi_nc, ierr) |
|---|
| 186 | IF( ln_timing ) CALL tic_tac(.FALSE.) |
|---|
| 187 | ! |
|---|
| 188 | ! ------------------------- ! |
|---|
| 189 | ! 4. Fill all halos ! |
|---|
| 190 | ! ------------------------- ! |
|---|
| 191 | ! |
|---|
| 192 | idx = 1 |
|---|
| 193 | ! MPI3 bug fix when domain decomposition has 2 columns/rows |
|---|
| 194 | IF (jpni .eq. 2) THEN |
|---|
| 195 | IF (jpnj .eq. 2) THEN |
|---|
| 196 | jnf(1:8) = (/ 2, 1, 4, 3, 8, 7, 6, 5 /) |
|---|
| 197 | ELSE |
|---|
| 198 | jnf(1:8) = (/ 2, 1, 3, 4, 6, 5, 8, 7 /) |
|---|
| 199 | ENDIF |
|---|
| 200 | ELSE |
|---|
| 201 | IF (jpnj .eq. 2) THEN |
|---|
| 202 | jnf(1:8) = (/ 1, 2, 4, 3, 7, 8, 5, 6 /) |
|---|
| 203 | ELSE |
|---|
| 204 | jnf(1:8) = (/ 1, 2, 3, 4, 5, 6, 7, 8 /) |
|---|
| 205 | ENDIF |
|---|
| 206 | ENDIF |
|---|
| 207 | |
|---|
| 208 | DO jn = 1, 8 |
|---|
| 209 | ishti = ishtRi(jnf(jn)) |
|---|
| 210 | ishtj = ishtRj(jnf(jn)) |
|---|
| 211 | SELECT CASE ( ifill(jnf(jn)) ) |
|---|
| 212 | CASE ( jpfillnothing ) ! no filling |
|---|
| 213 | CASE ( jpfillmpi ) ! fill with data received by MPI |
|---|
| 214 | DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1,isizej(jnf(jn)) ; DO ji = 1,isizei(jnf(jn)) |
|---|
| 215 | ptab(jf)%pt4d(ishti+ji,ishtj+jj,jk,jl) = BUFFRCV(idx) |
|---|
| 216 | idx = idx + 1 |
|---|
| 217 | END DO ; END DO ; END DO ; END DO ; END DO |
|---|
| 218 | CASE ( jpfillperio ) ! use periodicity |
|---|
| 219 | ishti2 = ishtPi(jnf(jn)) |
|---|
| 220 | ishtj2 = ishtPj(jnf(jn)) |
|---|
| 221 | DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1,isizej(jnf(jn)) ; DO ji = 1,isizei(jnf(jn)) |
|---|
| 222 | ptab(jf)%pt4d(ishti+ji,ishtj+jj,jk,jl) = ptab(jf)%pt4d(ishti2+ji,ishtj2+jj,jk,jl) |
|---|
| 223 | END DO ; END DO ; END DO ; END DO ; END DO |
|---|
| 224 | CASE ( jpfillcopy ) ! filling with inner domain values |
|---|
| 225 | ishti2 = ishtSi(jnf(jn)) |
|---|
| 226 | ishtj2 = ishtSj(jnf(jn)) |
|---|
| 227 | DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1,isizej(jnf(jn)) ; DO ji = 1,isizei(jnf(jn)) |
|---|
| 228 | ptab(jf)%pt4d(ishti+ji,ishtj+jj,jk,jl) = ptab(jf)%pt4d(ishti2+ji,ishtj2+jj,jk,jl) |
|---|
| 229 | END DO ; END DO ; END DO ; END DO ; END DO |
|---|
| 230 | CASE ( jpfillcst ) ! filling with constant value |
|---|
| 231 | DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1,isizej(jnf(jn)) ; DO ji = 1,isizei(jnf(jn)) |
|---|
| 232 | ptab(jf)%pt4d(ishti+ji,ishtj+jj,jk,jl) = zland |
|---|
| 233 | END DO ; END DO ; END DO ; END DO ; END DO |
|---|
| 234 | END SELECT |
|---|
| 235 | END DO |
|---|
| 236 | |
|---|
| 237 | DEALLOCATE( iScnt, iRcnt, iSdpl, iRdpl ) |
|---|
| 238 | IF( iszS > jpi*jpj ) DEALLOCATE(BUFFSND) ! blocking Send -> can directly deallocate |
|---|
| 239 | IF( iszR > jpi*jpj ) DEALLOCATE(BUFFRCV) ! blocking Recv -> can directly deallocate |
|---|
| 240 | |
|---|
| 241 | ! potential "indirect self-periodicity" for the corners |
|---|
| 242 | DO jn = 5, 8 |
|---|
| 243 | IF( .NOT. l_SelfPerio(jn) .AND. l_SelfPerio(jpwe) ) THEN ! no bi-perio but ew-perio: corners indirect definition |
|---|
| 244 | ishti = ishtRi(jn) |
|---|
| 245 | ishtj = ishtRj(jn) |
|---|
| 246 | ishti2 = ishtPi(jn) ! use i- shift periodicity |
|---|
| 247 | ishtj2 = ishtRj(jn) ! use j- shift recv location: use ew-perio -> ok as filling of the south and north halos now done |
|---|
| 248 | DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1,isizej(jn) ; DO ji = 1,isizei(jn) |
|---|
| 249 | ptab(jf)%pt4d(ishti+ji,ishtj+jj,jk,jl) = ptab(jf)%pt4d(ishti2+ji,ishtj2+jj,jk,jl) |
|---|
| 250 | END DO ; END DO ; END DO ; END DO ; END DO |
|---|
| 251 | ENDIF |
|---|
| 252 | IF( .NOT. l_SelfPerio(jn) .AND. l_SelfPerio(jpso) ) THEN ! no bi-perio but ns-perio: corners indirect definition |
|---|
| 253 | ishti = ishtRi(jn) |
|---|
| 254 | ishtj = ishtRj(jn) |
|---|
| 255 | ishti2 = ishtRi(jn) ! use i- shift recv location: use ns-perio -> ok as filling of the west and east halos now done |
|---|
| 256 | ishtj2 = ishtPj(jn) ! use j- shift periodicity |
|---|
| 257 | DO jf = 1, ipf ; DO jl = 1, ipl ; DO jk = 1, ipk ; DO jj = 1,isizej(jn) ; DO ji = 1,isizei(jn) |
|---|
| 258 | ptab(jf)%pt4d(ishti+ji,ishtj+jj,jk,jl) = ptab(jf)%pt4d(ishti2+ji,ishtj2+jj,jk,jl) |
|---|
| 259 | END DO ; END DO ; END DO ; END DO ; END DO |
|---|
| 260 | ENDIF |
|---|
| 261 | END DO |
|---|
| 262 | ! |
|---|
| 263 | ! ------------------------------- ! |
|---|
| 264 | ! 5. north fold treatment ! |
|---|
| 265 | ! ------------------------------- ! |
|---|
| 266 | ! |
|---|
| 267 | IF( l_IdoNFold ) THEN |
|---|
| 268 | IF( jpni == 1 ) THEN ; CALL lbc_nfd( ptab, cd_nat, psgn , ihls, ipf ) ! self NFold |
|---|
| 269 | ELSE ; CALL mpp_nfd( ptab, cd_nat, psgn, ifill_nfd, zland, ihls, ipf ) ! mpi NFold |
|---|
| 270 | ENDIF |
|---|
| 271 | ENDIF |
|---|
| 272 | ! |
|---|
| 273 | END SUBROUTINE lbc_lnk_neicoll_/**/PRECISION |
|---|
| 274 | |
|---|
