Index: NEMO/branches/2020/dev_r13333_KERNEL-08_techene_gm_HPG_SPG/src/OCE/LBC/lib_mpp.F90
===================================================================
--- NEMO/branches/2020/dev_r13333_KERNEL-08_techene_gm_HPG_SPG/src/OCE/LBC/lib_mpp.F90	(revision 13360)
+++ NEMO/branches/2020/dev_r13333_KERNEL-08_techene_gm_HPG_SPG/src/OCE/LBC/lib_mpp.F90	(revision 14037)
@@ -66,4 +66,5 @@
    PUBLIC   mppscatter, mppgather
    PUBLIC   mpp_ini_znl
+   PUBLIC   mpp_ini_nc
    PUBLIC   mppsend, mpprecv                          ! needed by TAM and ICB routines
    PUBLIC   mppsend_sp, mpprecv_sp                          ! needed by TAM and ICB routines
@@ -73,4 +74,5 @@
    PUBLIC   tic_tac
 #if ! defined key_mpp_mpi
+   PUBLIC MPI_wait
    PUBLIC MPI_Wtime
 #endif
@@ -115,4 +117,5 @@
 #else   
    INTEGER, PUBLIC, PARAMETER ::   MPI_STATUS_SIZE = 1
+   INTEGER, PUBLIC, PARAMETER ::   MPI_REAL = 4
    INTEGER, PUBLIC, PARAMETER ::   MPI_DOUBLE_PRECISION = 8
    LOGICAL, PUBLIC, PARAMETER ::   lk_mpp = .FALSE.    !: mpp flag
@@ -135,4 +138,8 @@
    INTEGER         ::   ndim_rank_znl   !  number of processors on the same zonal average
    INTEGER, DIMENSION(:), ALLOCATABLE, SAVE ::   nrank_znl  ! dimension ndim_rank_znl, number of the procs into the same znl domain
+
+   ! variables used for MPI3 neighbourhood collectives
+   INTEGER, PUBLIC :: mpi_nc_com                   ! MPI3 neighbourhood collectives communicator
+   INTEGER, PUBLIC :: mpi_nc_all_com               ! MPI3 neighbourhood collectives communicator (with diagionals)
 
    ! North fold condition in mpp_mpi with jpni > 1 (PUBLIC for TAM)
@@ -509,4 +516,5 @@
             ALLOCATE(todelay(idvar)%y1d(isz))
             todelay(idvar)%y1d(:) = CMPLX(todelay(idvar)%z1d(:), 0., wp)   ! create %y1d, complex variable needed by mpi_sumdd
+            ndelayid(idvar) = MPI_REQUEST_NULL                             ! initialised request to a valid value
          END IF
       ENDIF
@@ -516,8 +524,8 @@
          ALLOCATE(todelay(idvar)%z1d(isz), todelay(idvar)%y1d(isz))   ! allocate also %z1d as used for the restart
          CALL mpi_allreduce( y_in(:), todelay(idvar)%y1d(:), isz, MPI_DOUBLE_COMPLEX, mpi_sumdd, ilocalcomm, ierr )   ! get %y1d
-         todelay(idvar)%z1d(:) = REAL(todelay(idvar)%y1d(:), wp)      ! define %z1d from %y1d
-      ENDIF
-
-      IF( ndelayid(idvar) > 0 )   CALL mpp_delay_rcv( idvar )         ! make sure %z1d is received
+         ndelayid(idvar) = MPI_REQUEST_NULL
+      ENDIF
+
+      CALL mpp_delay_rcv( idvar )         ! make sure %z1d is received
 
       ! send back pout from todelay(idvar)%z1d defined at previous call
@@ -528,5 +536,5 @@
       IF( ln_timing ) CALL tic_tac( .TRUE., ld_global = .TRUE.)
       CALL  mpi_allreduce( y_in(:), todelay(idvar)%y1d(:), isz, MPI_DOUBLE_COMPLEX, mpi_sumdd, ilocalcomm, ierr )
-      ndelayid(idvar) = 1
+      ndelayid(idvar) = MPI_REQUEST_NULL
       IF( ln_timing ) CALL tic_tac(.FALSE., ld_global = .TRUE.)
 # else
@@ -589,4 +597,6 @@
             DEALLOCATE(todelay(idvar)%z1d)
             ndelayid(idvar) = -1                                      ! do as if we had no restart
+         ELSE
+            ndelayid(idvar) = MPI_REQUEST_NULL
          END IF
       ENDIF
@@ -596,7 +606,8 @@
          ALLOCATE(todelay(idvar)%z1d(isz))
          CALL mpi_allreduce( p_in(:), todelay(idvar)%z1d(:), isz, MPI_DOUBLE_PRECISION, mpi_max, ilocalcomm, ierr )   ! get %z1d
-      ENDIF
-
-      IF( ndelayid(idvar) > 0 )   CALL mpp_delay_rcv( idvar )         ! make sure %z1d is received
+         ndelayid(idvar) = MPI_REQUEST_NULL
+      ENDIF
+
+      CALL mpp_delay_rcv( idvar )         ! make sure %z1d is received
 
       ! send back pout from todelay(idvar)%z1d defined at previous call
@@ -604,7 +615,8 @@
 
       ! send p_in into todelay(idvar)%z1d with a non-blocking communication
+      ! (PM) Should we get rid of MPI2 option ? MPI3 was release in 2013. Who is still using MPI2 ?
 # if defined key_mpi2
       IF( ln_timing ) CALL tic_tac( .TRUE., ld_global = .TRUE.)
-      CALL  mpi_allreduce( p_in(:), todelay(idvar)%z1d(:), isz, MPI_TYPE, mpi_max, ilocalcomm, ndelayid(idvar), ierr )
+      CALL  mpi_allreduce( p_in(:), todelay(idvar)%z1d(:), isz, MPI_TYPE, mpi_max, ilocalcomm, ierr )
       IF( ln_timing ) CALL tic_tac(.FALSE., ld_global = .TRUE.)
 # else
@@ -629,13 +641,9 @@
       !!----------------------------------------------------------------------
 #if defined key_mpp_mpi
-      IF( ndelayid(kid) /= -2 ) THEN  
-#if ! defined key_mpi2
-         IF( ln_timing ) CALL tic_tac( .TRUE., ld_global = .TRUE.)
-         CALL mpi_wait( ndelayid(kid), MPI_STATUS_IGNORE, ierr )                        ! make sure todelay(kid) is received
-         IF( ln_timing ) CALL tic_tac(.FALSE., ld_global = .TRUE.)
-#endif
-         IF( ASSOCIATED(todelay(kid)%y1d) )   todelay(kid)%z1d(:) = REAL(todelay(kid)%y1d(:), wp)  ! define %z1d from %y1d
-         ndelayid(kid) = -2   ! add flag to know that mpi_wait was already called on kid
-      ENDIF
+      IF( ln_timing ) CALL tic_tac( .TRUE., ld_global = .TRUE.)
+      ! test on ndelayid(kid) useless as mpi_wait return immediatly if the request handle is MPI_REQUEST_NULL
+      CALL mpi_wait( ndelayid(kid), MPI_STATUS_IGNORE, ierr ) ! after this ndelayid(kid) = MPI_REQUEST_NULL
+      IF( ln_timing ) CALL tic_tac( .FALSE., ld_global = .TRUE.)
+      IF( ASSOCIATED(todelay(kid)%y1d) )   todelay(kid)%z1d(:) = REAL(todelay(kid)%y1d(:), wp)  ! define %z1d from %y1d
 #endif
    END SUBROUTINE mpp_delay_rcv
@@ -1064,4 +1072,151 @@
 
    END SUBROUTINE mpp_ini_znl
+
+   SUBROUTINE mpp_ini_nc
+      !!----------------------------------------------------------------------
+      !!               ***  routine mpp_ini_nc  ***
+      !!
+      !! ** Purpose :   Initialize special communicators for MPI3 neighbourhood
+      !!                collectives
+      !!
+      !! ** Method  : - Create graph communicators starting from the processes   
+      !!                distribution along i and j directions
+      !
+      !! ** output
+      !!         mpi_nc_com = MPI3 neighbourhood collectives communicator
+      !!         mpi_nc_all_com = MPI3 neighbourhood collectives communicator
+      !!                          (with diagonals)
+      !!
+      !!----------------------------------------------------------------------
+      INTEGER, DIMENSION(:), ALLOCATABLE :: ineigh, ineighalls, ineighallr
+      INTEGER :: ideg, idegalls, idegallr, icont, icont1
+      INTEGER :: ierr
+      LOGICAL, PARAMETER :: ireord = .FALSE.
+
+#if defined key_mpp_mpi
+
+      ideg = 0
+      idegalls = 0
+      idegallr = 0
+      icont = 0
+      icont1 = 0
+
+      IF (nbondi .eq. 1) THEN
+         ideg = ideg + 1
+      ELSEIF (nbondi .eq. -1) THEN
+         ideg = ideg + 1
+      ELSEIF (nbondi .eq. 0) THEN
+         ideg = ideg + 2
+      ENDIF
+
+      IF (nbondj .eq. 1) THEN
+         ideg = ideg + 1
+      ELSEIF (nbondj .eq. -1) THEN
+         ideg = ideg + 1
+      ELSEIF (nbondj .eq. 0) THEN
+         ideg = ideg + 2
+      ENDIF
+
+      idegalls = ideg
+      idegallr = ideg
+
+      IF (nones .ne. -1) idegalls = idegalls + 1
+      IF (nonws .ne. -1) idegalls = idegalls + 1
+      IF (noses .ne. -1) idegalls = idegalls + 1
+      IF (nosws .ne. -1) idegalls = idegalls + 1
+      IF (noner .ne. -1) idegallr = idegallr + 1
+      IF (nonwr .ne. -1) idegallr = idegallr + 1
+      IF (noser .ne. -1) idegallr = idegallr + 1
+      IF (noswr .ne. -1) idegallr = idegallr + 1
+
+      ALLOCATE(ineigh(ideg))
+      ALLOCATE(ineighalls(idegalls))
+      ALLOCATE(ineighallr(idegallr))
+
+      IF (nbondi .eq. 1) THEN
+         icont = icont + 1
+         ineigh(icont) = nowe
+         ineighalls(icont) = nowe
+         ineighallr(icont) = nowe
+      ELSEIF (nbondi .eq. -1) THEN
+         icont = icont + 1
+         ineigh(icont) = noea
+         ineighalls(icont) = noea
+         ineighallr(icont) = noea
+      ELSEIF (nbondi .eq. 0) THEN
+         icont = icont + 1
+         ineigh(icont) = nowe
+         ineighalls(icont) = nowe
+         ineighallr(icont) = nowe
+         icont = icont + 1
+         ineigh(icont) = noea
+         ineighalls(icont) = noea
+         ineighallr(icont) = noea
+      ENDIF
+
+      IF (nbondj .eq. 1) THEN
+         icont = icont + 1
+         ineigh(icont) = noso
+         ineighalls(icont) = noso
+         ineighallr(icont) = noso
+      ELSEIF (nbondj .eq. -1) THEN
+         icont = icont + 1
+         ineigh(icont) = nono
+         ineighalls(icont) = nono
+         ineighallr(icont) = nono
+      ELSEIF (nbondj .eq. 0) THEN
+         icont = icont + 1
+         ineigh(icont) = noso
+         ineighalls(icont) = noso
+         ineighallr(icont) = noso
+         icont = icont + 1
+         ineigh(icont) = nono
+         ineighalls(icont) = nono
+         ineighallr(icont) = nono
+      ENDIF
+
+      icont1 = icont
+      IF (nosws .ne. -1) THEN
+         icont = icont + 1
+         ineighalls(icont) = nosws
+      ENDIF
+      IF (noses .ne. -1) THEN
+         icont = icont + 1
+         ineighalls(icont) = noses
+      ENDIF
+      IF (nonws .ne. -1) THEN
+         icont = icont + 1
+         ineighalls(icont) = nonws
+      ENDIF
+      IF (nones .ne. -1) THEN
+         icont = icont + 1
+         ineighalls(icont) = nones
+      ENDIF
+      IF (noswr .ne. -1) THEN
+         icont1 = icont1 + 1
+         ineighallr(icont1) = noswr
+      ENDIF
+      IF (noser .ne. -1) THEN
+         icont1 = icont1 + 1
+         ineighallr(icont1) = noser
+      ENDIF
+      IF (nonwr .ne. -1) THEN
+         icont1 = icont1 + 1
+         ineighallr(icont1) = nonwr
+      ENDIF
+      IF (noner .ne. -1) THEN
+         icont1 = icont1 + 1
+         ineighallr(icont1) = noner
+      ENDIF
+
+      CALL MPI_Dist_graph_create_adjacent(mpi_comm_oce, ideg, ineigh, MPI_UNWEIGHTED, ideg, ineigh, MPI_UNWEIGHTED, MPI_INFO_NULL, ireord, mpi_nc_com, ierr)
+      CALL MPI_Dist_graph_create_adjacent(mpi_comm_oce, idegallr, ineighallr, MPI_UNWEIGHTED, idegalls, ineighalls, MPI_UNWEIGHTED, MPI_INFO_NULL, ireord, mpi_nc_all_com, ierr)
+
+      DEALLOCATE (ineigh)
+      DEALLOCATE (ineighalls)
+      DEALLOCATE (ineighallr)
+#endif
+   END SUBROUTINE mpp_ini_nc
+
 
 
