Changeset 11940 for NEMO/branches/2019/dev_r11470_HPC_12_mpi3/src/OCE/LBC/lib_mpp.F90

Timestamp:

2019-11-20T22:48:28+01:00 (4 years ago)

Author:

mocavero

Message:

Add MPI3 neighbourhood collectives halo exchange in LBC and call it in tracer advection FCT scheme #2011

File:

• NEMO/branches/2019/dev_r11470_HPC_12_mpi3/src/OCE/LBC/lib_mpp.F90 (modified) (5 diffs)

NEMO/branches/2019/dev_r11470_HPC_12_mpi3/src/OCE/LBC/lib_mpp.F90

@@ -25 +25 @@
    !!            4.0  !  2017  (G. Madec) automatique allocation of array argument (use any 3rd dimension)
    !!             -   !  2017  (G. Madec) create generic.h90 files to generate all lbc and north fold routines
+   !!             -   !  2019  (S. Mocavero, I. Epicoco - CMCC) add MPI3 neighbourhood collectives
    !!----------------------------------------------------------------------
 
@@ -50 +51 @@
    !!   mpp_ini_north : initialisation of north fold
    !!   mpp_lbc_north_icb : alternative to mpp_nfd for extra outer halo with icebergs
+   !!   mpp_ini_nc    : initialisation of MPI3 neighbourhood collectives communicator
    !!----------------------------------------------------------------------
    USE dom_oce        ! ocean space and time domain
@@ -64 +66 @@
    PUBLIC   mppscatter, mppgather
    PUBLIC   mpp_ini_znl
+   PUBLIC   mpp_ini_nc
    PUBLIC   mppsend, mpprecv                          ! needed by TAM and ICB routines
    PUBLIC   mpp_report
@@ -132 +135 @@
    INTEGER, PUBLIC ::   north_root        !: number (in the comm_opa) of proc 0 in the northern comm
    INTEGER, PUBLIC, DIMENSION(:), ALLOCATABLE, SAVE ::   nrank_north   !: dimension ndim_rank_north
+
+   ! variables used for MPI3 neighbourhood collectives
+   INTEGER, PUBLIC :: mpi_nc_com                   ! MPI3 neighbourhood collectives communicator
+   INTEGER, PUBLIC, DIMENSION(:), ALLOCATABLE :: nranks
 
    ! Communications summary report
@@ -800 +807 @@
 
    END SUBROUTINE mpp_ini_znl
+
+   SUBROUTINE mpp_ini_nc
+      !!----------------------------------------------------------------------
+      !!               ***  routine mpp_ini_nc  ***
+      !!
+      !! ** Purpose :   Initialize special communicator for MPI3 neighbourhood
+      !!                collectives
+      !!
+      !! ** Method  : - Create a cartesian communicator starting from the
+      !processes   
+      !!                distribution along i and j directions
+      !
+      !! ** output
+      !!         mpi_nc_com = MPI3 neighbourhood collectives communicator
+      !!
+      !!----------------------------------------------------------------------
+      INTEGER, DIMENSION(2) :: ipdims
+      LOGICAL, PARAMETER :: ireord = .FALSE.
+      LOGICAL :: nperiod_nc ( 2 ) = .FALSE.
+      INTEGER :: mpi_nc_com_re, igroup, iprc, ingroup, iii, jjj
+      INTEGER :: ierr
+
+#if defined key_mpp_mpi
+      ipdims(1) = jpni
+      ipdims(2) = jpnj
+      nperiod_nc(1) = .TRUE.
+
+      iprc = jpni*jpnj
+
+      ! Create a group from mpi_comm_oce
+      CALL MPI_Comm_group(mpi_comm_oce, igroup, ierr)
+
+      ! Create a group to reorder MPI rank
+      CALL MPI_Group_incl(igroup, iprc, nranks, ingroup, ierr)
+
+      ! Create reordered communicator
+      CALL MPI_Comm_create(mpi_comm_oce, ingroup, mpi_nc_com_re, ierr)
+
+      ! Create the cartesian communicator
+      CALL MPI_Cart_create(mpi_nc_com_re, 2, ipdims, nperiod_nc, ireord, mpi_nc_com, ierr)
+      DEALLOCATE (nranks)
+#endif
+   END SUBROUTINE mpp_ini_nc