Index: branches/2015/dev_r5546_CNRS19_HPC_scalability/NEMOGCM/NEMO/OPA_SRC/LBC/lib_mpp.F90
===================================================================
--- branches/2015/dev_r5546_CNRS19_HPC_scalability/NEMOGCM/NEMO/OPA_SRC/LBC/lib_mpp.F90	(revision 5549)
+++ branches/2015/dev_r5546_CNRS19_HPC_scalability/NEMOGCM/NEMO/OPA_SRC/LBC/lib_mpp.F90	(revision 5579)
@@ -62,4 +62,5 @@
    USE lbcnfd         ! north fold treatment
    USE in_out_manager ! I/O manager
+   USE wrk_nemo       ! work arrays
 
    IMPLICIT NONE
@@ -70,6 +71,7 @@
    PUBLIC   mpp_ini_north, mpp_lbc_north, mpp_lbc_north_e
    PUBLIC   mpp_min, mpp_max, mpp_sum, mpp_minloc, mpp_maxloc
+   PUBLIC   mpp_max_multiple
    PUBLIC   mpp_lnk_3d, mpp_lnk_3d_gather, mpp_lnk_2d, mpp_lnk_2d_e
-   PUBLIC   mpp_lnk_2d_9 
+   PUBLIC   mpp_lnk_2d_9 , mpp_lnk_2d_multiple 
    PUBLIC   mppscatter, mppgather
    PUBLIC   mpp_ini_ice, mpp_ini_znl
@@ -78,8 +80,10 @@
    PUBLIC   mpp_lnk_bdy_2d, mpp_lnk_bdy_3d
    PUBLIC   mpp_lbc_north_icb, mpp_lnk_2d_icb
+   PUBLIC   mpprank
 
    TYPE arrayptr
       REAL , DIMENSION (:,:),  POINTER :: pt2d
    END TYPE arrayptr
+   PUBLIC   arrayptr
    
    !! * Interfaces
@@ -105,4 +109,8 @@
    INTERFACE mpp_maxloc
       MODULE PROCEDURE mpp_maxloc2d ,mpp_maxloc3d
+   END INTERFACE
+
+   INTERFACE mpp_max_multiple
+      MODULE PROCEDURE mppmax_real_multiple
    END INTERFACE
 
@@ -724,16 +732,17 @@
       ! -----------------------
       !
-      DO ii = 1 , num_fields
          !First Array
-         IF( npolj /= 0 .AND. .NOT. PRESENT(cd_mpp) ) THEN
-            !
-            SELECT CASE ( jpni )
-            CASE ( 1 )     ;   CALL lbc_nfd      ( pt2d_array(ii)%pt2d( : , : ), type_array(ii) , psgn_array(ii) )   ! only 1 northern proc, no mpp
-            CASE DEFAULT   ;   CALL mpp_lbc_north( pt2d_array(ii)%pt2d( : , : ), type_array(ii), psgn_array(ii) )   ! for all northern procs.
-            END SELECT
-            !
-         ENDIF
-         !
-      END DO
+      IF( npolj /= 0 .AND. .NOT. PRESENT(cd_mpp) ) THEN
+         !
+         SELECT CASE ( jpni )
+         CASE ( 1 )     ;   
+             DO ii = 1 , num_fields  
+                       CALL lbc_nfd      ( pt2d_array(ii)%pt2d( : , : ), type_array(ii) , psgn_array(ii) )   ! only 1 northern proc, no mpp
+             END DO
+         CASE DEFAULT   ;   CALL mpp_lbc_north_2d_multiple( pt2d_array, type_array, psgn_array, num_fields )   ! for all northern procs.
+         END SELECT
+         !
+      ENDIF
+        !
       
       DEALLOCATE( zt2ns, zt2sn, zt2ew, zt2we )
@@ -1681,4 +1690,29 @@
    END SUBROUTINE mppmax_real
 
+   SUBROUTINE mppmax_real_multiple( ptab, NUM , kcom  )
+      !!----------------------------------------------------------------------
+      !!                  ***  routine mppmax_real  ***
+      !!
+      !! ** Purpose :   Maximum
+      !!
+      !!----------------------------------------------------------------------
+      REAL(wp), DIMENSION(:) ,  INTENT(inout)           ::   ptab   ! ???
+      INTEGER , INTENT(in   )           ::   NUM
+      INTEGER , INTENT(in   ), OPTIONAL ::   kcom   ! ???
+      !!
+      INTEGER  ::   ierror, localcomm
+      REAL(wp) , POINTER , DIMENSION(:) ::   zwork
+      !!----------------------------------------------------------------------
+      !
+      CALL wrk_alloc(NUM , zwork)
+      localcomm = mpi_comm_opa
+      IF( PRESENT(kcom) )   localcomm = kcom
+      !
+      CALL mpi_allreduce( ptab, zwork, NUM, mpi_double_precision, mpi_max, localcomm, ierror )
+      ptab = zwork
+      CALL wrk_dealloc(NUM , zwork)
+      !
+   END SUBROUTINE mppmax_real_multiple
+
 
    SUBROUTINE mppmin_a_real( ptab, kdim, kcom )
@@ -2575,4 +2609,165 @@
    END SUBROUTINE mpp_lbc_north_2d
 
+   SUBROUTINE mpp_lbc_north_2d_multiple( pt2d_array, cd_type, psgn, num_fields)
+      !!---------------------------------------------------------------------
+      !!                   ***  routine mpp_lbc_north_2d  ***
+      !!
+      !! ** Purpose :   Ensure proper north fold horizontal bondary condition
+      !!              in mpp configuration in case of jpn1 > 1
+      !!              (for multiple 2d arrays )
+      !!
+      !! ** Method  :   North fold condition and mpp with more than one proc
+      !!              in i-direction require a specific treatment. We gather
+      !!              the 4 northern lines of the global domain on 1 processor
+      !!              and apply lbc north-fold on this sub array. Then we
+      !!              scatter the north fold array back to the processors.
+      !!
+      !!----------------------------------------------------------------------
+      INTEGER ,  INTENT (in   ) ::   num_fields  ! number of variables contained in pt2d
+      TYPE( arrayptr ), DIMENSION(:) :: pt2d_array
+      CHARACTER(len=1), DIMENSION(:), INTENT(in   ) ::   cd_type   ! nature of pt2d grid-points
+      !                                                          !   = T ,  U , V , F or W  gridpoints
+      REAL(wp), DIMENSION(:), INTENT(in   ) ::   psgn      ! = -1. the sign change across the north fold 
+      !!                                                             ! =  1. , the sign is kept
+      INTEGER ::   ji, jj, jr, jk
+      INTEGER ::   ierr, itaille, ildi, ilei, iilb
+      INTEGER ::   ijpj, ijpjm1, ij, iproc
+      INTEGER, DIMENSION (jpmaxngh)      ::   ml_req_nf          !for mpi_isend when avoiding mpi_allgather
+      INTEGER                            ::   ml_err             ! for mpi_isend when avoiding mpi_allgather
+      INTEGER, DIMENSION(MPI_STATUS_SIZE)::   ml_stat            ! for mpi_isend when avoiding mpi_allgather
+      !                                                              ! Workspace for message transfers avoiding mpi_allgather
+      REAL(wp), DIMENSION(:,:,:)  , ALLOCATABLE   :: ztab
+      REAL(wp), DIMENSION(:,:,:)  , ALLOCATABLE   :: znorthloc, zfoldwk
+      REAL(wp), DIMENSION(:,:,:,:), ALLOCATABLE   :: znorthgloio
+      REAL(wp), DIMENSION(:,:,:)  , ALLOCATABLE   :: ztabl, ztabr
+      INTEGER :: istatus(mpi_status_size)
+      INTEGER :: iflag
+      !!----------------------------------------------------------------------
+      !
+      ALLOCATE( ztab(jpiglo,4,num_fields), znorthloc(jpi,4,num_fields), zfoldwk(jpi,4,num_fields), znorthgloio(jpi,4,num_fields,jpni) )   ! expanded to 3 dimensions
+      ALLOCATE( ztabl(jpi,4,num_fields), ztabr(jpi*jpmaxngh, 4,num_fields) )
+      !
+      ijpj   = 4
+      ijpjm1 = 3
+      !
+      
+      DO jk = 1, num_fields
+         DO jj = nlcj-ijpj+1, nlcj             ! put in znorthloc the last 4 jlines of pt2d (for every variable)
+            ij = jj - nlcj + ijpj
+            znorthloc(:,ij,jk) = pt2d_array(jk)%pt2d(:,jj)
+         END DO
+      END DO
+      !                                     ! Build in procs of ncomm_north the znorthgloio
+      itaille = jpi * ijpj
+                                                                  
+      IF ( l_north_nogather ) THEN
+         !
+         ! Avoid the use of mpi_allgather by exchanging only with the processes already identified 
+         ! (in nemo_northcomms) as being  involved in this process' northern boundary exchange
+         !
+         ztabr(:,:,:) = 0
+         ztabl(:,:,:) = 0
+
+         DO jk = 1, num_fields
+            DO jj = nlcj-ijpj+1, nlcj          ! First put local values into the global array
+               ij = jj - nlcj + ijpj
+               DO ji = nfsloop, nfeloop
+                  ztabl(ji,ij,jk) = pt2d_array(jk)%pt2d(ji,jj)
+               END DO
+            END DO
+         END DO
+
+         DO jr = 1,nsndto
+            IF ((nfipproc(isendto(jr),jpnj) .ne. (narea-1)) .and. (nfipproc(isendto(jr),jpnj) .ne. -1)) THEN
+               CALL mppsend(5, znorthloc, itaille*num_fields, nfipproc(isendto(jr),jpnj), ml_req_nf(jr)) ! Buffer expanded "num_fields" times
+            ENDIF
+         END DO
+         DO jr = 1,nsndto
+            iproc = nfipproc(isendto(jr),jpnj)
+            IF(iproc .ne. -1) THEN
+               ilei = nleit (iproc+1)
+               ildi = nldit (iproc+1)
+               iilb = nfiimpp(isendto(jr),jpnj) - nfiimpp(isendto(1),jpnj)
+            ENDIF
+            IF((iproc .ne. (narea-1)) .and. (iproc .ne. -1)) THEN
+              CALL mpprecv(5, zfoldwk, itaille*num_fields, iproc) ! Buffer expanded "num_fields" times
+              DO jk = 1 , num_fields
+                 DO jj = 1, ijpj
+                    DO ji = ildi, ilei
+                       ztabr(iilb+ji,jj,jk) = zfoldwk(ji,jj,jk)       ! Modified to 3D
+                    END DO
+                 END DO
+              END DO
+            ELSE IF (iproc .eq. (narea-1)) THEN
+              DO jk = 1, num_fields
+                 DO jj = 1, ijpj
+                    DO ji = ildi, ilei
+                          ztabr(iilb+ji,jj,jk) = pt2d_array(jk)%pt2d(ji,nlcj-ijpj+jj)       ! Modified to 3D
+                    END DO
+                 END DO
+              END DO
+            ENDIF
+         END DO
+         IF (l_isend) THEN
+            DO jr = 1,nsndto
+               IF ((nfipproc(isendto(jr),jpnj) .ne. (narea-1)) .and. (nfipproc(isendto(jr),jpnj) .ne. -1)) THEN
+                  CALL mpi_wait(ml_req_nf(jr), ml_stat, ml_err)
+               ENDIF
+            END DO
+         ENDIF
+         !
+         DO ji = 1, num_fields     ! Loop to manage 3D variables
+            CALL mpp_lbc_nfd( ztabl(:,:,ji), ztabr(:,:,ji), cd_type(ji), psgn(ji) )  ! North fold boundary condition
+         END DO
+         !
+         DO jk = 1, num_fields
+            DO jj = nlcj-ijpj+1, nlcj             ! Scatter back to pt2d
+               ij = jj - nlcj + ijpj
+               DO ji = 1, nlci
+                  pt2d_array(jk)%pt2d(ji,jj) = ztabl(ji,ij,jk)       ! Modified to 3D
+               END DO
+            END DO
+         END DO
+         
+         !
+      ELSE
+         !
+         CALL MPI_ALLGATHER( znorthloc  , itaille*num_fields, MPI_DOUBLE_PRECISION,        &
+            &                znorthgloio, itaille*num_fields, MPI_DOUBLE_PRECISION, ncomm_north, ierr )
+         !
+         ztab(:,:,:) = 0.e0
+         DO jk = 1, num_fields
+            DO jr = 1, ndim_rank_north            ! recover the global north array
+               iproc = nrank_north(jr) + 1
+               ildi = nldit (iproc)
+               ilei = nleit (iproc)
+               iilb = nimppt(iproc)
+               DO jj = 1, ijpj
+                  DO ji = ildi, ilei
+                     ztab(ji+iilb-1,jj,jk) = znorthgloio(ji,jj,jk,jr)
+                  END DO
+               END DO
+            END DO
+         END DO
+         
+         DO ji = 1, num_fields
+            CALL lbc_nfd( ztab(:,:,ji), cd_type(ji), psgn(ji) )   ! North fold boundary condition
+         END DO
+         !
+         DO jk = 1, num_fields
+            DO jj = nlcj-ijpj+1, nlcj             ! Scatter back to pt2d
+               ij = jj - nlcj + ijpj
+               DO ji = 1, nlci
+                  pt2d_array(jk)%pt2d(ji,jj) = ztab(ji+nimpp-1,ij,jk)
+               END DO
+            END DO
+         END DO
+         !
+         !
+      ENDIF
+      DEALLOCATE( ztab, znorthloc, zfoldwk, znorthgloio )
+      DEALLOCATE( ztabl, ztabr )
+      !
+   END SUBROUTINE mpp_lbc_north_2d_multiple
 
    SUBROUTINE mpp_lbc_north_e( pt2d, cd_type, psgn)
