Index: branches/2011/dev_r2855_NOCS_mppsca/DOC/TexFiles/Chapters/Chap_MISC.tex
===================================================================
--- branches/2011/dev_r2855_NOCS_mppsca/DOC/TexFiles/Chapters/Chap_MISC.tex	(revision 2898)
+++ branches/2011/dev_r2855_NOCS_mppsca/DOC/TexFiles/Chapters/Chap_MISC.tex	(revision 2899)
@@ -253,4 +253,24 @@
 Note this implementation may be sensitive to the optimization level. 
 
+\subsection{MPP scalability}
+\label{MISC_mppsca}
+
+The default method of communicating values across the north-fold in distributed memory applications
+(\key{mpp\_mpi}) uses a \textsc{MPI\_ALLGATHER} function to exchange values from each processing
+region in the northern row with every other processing region in the northern row. This enables a
+global width array containing the top 4 rows to be collated on every northern row processor and then
+folded with a simple algorithm. Although conceptually simple, this "All to All" communication will
+hamper performance scalability for large numbers of northern row processors. From version 3.4
+onwards an alternative method is available which only performs direct "Peer to Peer" communications
+between each processor and its immediate "neighbours" across the fold line. This is achieved by
+using the default \textsc{MPI\_ALLGATHER} method during initialisation to help identify the "active"
+neighbours. Stored lists of these neighbours are then used in all subsequent north-fold exchanges to
+restrict exchanges to those between associated regions. The collated global width array for each
+region is thus only partially filled but is guaranteed to be set at all the locations actually
+required by each individual for the fold operation. This alternative method should give identical
+results to the default \textsc{ALLGATHER} method and is recommended for large values of \np{jpni}.
+The new method is activated by setting \np{ln\_nnogather} to be true ({\bf nammpp}). The
+reproducibility of results using the two methods should be confirmed for each new, non-reference
+configuration.
 
 % ================================================================
Index: branches/2011/dev_r2855_NOCS_mppsca/NEMOGCM/NEMO/OPA_SRC/LBC/lib_mpp.F90
===================================================================
--- branches/2011/dev_r2855_NOCS_mppsca/NEMOGCM/NEMO/OPA_SRC/LBC/lib_mpp.F90	(revision 2898)
+++ branches/2011/dev_r2855_NOCS_mppsca/NEMOGCM/NEMO/OPA_SRC/LBC/lib_mpp.F90	(revision 2899)
@@ -2233,5 +2233,5 @@
       INTEGER ::   ierr, itaille, ildi, ilei, iilb
       INTEGER ::   ijpj, ijpjm1, ij, iproc
-      INTEGER, DIMENSION (jpmaxngh)          ::   ml_req5            ! for mpi_isend when avoiding mpi_allgather
+      INTEGER, DIMENSION (jpmaxngh)          ::   ml_req_nf          ! for mpi_isend when avoiding mpi_allgather
       INTEGER                                ::   ml_err             ! for mpi_isend when avoiding mpi_allgather
       INTEGER, DIMENSION(MPI_STATUS_SIZE)    ::   ml_stat            ! for mpi_isend when avoiding mpi_allgather
@@ -2251,61 +2251,57 @@
       itaille = jpi * jpk * ijpj
       IF ( l_north_nogather ) THEN
-!
-! Avoid the use of mpi_allgather by exchanging only with the processes already identified (in nemo_northcomms)
-! as being  involved in this process' northern boundary exchange
-!
-! First put local values into the global arraay
-         DO jj = nlcj-ijpj+1, nlcj
-           ij = jj - nlcj + ijpj
-           DO ji = 1, nlci
-             ztab(ji+nimpp-1,ij,:) = pt3d(ji,jj,:)
-           END DO
-         END DO
-
-!
-! Set the exchange type in order to access the correct list of active neighbours
-!
+         !
+         ! Avoid the use of mpi_allgather by exchanging only with the processes already identified 
+         ! (in nemo_northcomms) as being  involved in this process' northern boundary exchange
+         !
+         DO jj = nlcj-ijpj+1, nlcj          ! First put local values into the global array
+            ij = jj - nlcj + ijpj
+            DO ji = 1, nlci
+               ztab(ji+nimpp-1,ij,:) = pt3d(ji,jj,:)
+            END DO
+         END DO
+
+         !
+         ! Set the exchange type in order to access the correct list of active neighbours
+         !
          SELECT CASE ( cd_type )
             CASE ( 'T' , 'W' )
-             ityp = 1
+               ityp = 1
             CASE ( 'U' )
-             ityp = 2
+               ityp = 2
             CASE ( 'V' )
-             ityp = 3
+               ityp = 3
             CASE ( 'F' )
-             ityp = 4
+               ityp = 4
             CASE ( 'I' )
-             ityp = 5
+               ityp = 5
             CASE DEFAULT
-!
-! Set a default value for unsupported types which will cause a fallback to
-! the mpi_allgather method
-!
-             ityp = -1
-          END SELECT
-          IF ( ityp .gt. 0 ) THEN
-
-           DO jr = 1,nsndto(ityp)
-            CALL mppsend(5, znorthloc, itaille, isendto(jr,ityp), ml_req5(jr) )
-           END DO
-           DO jr = 1,nsndto(ityp)
-            CALL mpprecv(5, zfoldwk, itaille, isendto(jr,ityp))
-            iproc = isendto(jr,ityp) + 1
-            ildi=nldit (iproc)
-            ilei=nleit (iproc)
-            iilb=nimppt(iproc)
-            DO jj = 1, 4
-               DO ji = ildi, ilei
-                  ztab(ji+iilb-1,jj,:) = zfoldwk(ji,jj,:)
+               ityp = -1                    ! Set a default value for unsupported types which 
+                                            ! will cause a fallback to the mpi_allgather method
+         END SELECT
+         IF ( ityp .gt. 0 ) THEN
+
+            DO jr = 1,nsndto(ityp)
+               CALL mppsend(5, znorthloc, itaille, isendto(jr,ityp), ml_req_nf(jr) )
+            END DO
+            DO jr = 1,nsndto(ityp)
+               CALL mpprecv(5, zfoldwk, itaille, isendto(jr,ityp))
+               iproc = isendto(jr,ityp) + 1
+               ildi = nldit (iproc)
+               ilei = nleit (iproc)
+               iilb = nimppt(iproc)
+               DO jj = 1, ijpj
+                  DO ji = ildi, ilei
+                     ztab(ji+iilb-1,jj,:) = zfoldwk(ji,jj,:)
+                  END DO
                END DO
             END DO
-           END DO
-           IF(l_isend) THEN
-              DO jr = 1,nsndto(ityp)
-                CALL mpi_wait(ml_req5(jr), ml_stat, ml_err)
-              END DO
-           ENDIF
-
-          ENDIF
+            IF (l_isend) THEN
+               DO jr = 1,nsndto(ityp)
+                  CALL mpi_wait(ml_req_nf(jr), ml_stat, ml_err)
+               END DO
+            ENDIF
+
+         ENDIF
 
       ENDIF
@@ -2314,12 +2310,11 @@
          CALL MPI_ALLGATHER( znorthloc  , itaille, MPI_DOUBLE_PRECISION,                &
             &                znorthgloio, itaille, MPI_DOUBLE_PRECISION, ncomm_north, ierr )
-      !
-      !                                     ! recover the global north array
-         DO jr = 1, ndim_rank_north
+         !
+         DO jr = 1, ndim_rank_north         ! recover the global north array
             iproc = nrank_north(jr) + 1
             ildi  = nldit (iproc)
             ilei  = nleit (iproc)
             iilb  = nimppt(iproc)
-            DO jj = 1, 4
+            DO jj = 1, ijpj
                DO ji = ildi, ilei
                   ztab(ji+iilb-1,jj,:) = znorthgloio(ji,jj,:,jr)
@@ -2370,5 +2365,5 @@
       INTEGER ::   ierr, itaille, ildi, ilei, iilb
       INTEGER ::   ijpj, ijpjm1, ij, iproc
-      INTEGER, DIMENSION (jpmaxngh)      ::   ml_req5            ! for mpi_isend when avoiding mpi_allgather
+      INTEGER, DIMENSION (jpmaxngh)      ::   ml_req_nf          ! for mpi_isend when avoiding mpi_allgather
       INTEGER                            ::   ml_err             ! for mpi_isend when avoiding mpi_allgather
       INTEGER, DIMENSION(MPI_STATUS_SIZE)::   ml_stat            ! for mpi_isend when avoiding mpi_allgather
@@ -2388,81 +2383,76 @@
       itaille = jpi * ijpj
       IF ( l_north_nogather ) THEN
-!
-! Avoid the use of mpi_allgather by exchanging only with the processes already identified (in nemo_northcomms) 
-! as being  involved in this process' northern boundary exchange
-!
-! First put local values into the global array
-!
-         DO jj = nlcj-ijpj+1, nlcj
-           ij = jj - nlcj + ijpj
-           DO ji = 1, nlci
-             ztab_2d(ji+nimpp-1,ij) = pt2d(ji,jj)
-           END DO
-         END DO
-
-!
-! Set the exchange type in order to access the correct list of active neighbours
-!
+         !
+         ! Avoid the use of mpi_allgather by exchanging only with the processes already identified 
+         ! (in nemo_northcomms) as being  involved in this process' northern boundary exchange
+         !
+         DO jj = nlcj-ijpj+1, nlcj          ! First put local values into the global array
+            ij = jj - nlcj + ijpj
+            DO ji = 1, nlci
+               ztab_2d(ji+nimpp-1,ij) = pt2d(ji,jj)
+            END DO
+         END DO
+
+         !
+         ! Set the exchange type in order to access the correct list of active neighbours
+         !
          SELECT CASE ( cd_type )
             CASE ( 'T' , 'W' )
-             ityp = 1
+               ityp = 1
             CASE ( 'U' )
-             ityp = 2
+               ityp = 2
             CASE ( 'V' )
-             ityp = 3
+               ityp = 3
             CASE ( 'F' )
-             ityp = 4
+               ityp = 4
             CASE ( 'I' )
-             ityp = 5
+               ityp = 5
             CASE DEFAULT
-!
-! Set a default value for unsupported types which will cause a fallback to
-! the mpi_allgather method
-!
-             ityp = -1
-          END SELECT
-
-          IF ( ityp .gt. 0 ) THEN
-
-           DO jr = 1,nsndto(ityp)
-            CALL mppsend(5, znorthloc_2d, itaille, isendto(jr,ityp), ml_req5(jr) )
-           END DO
-           DO jr = 1,nsndto(ityp)
-            CALL mpprecv(5, zfoldwk_2d, itaille, isendto(jr,ityp))
-            iproc = isendto(jr,ityp) + 1
-            ildi=nldit (iproc)
-            ilei=nleit (iproc)
-            iilb=nimppt(iproc)
-            DO jj = 1, 4
-               DO ji = ildi, ilei
-                  ztab_2d(ji+iilb-1,jj) = zfoldwk_2d(ji,jj)
+               ityp = -1                    ! Set a default value for unsupported types which 
+                                            ! will cause a fallback to the mpi_allgather method
+         END SELECT
+
+         IF ( ityp .gt. 0 ) THEN
+
+            DO jr = 1,nsndto(ityp)
+               CALL mppsend(5, znorthloc_2d, itaille, isendto(jr,ityp), ml_req_nf(jr) )
+            END DO
+            DO jr = 1,nsndto(ityp)
+               CALL mpprecv(5, zfoldwk_2d, itaille, isendto(jr,ityp))
+               iproc = isendto(jr,ityp) + 1
+               ildi = nldit (iproc)
+               ilei = nleit (iproc)
+               iilb = nimppt(iproc)
+               DO jj = 1, ijpj
+                  DO ji = ildi, ilei
+                     ztab_2d(ji+iilb-1,jj) = zfoldwk_2d(ji,jj)
+                  END DO
                END DO
             END DO
-           END DO
-           IF(l_isend) THEN
-              DO jr = 1,nsndto(ityp)
-                CALL mpi_wait(ml_req5(jr), ml_stat, ml_err)
-              END DO
-           ENDIF
-
-          ENDIF
+            IF (l_isend) THEN
+               DO jr = 1,nsndto(ityp)
+                  CALL mpi_wait(ml_req_nf(jr), ml_stat, ml_err)
+               END DO
+            ENDIF
+
+         ENDIF
 
       ENDIF
 
       IF ( ityp .lt. 0 ) THEN
-       CALL MPI_ALLGATHER( znorthloc_2d  , itaille, MPI_DOUBLE_PRECISION,        &
-          &                znorthgloio_2d, itaille, MPI_DOUBLE_PRECISION, ncomm_north, ierr )
-      !
-       DO jr = 1, ndim_rank_north            ! recover the global north array
-          iproc = nrank_north(jr) + 1
-          ildi=nldit (iproc)
-          ilei=nleit (iproc)
-          iilb=nimppt(iproc)
-          DO jj = 1, 4
-             DO ji = ildi, ilei
-                ztab_2d(ji+iilb-1,jj) = znorthgloio_2d(ji,jj,jr)
-             END DO
-          END DO
-       END DO
+         CALL MPI_ALLGATHER( znorthloc_2d  , itaille, MPI_DOUBLE_PRECISION,        &
+            &                znorthgloio_2d, itaille, MPI_DOUBLE_PRECISION, ncomm_north, ierr )
+         !
+         DO jr = 1, ndim_rank_north            ! recover the global north array
+            iproc = nrank_north(jr) + 1
+            ildi = nldit (iproc)
+            ilei = nleit (iproc)
+            iilb = nimppt(iproc)
+            DO jj = 1, ijpj
+               DO ji = ildi, ilei
+                  ztab_2d(ji+iilb-1,jj) = znorthgloio_2d(ji,jj,jr)
+               END DO
+            END DO
+         END DO
       ENDIF
       !
Index: branches/2011/dev_r2855_NOCS_mppsca/NEMOGCM/NEMO/OPA_SRC/nemogcm.F90
===================================================================
--- branches/2011/dev_r2855_NOCS_mppsca/NEMOGCM/NEMO/OPA_SRC/nemogcm.F90	(revision 2898)
+++ branches/2011/dev_r2855_NOCS_mppsca/NEMOGCM/NEMO/OPA_SRC/nemogcm.F90	(revision 2899)
@@ -652,71 +652,59 @@
       isendto = -1
       ijpj   = 4
-!
-! This routine has been called because ln_nnogather has been set true ( nammpp )
-! However, these first few exchanges have to use the mpi_allgather method to
-! establish the neighbour lists to use in subsequent peer to peer exchanges.
-! Consequently, set l_north_nogather to be false here and set it true only after
-! the lists have been established.
-!
+      !
+      ! This routine has been called because ln_nnogather has been set true ( nammpp )
+      ! However, these first few exchanges have to use the mpi_allgather method to
+      ! establish the neighbour lists to use in subsequent peer to peer exchanges.
+      ! Consequently, set l_north_nogather to be false here and set it true only after
+      ! the lists have been established.
+      !
       l_north_nogather = .FALSE.
       !
-! Exchange and store ranks on northern rows
-
-     DO jtyp = 1,4
-
-        lrankset = .FALSE.
-        znnbrs = narea
-        SELECT CASE (jtyp)
-        CASE(1)
-           !
-           ! Type 1: T,W-points
-           !
-           CALL lbc_lnk( znnbrs, 'T', 1. )
-        CASE(2)
-           !
-           ! Type 2: U-point
-           !
-           CALL lbc_lnk( znnbrs, 'U', 1. )
-        CASE(3)
-           !
-           ! Type 3: V-point
-           !
-           CALL lbc_lnk( znnbrs, 'V', 1. )
-        CASE(4)
-           !
-           ! Type 5: F-point
-           !
-           CALL lbc_lnk( znnbrs, 'F', 1. )
-        END SELECT
-
-        IF ( njmppt(narea) .eq. MAXVAL( njmppt )) THEN
-          do jj = nlcj-ijpj+1, nlcj
-           ij = jj - nlcj + ijpj
-           do ji = 1,jpi
-            if(int(znnbrs(ji,jj)) .ne. 0 .and. int(znnbrs(ji,jj)) .ne. narea ) &
-         &         lrankset(int(znnbrs(ji,jj))) = .true.
-           end do
-          end do
-
-          do jj = 1,jpnij
-           IF (lrankset(jj)) THEN
-            nsndto(jtyp) = nsndto(jtyp) + 1
-            IF(nsndto(jtyp) .gt. jpmaxngh ) THEN
-             CALL ctl_stop( ' Too many neighbours in nemo_northcomms ', &
-           &                ' jpmaxngh will need to be increased ')
-            ENDIF
-            isendto(nsndto(jtyp),jtyp) = jj-1   ! narea converted to MPI rank
-           ENDIF
-          end do
-        ENDIF
-
-     END DO
-
-!
-! Type 5: I-point
-!
-! ICE point exchanges may involve some averaging. The neighbours list is
-! built up using two exchanges to ensure that the whole stencil is covered.
-! lrankset should not be reset between these 'J' and 'K' point exchanges
+      ! Exchange and store ranks on northern rows
+
+      DO jtyp = 1,4
+
+         lrankset = .FALSE.
+         znnbrs = narea
+         SELECT CASE (jtyp)
+            CASE(1)
+               CALL lbc_lnk( znnbrs, 'T', 1. )      ! Type 1: T,W-points
+            CASE(2)
+               CALL lbc_lnk( znnbrs, 'U', 1. )      ! Type 2: U-point
+            CASE(3)
+               CALL lbc_lnk( znnbrs, 'V', 1. )      ! Type 3: V-point
+            CASE(4)
+               CALL lbc_lnk( znnbrs, 'F', 1. )      ! Type 4: F-point
+         END SELECT
+
+         IF ( njmppt(narea) .EQ. MAXVAL( njmppt ) ) THEN
+            DO jj = nlcj-ijpj+1, nlcj
+               ij = jj - nlcj + ijpj
+               DO ji = 1,jpi
+                  IF ( INT(znnbrs(ji,jj)) .NE. 0 .AND. INT(znnbrs(ji,jj)) .NE. narea ) &
+               &     lrankset(INT(znnbrs(ji,jj))) = .true.
+               END DO
+            END DO
+
+            DO jj = 1,jpnij
+               IF ( lrankset(jj) ) THEN
+                  nsndto(jtyp) = nsndto(jtyp) + 1
+                  IF ( nsndto(jtyp) .GT. jpmaxngh ) THEN
+                     CALL ctl_stop( ' Too many neighbours in nemo_northcomms ', &
+                  &                 ' jpmaxngh will need to be increased ')
+                  ENDIF
+                  isendto(nsndto(jtyp),jtyp) = jj-1   ! narea converted to MPI rank
+               ENDIF
+            END DO
+         ENDIF
+
+      END DO
+
+      !
+      ! Type 5: I-point
+      !
+      ! ICE point exchanges may involve some averaging. The neighbours list is
+      ! built up using two exchanges to ensure that the whole stencil is covered.
+      ! lrankset should not be reset between these 'J' and 'K' point exchanges
 
       jtyp = 5
@@ -725,12 +713,12 @@
       CALL lbc_lnk( znnbrs, 'J', 1. ) ! first ice U-V point
 
-      IF ( njmppt(narea) .eq. MAXVAL( njmppt )) THEN
-        do jj = nlcj-ijpj+1, nlcj
-         ij = jj - nlcj + ijpj
-         do ji = 1,jpi
-          if(int(znnbrs(ji,jj)) .ne. 0 .and. int(znnbrs(ji,jj)) .ne. narea ) &
-         &       lrankset(int(znnbrs(ji,jj))) = .true.
-         end do
-        end do
+      IF ( njmppt(narea) .EQ. MAXVAL( njmppt ) ) THEN
+         DO jj = nlcj-ijpj+1, nlcj
+            ij = jj - nlcj + ijpj
+            DO ji = 1,jpi
+               IF ( INT(znnbrs(ji,jj)) .NE. 0 .AND. INT(znnbrs(ji,jj)) .NE. narea ) &
+            &     lrankset(INT(znnbrs(ji,jj))) = .true.
+         END DO
+        END DO
       ENDIF
 
@@ -738,29 +726,29 @@
       CALL lbc_lnk( znnbrs, 'K', 1. ) ! second ice U-V point
 
-      IF ( njmppt(narea) .eq. MAXVAL( njmppt )) THEN
-        do jj = nlcj-ijpj+1, nlcj
-         ij = jj - nlcj + ijpj
-         do ji = 1,jpi
-          if(int(znnbrs(ji,jj)) .ne. 0 .and. int(znnbrs(ji,jj)) .ne. narea ) &
-         &       lrankset(int(znnbrs(ji,jj))) = .true.
-         end do
-        end do
-
-        do jj = 1,jpnij
-         IF (lrankset(jj)) THEN
-          nsndto(jtyp) = nsndto(jtyp) + 1
-          IF(nsndto(jtyp) .gt. jpmaxngh ) THEN
-           CALL ctl_stop( ' Too many neighbours in nemo_northcomms ', &
-           &              ' jpmaxngh will need to be increased ')
-          ENDIF
-          isendto(nsndto(jtyp),jtyp) = jj-1   ! narea converted to MPI rank
-         ENDIF
-        end do
-!
-! For northern row areas, set l_north_nogather so that all subsequent exchanges can use
-! peer to peer communications at the north fold
-!
-        l_north_nogather = .TRUE.
-!
+      IF ( njmppt(narea) .EQ. MAXVAL( njmppt )) THEN
+         DO jj = nlcj-ijpj+1, nlcj
+            ij = jj - nlcj + ijpj
+            DO ji = 1,jpi
+               IF ( INT(znnbrs(ji,jj)) .NE. 0 .AND.  INT(znnbrs(ji,jj)) .NE. narea ) &
+            &       lrankset( INT(znnbrs(ji,jj))) = .true.
+            END DO
+         END DO
+
+         DO jj = 1,jpnij
+            IF ( lrankset(jj) ) THEN
+               nsndto(jtyp) = nsndto(jtyp) + 1
+               IF ( nsndto(jtyp) .GT. jpmaxngh ) THEN
+                  CALL ctl_stop( ' Too many neighbours in nemo_northcomms ', &
+               &                 ' jpmaxngh will need to be increased ')
+               ENDIF
+               isendto(nsndto(jtyp),jtyp) = jj-1   ! narea converted to MPI rank
+            ENDIF
+         END DO
+         !
+         ! For northern row areas, set l_north_nogather so that all subsequent exchanges 
+         ! can use peer to peer communications at the north fold
+         !
+         l_north_nogather = .TRUE.
+         !
       ENDIF
       DEALLOCATE( znnbrs )
