Index: NEMO/branches/2020/dev_r13383_HPC-02_Daley_Tiling/src/OCE/TRA/traadv_fct.F90
===================================================================
--- NEMO/branches/2020/dev_r13383_HPC-02_Daley_Tiling/src/OCE/TRA/traadv_fct.F90	(revision 13410)
+++ NEMO/branches/2020/dev_r13383_HPC-02_Daley_Tiling/src/OCE/TRA/traadv_fct.F90	(revision 13516)
@@ -15,4 +15,6 @@
    USE oce            ! ocean dynamics and active tracers
    USE dom_oce        ! ocean space and time domain
+   ! TEMP: This change not necessary after trd_tra is tiled
+   USE domain, ONLY : dom_tile
    USE trc_oce        ! share passive tracers/Ocean variables
    USE trd_oce        ! trends: ocean variables
@@ -79,22 +81,30 @@
       INTEGER                                  , INTENT(in   ) ::   kn_fct_v        ! order of the FCT scheme (=2 or 4)
       REAL(wp)                                 , INTENT(in   ) ::   p2dt            ! tracer time-step
+      ! TEMP: This can be ST_2D(nn_hls) after trd_tra is tiled
       REAL(wp), DIMENSION(jpi,jpj,jpk         ), INTENT(in   ) ::   pU, pV, pW      ! 3 ocean volume flux components
       REAL(wp), DIMENSION(jpi,jpj,jpk,kjpt,jpt), INTENT(inout) ::   pt              ! tracers and RHS of tracer equation
       !
       INTEGER  ::   ji, jj, jk, jn                           ! dummy loop indices  
+      ! TEMP: This change not necessary after trd_tra is tiled
+      INTEGER  ::   itile
       REAL(wp) ::   ztra                                     ! local scalar
       REAL(wp) ::   zfp_ui, zfp_vj, zfp_wk, zC2t_u, zC4t_u   !   -      -
       REAL(wp) ::   zfm_ui, zfm_vj, zfm_wk, zC2t_v, zC4t_v   !   -      -
-      REAL(wp), DIMENSION(jpi,jpj,jpk)        ::   zwi, zwx, zwy, zwz, ztu, ztv, zltu, zltv, ztw
-      REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::   ztrdx, ztrdy, ztrdz, zptry
-      REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::   zwinf, zwdia, zwsup
+      REAL(wp), DIMENSION(ST_2D(nn_hls),jpk)        ::   zwi, zwx, zwy, zwz, ztu, ztv, zltu, zltv, ztw
+      ! TEMP: This change not necessary after trd_tra is tiled
+      REAL(wp), DIMENSION(:,:,:), SAVE, ALLOCATABLE ::   ztrdx, ztrdy, ztrdz
+      REAL(wp), DIMENSION(:,:,:)      , ALLOCATABLE ::   zptry
+      REAL(wp), DIMENSION(:,:,:)      , ALLOCATABLE ::   zwinf, zwdia, zwsup
       LOGICAL  ::   ll_zAimp                                 ! flag to apply adaptive implicit vertical advection
       !!----------------------------------------------------------------------
-      !
-      IF( kt == kit000 )  THEN
-         IF(lwp) WRITE(numout,*)
-         IF(lwp) WRITE(numout,*) 'tra_adv_fct : FCT advection scheme on ', cdtype
-         IF(lwp) WRITE(numout,*) '~~~~~~~~~~~'
-      ENDIF
+      ! TEMP: This change not necessary after trd_tra is tiled
+      itile = ntile
+      !
+      IF( ntile == 0 .OR. ntile == 1 )  THEN                       ! Do only on the first tile
+         IF( kt == kit000 )  THEN
+            IF(lwp) WRITE(numout,*)
+            IF(lwp) WRITE(numout,*) 'tra_adv_fct : FCT advection scheme on ', cdtype
+            IF(lwp) WRITE(numout,*) '~~~~~~~~~~~'
+         ENDIF
       !! -- init to 0
       zwi(:,:,:) = 0._wp
@@ -107,21 +117,22 @@
       zltv(:,:,:) = 0._wp
       ztw(:,:,:) = 0._wp
-      !
-      l_trd = .FALSE.            ! set local switches
-      l_hst = .FALSE.
-      l_ptr = .FALSE.
-      ll_zAimp = .FALSE.
-      IF( ( cdtype == 'TRA' .AND. l_trdtra  ) .OR. ( cdtype =='TRC' .AND. l_trdtrc ) )      l_trd = .TRUE.
-      IF(   cdtype == 'TRA' .AND. ( iom_use( 'sophtadv' ) .OR. iom_use( 'sophtadv' ) ) )    l_ptr = .TRUE. 
-      IF(   cdtype == 'TRA' .AND. ( iom_use("uadv_heattr") .OR. iom_use("vadv_heattr") .OR.  &
-         &                         iom_use("uadv_salttr") .OR. iom_use("vadv_salttr")  ) )  l_hst = .TRUE.
-      !
-      IF( l_trd .OR. l_hst )  THEN
-         ALLOCATE( ztrdx(jpi,jpj,jpk), ztrdy(jpi,jpj,jpk), ztrdz(jpi,jpj,jpk) )
-         ztrdx(:,:,:) = 0._wp   ;    ztrdy(:,:,:) = 0._wp   ;   ztrdz(:,:,:) = 0._wp
+         !
+         l_trd = .FALSE.            ! set local switches
+         l_hst = .FALSE.
+         l_ptr = .FALSE.
+         ll_zAimp = .FALSE.
+         IF( ( cdtype == 'TRA' .AND. l_trdtra  ) .OR. ( cdtype =='TRC' .AND. l_trdtrc ) )      l_trd = .TRUE.
+         IF(   cdtype == 'TRA' .AND. ( iom_use( 'sophtadv' ) .OR. iom_use( 'sophtadv' ) ) )    l_ptr = .TRUE.
+         IF(   cdtype == 'TRA' .AND. ( iom_use("uadv_heattr") .OR. iom_use("vadv_heattr") .OR.  &
+            &                         iom_use("uadv_salttr") .OR. iom_use("vadv_salttr")  ) )  l_hst = .TRUE.
+         !
+         ! TEMP: This can be ST_2D(nn_hls) after trd_tra is tiled
+         IF( kt == kit000 .AND. (l_trd .OR. l_hst) )  THEN
+            ALLOCATE( ztrdx(jpi,jpj,jpk), ztrdy(jpi,jpj,jpk), ztrdz(jpi,jpj,jpk) )
+         ENDIF
       ENDIF
       !
       IF( l_ptr ) THEN  
-         ALLOCATE( zptry(jpi,jpj,jpk) )
+         ALLOCATE( zptry(ST_2D(nn_hls),jpk) )
          zptry(:,:,:) = 0._wp
       ENDIF
@@ -134,9 +145,9 @@
       ! If adaptive vertical advection, check if it is needed on this PE at this time
       IF( ln_zad_Aimp ) THEN
-         IF( MAXVAL( ABS( wi(:,:,:) ) ) > 0._wp ) ll_zAimp = .TRUE.
+         IF( MAXVAL( ABS( wi(ST_2D(nn_hls),:) ) ) > 0._wp ) ll_zAimp = .TRUE.
       END IF
       ! If active adaptive vertical advection, build tridiagonal matrix
       IF( ll_zAimp ) THEN
-         ALLOCATE(zwdia(jpi,jpj,jpk), zwinf(jpi,jpj,jpk),zwsup(jpi,jpj,jpk))
+         ALLOCATE(zwdia(ST_2D(nn_hls),jpk), zwinf(ST_2D(nn_hls),jpk), zwsup(ST_2D(nn_hls),jpk))
          DO_3D( 0, 0, 0, 0, 1, jpkm1 )
             zwdia(ji,jj,jk) =  1._wp + p2dt * ( MAX( wi(ji,jj,jk) , 0._wp ) - MIN( wi(ji,jj,jk+1) , 0._wp ) )   &
@@ -167,4 +178,5 @@
          END_3D
          IF( ln_linssh ) THEN    ! top ocean value (only in linear free surface as zwz has been w-masked)
+            ! TODO: NOT TESTED- requires isf
             IF( ln_isfcav ) THEN             ! top of the ice-shelf cavities and at the ocean surface
                DO_2D( 1, 1, 1, 1 )
@@ -207,6 +219,9 @@
          END IF
          !                
+         ! TEMP: This change not necessary after trd_tra is tiled
          IF( l_trd .OR. l_hst )  THEN             ! trend diagnostics (contribution of upstream fluxes)
-            ztrdx(:,:,:) = zwx(:,:,:)   ;   ztrdy(:,:,:) = zwy(:,:,:)   ;   ztrdz(:,:,:) = zwz(:,:,:)
+            DO_3D( 1, 0, 1, 0, 1, jpk )
+               ztrdx(ji,jj,jk) = zwx(ji,jj,jk)   ;   ztrdy(ji,jj,jk) = zwy(ji,jj,jk)   ;   ztrdz(ji,jj,jk) = zwz(ji,jj,jk)
+            END_3D
          END IF
          !                             ! "Poleward" heat and salt transports (contribution of upstream fluxes)
@@ -336,16 +351,26 @@
          END IF         
          !
+         ! TEMP: These changes not necessary after trd_tra is tiled
          IF( l_trd .OR. l_hst ) THEN   ! trend diagnostics // heat/salt transport
-            ztrdx(:,:,:) = ztrdx(:,:,:) + zwx(:,:,:)  ! <<< add anti-diffusive fluxes 
-            ztrdy(:,:,:) = ztrdy(:,:,:) + zwy(:,:,:)  !     to upstream fluxes
-            ztrdz(:,:,:) = ztrdz(:,:,:) + zwz(:,:,:)  !
-            !
-            IF( l_trd ) THEN              ! trend diagnostics
-               CALL trd_tra( kt, Kmm, Krhs, cdtype, jn, jptra_xad, ztrdx, pU, pt(:,:,:,jn,Kmm) )
-               CALL trd_tra( kt, Kmm, Krhs, cdtype, jn, jptra_yad, ztrdy, pV, pt(:,:,:,jn,Kmm) )
-               CALL trd_tra( kt, Kmm, Krhs, cdtype, jn, jptra_zad, ztrdz, pW, pt(:,:,:,jn,Kmm) )
+            DO_3D( 1, 0, 1, 0, 1, jpk )
+               ztrdx(ji,jj,jk) = ztrdx(ji,jj,jk) + zwx(ji,jj,jk)  ! <<< add anti-diffusive fluxes
+               ztrdy(ji,jj,jk) = ztrdy(ji,jj,jk) + zwy(ji,jj,jk)  !     to upstream fluxes
+               ztrdz(ji,jj,jk) = ztrdz(ji,jj,jk) + zwz(ji,jj,jk)  !
+            END_3D
+            !
+            IF( ntile == 0 .OR. ntile == nijtile )  THEN                ! Do only for the full domain
+               IF( l_trd ) THEN              ! trend diagnostics
+                  IF( ln_tile ) CALL dom_tile( ntsi, ntsj, ntei, ntej, ktile = 0 )         ! Use full domain
+
+                  ! TODO: TO BE TILED- trd_tra
+                  CALL trd_tra( kt, Kmm, Krhs, cdtype, jn, jptra_xad, ztrdx, pU, pt(:,:,:,jn,Kmm) )
+                  CALL trd_tra( kt, Kmm, Krhs, cdtype, jn, jptra_yad, ztrdy, pV, pt(:,:,:,jn,Kmm) )
+                  CALL trd_tra( kt, Kmm, Krhs, cdtype, jn, jptra_zad, ztrdz, pW, pt(:,:,:,jn,Kmm) )
+
+                  IF( ln_tile ) CALL dom_tile( ntsi, ntsj, ntei, ntej, ktile = itile )     ! Revert to tile domain
+               ENDIF
             ENDIF
             !                             ! heat/salt transport
-            IF( l_hst )   CALL dia_ar5_hst( jn, 'adv', ztrdx(:,:,:), ztrdy(:,:,:) )
+            IF( l_hst )   CALL dia_ar5_hst( jn, 'adv', ztrdx(ST_2D(nn_hls),:), ztrdy(ST_2D(nn_hls),:) )
             !
          ENDIF
@@ -360,7 +385,8 @@
          DEALLOCATE( zwdia, zwinf, zwsup )
       ENDIF
-      IF( l_trd .OR. l_hst ) THEN 
-         DEALLOCATE( ztrdx, ztrdy, ztrdz )
-      ENDIF
+      ! TEMP: These changes not necessary after trd_tra is tiled
+!      IF( l_trd .OR. l_hst ) THEN
+!         DEALLOCATE( ztrdx, ztrdy, ztrdz )
+!      ENDIF
       IF( l_ptr ) THEN 
          DEALLOCATE( zptry )
@@ -383,8 +409,9 @@
       !!       in-space based differencing for fluid
       !!----------------------------------------------------------------------
-      INTEGER                          , INTENT(in   ) ::   Kmm             ! time level index 
-      REAL(wp)                         , INTENT(in   ) ::   p2dt            ! tracer time-step
-      REAL(wp), DIMENSION (jpi,jpj,jpk), INTENT(in   ) ::   pbef, paft      ! before & after field
-      REAL(wp), DIMENSION (jpi,jpj,jpk), INTENT(inout) ::   paa, pbb, pcc   ! monotonic fluxes in the 3 directions
+      INTEGER                         , INTENT(in   ) ::   Kmm             ! time level index
+      REAL(wp)                        , INTENT(in   ) ::   p2dt            ! tracer time-step
+      REAL(wp), DIMENSION(jpi,jpj,jpk), INTENT(in   ) ::   pbef            ! before field
+      REAL(wp), DIMENSION(ST_2D(nn_hls)    ,jpk), INTENT(in   ) ::   paft            ! after field
+      REAL(wp), DIMENSION(ST_2D(nn_hls)    ,jpk), INTENT(inout) ::   paa, pbb, pcc   ! monotonic fluxes in the 3 directions
       !
       INTEGER  ::   ji, jj, jk   ! dummy loop indices
@@ -392,5 +419,5 @@
       REAL(dp) ::   zpos, zneg, zbt, za, zb, zc, zbig, zrtrn    ! local scalars
       REAL(dp) ::   zau, zbu, zcu, zav, zbv, zcv, zup, zdo            !   -      -
-      REAL(dp), DIMENSION(jpi,jpj,jpk) :: zbetup, zbetdo, zbup, zbdo
+      REAL(dp), DIMENSION(ST_2D(nn_hls),jpk) :: zbetup, zbetdo, zbup, zbdo
       !!----------------------------------------------------------------------
       !
@@ -402,8 +429,10 @@
       ! --------------------
       ! max/min of pbef & paft with large negative/positive value (-/+zbig) inside land
-      zbup = MAX( pbef * tmask - zbig * ( 1._wp - tmask ),   &
-         &        paft * tmask - zbig * ( 1._wp - tmask )  )
-      zbdo = MIN( pbef * tmask + zbig * ( 1._wp - tmask ),   &
-         &        paft * tmask + zbig * ( 1._wp - tmask )  )
+      DO_3D( 1, 1, 1, 1, 1, jpk )
+         zbup(ji,jj,jk) = MAX( pbef(ji,jj,jk) * tmask(ji,jj,jk) - zbig * ( 1._wp - tmask(ji,jj,jk) ),   &
+            &                  paft(ji,jj,jk) * tmask(ji,jj,jk) - zbig * ( 1._wp - tmask(ji,jj,jk) )  )
+         zbdo(ji,jj,jk) = MIN( pbef(ji,jj,jk) * tmask(ji,jj,jk) + zbig * ( 1._wp - tmask(ji,jj,jk) ),   &
+            &                  paft(ji,jj,jk) * tmask(ji,jj,jk) + zbig * ( 1._wp - tmask(ji,jj,jk) )  )
+      END_3D
 
       DO jk = 1, jpkm1
@@ -537,9 +566,9 @@
       !!----------------------------------------------------------------------
       REAL(wp),DIMENSION(jpi,jpj,jpk), INTENT(in   ) ::   pt_in    ! field at t-point
-      REAL(wp),DIMENSION(jpi,jpj,jpk), INTENT(  out) ::   pt_out   ! field interpolated at w-point
+      REAL(wp),DIMENSION(ST_2D(nn_hls)    ,jpk), INTENT(  out) ::   pt_out   ! field interpolated at w-point
       !
       INTEGER ::   ji, jj, jk   ! dummy loop integers
       INTEGER ::   ikt, ikb     ! local integers
-      REAL(wp),DIMENSION(jpi,jpj,jpk) :: zwd, zwi, zws, zwrm, zwt
+      REAL(wp),DIMENSION(ST_2D(nn_hls),jpk) :: zwd, zwi, zws, zwrm, zwt
       !!----------------------------------------------------------------------
       !
@@ -561,4 +590,5 @@
 !!gm  
       !
+      ! TODO: NOT TESTED- requires isf
       IF ( ln_isfcav ) THEN            ! set level two values which may not be set in ISF case
          zwd(:,:,2) = 1._wp  ;  zwi(:,:,2) = 0._wp  ;  zws(:,:,2) = 0._wp  ;  zwrm(:,:,2) = 0._wp
@@ -626,12 +656,12 @@
       !!        The 3d array zwt is used as a work space array.
       !!----------------------------------------------------------------------
-      REAL(wp),DIMENSION(:,:,:), INTENT(in   ) ::   pD, pU, PL    ! 3-diagonal matrix
-      REAL(wp),DIMENSION(:,:,:), INTENT(in   ) ::   pRHS          ! Right-Hand-Side
-      REAL(wp),DIMENSION(:,:,:), INTENT(  out) ::   pt_out        !!gm field at level=F(klev)
-      INTEGER                  , INTENT(in   ) ::   klev          ! =1 pt_out at w-level 
-      !                                                           ! =0 pt at t-level
+      REAL(wp),DIMENSION(ST_2D(nn_hls),jpk), INTENT(in   ) ::   pD, pU, PL    ! 3-diagonal matrix
+      REAL(wp),DIMENSION(ST_2D(nn_hls),jpk), INTENT(in   ) ::   pRHS          ! Right-Hand-Side
+      REAL(wp),DIMENSION(ST_2D(nn_hls),jpk), INTENT(  out) ::   pt_out        !!gm field at level=F(klev)
+      INTEGER                    , INTENT(in   ) ::   klev          ! =1 pt_out at w-level
+      !                                                             ! =0 pt at t-level
       INTEGER ::   ji, jj, jk   ! dummy loop integers
       INTEGER ::   kstart       ! local indices
-      REAL(wp),DIMENSION(jpi,jpj,jpk) ::   zwt   ! 3D work array
+      REAL(wp),DIMENSION(ST_2D(nn_hls),jpk) ::   zwt   ! 3D work array
       !!----------------------------------------------------------------------
       !
