Index: /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/DIA/diadimg.F90
===================================================================
--- /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/DIA/diadimg.F90	(revision 5283)
+++ /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/DIA/diadimg.F90	(revision 5284)
@@ -124,6 +124,7 @@
 
     CASE DEFAULT
-       IF(lwp) WRITE(numout,*) ' E R R O R : bad cd_type in dia_wri_dimg '
-       STOP 'dia_wri_dimg'
+    
+       WRITE(numout,*) 'dia_wri_dimg : E R R O R : bad cd_type in dia_wri_dimg'
+       CALL ctl_stop( 'STOP', 'dia_wri_dimg :bad cd_type in dia_wri_dimg ' )
 
     END SELECT
Index: /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/DYN/dynldf_bilapg.F90
===================================================================
--- /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/DYN/dynldf_bilapg.F90	(revision 5283)
+++ /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/DYN/dynldf_bilapg.F90	(revision 5284)
@@ -465,7 +465,9 @@
             END DO
          ELSE
-            IF(lwp)WRITE(numout,*) ' ldfguv: kahm= 1 or 2, here =', kahm
-            IF(lwp)WRITE(numout,*) '         We stop'
-            STOP 'ldfguv'
+            
+            WRITE(numout,*) ' ldfguv: kahm= 1 or 2, here =', kahm
+            WRITE(numout,*) '         We stop'
+            CALL ctl_stop('STOP', 'ldfguv: Unexpected kahm value')
+
          ENDIF
          !                                             ! ===============
Index: /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/LBC/lib_mpp.F90
===================================================================
--- /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/LBC/lib_mpp.F90	(revision 5283)
+++ /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/LBC/lib_mpp.F90	(revision 5284)
@@ -1714,4 +1714,6 @@
 
    SUBROUTINE mppstop
+   
+
       !!----------------------------------------------------------------------
       !!                  ***  routine mppstop  ***
@@ -1723,6 +1725,19 @@
       !!----------------------------------------------------------------------
       !
+      
+#if defined key_oasis3 || defined key_oasis3mct
+      ! If we're trying to shut down cleanly then we need to consider the fact
+      ! that this could be part of an MPMD configuration - we don't want to
+      ! leave other components deadlocked.
+
+      CALL oasis_abort(nproc,"mppstop","NEMO initiated abort")
+
+
+#else
+      
       CALL mppsync
       CALL mpi_finalize( info )
+#endif
+
       !
    END SUBROUTINE mppstop
Index: /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/SBC/cpl_oasis3.F90
===================================================================
--- /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/SBC/cpl_oasis3.F90	(revision 5283)
+++ /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/SBC/cpl_oasis3.F90	(revision 5284)
@@ -23,5 +23,5 @@
    !!   cpl_finalize : finalize the coupled mode communication
    !!----------------------------------------------------------------------
-#if defined key_oasis3
+#if defined key_oasis3 || defined key_oasis3mct
    USE mod_oasis                    ! OASIS3-MCT module
 #endif
@@ -30,4 +30,9 @@
    USE in_out_manager               ! I/O manager
    USE lbclnk                       ! ocean lateral boundary conditions (or mpp link)
+   
+#if defined key_cpl_rootexchg
+   USE lib_mpp, only : mppsync
+   USE lib_mpp, only : mppscatter,mppgather
+#endif 
 
    IMPLICIT NONE
@@ -40,4 +45,15 @@
    PUBLIC   cpl_freq
    PUBLIC   cpl_finalize
+#if defined key_mpp_mpi
+   INCLUDE 'mpif.h'
+#endif
+   
+   INTEGER, PARAMETER         :: localRoot  = 0
+   LOGICAL                    :: commRank            ! true for ranks doing OASIS communication
+#if defined key_cpl_rootexchg
+   LOGICAL                    :: rootexchg =.true.   ! logical switch 
+#else
+   LOGICAL                    :: rootexchg =.false.  ! logical switch 
+#endif 
 
    INTEGER, PUBLIC            ::   OASIS_Rcv  = 1    !: return code if received field
@@ -78,5 +94,7 @@
 
    REAL(wp), DIMENSION(:,:), ALLOCATABLE ::   exfld   ! Temporary buffer for receiving
-
+   REAL(wp), DIMENSION(:,:,:), ALLOCATABLE ::   tbuf  ! Temporary buffer for sending / receiving 
+   INTEGER, PUBLIC :: localComm 
+      
    !!----------------------------------------------------------------------
    !! NEMO/OPA 3.3 , NEMO Consortium (2010)
@@ -104,5 +122,5 @@
       ! 1st Initialize the OASIS system for the application
       !------------------------------------------------------------------
-      CALL oasis_init_comp ( ncomp_id, 'oceanx', nerror )
+      CALL oasis_init_comp ( ncomp_id, 'toyoce', nerror )
       IF ( nerror /= OASIS_Ok ) &
          CALL oasis_abort (ncomp_id, 'cpl_init', 'Failure in oasis_init_comp')
@@ -115,4 +133,5 @@
       IF ( nerror /= OASIS_Ok ) &
          CALL oasis_abort (ncomp_id, 'cpl_init','Failure in oasis_get_localcomm' )
+      localComm = kl_comm 
       !
    END SUBROUTINE cpl_init
@@ -143,4 +162,11 @@
       IF(lwp) WRITE(numout,*) '~~~~~~~~~~~~~~~~~'
       IF(lwp) WRITE(numout,*)
+      
+      commRank = .false.
+      IF ( rootexchg ) THEN
+         IF ( nproc == localRoot ) commRank = .true.
+      ELSE
+         commRank = .true.
+      ENDIF
 
       IF( kcplmodel > nmaxcpl ) THEN
@@ -155,14 +181,18 @@
       ishape(:,2) = (/ 1, nlej-nldj+1 /)
       !
-      ! ... Allocate memory for data exchange
-      !
-      ALLOCATE(exfld(nlei-nldi+1, nlej-nldj+1), stat = nerror)
-      IF( nerror > 0 ) THEN
-         CALL oasis_abort ( ncomp_id, 'cpl_define', 'Failure in allocating exfld')   ;   RETURN
-      ENDIF
       !
       ! -----------------------------------------------------------------
       ! ... Define the partition 
       ! -----------------------------------------------------------------
+      
+      IF ( rootexchg ) THEN
+     
+      paral(1) = 2              ! box partitioning
+      paral(2) = 0              ! NEMO lower left corner global offset     
+      paral(3) = jpiglo         ! local extent in i 
+      paral(4) = jpjglo         ! local extent in j
+      paral(5) = jpiglo         ! global extent in x
+
+      ELSE 
       
       paral(1) = 2                                              ! box partitioning
@@ -179,5 +209,22 @@
       ENDIF
       
-      CALL oasis_def_partition ( id_part, paral, nerror )
+      ENDIF
+      IF ( commRank )  CALL oasis_def_partition ( id_part, paral, nerror )
+     
+      ! ... Allocate memory for data exchange
+      !
+      ALLOCATE(exfld(paral(3), paral(4)), stat = nerror)
+      IF( nerror > 0 ) THEN
+         CALL oasis_abort ( ncomp_id, 'cpl_define', 'Failure in allocating exfld')   ;   RETURN
+      ENDIF
+      IF ( rootexchg ) THEN
+! Should possibly use one of the work arrays for tbuf really
+         ALLOCATE(tbuf(jpi, jpj, jpnij), stat = nerror)
+         IF( nerror > 0 ) THEN
+             CALL oasis_abort ( ncomp_id, 'cpl_define', 'Failure in allocating tbuf') ; RETURN
+         ENDIF
+       ENDIF             
+       !
+       IF (commRank ) THEN
       !
       ! ... Announce send variables. 
@@ -224,5 +271,5 @@
             END DO
          ENDIF
-      END DO
+      END DO      
       !
       ! ... Announce received variables. 
@@ -271,4 +318,6 @@
          ENDIF
       END DO
+      !
+      ENDIF  ! commRank=true 
       
       !------------------------------------------------------------------
@@ -276,6 +325,9 @@
       !------------------------------------------------------------------
       
-      CALL oasis_enddef(nerror)
-      IF( nerror /= OASIS_Ok )   CALL oasis_abort ( ncomp_id, 'cpl_define', 'Failure in oasis_enddef')
+      IF ( commRank ) THEN     
+      
+         CALL oasis_enddef(nerror)
+         IF( nerror /= OASIS_Ok )   CALL oasis_abort ( ncomp_id, 'cpl_define', 'Failure in oasis_enddef')
+      ENDIF
       !
    END SUBROUTINE cpl_define
@@ -294,5 +346,5 @@
       REAL(wp), DIMENSION(:,:,:), INTENT(in   ) ::   pdata
       !!
-      INTEGER                                   ::   jc,jm     ! local loop index
+      INTEGER                                   ::   jn,jc,jm     ! local loop index
       !!--------------------------------------------------------------------
       !
@@ -303,5 +355,28 @@
         
             IF( ssnd(kid)%nid(jc,jm) /= -1 ) THEN
-               CALL oasis_put ( ssnd(kid)%nid(jc,jm), kstep, pdata(nldi:nlei, nldj:nlej,jc), kinfo )
+               IF ( rootexchg ) THEN
+                  !
+                  ! collect data on the local root process
+                  !
+                  CALL mppgather (pdata(:,:,jc),localRoot,tbuf) 
+                  CALL mppsync 
+           
+                  IF ( nproc == localRoot ) THEN
+                               
+                     DO jn = 1, jpnij
+                        exfld(nimppt(jn)-1+nldit(jn):nimppt(jn)+nleit(jn)-1,njmppt(jn)-1+nldjt(jn):njmppt(jn)+nlejt(jn)-1)= &
+                          tbuf(nldit(jn):nleit(jn),nldjt(jn):nlejt(jn),jn)
+                     ENDDO
+               
+                     ! snd data to OASIS3
+                     CALL oasis_put ( ssnd(kid)%nid(jc,jm), kstep, exfld, kinfo )
+           
+                  ENDIF
+           
+               ELSE
+
+                   ! snd data to OASIS3
+                   CALL oasis_put ( ssnd(kid)%nid(jc,jm), kstep, pdata(nldi:nlei, nldj:nlej,jc), kinfo )
+               ENDIF
                
                IF ( ln_ctl ) THEN        
@@ -341,5 +416,5 @@
       INTEGER                   , INTENT(  out) ::   kinfo     ! OASIS3 info argument
       !!
-      INTEGER                                   ::   jc,jm     ! local loop index
+      INTEGER                                   ::   jn,jc,jm     ! local loop index
       LOGICAL                                   ::   llaction, llfisrt
       !!--------------------------------------------------------------------
@@ -355,6 +430,11 @@
 
             IF( srcv(kid)%nid(jc,jm) /= -1 ) THEN
-
-               CALL oasis_get ( srcv(kid)%nid(jc,jm), kstep, exfld, kinfo )         
+               
+               !
+               ! receive data from OASIS3
+               !
+               IF ( commRank )  CALL oasis_get ( srcv(kid)%nid(jc,jm), kstep, exfld, kinfo )
+       
+               IF ( rootexchg )  CALL MPI_BCAST ( kinfo, 1, MPI_INTEGER, localRoot, localComm, nerror )                      
                
                llaction =  kinfo == OASIS_Recvd   .OR. kinfo == OASIS_FromRest .OR.   &
@@ -367,5 +447,28 @@
                   kinfo = OASIS_Rcv
                   IF( llfisrt ) THEN 
-                     pdata(nldi:nlei,nldj:nlej,jc) =                                 exfld(:,:) * pmask(nldi:nlei,nldj:nlej,jm)
+                     
+                     IF ( rootexchg ) THEN
+           
+                        ! distribute data to processes
+                        !
+                        IF ( nproc == localRoot ) THEN
+
+                           DO jn = 1, jpnij
+                              tbuf(nldit(jn):nleit(jn),nldjt(jn):nlejt(jn),jn)=          &
+                              exfld(nimppt(jn)-1+nldit(jn):nimppt(jn)+nleit(jn)-1,njmppt(jn)-1+nldjt(jn):njmppt(jn)+nlejt(jn)-1)
+                              ! NOTE: we are missing combining this with pmask (see else below)
+                           ENDDO
+               
+                        ENDIF
+
+                        CALL mppscatter (tbuf,localRoot,pdata(:,:,jc)) 
+                        CALL mppsync
+
+                     ELSE
+           
+                        pdata(nldi:nlei, nldj:nlej, jc) = exfld(:,:) * pmask(nldi:nlei,nldj:nlej,jm)
+               
+                     ENDIF
+                     
                      llfisrt = .FALSE.
                   ELSE
@@ -411,6 +514,12 @@
       INTEGER, DIMENSION(1) :: itmp
       !!----------------------------------------------------------------------
+#if defined key_oasis3 
       CALL oasis_get_freqs(kid, 1, itmp, info)
       cpl_freq = itmp(1)
+#endif
+
+#if defined key_oasis3mct
+      cpl_freq = namflddti( kid )
+#endif
       !
    END FUNCTION cpl_freq
@@ -427,4 +536,5 @@
       !
       DEALLOCATE( exfld )
+      IF ( rootexchg ) DEALLOCATE ( tbuf )
       IF (nstop == 0) THEN
          CALL oasis_terminate( nerror )         
Index: /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/SBC/sbccpl.F90
===================================================================
--- /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/SBC/sbccpl.F90	(revision 5283)
+++ /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/SBC/sbccpl.F90	(revision 5284)
@@ -47,4 +47,9 @@
    USE ice_domain_size, only: ncat
 #endif
+
+#if defined key_oasis3 || defined key_oasis3mct
+   USE mod_oasis                    ! OASIS3-MCT module
+#endif
+
    IMPLICIT NONE
    PRIVATE
Index: /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/TRA/traldf_bilapg.F90
===================================================================
--- /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/TRA/traldf_bilapg.F90	(revision 5283)
+++ /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/TRA/traldf_bilapg.F90	(revision 5284)
@@ -330,7 +330,7 @@
                END DO
             ELSE
-               IF(lwp) WRITE(numout,*) ' ldfght: kaht= 1 or 2, here =', kaht
-               IF(lwp) WRITE(numout,*) '         We stop'
-               STOP 'ldfght'
+               WRITE(numout,*) ' ldfght: kaht= 1 or 2, here =', kaht
+               WRITE(numout,*) '         We stop'
+               CALL ctl_stop( 'STOP', 'ldfght : unexpected kaht value')
             ENDIF
             !                                             ! ===============
Index: /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/step.F90
===================================================================
--- /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/step.F90	(revision 5283)
+++ /branches/UKMO/dev_r5107_hadgem3_mct/NEMOGCM/NEMO/OPA_SRC/step.F90	(revision 5284)
@@ -318,4 +318,5 @@
                                CALL ctl_stop( 'step: indic < 0' )
                                CALL dia_wri_state( 'output.abort', kstp )
+                               CALL ctl_stop('STOP','NEMO failure in stp')
       ENDIF
       IF( kstp == nit000   )   THEN
