Changeset 15072

Timestamp:

2021-07-02T15:54:03+02:00 (3 years ago)

Author:

girrmann

Message:

add structure for new RMA communications, small asynchronous communications debugging

Location:

NEMO/branches/2021/dev_r14447_HPC-07_Irrmann_try_new_pt2pt

Files:

• cfgs/SHARED/namelist_ref (modified) (1 diff)
• src/OCE/DYN/dynspg_ts.F90 (modified) (2 diffs)
• src/OCE/LBC/lbc_lnk_call_generic.h90 (modified) (2 diffs)
• src/OCE/LBC/lbc_lnk_newpt2pt_generic.h90 (modified) (4 diffs)
• src/OCE/LBC/lbc_lnk_pt2pt_RMA.h90 (added)
• src/OCE/LBC/lbc_lnk_pt2pt_async.h90 (modified) (3 diffs)
• src/OCE/LBC/lbclnk.F90 (modified) (3 diffs)
• src/OCE/LBC/lib_mpp.F90 (modified) (1 diff)
• src/OCE/LBC/mppini.F90 (modified) (2 diffs)
• tests/BENCH/MY_SRC/usrdef_nam.F90 (modified) (1 diff)

NEMO/branches/2021/dev_r14447_HPC-07_Irrmann_try_new_pt2pt/cfgs/SHARED/namelist_ref

@@ -1501 +1501 @@
    ln_tspers   = .false.   !  change to persistent communications in time splitting
    ln_async    = .false.   !  change to asynchronous communications whenever possible
+   ln_RMA      = .false.   !  change to RMA communications in time splitting
 /
 !-----------------------------------------------------------------------

NEMO/branches/2021/dev_r14447_HPC-07_Irrmann_try_new_pt2pt/src/OCE/DYN/dynspg_ts.F90

@@ -524 +524 @@
          !-------------------------------------------------------------------------!
          !
-         IF( ln_async )  THEN
+         IF(     ln_async .OR. ln_RMA )  THEN
             CALL lbc_lnk( 'dynspg_ts', ssha_e, 'T', 1._wp,  zhU, 'U', -1._wp,  zhV, 'V', -1._wp, loop_fct=loop_ssha_e )
          ELSE
@@ -625 +625 @@
          !------------------------------------------------------------------------------------------------------------------------!
 
-         IF( ln_async )  THEN
+         IF( ln_async .OR. ln_RMA )  THEN
             IF( .NOT.ln_linssh ) THEN   !* Update ocean depth (variable volume case only)
                CALL lbc_lnk( 'dynspg_ts', ua_e , 'U', -1._wp, va_e , 'V', -1._wp  &

NEMO/branches/2021/dev_r14447_HPC-07_Irrmann_try_new_pt2pt/src/OCE/LBC/lbc_lnk_call_generic.h90

@@ -80 +80 @@
       IF( PRESENT(psgn16) )   CALL load_ptr_/**/XD/**/_/**/PRECISION( pt16, cdna16, psgn16, ptab_ptr, cdna_ptr, psgn_ptr, kfld )
       !
-
+      
       IF( lints ) THEN   ! in time splitting
          IF( ln_tspers ) THEN
@@ -86 +86 @@
          ELSE IF( PRESENT(loop_fct) .AND. ln_async ) THEN
             CALL lbc_lnk_async( cdname, ptab_ptr, cdna_ptr, psgn_ptr, kfld, kfillmode, pfillval, khls, lsend, lrecv, ld4only, loop_fct )
+         ELSE IF( PRESENT(loop_fct) .AND. ln_RMA ) THEN
+            CALL lbc_lnk_RMA(   cdname, ptab_ptr, cdna_ptr, psgn_ptr, kfld, kfillmode, pfillval, khls, lsend, lrecv, ld4only, loop_fct )
          ELSE
             SELECT CASE (nn_comm)

NEMO/branches/2021/dev_r14447_HPC-07_Irrmann_try_new_pt2pt/src/OCE/LBC/lbc_lnk_newpt2pt_generic.h90

@@ -174 +174 @@
       ! ----------------------------- !
       !
-      !IF( ln_timing ) CALL tic_tac(.TRUE.)
-      DO jn = 1, 8
-#define NBLOCK_ISEND
-#  include "lbc_lnk_newpt2pt_generic.h90"         
-#undef NBLOCK_ISEND
-      END DO
-      !IF( ln_timing ) CALL tic_tac(.FALSE.)
+      DO jn = 1, 8
+         IF( llsend(jn) ) THEN
+            ishti = ishtSi(jn)
+            ishtj = ishtSj(jn)
+            DO jf = 1, ipf  ;  DO jl = 1, ipl  ;  DO jk = 1, ipk  ;  DO jj = 1,isizej(jn)  ;  DO ji = 1,isizei(jn)
+               BUFFSND(idxs) = ptab(jf)%pt4d(ishti+ji,ishtj+jj,jk,jl)
+               idxs = idxs + 1
+            END DO;   END DO   ;   END DO   ;   END DO   ;   END DO
+         ENDIF
+      END DO
+      IF( ln_timing ) CALL tic_tac(.TRUE.)
+      IF( llsend(jpso) ) THEN   ! jpso = 3, send south first for north fold
+         CALL MPI_ISEND( BUFFSND(ishtS(jpso)+1), iszall(jpso), MPI_TYPE, mpiSnei(ihls,jpso), iStag(jpso), icomm, nreq_p2p(jpso), ierr )
+      ENDIF
+      DO jn = 1, 2
+         IF( llsend(jn) ) THEN
+            CALL MPI_ISEND( BUFFSND(ishtS(jn)+1), iszall(jn), MPI_TYPE, mpiSnei(ihls,jn), iStag(jn), icomm, nreq_p2p(jn), ierr )
+         ENDIF
+      END DO
+      DO jn = 4, 8
+         IF( llsend(jn) ) THEN
+            CALL MPI_ISEND( BUFFSND(ishtS(jn)+1), iszall(jn), MPI_TYPE, mpiSnei(ihls,jn), iStag(jn), icomm, nreq_p2p(jn), ierr )
+         ENDIF
+      END DO
+      IF( ln_timing ) CALL tic_tac(.FALSE.)
       !
 
@@ -187 +205 @@
       ! ------------------------------ !
       !
-      !IF( ln_timing ) CALL tic_tac(.TRUE.)
+      IF( ln_timing ) CALL tic_tac(.TRUE.)
       DO jn = 1, 8
          IF ( ifill(jn) == jpfillmpi   ) THEN                ! fill with data received earlier by MPI
@@ -193 +211 @@
          END IF
       END DO
-      !IF( ln_timing ) CALL tic_tac(.FALSE.)
+      IF( ln_timing ) CALL tic_tac(.FALSE.)
 
       ! ----------------------- !
@@ -260 +278 @@
 
 
-#if defined NBLOCK_ISEND
-   IF( llsend(jn) ) THEN
-      ishti = ishtSi(jn)
-      ishtj = ishtSj(jn)
-      DO jf = 1, ipf  ;  DO jl = 1, ipl  ;  DO jk = 1, ipk  ;  DO jj = 1,isizej(jn)  ;  DO ji = 1,isizei(jn)
-         BUFFSND(idxs) = ptab(jf)%pt4d(ishti+ji,ishtj+jj,jk,jl)
-         idxs = idxs + 1
-      END DO   ;   END DO   ;   END DO   ;   END DO   ;   END DO
-      IF( ln_timing ) CALL tic_tac(.TRUE.)
-      CALL MPI_ISEND( BUFFSND(ishtS(jn)+1), iszall(jn), MPI_TYPE, mpiSnei(ihls,jn), iStag(jn), icomm, nreq_p2p(jn), ierr )
-      IF( ln_timing ) CALL tic_tac(.FALSE.)
-   ENDIF
-#endif
    
 #if defined NMPI_FILL

NEMO/branches/2021/dev_r14447_HPC-07_Irrmann_try_new_pt2pt/src/OCE/LBC/lbc_lnk_pt2pt_async.h90

@@ -101 +101 @@
          ENDIF
       END DO
-      DO jn = 5, 8
-         IF(             llrecv(jn) ) THEN   ;   ifill(jn) = jpfillmpi    ! with an mpi communication
-         ELSE                                ;   ifill(jn) = jpfillnothing! do nothing
-         ENDIF
-      END DO
       ! north fold treatment
       ll_IdoNFold = l_IdoNFold .AND. ifill(jpno) /= jpfillnothing
@@ -270 +265 @@
 
 #if defined BLOCK_IRECV
-   ishti = ishtRi(jn)
-   ishtj = ishtRj(jn)
-   SELECT CASE ( ifill(jn) )
-   CASE ( jpfillnothing )               ! no filling
-   CASE ( jpfillmpi   )                 ! fill with data received by MPI
+   IF( ifill(jn) == jpfillmpi   ) THEN                ! fill with data received by MPI
 #if ! defined key_mpi_off
       IF( ln_timing ) CALL tic_tac(.TRUE.)
@@ -280 +271 @@
       IF( ln_timing ) CALL tic_tac(.FALSE.)
 #endif
-END SELECT
+   END IF
 #endif
 

NEMO/branches/2021/dev_r14447_HPC-07_Irrmann_try_new_pt2pt/src/OCE/LBC/lbclnk.F90

@@ -58 +58 @@
       MODULE PROCEDURE   lbc_lnk_async_sp, lbc_lnk_async_dp
    END INTERFACE lbc_lnk_async
+
+   INTERFACE lbc_lnk_RMA
+      MODULE PROCEDURE   lbc_lnk_RMA_sp, lbc_lnk_RMA_dp
+   END INTERFACE lbc_lnk_RMA
    !
    INTERFACE lbc_lnk_icb
@@ -160 +164 @@
 #  define BUFFRCV buffrcv_async_sp
 #  include "lbc_lnk_pt2pt_async.h90"
+#  include "lbc_lnk_pt2pt_RMA.h90"
 #  undef BUFFSND
 #  undef BUFFRCV
@@ -181 +186 @@
 #  define BUFFRCV buffrcv_async_dp
 #  include "lbc_lnk_pt2pt_async.h90"
+#  include "lbc_lnk_pt2pt_RMA.h90"
 #  undef BUFFSND
 #  undef BUFFRCV

NEMO/branches/2021/dev_r14447_HPC-07_Irrmann_try_new_pt2pt/src/OCE/LBC/lib_mpp.F90

@@ -223 +223 @@
    LOGICAL                            , PUBLIC :: ln_tspers         ! indicate if persistent call enabled in time-splitting
    LOGICAL                            , PUBLIC :: ln_async          ! indicate if asynchronous communications enabled
+   LOGICAL                            , PUBLIC :: ln_RMA            ! indicate if RMA communications enabled
    
    !! * Substitutions

NEMO/branches/2021/dev_r14447_HPC-07_Irrmann_try_new_pt2pt/src/OCE/LBC/mppini.F90

@@ -145 +145 @@
            &             cn_ice, nn_ice_dta,                                     &
            &             ln_vol, nn_volctl, nn_rimwidth
-      NAMELIST/nammpp/ jpni, jpnj, nn_hls, ln_nnogather, ln_listonly, nn_comm, ln_tspers, ln_async
+      NAMELIST/nammpp/ jpni, jpnj, nn_hls, ln_nnogather, ln_listonly, nn_comm, ln_tspers, ln_async, ln_RMA
       !!----------------------------------------------------------------------
       !
@@ -171 +171 @@
             WRITE(numout,*) '      communication type                                 nn_comm = ', nn_comm
             WRITE(numout,*) '      switch to persistent calls in time-splitting    ln_tspers = ', ln_tspers
+            WRITE(numout,*) '      switch to asynchronous com in time-splitting    ln_async = ' , ln_async
+            WRITE(numout,*) '      switch to RMA communications in time-splitting    ln_RMA = ' , ln_RMA
       ENDIF
       !

NEMO/branches/2021/dev_r14447_HPC-07_Irrmann_try_new_pt2pt/tests/BENCH/MY_SRC/usrdef_nam.F90

@@ -61 +61 @@
       !!
       NAMELIST/namusr_def/ nn_isize, nn_jsize, nn_ksize, ln_Iperio, ln_Jperio, ln_NFold, cn_NFtype
-      NAMELIST/nammpp/ jpni, jpnj, nn_hls, ln_nnogather, ln_listonly, nn_comm
+      NAMELIST/nammpp/ jpni, jpnj, nn_hls, ln_nnogather, ln_listonly, nn_comm, ln_tspers, ln_async, ln_RMA
       !!----------------------------------------------------------------------     
       !