MODULE nemogcm !!====================================================================== !! *** MODULE nemogcm *** !! Ocean system : NEMO GCM (ocean dynamics, on-line tracers, biochemistry and sea-ice) !!====================================================================== !! History : OPA ! 1990-10 (C. Levy, G. Madec) Original code !! 7.0 ! 1991-11 (M. Imbard, C. Levy, G. Madec) !! 7.1 ! 1993-03 (M. Imbard, C. Levy, G. Madec, O. Marti, M. Guyon, A. Lazar, !! P. Delecluse, C. Perigaud, G. Caniaux, B. Colot, C. Maes) release 7.1 !! - ! 1992-06 (L.Terray) coupling implementation !! - ! 1993-11 (M.A. Filiberti) IGLOO sea-ice !! 8.0 ! 1996-03 (M. Imbard, C. Levy, G. Madec, O. Marti, M. Guyon, A. Lazar, !! P. Delecluse, L.Terray, M.A. Filiberti, J. Vialar, A.M. Treguier, M. Levy) release 8.0 !! 8.1 ! 1997-06 (M. Imbard, G. Madec) !! 8.2 ! 1999-11 (M. Imbard, H. Goosse) LIM sea-ice model !! ! 1999-12 (V. Thierry, A-M. Treguier, M. Imbard, M-A. Foujols) OPEN-MP !! ! 2000-07 (J-M Molines, M. Imbard) Open Boundary Conditions (CLIPPER) !! NEMO 1.0 ! 2002-08 (G. Madec) F90: Free form and modules !! - ! 2004-06 (R. Redler, NEC CCRLE, Germany) add OASIS[3/4] coupled interfaces !! - ! 2004-08 (C. Talandier) New trends organization !! - ! 2005-06 (C. Ethe) Add the 1D configuration possibility !! - ! 2005-11 (V. Garnier) Surface pressure gradient organization !! - ! 2006-03 (L. Debreu, C. Mazauric) Agrif implementation !! - ! 2006-04 (G. Madec, R. Benshila) Step reorganization !! - ! 2007-07 (J. Chanut, A. Sellar) Unstructured open boundaries (BDY) !! 3.2 ! 2009-08 (S. Masson) open/write in the listing file in mpp !! 3.3 ! 2010-05 (K. Mogensen, A. Weaver, M. Martin, D. Lea) Assimilation interface !! - ! 2010-10 (C. Ethe, G. Madec) reorganisation of initialisation phase !! 4.0 ! 2011-01 (A. R. Porter, STFC Daresbury) dynamical allocation !!---------------------------------------------------------------------- !!---------------------------------------------------------------------- !! nemo_gcm : solve ocean dynamics, tracer, biogeochemistry and/or sea-ice !! nemo_init : initialization of the NEMO system !! nemo_ctl : initialisation of the contol print !! nemo_closefile : close remaining open files !! nemo_alloc : dynamical allocation !! nemo_partition : calculate MPP domain decomposition !! sqfact : calculate factors of the no. of MPI processes !!---------------------------------------------------------------------- USE step_oce ! module used in the ocean time stepping module USE sbc_oce ! surface boundary condition: ocean USE cla ! cross land advection (tra_cla routine) USE domcfg ! domain configuration (dom_cfg routine) USE mppini ! shared/distributed memory setting (mpp_init routine) USE domain ! domain initialization (dom_init routine) USE obcini ! open boundary cond. initialization (obc_ini routine) USE bdyini ! unstructured open boundary cond. initialization (bdy_init routine) USE istate ! initial state setting (istate_init routine) USE ldfdyn ! lateral viscosity setting (ldfdyn_init routine) USE ldftra ! lateral diffusivity setting (ldftra_init routine) USE zdfini ! vertical physics setting (zdf_init routine) USE phycst ! physical constant (par_cst routine) USE trdmod ! momentum/tracers trends (trd_mod_init routine) USE asminc ! assimilation increments (asm_inc_init routine) USE asmtrj ! writing out state trajectory USE sshwzv ! vertical velocity used in asm USE diaptr ! poleward transports (dia_ptr_init routine) USE diaobs ! Observation diagnostics (dia_obs_init routine) USE step ! NEMO time-stepping (stp routine) #if defined key_oasis3 USE cpl_oasis3 ! OASIS3 coupling #elif defined key_oasis4 USE cpl_oasis4 ! OASIS4 coupling (not working) #endif USE c1d ! 1D configuration USE step_c1d ! Time stepping loop for the 1D configuration #if defined key_top USE trcini ! passive tracer initialisation #endif USE lib_mpp ! distributed memory computing #if defined key_iomput USE mod_ioclient #endif USE partition_mod ! irregular domain partitioning USE timing, ONLY: timing_init, timing_finalize, timing_disable, timing_enable #define ARPDEBUG IMPLICIT NONE PRIVATE PUBLIC nemo_gcm ! called by model.F90 PUBLIC nemo_init ! needed by AGRIF CHARACTER(lc) :: cform_aaa="( /, 'AAAAAAAA', / ) " ! flag for output listing !!---------------------------------------------------------------------- !! NEMO/OPA 4.0 , NEMO Consortium (2011) !! $Id$ !! Software governed by the CeCILL licence (NEMOGCM/NEMO_CeCILL.txt) !!---------------------------------------------------------------------- CONTAINS SUBROUTINE nemo_gcm !!---------------------------------------------------------------------- !! *** ROUTINE nemo_gcm *** !! !! ** Purpose : NEMO solves the primitive equations on an orthogonal !! curvilinear mesh on the sphere. !! !! ** Method : - model general initialization !! - launch the time-stepping (stp routine) !! - finalize the run by closing files and communications !! !! References : Madec, Delecluse, Imbard, and Levy, 1997: internal report, IPSL. !! Madec, 2008, internal report, IPSL. !!---------------------------------------------------------------------- INTEGER :: istp ! time step index !!---------------------------------------------------------------------- ! #if defined key_agrif CALL Agrif_Init_Grids() ! AGRIF: set the meshes #endif ! !-----------------------! CALL nemo_init !== Initialisations ==! ! !-----------------------! #if defined key_agrif CALL Agrif_Declare_Var ! AGRIF: set the meshes # if defined key_top CALL Agrif_Declare_Var_Top ! AGRIF: set the meshes # endif #endif ! check that all process are still there... If some process have an error, ! they will never enter in step and other processes will wait until the end of the cpu time! IF( lk_mpp ) CALL mpp_max( nstop ) IF(lwp) WRITE(numout,cform_aaa) ! Flag AAAAAAA CALL timing_enable() ! !-----------------------! ! !== time stepping ==! ! !-----------------------! istp = nit000 #if defined key_c1d DO WHILE ( istp <= nitend .AND. nstop == 0 ) CALL stp_c1d( istp ) istp = istp + 1 END DO #else IF( lk_asminc ) THEN IF( ln_bkgwri ) CALL asm_bkg_wri( nit000 - 1 ) ! Output background fields IF( ln_trjwri ) CALL asm_trj_wri( nit000 - 1 ) ! Output trajectory fields IF( ln_asmdin ) THEN ! Direct initialization IF( ln_trainc ) CALL tra_asm_inc( nit000 - 1 ) ! Tracers IF( ln_dyninc ) THEN CALL dyn_asm_inc( nit000 - 1 ) ! Dynamics IF ( ln_asmdin ) CALL ssh_wzv ( nit000 - 1 ) ! update vertical velocity ENDIF IF( ln_sshinc ) CALL ssh_asm_inc( nit000 - 1 ) ! SSH ENDIF ENDIF DO WHILE ( istp <= nitend .AND. nstop == 0 ) #if defined key_agrif CALL Agrif_Step( stp ) ! AGRIF: time stepping #else CALL stp( istp ) ! standard time stepping #endif istp = istp + 1 IF( lk_mpp ) CALL mpp_max( nstop ) END DO #endif IF( lk_diaobs ) CALL dia_obs_wri ! !------------------------! ! !== finalize the run ==! ! !------------------------! IF(lwp) WRITE(numout,cform_aaa) ! Flag AAAAAAA ! IF( nstop /= 0 .AND. lwp ) THEN ! error print WRITE(numout,cform_err) WRITE(numout,*) nstop, ' error have been found' ENDIF ! CALL timing_finalize ! Timing report CALL nemo_closefile #if defined key_oasis3 || defined key_oasis4 CALL cpl_prism_finalize ! end coupling and mpp communications with OASIS #else IF( lk_mpp ) CALL mppstop ! end mpp communications #endif ! END SUBROUTINE nemo_gcm SUBROUTINE nemo_init !!---------------------------------------------------------------------- !! *** ROUTINE nemo_init *** !! !! ** Purpose : initialization of the NEMO GCM !!---------------------------------------------------------------------- INTEGER :: ji ! dummy loop indices INTEGER :: ilocal_comm ! local integer CHARACTER(len=80), DIMENSION(24) :: cltxt !! NAMELIST/namctl/ ln_ctl , nn_print, nn_ictls, nn_ictle, & & nn_isplt, nn_jsplt, nn_jctls, nn_jctle, nn_bench !!---------------------------------------------------------------------- ! cltxt(:) = '' ! ! ! open Namelist file CALL ctl_opn( numnam, 'namelist', 'OLD', 'FORMATTED', 'SEQUENTIAL', -1, 6, .FALSE. ) ! READ( numnam, namctl ) ! Namelist namctl : Control prints & Benchmark ! ! !--------------------------------------------! ! ! set communicator & select the local node ! ! !--------------------------------------------! #if defined key_iomput IF( Agrif_Root() ) THEN # if defined key_oasis3 || defined key_oasis4 CALL cpl_prism_init( ilocal_comm ) ! nemo local communicator given by oasis # endif CALL init_ioclient( ilocal_comm ) ! exchange io_server nemo local communicator with the io_server ENDIF narea = mynode( cltxt, numnam, nstop, ilocal_comm ) ! Nodes selection #else # if defined key_oasis3 || defined key_oasis4 IF( Agrif_Root() ) THEN CALL cpl_prism_init( ilocal_comm ) ! nemo local communicator given by oasis ENDIF narea = mynode( cltxt, numnam, nstop, ilocal_comm ) ! Nodes selection (control print return in cltxt) # else ilocal_comm = 0 narea = mynode( cltxt, numnam, nstop ) ! Nodes selection (control print return in cltxt) # endif #endif narea = narea + 1 ! mynode return the rank of proc (0 --> jpnij -1 ) lwp = (narea == 1) .OR. ln_ctl ! control of all listing output print ! Calculate domain z dimensions as needed when partitioning. ! This used to be done in par_oce.F90 when they were parameters rather ! than variables IF( Agrif_Root() ) THEN jpk = jpkdta ! third dim jpkm1 = jpk-1 ! inner domain indices jpkorig = jpk ! Copy of jpk that is NOT modified jpkf = jpk ! Max depth of this sub-domain. Initially set to jpk here ! but altered later in domzgr ENDIF CALL timing_init ! Init timing module CALL timing_disable ! but disable during startup ! If dimensions of processor grid weren't specified in the namelist file ! then we calculate them here now that we have our communicator size IF( (jpni < 1) .OR. (jpnj < 1) )THEN #if defined key_mpp_mpi #if defined key_mpp_rkpart IF( Agrif_Root() ) CALL nemo_recursive_partition(mppsize) #else IF( Agrif_Root() ) CALL nemo_partition(mppsize) #endif #else jpni = 1 jpnj = 1 jpnij = jpni*jpnj #endif #if defined key_mpp_rkpart ELSE CALL ctl_stop( 'STOP', & 'nemo_init : invalid inputs in namelist - cannot specify jpn{i,j}>0 & & when using recursive k-section paritioning!' ) #endif END IF ! Calculate domain dimensions given calculated jpni and jpnj ! This used to be done in par_oce.F90 when they were parameters rather ! than variables IF( Agrif_Root() ) THEN #if ! defined key_mpp_rkpart jpi = ( jpiglo-2*jpreci + (jpni-1) ) / jpni + 2*jpreci ! first dim. jpj = ( jpjglo-2*jprecj + (jpnj-1) ) / jpnj + 2*jprecj ! second dim. jpim1 = jpi-1 ! inner domain indices jpjm1 = jpj-1 ! " " jpij = jpi*jpj ! jpi x j #endif ENDIF IF(lwp) THEN ! open listing units ! CALL ctl_opn( numout, 'ocean.output', 'REPLACE', 'FORMATTED', 'SEQUENTIAL', -1, 6, .FALSE., narea ) ! WRITE(numout,*) WRITE(numout,*) ' CNRS - NERC - Met OFFICE - MERCATOR-ocean' WRITE(numout,*) ' NEMO team' WRITE(numout,*) ' Ocean General Circulation Model' WRITE(numout,*) ' version 3.3 (2010) ' WRITE(numout,*) WRITE(numout,*) DO ji = 1, SIZE(cltxt,1) IF( TRIM(cltxt(ji)) /= '' ) WRITE(numout,*) cltxt(ji) ! control print of mynode END DO WRITE(numout,cform_aaa) ! Flag AAAAAAA ! ENDIF ! Now we know the dimensions of the grid and numout has been set we can ! allocate arrays CALL nemo_alloc() ! !-------------------------------! ! ! NEMO general initialization ! ! !-------------------------------! CALL nemo_ctl ! Control prints & Benchmark ! ! Domain decomposition #if defined key_mpp_rkpart CALL mpp_init3 ! Remainder of set-up for ! recursive partitioning #else IF( jpni*jpnj == jpnij ) THEN ; CALL mpp_init ! standard cutting out ELSE ; CALL mpp_init2 ! eliminate land processors ENDIF #endif ! ! ! General initialization ! CALL timing_init! Timing module CALL phy_cst ! Physical constants CALL eos_init ! Equation of state CALL dom_cfg ! Domain configuration CALL dom_init ! Domain IF( ln_ctl ) CALL prt_ctl_init ! Print control IF( lk_obc ) CALL obc_init ! Open boundaries IF( lk_bdy ) CALL bdy_init ! Unstructured open boundaries CALL istate_init ! ocean initial state (Dynamics and tracers) ! ! Ocean physics CALL sbc_init ! Forcings : surface module ! ! Vertical physics CALL zdf_init ! namelist read CALL zdf_bfr_init ! bottom friction IF( lk_zdfric ) CALL zdf_ric_init ! Richardson number dependent Kz IF( lk_zdftke ) CALL zdf_tke_init ! TKE closure scheme IF( lk_zdfgls ) CALL zdf_gls_init ! GLS closure scheme IF( lk_zdfkpp ) CALL zdf_kpp_init ! KPP closure scheme IF( lk_zdftmx ) CALL zdf_tmx_init ! tidal vertical mixing IF( lk_zdfddm .AND. .NOT. lk_zdfkpp ) & & CALL zdf_ddm_init ! double diffusive mixing ! ! Lateral physics CALL ldf_tra_init ! Lateral ocean tracer physics CALL ldf_dyn_init ! Lateral ocean momentum physics IF( lk_ldfslp ) CALL ldf_slp_init ! slope of lateral mixing ! ! Active tracers CALL tra_qsr_init ! penetrative solar radiation qsr CALL tra_bbc_init ! bottom heat flux IF( lk_trabbl ) CALL tra_bbl_init ! advective (and/or diffusive) bottom boundary layer scheme IF( lk_tradmp ) CALL tra_dmp_init ! internal damping trends CALL tra_adv_init ! horizontal & vertical advection CALL tra_ldf_init ! lateral mixing CALL tra_zdf_init ! vertical mixing and after tracer fields ! ! Dynamics CALL dyn_adv_init ! advection (vector or flux form) CALL dyn_vor_init ! vorticity term including Coriolis CALL dyn_ldf_init ! lateral mixing CALL dyn_hpg_init ! horizontal gradient of Hydrostatic pressure CALL dyn_zdf_init ! vertical diffusion CALL dyn_spg_init ! surface pressure gradient ! ! Misc. options IF( nn_cla == 1 ) CALL cla_init ! Cross Land Advection #if defined key_top ! ! Passive tracers CALL trc_init #endif ! ! Diagnostics CALL iom_init ! iom_put initialization IF( lk_floats ) CALL flo_init ! drifting Floats IF( lk_diaar5 ) CALL dia_ar5_init ! ar5 diag CALL dia_ptr_init ! Poleward TRansports initialization CALL dia_hsb_init ! heat content, salt content and volume budgets CALL trd_mod_init ! Mixed-layer/Vorticity/Integral constraints trends IF( lk_diaobs ) THEN ! Observation & model comparison CALL dia_obs_init ! Initialize observational data CALL dia_obs( nit000 - 1 ) ! Observation operator for restart ENDIF ! ! Assimilation increments IF( lk_asminc ) CALL asm_inc_init ! Initialize assimilation increments IF(lwp) WRITE(numout,*) 'Euler time step switch is ', neuler ! END SUBROUTINE nemo_init SUBROUTINE nemo_ctl !!---------------------------------------------------------------------- !! *** ROUTINE nemo_ctl *** !! !! ** Purpose : control print setting !! !! ** Method : - print namctl information and check some consistencies !!---------------------------------------------------------------------- ! IF(lwp) THEN ! control print WRITE(numout,*) WRITE(numout,*) 'nemo_ctl: Control prints & Benchmark' WRITE(numout,*) '~~~~~~~ ' WRITE(numout,*) ' Namelist namctl' WRITE(numout,*) ' run control (for debugging) ln_ctl = ', ln_ctl WRITE(numout,*) ' level of print nn_print = ', nn_print WRITE(numout,*) ' Start i indice for SUM control nn_ictls = ', nn_ictls WRITE(numout,*) ' End i indice for SUM control nn_ictle = ', nn_ictle WRITE(numout,*) ' Start j indice for SUM control nn_jctls = ', nn_jctls WRITE(numout,*) ' End j indice for SUM control nn_jctle = ', nn_jctle WRITE(numout,*) ' number of proc. following i nn_isplt = ', nn_isplt WRITE(numout,*) ' number of proc. following j nn_jsplt = ', nn_jsplt WRITE(numout,*) ' benchmark parameter (0/1) nn_bench = ', nn_bench ENDIF ! nprint = nn_print ! convert DOCTOR namelist names into OLD names nictls = nn_ictls nictle = nn_ictle njctls = nn_jctls njctle = nn_jctle isplt = nn_isplt jsplt = nn_jsplt nbench = nn_bench ! ! Parameter control ! IF( ln_ctl ) THEN ! sub-domain area indices for the control prints IF( lk_mpp ) THEN isplt = jpni ; jsplt = jpnj ; ijsplt = jpni*jpnj ! the domain is forced to the real split domain ELSE IF( isplt == 1 .AND. jsplt == 1 ) THEN CALL ctl_warn( ' - isplt & jsplt are equal to 1', & & ' - the print control will be done over the whole domain' ) ENDIF ijsplt = isplt * jsplt ! total number of processors ijsplt ENDIF IF(lwp) WRITE(numout,*)' - The total number of processors over which the' IF(lwp) WRITE(numout,*)' print control will be done is ijsplt : ', ijsplt ! ! ! indices used for the SUM control IF( nictls+nictle+njctls+njctle == 0 ) THEN ! print control done over the default area lsp_area = .FALSE. ELSE ! print control done over a specific area lsp_area = .TRUE. IF( nictls < 1 .OR. nictls > jpiglo ) THEN CALL ctl_warn( ' - nictls must be 1<=nictls>=jpiglo, it is forced to 1' ) nictls = 1 ENDIF IF( nictle < 1 .OR. nictle > jpiglo ) THEN CALL ctl_warn( ' - nictle must be 1<=nictle>=jpiglo, it is forced to jpiglo' ) nictle = jpiglo ENDIF IF( njctls < 1 .OR. njctls > jpjglo ) THEN CALL ctl_warn( ' - njctls must be 1<=njctls>=jpjglo, it is forced to 1' ) njctls = 1 ENDIF IF( njctle < 1 .OR. njctle > jpjglo ) THEN CALL ctl_warn( ' - njctle must be 1<=njctle>=jpjglo, it is forced to jpjglo' ) njctle = jpjglo ENDIF ENDIF ENDIF ! IF( nbench == 1 ) THEN ! Benchmark SELECT CASE ( cp_cfg ) CASE ( 'gyre' ) ; CALL ctl_warn( ' The Benchmark is activated ' ) CASE DEFAULT ; CALL ctl_stop( ' The Benchmark is based on the GYRE configuration:', & & ' key_gyre must be used or set nbench = 0' ) END SELECT ENDIF ! IF( lk_c1d .AND. .NOT.lk_iomput ) CALL ctl_stop( 'nemo_ctl: The 1D configuration must be used ', & & 'with the IOM Input/Output manager. ' , & & 'Compile with key_iomput enabled' ) ! END SUBROUTINE nemo_ctl SUBROUTINE nemo_closefile !!---------------------------------------------------------------------- !! *** ROUTINE nemo_closefile *** !! !! ** Purpose : Close the files !!---------------------------------------------------------------------- ! IF( lk_mpp ) CALL mppsync ! CALL iom_close ! close all input/output files managed by iom_* ! IF( numstp /= -1 ) CLOSE( numstp ) ! time-step file IF( numsol /= -1 ) CLOSE( numsol ) ! solver file IF( numnam /= -1 ) CLOSE( numnam ) ! oce namelist IF( numnam_ice /= -1 ) CLOSE( numnam_ice ) ! ice namelist IF( numevo_ice /= -1 ) CLOSE( numevo_ice ) ! ice variables (temp. evolution) IF( numout /= 6 ) CLOSE( numout ) ! standard model output file ! numout = 6 ! redefine numout in case it is used after this point... ! END SUBROUTINE nemo_closefile SUBROUTINE nemo_alloc !!---------------------------------------------------------------------- !! *** ROUTINE nemo_alloc *** !! !! ** Purpose : Allocate all the dynamic arrays of the OPA modules !! !! ** Method : !!---------------------------------------------------------------------- USE diawri , ONLY: dia_wri_alloc USE dom_oce , ONLY: dom_oce_alloc USE ldfdyn_oce, ONLY: ldfdyn_oce_alloc USE ldftra_oce, ONLY: ldftra_oce_alloc USE trc_oce , ONLY: trc_oce_alloc USE wrk_nemo , ONLY: wrk_alloc USE exchmod , ONLY: exchmod_alloc ! INTEGER :: ierr !!---------------------------------------------------------------------- ! ierr = oce_alloc () ! ocean ierr = ierr + dia_wri_alloc () ierr = ierr + dom_oce_alloc () ! ocean domain ierr = ierr + ldfdyn_oce_alloc() ! ocean lateral physics : dynamics ierr = ierr + ldftra_oce_alloc() ! ocean lateral physics : tracers ierr = ierr + zdf_oce_alloc () ! ocean vertical physics ! ierr = ierr + lib_mpp_alloc (numout) ! mpp exchanges ierr = ierr + trc_oce_alloc () ! shared TRC / TRA arrays ! ierr = ierr + wrk_alloc(numout, lwp) ! workspace ! ierr = ierr + exchmod_alloc() ! New mpp msg framework ! IF( lk_mpp ) CALL mpp_sum( ierr ) IF( ierr /= 0 ) CALL ctl_stop( 'STOP', 'nemo_alloc : unable to allocate standard ocean arrays' ) ! END SUBROUTINE nemo_alloc SUBROUTINE nemo_partition( num_pes ) USE mapcomm_mod, ONLY: trimmed !!---------------------------------------------------------------------- !! *** ROUTINE nemo_partition *** !! !! ** Purpose : Work out a sensible factorisation of the number of !! processors for the x and y dimensions. !! ** Method : !!---------------------------------------------------------------------- INTEGER, INTENT(in) :: num_pes ! The number of MPI processes we have ! INTEGER :: ifact1, ifact2 ! factors of num_pes, ifact1 <= ifact2 !!---------------------------------------------------------------------- ! Factorise the number of processors into ifact1*ifact2, such that ! ifact1 and ifact2 are as nearly equal as possible. CALL sqfact( num_pes, ifact1, ifact2 ) ! Make sure that the smaller dimension of the processor grid ! is given the smaller dimension of the global domain IF( jpiglo <= jpjglo) THEN jpni = ifact1 jpnj = ifact2 ELSE jpni = ifact2 jpnj = ifact1 ENDIF ! This should never happen IF( (jpni*jpnj) /= num_pes) THEN WRITE (numout, *) 'WARNING: internal error - factorisation of number of PEs failed' ENDIF ! This should only happen if num_pes is prime IF( ifact1 <= 1 ) THEN WRITE (numout, *) 'WARNING: factorisation of number of PEs failed' WRITE (numout, *) ' : using grid of ',jpni,' x ',jpnj ENDIF ! jpnij = jpni*jpnj ! ! Array that stores whether domain boundaries have been trimmed. Not used in ! this case (regular domain decomp.) so set all to false. ALLOCATE(trimmed(4,jpnij)) trimmed(:,:) = .FALSE. END SUBROUTINE nemo_partition SUBROUTINE nemo_recursive_partition( num_pes ) USE in_out_manager, ONLY: numnam USE dom_oce, ONLY: ln_zco USE dom_oce, ONLY: gdepw_0, gdept_0, e3w_0, e3t_0, & mig, mjg, mi0, mi1, mj0, mj1, mbathy, bathy USE domzgr, ONLY: zgr_z, zgr_bat, namzgr, zgr_zco, zgr_zps USE closea, ONLY: dom_clo USE domain, ONLY: dom_nam USE iom, ONLY: jpiglo, jpjglo, wp, jpdom_unknown, & iom_open, iom_get, iom_close USE mapcomm_mod, ONLY: ielb, ieub, pielb, pjelb, pieub, pjeub, & iesub, jesub, jeub, ilbext, iubext, jubext, & jlbext, pnactive, piesub, pjesub, jelb, pilbext, & piubext, pjlbext, pjubext, LAND, trimmed, & msgtrim_z, set_num_subdomains USE partition_mod, ONLY: partition_rk, partition_mca_rk, read_partition, & imask, ibotlevel, partition_mask_alloc, & smooth_global_bathy, global_bot_level USE par_oce, ONLY: do_exchanges #if defined key_mpp_mpi USE mpi #endif !!---------------------------------------------------------------------- !! *** ROUTINE nemo_recursive_partition *** !! !! ** Purpose : Work out a sensible factorisation of the number of !! processors for the x and y dimensions. !! ** Method : !!---------------------------------------------------------------------- IMPLICIT none INTEGER, INTENT(in) :: num_pes ! The number of MPI processes we have ! Local vars INTEGER :: ierr ! Error flag INTEGER :: inum ! temporary logical unit INTEGER :: ii,jj,iproc ! Loop index CHARACTER(LEN=8) :: lstr ! Local string for reading env. var. INTEGER :: lztrim ! Local int for " " " REAL(wp), ALLOCATABLE, DIMENSION(:,:) :: zdta ! temporary data workspace !!---------------------------------------------------------------------- ! Allocate masking arrays used in partitioning CALL partition_mask_alloc(jpiglo,jpjglo,ierr) IF(ierr /= 0)THEN CALL ctl_stop('nemo_recursive_partition: failed to allocate masking arrays') RETURN END IF ! Allocate local workspace array for this routine ALLOCATE(zdta(jpiglo,jpjglo), Stat=ierr) IF(ierr /= 0)THEN CALL ctl_stop('nemo_recursive_partition: failed to allocate workspace arrays') RETURN END IF ! Check whether user has specified halo trimming in z via environment ! variable. ! Halo trimming in z is on by default msgtrim_z = .TRUE. CALL GET_ENVIRONMENT_VARIABLE(NAME='NEMO_MSGTRIM_Z', VALUE=lstr, & STATUS=ierr) IF( ierr == 0)THEN READ(lstr,FMT="(I10)",IOSTAT=ierr) lztrim IF(ierr == 0)THEN IF (lztrim == 0) msgtrim_z = .FALSE. ELSE CALL ctl_warn('nemo_recursive_partition: failed to parse value of NEMO_MSGTRIM_Z environment variable: '//TRIM(lstr)) END IF END IF IF(lwp) WRITE(*,*) 'ARPDBG: msgtrim_z = ',msgtrim_z ! ============================ ! Generate a global mask from the model bathymetry ! ============================ ! Read the z-coordinate options from the namelist file REWIND(numnam) READ (numnam, namzgr) ! Read domain options from namelist file CALL dom_nam() ! Allocate these arrays so we can use domzgr::zgr_z routine; free them ! when we're done so as not to upset the 'official' allocation once ! the domain decomposition is done. ALLOCATE(gdepw_0(jpk), gdept_0(jpk), e3w_0(jpk), e3t_0(jpk), & mig(jpiglo), mjg(jpjglo), & mbathy(jpiglo,jpjglo), bathy(jpiglo,jpjglo), Stat=ierr) IF(ierr /= 0)THEN CALL ctl_stop('STOP', & 'nemo_recursive_partition: failed to allocate zgr_z() arrays') RETURN END IF ! Set-up reference depth coordinates CALL zgr_z() ! Set-up sub-domain limits as global domain for zgr_bat() nldi = 2 ; nlci = jpiglo - 1 nldj = 2 ; nlcj = jpjglo - 1 jpi = jpiglo jpj = jpjglo ! Set-up fake m{i,j}g arrays for zgr_bat() call DO ii = 1, jpiglo, 1 mig(ii) = ii mi0(ii) = ii mi1(ii) = ii END DO DO jj = 1, jpjglo, 1 mjg(jj) = jj mj0(jj) = jj mj1(jj) = jj END DO ! Initialise closed seas so loop over closed seas in zgr_bat works CALL dom_clo() ! Read-in bathy (if required) of global domain CALL zgr_bat(.TRUE.) ! land/sea mask (zero on land, 1 otherwise) over the global/zoom domain imask(:,:)=1 ! Copy bathymetry in case we need to smooth it zdta(:,:) = bathy(:,:) IF(ln_sco)THEN ! If ln_sco defined then the bathymetry gets smoothed before the ! simulation begins and that process can alter the coastlines (bug!) ! therefore we do it here too before calculating our mask. CALL smooth_global_bathy(zdta, mbathy) ELSE IF(ln_zps)THEN CALL zgr_zps(.TRUE.) ELSE IF(ln_zco)THEN ! Not certain this is required since mbathy computed in zgr_bat() ! in this case. !CALL zgr_zco() END IF ! Compute the deepest/last ocean level for every point on the grid ibotlevel(:,:) = mbathy(:,:) CALL global_bot_level(ibotlevel) ! Comment-out line below to achieve a regular partition WHERE ( zdta(:,:) <= 1.0E-20 ) imask = LAND ! Allocate partitioning arrays. IF ( .NOT. ALLOCATED(pielb) ) THEN ALLOCATE (pielb(num_pes), pieub(num_pes), piesub(num_pes), & pilbext(num_pes), piubext(num_pes), & pjelb(num_pes), pjeub(num_pes), pjesub(num_pes), & pjlbext(num_pes), pjubext(num_pes), pnactive(num_pes), & trimmed(4,num_pes), Stat = ierr) IF(ierr /= 0)THEN CALL ctl_stop('STOP', & 'nemo_recursive_partition: failed to allocate partitioning arrays') RETURN END IF ENDIF ! Set error flag so that we calculate domain decomp if not reading ! existing decomposition or if read fails. ierr = 1 IF( nn_readpart )THEN ! Read the partitioning to use from disk CALL read_partition(ierr) IF ( ierr /= 0 ) THEN CALL ctl_warn('Read of pre-calculated domain decomposition failed - will calculate one instead.') END IF END IF ! Set the number of sub-domains for which we are to partition ! (module var in mapcomm_mod) CALL set_num_subdomains(num_pes) IF(ierr /= 0)THEN ! Multi-core aware version of recursive k-section partitioning. ! Currently only accounts for whether a grid point is wet or dry. ! It has no knowledge of the number of wet levels at a point. CALL partition_mca_rk ( imask, 1, jpiglo, 1, jpjglo, ierr ) ! Now we can do recursive k-section partitioning ! ARPDBG - BUG if limits on array below are set to anything other than ! 1 and jp{i,j}glo then check for external boundaries in a few lines ! time WILL FAIL! ! CALL partition_rk ( imask, 1, jpiglo, 1, jpjglo, ierr ) END IF ! Check the error code from partitioning. IF ( ierr /= 0 ) THEN CALL ctl_stop('STOP','nemo_recursive_partition: Partitioning failed') RETURN ENDIF ! If we used generate_fake_land() above then we must set ! the mask correctly now we've partitioned. This is only ! necessary when testing. !WHERE ( zdta(:,:) <= 0. ) imask = 0 ! ARPDBG Quick and dirty dump to stdout in gnuplot form IF(narea == 1)THEN OPEN(UNIT=998, FILE="imask.dat", & STATUS='REPLACE', ACTION='WRITE', IOSTAT=jj) IF( jj == 0 )THEN WRITE (998,*) '# Depth map' WRITE (998,*) '# i j bathy imask ibotlevel mbathy' DO jj = 1, jpjglo, 1 DO ii = 1, jpiglo, 1 WRITE (998,"(I4,1x,I4,1x,E16.6,1x,I4,1x,I4,1x,I4)") & ii, jj, zdta(ii,jj), imask(ii,jj), ibotlevel(ii,jj), mbathy(ii,jj) END DO WRITE (998,*) END DO CLOSE(998) END IF END IF jpkm1 = jpk - 1 ! This chunk taken directly from original mpp_ini - not sure why nbondi ! is reset? However, if it isn't reset then bad things happen in dommsk ! so I'm doing what the original code does... nperio = 0 nbondi = 0 IF( jperio == 1 .OR. jperio == 4 .OR. jperio == 6 ) THEN IF( jpni == 1 )THEN nbondi = 2 nperio = 1 END IF END IF #if defined ARPDEBUG ! This output is REQUIRED by the check_nemo_comms.pl test script WRITE (*,FMT="(I4,' : ARPDBG: ielb, ieub, iesub = ',3I5)") narea-1,& ielb, ieub, iesub WRITE (*,FMT="(I4,' : ARPDBG: jelb, jeub, jesub = ',3I5)") narea-1,& jelb, jeub, jesub WRITE (*,FMT="(I4,' : ARPDBG: nldi, nlei, nlci = ',3I5)") narea-1, & nldi, nlei, nlci WRITE (*,FMT="(I4,' : ARPDBG: nldj, nlej, nlcj = ',3I5)") narea-1, & nldj, nlej, nlcj WRITE (*,FMT="(I4,' : ARPDBG: jpi, jpj = ',2I5)") narea-1, jpi, jpj WRITE (*,FMT="(I4,' : ARPDBG: nimpp, njmpp = ',2I5)") narea-1, & nimpp, njmpp #endif ! Debugging option - can turn off all halo exchanges by setting this to ! false. do_exchanges = .TRUE. ! Free the domzgr/_oce member arrays that we used earlier in zgr_z() and ! zgr_bat(). DEALLOCATE(gdepw_0, gdept_0, e3w_0, e3t_0, mig, mjg, & mbathy, bathy) END SUBROUTINE nemo_recursive_partition SUBROUTINE sqfact ( kn, kna, knb ) !!---------------------------------------------------------------------- !! *** ROUTINE sqfact *** !! !! ** Purpose : return factors (kna, knb) of kn, such that !! (1) kna*knb=kn !! (2) kna and knb are as near equal as possible !! (3) kna < knb !! ** Method : Search backwards from the square root of kn, !! until we find an integer that cleanly divides kn !! ** Preconditions : kn must be positive !!---------------------------------------------------------------------- INTEGER, INTENT(in ) :: kn INTEGER, INTENT( out) :: kna, knb ! Search backwards from the square root of n. fact_loop: DO kna=INT(SQRT(REAL(kn))),1,-1 IF ( kn/kna*kna == kn ) THEN EXIT fact_loop ENDIF END DO fact_loop IF( kna < 1 ) kna = 1 ! kna divides kn cleanly. Work out the other factor. knb = kn/kna END SUBROUTINE sqfact SUBROUTINE generate_fake_land(imask) !!---------------------------------------------------------------------- !! Generate a fake land mass to test the decomposition code !!---------------------------------------------------------------------- USE par_oce, ONLY: jpiglo, jpjglo USE partition_mod, ONLY: write_partition_map IMPLICIT none INTEGER, DIMENSION(jpiglo,jpjglo), INTENT(inout) :: imask ! Locals INTEGER :: ii, jj INTEGER :: icentre, jcentre INTEGER :: iwidth, iheight INTEGER :: istart, istop ! imask is zero on land points , unity on ocean points iwidth = jpiglo/8 iheight = jpjglo/8 icentre = jpiglo/2 jcentre = jpjglo/2 istart = icentre - iwidth istop = icentre + iwidth DO jj = jcentre, jcentre - iheight, -1 imask(istart:istop,jj) = 0 istart = istart + 1 istop = istop - 1 END DO istart = icentre - iwidth istop = icentre + iwidth DO jj = jcentre+1, jcentre + iheight, 1 imask(istart:istop,jj) = 0 istart = istart + 1 istop = istop - 1 END DO ! Quick and dirty dump to stdout in gnuplot form !!$ WRITE (*,*) 'GNUPLOT MAP' !!$ DO jj = 1, jpjglo, 1 !!$ DO ii = 1, jpiglo, 1 !!$ WRITE (*,*) ii, jj, imask(ii,jj) !!$ END DO !!$ WRITE (*,*) !!$ END DO !!$ WRITE (*,*) 'END GNUPLOT MAP' END SUBROUTINE generate_fake_land !!====================================================================== END MODULE nemogcm