[615] | 1 | import time |
---|
| 2 | import math |
---|
| 3 | import numpy as np |
---|
| 4 | import dynamico.wrap as wrap |
---|
| 5 | from ctypes import c_void_p, c_int, c_double, c_bool |
---|
| 6 | |
---|
[640] | 7 | from libc.time cimport time as ctime, time_t |
---|
| 8 | cimport numpy as np |
---|
| 9 | |
---|
[615] | 10 | #------------- direct Cython interface to DYNAMICO routines -------------# |
---|
| 11 | |
---|
[640] | 12 | cdef enum: max_nb_stage=5 |
---|
| 13 | cdef extern : |
---|
| 14 | cdef double tauj[max_nb_stage] |
---|
| 15 | cdef double cslj[max_nb_stage][max_nb_stage] |
---|
| 16 | cdef double cflj[max_nb_stage][max_nb_stage] |
---|
| 17 | cdef int nb_stage[1] |
---|
| 18 | |
---|
[615] | 19 | cdef extern from "functions.h": |
---|
[640] | 20 | cdef void dynamico_ARK_step(double *mass_col, double *rhodz, double *theta_rhodz, |
---|
| 21 | double *u, double *geopot, double *w, |
---|
| 22 | double *theta, double *ps, double *pk, double *hflux, double *qv, |
---|
| 23 | double *dmass_col, double *drhodz, double *dtheta_rhodz, |
---|
| 24 | double *du_fast, double *du_slow, |
---|
| 25 | double *dPhi_fast, double *dPhi_slow, |
---|
| 26 | double *dW_fast, double *dW_slow) |
---|
[618] | 27 | cdef void dynamico_init_params() |
---|
| 28 | cpdef void dynamico_setup_xios() |
---|
| 29 | cpdef void dynamico_xios_set_timestep(double) |
---|
| 30 | cpdef void dynamico_xios_update_calendar(int) |
---|
[615] | 31 | |
---|
| 32 | #------------- import and wrap DYNAMICO routines -------------# |
---|
| 33 | |
---|
| 34 | ker=wrap.Struct() # store imported fun X as funs.X |
---|
| 35 | |
---|
| 36 | check_args = False # use True instead of False for debugging, probably with some overhead |
---|
| 37 | |
---|
| 38 | try: |
---|
[618] | 39 | kernels = wrap.SharedLib(vars(ker), 'libicosa.so', check_args=check_args) |
---|
[615] | 40 | setvar, setvars, getvar, getvars = kernels.setvar, kernels.setvars, kernels.getvar, kernels.getvars |
---|
| 41 | except OSError: |
---|
| 42 | print """ |
---|
| 43 | Unable to load shared library 'libkernels.so' ! |
---|
| 44 | """ |
---|
| 45 | raise |
---|
| 46 | |
---|
| 47 | # providing a full prototype enables type-checking when calling |
---|
| 48 | # if a number n is present in the prototype, the previous type is repeated n times |
---|
| 49 | kernels.import_funs([ |
---|
[622] | 50 | ['dynamico_setup_xios',None], |
---|
| 51 | ['dynamico_xios_set_timestep',c_double], |
---|
| 52 | ['dynamico_xios_update_calendar',c_int], |
---|
[618] | 53 | ['dynamico_init_mesh',c_void_p,13], |
---|
| 54 | ['dynamico_init_metric', c_void_p,6], |
---|
| 55 | ['dynamico_init_hybrid', c_void_p,3], |
---|
| 56 | ['dynamico_caldyn_unstructured', c_double, c_void_p,20], |
---|
[620] | 57 | ['dynamico_partition_graph', c_int,2, c_void_p,3, c_int, c_void_p], |
---|
[615] | 58 | ]) |
---|
| 59 | |
---|
| 60 | # set/get global variables |
---|
| 61 | eta_mass,eta_lag=(1,2) |
---|
| 62 | thermo_theta,thermo_entropy,thermo_moist,thermo_boussinesq=(1,2,3,4) |
---|
| 63 | |
---|
| 64 | kernels.addvars( |
---|
[640] | 65 | c_bool,'hydrostatic','debug_hevi_solver', |
---|
[615] | 66 | c_int,'llm','nqdyn','primal_num','max_primal_deg', |
---|
| 67 | 'dual_num','max_dual_deg','edge_num','max_trisk_deg', |
---|
| 68 | 'caldyn_thermo','caldyn_eta','nb_threads', |
---|
| 69 | c_double,'elapsed','g', 'ptop', 'cpp', 'cppv', |
---|
| 70 | 'Rd', 'Rv', 'preff', 'Treff', 'pbot', 'rho_bot', 'Phi_bot') |
---|
| 71 | |
---|
| 72 | elapsed=0. |
---|
| 73 | |
---|
[640] | 74 | #------------------------ Extension type performing a full ARK time step ---------------------- |
---|
| 75 | |
---|
| 76 | ctypedef double *dbl_ptr |
---|
| 77 | cdef dbl_ptr ptr1(double[:] data) : return &data[0] |
---|
| 78 | cdef dbl_ptr ptr2(double[:,:] data) : return &data[0,0] |
---|
| 79 | cdef dbl_ptr ptr3(double[:,:,:] data) : return &data[0,0,0] |
---|
| 80 | cdef dbl_ptr ptr(data): |
---|
| 81 | n=data.ndim |
---|
| 82 | if n==1 : return ptr1(data) |
---|
| 83 | if n==2 : return ptr2(data) |
---|
| 84 | if n==3 : return ptr3(data) |
---|
| 85 | |
---|
| 86 | cdef alloc(dbl_ptr *p, allocator, n=1): |
---|
| 87 | data=allocator(n) |
---|
| 88 | p[0]=ptr(data) |
---|
| 89 | return data |
---|
| 90 | |
---|
| 91 | cdef check_ptr(name, dbl_ptr p, np.ndarray data): |
---|
| 92 | if p != ptr(data) : print name, 'p <> ptr(data) !!' |
---|
| 93 | |
---|
| 94 | cdef class Caldyn_step: |
---|
| 95 | # pointer to allocated arrays |
---|
| 96 | cdef dbl_ptr p_mass, p_theta_rhodz, p_u, p_geopot, p_w # prognostic |
---|
| 97 | cdef dbl_ptr p_mass_col, p_dmass_col, p_ps, p_theta, p_pk, p_hflux, p_qv # diagnostic |
---|
| 98 | cdef dbl_ptr p_drhodz, p_dtheta_rhodz, p_du_fast, p_du_slow # tendencies |
---|
| 99 | cdef dbl_ptr p_dPhi_fast, p_dPhi_slow, p_dW_fast, p_dW_slow # tendencies |
---|
| 100 | # allocated arrays, must remain referenced or segfault |
---|
| 101 | cdef readonly np.ndarray mass, theta_rhodz, u, geopot, w |
---|
| 102 | cdef readonly np.ndarray mass_col, dmass_col, ps, theta, pk, hflux, qv |
---|
| 103 | cdef readonly np.ndarray drhodz, dtheta_rhodz, du_fast, du_slow |
---|
| 104 | cdef readonly np.ndarray dPhi_fast, dPhi_slow, dW_fast, dW_slow |
---|
| 105 | |
---|
| 106 | def __init__(self,mesh,time_scheme): |
---|
| 107 | # self.mesh=mesh |
---|
| 108 | fps, ftheta, fmass = mesh.field_ps, mesh.field_theta, mesh.field_mass |
---|
| 109 | fw, fu, fz = mesh.field_w, mesh.field_u, mesh.field_z |
---|
| 110 | # collect coefficients of time scheme |
---|
| 111 | cdef double[:] tauj_ = time_scheme.tauj |
---|
| 112 | cdef double[:,:] cslj_ = time_scheme.csjl |
---|
| 113 | cdef double[:,:] cflj_ = time_scheme.cfjl |
---|
| 114 | ns = time_scheme.nstage |
---|
| 115 | nb_stage[0]= ns |
---|
| 116 | cdef int i,j |
---|
| 117 | for i in range(ns): |
---|
| 118 | tauj[i]=tauj_[i] |
---|
| 119 | for j in range(ns): |
---|
| 120 | cslj[i][j]=cslj_[i,j] |
---|
| 121 | cflj[i][j]=cflj_[i,j] |
---|
| 122 | # allocate arrays, store pointers to avoid overhead when calling dynamico |
---|
| 123 | # prognostic/diagnostic |
---|
| 124 | self.ps = alloc(&self.p_ps, fps) |
---|
| 125 | self.mass_col, self.dmass_col = alloc(&self.p_mass_col, fps), alloc(&self.p_dmass_col, fps), |
---|
| 126 | self.mass, self.theta_rhodz = alloc(&self.p_mass, fmass), alloc(&self.p_theta_rhodz, fmass), |
---|
| 127 | self.theta, self.pk = alloc(&self.p_theta, fmass), alloc(&self.p_pk, fmass), |
---|
| 128 | self.geopot, self.w = alloc(&self.p_geopot, fw), alloc(&self.p_w, fw), |
---|
| 129 | self.hflux, self.u = alloc(&self.p_hflux, fu), alloc(&self.p_u, fu) |
---|
| 130 | self.qv = alloc(&self.p_qv,fz) |
---|
| 131 | # tendencies |
---|
| 132 | self.drhodz, self.dtheta_rhodz = alloc(&self.p_drhodz,fmass,ns), alloc(&self.p_dtheta_rhodz,fmass,ns) |
---|
| 133 | self.du_fast, self.du_slow = alloc(&self.p_du_fast,fu,ns), alloc(&self.p_du_slow,fu,ns) |
---|
| 134 | self.dPhi_fast, self.dPhi_slow = alloc(&self.p_dPhi_fast,fw,ns), alloc(&self.p_dPhi_slow,fw,ns) |
---|
| 135 | self.dW_fast, self.dW_slow = alloc(&self.p_dW_fast,fw,ns), alloc(&self.p_dW_slow,fw,ns) |
---|
| 136 | def next(self): |
---|
| 137 | # global elapsed |
---|
| 138 | # time1=time.time() |
---|
| 139 | check_ptr('mass', self.p_mass, self.mass) |
---|
| 140 | check_ptr('u', self.p_u, self.u) |
---|
| 141 | dynamico_ARK_step(self.p_mass_col, self.p_mass, self.p_theta_rhodz, |
---|
| 142 | self.p_u, self.p_geopot, self.p_w, |
---|
| 143 | self.p_theta, self.p_ps, self.p_pk, self.p_hflux, self.p_qv, |
---|
| 144 | self.p_dmass_col, self.p_drhodz, self.p_dtheta_rhodz, |
---|
| 145 | self.p_du_fast, self.p_du_slow, |
---|
| 146 | self.p_dPhi_fast, self.p_dPhi_slow, |
---|
| 147 | self.p_dW_fast, self.p_dW_slow) |
---|
| 148 | #time2=time.time() |
---|
| 149 | #if time2>time1: elapsed=elapsed+time2-time1 |
---|
| 150 | def setup_TRSW(self): |
---|
| 151 | setvars(('hydrostatic','caldyn_thermo','caldyn_eta'), |
---|
| 152 | (True,thermo_boussinesq,eta_lag)) |
---|
| 153 | dynamico_init_params() |
---|
| 154 | def setup_HPE(self, caldyn_thermo, caldyn_eta, g, BC,thermo): |
---|
| 155 | setvars(('hydrostatic','caldyn_thermo','caldyn_eta', |
---|
| 156 | 'g','ptop','Rd','cpp','preff','Treff'), |
---|
| 157 | (True,caldyn_thermo,caldyn_eta, |
---|
| 158 | g,BC.ptop,thermo.Rd,thermo.Cpd,thermo.p0,thermo.T0)) |
---|
| 159 | dynamico_init_params() |
---|
| 160 | def setup_NH(self, caldyn_thermo, caldyn_eta, g, BC,thermo): |
---|
| 161 | setvars(('hydrostatic','caldyn_thermo','caldyn_eta', |
---|
| 162 | 'g','ptop','Rd','cpp','preff','Treff', |
---|
| 163 | 'pbot','rho_bot'), |
---|
| 164 | (False,caldyn_thermo,caldyn_eta, |
---|
| 165 | g,BC.ptop,thermo.Rd,thermo.Cpd,thermo.p0,thermo.T0, |
---|
| 166 | BC.pbot.max(), BC.rho_bot.max())) |
---|
| 167 | dynamico_init_params() |
---|
| 168 | |
---|
[615] | 169 | #----------------------------- Base class for dynamics ------------------------ |
---|
| 170 | |
---|
| 171 | class Caldyn: |
---|
| 172 | def __init__(self,mesh): |
---|
| 173 | self.mesh=mesh |
---|
| 174 | fps, ftheta, fmass = mesh.field_ps, mesh.field_theta, mesh.field_mass |
---|
| 175 | fw, fu, fz = mesh.field_w, mesh.field_u, mesh.field_z |
---|
| 176 | self.ps, self.ms, self.dms = fps(), fps(), fps() |
---|
| 177 | self.s, self.hs, self.dhs = ftheta(), ftheta(), ftheta() |
---|
| 178 | self.pk, self.berni, self.geopot, self.hflux = fmass(),fmass(),fw(),fu() |
---|
| 179 | self.qu, self.qv = fu(),fz() |
---|
| 180 | self.fmass, self.ftheta, self.fu, self.fw = fmass, ftheta, fu, fw |
---|
| 181 | def bwd_fast_slow(self, flow, tau): |
---|
| 182 | global elapsed |
---|
| 183 | time1=time.time() |
---|
| 184 | flow,fast,slow = self._bwd_fast_slow_(flow,tau) |
---|
| 185 | time2=time.time() |
---|
| 186 | elapsed=elapsed+time2-time1 |
---|
| 187 | return flow,fast,slow |
---|
| 188 | |
---|
[618] | 189 | # when calling caldyn_unstructured, arrays for tendencies must be re-created each time |
---|
[615] | 190 | # to avoid overwriting in the same memory space when time scheme is multi-stage |
---|
| 191 | |
---|
| 192 | #-------------------------- Shallow-water dynamics --------------------- |
---|
| 193 | |
---|
| 194 | class Caldyn_RSW(Caldyn): |
---|
| 195 | def __init__(self,mesh): |
---|
| 196 | Caldyn.__init__(self,mesh) |
---|
| 197 | setvars(('hydrostatic','caldyn_thermo','caldyn_eta'), |
---|
| 198 | (True,thermo_boussinesq,eta_lag)) |
---|
| 199 | self.dhs = self.fmass() |
---|
[618] | 200 | dynamico_init_params() |
---|
[615] | 201 | def _bwd_fast_slow_(self, flow, tau): |
---|
| 202 | h,u = flow |
---|
| 203 | # h*s = h => uniform buoyancy s=1 => shallow-water |
---|
| 204 | dh, du_slow, du_fast, hs, buf = self.fmass(), self.fu(), self.fu(), h.copy(), self.geopot |
---|
[618] | 205 | ker.dynamico_caldyn_unstructured(tau, self.ms, h, hs, u, self.geopot, buf, |
---|
[615] | 206 | self.s, self.ps, self.pk, self.hflux, self.qv, |
---|
| 207 | self.dms, dh, self.dhs, du_fast, du_slow, |
---|
| 208 | buf, buf, buf, buf) |
---|
| 209 | return (h,u), (0.,du_fast), (dh,du_slow) |
---|
| 210 | |
---|
| 211 | #----------------------------------- HPE ------------------------------------ |
---|
| 212 | |
---|
| 213 | class Caldyn_HPE(Caldyn): |
---|
| 214 | def __init__(self,caldyn_thermo,caldyn_eta, mesh,metric,thermo,BC,g): |
---|
| 215 | Caldyn.__init__(self,mesh) |
---|
| 216 | setvars(('hydrostatic','caldyn_thermo','caldyn_eta', |
---|
| 217 | 'g','ptop','Rd','cpp','preff','Treff'), |
---|
| 218 | (True,caldyn_thermo,caldyn_eta, |
---|
| 219 | g,BC.ptop,thermo.Rd,thermo.Cpd,thermo.p0,thermo.T0)) |
---|
[618] | 220 | dynamico_init_params() |
---|
[615] | 221 | def _bwd_fast_slow_(self, flow, tau): |
---|
| 222 | dm, dS, du_slow, du_fast, buf = self.fmass(), self.ftheta(), self.fu(), self.fu(), self.geopot |
---|
| 223 | m,S,u = flow |
---|
[618] | 224 | ker.dynamico_caldyn_unstructured(tau, self.ms, m, S, u, self.geopot, buf, |
---|
[615] | 225 | self.s, self.ps, self.pk, self.hflux, self.qv, |
---|
| 226 | self.dms, dm, dS, du_fast, du_slow, |
---|
| 227 | buf, buf, buf, buf) |
---|
| 228 | return (m,S,u), (0.,0.,du_fast), (dm,dS,du_slow) |
---|
| 229 | |
---|
| 230 | #----------------------------------- NH ------------------------------------ |
---|
| 231 | |
---|
| 232 | class Caldyn_NH(Caldyn): |
---|
| 233 | def __init__(self,caldyn_thermo,caldyn_eta, mesh,metric,thermo,BC,g): |
---|
| 234 | Caldyn.__init__(self,mesh) |
---|
| 235 | setvars(('hydrostatic','caldyn_thermo','caldyn_eta', |
---|
| 236 | 'g','ptop','Rd','cpp','preff','Treff', |
---|
| 237 | 'pbot','rho_bot'), |
---|
| 238 | (False,caldyn_thermo,caldyn_eta, |
---|
| 239 | g,BC.ptop,thermo.Rd,thermo.Cpd,thermo.p0,thermo.T0, |
---|
| 240 | BC.pbot.max(), BC.rho_bot.max())) |
---|
[618] | 241 | dynamico_init_params() |
---|
[615] | 242 | def bwd_fast_slow(self, flow, tau): |
---|
| 243 | ftheta, fmass, fu, fw = self.ftheta, self.fmass, self.fu, self.fw |
---|
| 244 | dm, dS, du_slow, du_fast = fmass(), ftheta(), fu(), fu() |
---|
| 245 | dPhi_slow, dPhi_fast, dW_slow, dW_fast = fw(), fw(), fw(), fw() |
---|
| 246 | m,S,u,Phi,W = flow |
---|
[618] | 247 | ker.dynamico_caldyn_unstructured(tau, self.ms, m, S, u, Phi, W, |
---|
[615] | 248 | self.s, self.ps, self.pk, self.hflux, self.qv, |
---|
| 249 | self.dms, dm, dS, du_fast, du_slow, |
---|
| 250 | dPhi_fast, dPhi_slow, dW_fast, dW_slow) |
---|
| 251 | return ((m,S,u,Phi,W), (0.,0.,du_fast,dPhi_fast,dW_fast), |
---|
| 252 | (dm,dS,du_slow,dPhi_slow,dW_slow)) |
---|
| 253 | |
---|
[631] | 254 | #------------------------ Copy mesh info to Fortran side ------------------- |
---|
[615] | 255 | |
---|
[631] | 256 | def init_mesh(llm, nqdyn, edge_num, primal_num, dual_num, |
---|
| 257 | max_trisk_deg, max_primal_deg, max_dual_deg, |
---|
| 258 | primal_nb, primal_edge, primal_ne, |
---|
| 259 | dual_nb,dual_edge,dual_ne,dual_vertex, |
---|
| 260 | left,right,down,up,trisk_deg,trisk, |
---|
| 261 | Ai, Av, fv, le_de, Riv2, wee): |
---|
| 262 | setvars( ('llm','nqdyn','edge_num','primal_num','dual_num', |
---|
| 263 | 'max_trisk_deg','max_primal_deg','max_dual_deg'), |
---|
| 264 | (llm, nqdyn, edge_num, primal_num, dual_num, |
---|
| 265 | max_trisk_deg, max_primal_deg, max_dual_deg) ) |
---|
| 266 | print('init_mesh ...') |
---|
| 267 | ker.dynamico_init_mesh(primal_nb,primal_edge,primal_ne, |
---|
| 268 | dual_nb,dual_edge,dual_ne,dual_vertex, |
---|
| 269 | left,right,down,up,trisk_deg,trisk) |
---|
| 270 | print ('...done') |
---|
| 271 | print('init_metric ...') |
---|
| 272 | ker.dynamico_init_metric(Ai,Av,fv,le_de,Riv2,wee) |
---|
| 273 | print ('...done') |
---|
[615] | 274 | |
---|
[631] | 275 | #------------------------ Mesh partitioning ------------------------ |
---|
[615] | 276 | |
---|
| 277 | # Helper functions and interface to ParMETIS |
---|
| 278 | # list_stencil converts an adjacency graph from array format index[num_cells, MAX_EDGES] to compressed format |
---|
| 279 | # loc_stencil returns the start/end indices (vtxdist) expected by ParMETIS |
---|
| 280 | # i.e. index[start:end] with start=vtxdist[cell], end=vtxdist[cell+1] lists the edges of cell 'cell' |
---|
| 281 | |
---|
| 282 | def list_stencil(degree, stencil, cond=lambda x:True): |
---|
| 283 | for i in range(degree.size): |
---|
| 284 | for j in range(degree[i]): |
---|
| 285 | s=stencil[i,j] |
---|
| 286 | if cond(s): yield stencil[i,j] |
---|
| 287 | |
---|
| 288 | def loc_stencil(degree, stencil): |
---|
| 289 | loc=0 |
---|
| 290 | for i in range(degree.size): |
---|
| 291 | yield loc |
---|
| 292 | loc=loc+degree[i] |
---|
| 293 | yield loc |
---|
| 294 | |
---|
| 295 | def partition_mesh(degree, stencil, nparts): |
---|
| 296 | # arguments : PArray1D and PArray2D describing mesh, number of desired partitions |
---|
| 297 | dim_cell, degree, stencil = degree.dim, degree.data, stencil.data |
---|
| 298 | comm, vtxdist, idx_start, idx_end = dim_cell.comm, dim_cell.vtxdist, dim_cell.start, dim_cell.end |
---|
| 299 | mpi_rank, mpi_size = comm.Get_rank(), comm.Get_size() |
---|
| 300 | adjncy_loc, xadj_loc = list_stencil(degree, stencil), loc_stencil(degree, stencil) |
---|
| 301 | adjncy_loc, xadj_loc = [np.asarray(list(x), dtype=np.int32) for x in (adjncy_loc, xadj_loc)] |
---|
| 302 | owner = np.zeros(idx_end-idx_start, dtype=np.int32); |
---|
[618] | 303 | ker.dynamico_partition_graph(mpi_rank, mpi_size, vtxdist, xadj_loc, adjncy_loc, nparts, owner) |
---|
[615] | 304 | return owner |
---|