- Timestamp:
- 07/25/19 19:53:31 (5 years ago)
- Location:
- codes/icosagcm/trunk/src/parallel
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
codes/icosagcm/trunk/src/parallel/transfert.F90
r964 r965 15 15 wait_message, & 16 16 test_message 17 17 18 18 #else 19 19 use transfert_mpi_legacy_mod, only : t_message, t_request, & … … 49 49 req_i1, req_e1_scal, req_e1_vect, & 50 50 req_i0, req_e0_scal, req_e0_vect, & 51 #if !defined(CPP_USING_MPI_NEW)52 51 req_z1_scal, & 53 #endif54 52 init_transfert, & 55 53 init_message, & … … 99 97 IMPLICIT NONE 100 98 CHARACTER(LEN=*),INTENT(INOUT) :: Var 101 102 !$OMP MASTER 103 CALL bcast_mpi(Var) 104 !$OMP END MASTER 105 CALL bcast_omp(Var) 106 99 100 !$OMP MASTER 101 CALL bcast_mpi(Var) 102 !$OMP END MASTER 103 CALL bcast_omp(Var) 104 107 105 END SUBROUTINE bcast_c 108 106 109 107 !! -- Les entiers -- !! 110 108 111 109 SUBROUTINE bcast_i(var) 112 110 IMPLICIT NONE … … 116 114 !$OMP END MASTER 117 115 CALL bcast_omp(Var) 118 116 119 117 END SUBROUTINE bcast_i 120 118 … … 122 120 IMPLICIT NONE 123 121 INTEGER,INTENT(INOUT) :: Var(:) 124 125 !$OMP MASTER 126 CALL bcast_mpi(Var) 127 !$OMP END MASTER 128 CALL bcast_omp(Var) 129 122 123 !$OMP MASTER 124 CALL bcast_mpi(Var) 125 !$OMP END MASTER 126 CALL bcast_omp(Var) 127 130 128 END SUBROUTINE bcast_i1 131 129 … … 134 132 IMPLICIT NONE 135 133 INTEGER,INTENT(INOUT) :: Var(:,:) 136 137 !$OMP MASTER 138 CALL bcast_mpi(Var) 139 !$OMP END MASTER 140 CALL bcast_omp(Var) 141 134 135 !$OMP MASTER 136 CALL bcast_mpi(Var) 137 !$OMP END MASTER 138 CALL bcast_omp(Var) 139 142 140 END SUBROUTINE bcast_i2 143 141 … … 146 144 IMPLICIT NONE 147 145 INTEGER,INTENT(INOUT) :: Var(:,:,:) 148 149 !$OMP MASTER 150 CALL bcast_mpi(Var) 151 !$OMP END MASTER 152 CALL bcast_omp(Var) 153 146 147 !$OMP MASTER 148 CALL bcast_mpi(Var) 149 !$OMP END MASTER 150 CALL bcast_omp(Var) 151 154 152 END SUBROUTINE bcast_i3 155 153 … … 158 156 IMPLICIT NONE 159 157 INTEGER,INTENT(INOUT) :: Var(:,:,:,:) 160 161 !$OMP MASTER 162 CALL bcast_mpi(Var) 163 !$OMP END MASTER 164 CALL bcast_omp(Var) 165 158 159 !$OMP MASTER 160 CALL bcast_mpi(Var) 161 !$OMP END MASTER 162 CALL bcast_omp(Var) 163 166 164 END SUBROUTINE bcast_i4 167 165 168 166 169 167 !! -- Les reels -- !! 170 168 171 169 SUBROUTINE bcast_r(var) 172 170 IMPLICIT NONE … … 177 175 !$OMP END MASTER 178 176 CALL bcast_omp(Var) 179 177 180 178 END SUBROUTINE bcast_r 181 179 … … 183 181 IMPLICIT NONE 184 182 REAL,INTENT(INOUT) :: Var(:) 185 186 !$OMP MASTER 187 CALL bcast_mpi(Var) 188 !$OMP END MASTER 189 CALL bcast_omp(Var) 190 183 184 !$OMP MASTER 185 CALL bcast_mpi(Var) 186 !$OMP END MASTER 187 CALL bcast_omp(Var) 188 191 189 END SUBROUTINE bcast_r1 192 190 … … 195 193 IMPLICIT NONE 196 194 REAL,INTENT(INOUT) :: Var(:,:) 197 198 !$OMP MASTER 199 CALL bcast_mpi(Var) 200 !$OMP END MASTER 201 CALL bcast_omp(Var) 202 195 196 !$OMP MASTER 197 CALL bcast_mpi(Var) 198 !$OMP END MASTER 199 CALL bcast_omp(Var) 200 203 201 END SUBROUTINE bcast_r2 204 202 … … 207 205 IMPLICIT NONE 208 206 REAL,INTENT(INOUT) :: Var(:,:,:) 209 210 !$OMP MASTER 211 CALL bcast_mpi(Var) 212 !$OMP END MASTER 213 CALL bcast_omp(Var) 214 207 208 !$OMP MASTER 209 CALL bcast_mpi(Var) 210 !$OMP END MASTER 211 CALL bcast_omp(Var) 212 215 213 END SUBROUTINE bcast_r3 216 214 … … 219 217 IMPLICIT NONE 220 218 REAL,INTENT(INOUT) :: Var(:,:,:,:) 221 222 !$OMP MASTER 223 CALL bcast_mpi(Var) 224 !$OMP END MASTER 225 CALL bcast_omp(Var) 226 227 END SUBROUTINE bcast_r4 219 220 !$OMP MASTER 221 CALL bcast_mpi(Var) 222 !$OMP END MASTER 223 CALL bcast_omp(Var) 224 225 END SUBROUTINE bcast_r4 228 226 229 227 230 228 !! -- Les booleens -- !! 231 229 232 230 SUBROUTINE bcast_l(var) 233 231 IMPLICIT NONE … … 237 235 !$OMP END MASTER 238 236 CALL bcast_omp(Var) 239 237 240 238 END SUBROUTINE bcast_l 241 239 … … 243 241 IMPLICIT NONE 244 242 LOGICAL,INTENT(INOUT) :: Var(:) 245 246 !$OMP MASTER 247 CALL bcast_mpi(Var) 248 !$OMP END MASTER 249 CALL bcast_omp(Var) 250 243 244 !$OMP MASTER 245 CALL bcast_mpi(Var) 246 !$OMP END MASTER 247 CALL bcast_omp(Var) 248 251 249 END SUBROUTINE bcast_l1 252 250 … … 255 253 IMPLICIT NONE 256 254 LOGICAL,INTENT(INOUT) :: Var(:,:) 257 258 !$OMP MASTER 259 CALL bcast_mpi(Var) 260 !$OMP END MASTER 261 CALL bcast_omp(Var) 262 255 256 !$OMP MASTER 257 CALL bcast_mpi(Var) 258 !$OMP END MASTER 259 CALL bcast_omp(Var) 260 263 261 END SUBROUTINE bcast_l2 264 262 … … 267 265 IMPLICIT NONE 268 266 LOGICAL,INTENT(INOUT) :: Var(:,:,:) 269 270 !$OMP MASTER 271 CALL bcast_mpi(Var) 272 !$OMP END MASTER 273 CALL bcast_omp(Var) 274 267 268 !$OMP MASTER 269 CALL bcast_mpi(Var) 270 !$OMP END MASTER 271 CALL bcast_omp(Var) 272 275 273 END SUBROUTINE bcast_l3 276 274 … … 279 277 IMPLICIT NONE 280 278 LOGICAL,INTENT(INOUT) :: Var(:,:,:,:) 281 282 !$OMP MASTER 283 CALL bcast_mpi(Var) 284 !$OMP END MASTER 285 CALL bcast_omp(Var) 286 279 280 !$OMP MASTER 281 CALL bcast_mpi(Var) 282 !$OMP END MASTER 283 CALL bcast_omp(Var) 284 287 285 END SUBROUTINE bcast_l4 288 286 289 287 290 288 END MODULE transfert_mod -
codes/icosagcm/trunk/src/parallel/transfert_mpi.f90
r963 r965 5 5 use profiling_mod, only : enter_profile, exit_profile, register_id 6 6 use domain_mod, only : ndomain, ndomain_glo, domain, domain_glo, domloc_glo_ind, domglo_rank, domglo_loc_ind 7 use field_mod, only : t_field, field_T, field_U 7 use field_mod, only : t_field, field_T, field_U, field_Z 8 8 use transfert_request_mod 9 9 implicit none … … 103 103 type(t_submessage), allocatable :: message_in_tmp(:), message_out_tmp(:) 104 104 type(t_submessage), pointer :: submessage 105 integer :: field_type 105 106 106 107 !$omp barrier … … 110 111 message%send_seq = 0 111 112 message%wait_seq = 0 113 114 if( request(1)%field_type /= field(1)%field_type ) call dynamico_abort( "init_message : field_type/request mismatch" ) 115 field_type = request(1)%field_type 116 112 117 ! Set field%rval4d pointer to always use 4d array 113 118 do ind = 1, ndomain … … 134 139 if(request(ind_loc)%points_HtoB(remote_ind_glo)%npoints > 0 ) then ! Add only non-empty messages 135 140 ! Add local message ind_loc -> remote_ind_glo, aggregarting submessage_in and submessage_out into submessage_local 136 submessage_out = make_submessage( request(ind_loc)%points_HtoB(remote_ind_glo), &141 submessage_out = make_submessage( field_type, request(ind_loc)%points_HtoB(remote_ind_glo), & 137 142 ind_loc, remote_ind_glo, dim3, dim4, request(1)%vector ) 138 submessage_in = make_submessage( request(domglo_loc_ind(remote_ind_glo))%points_BtoH(domloc_glo_ind(ind_loc)), &143 submessage_in = make_submessage( field_type, request(domglo_loc_ind(remote_ind_glo))%points_BtoH(domloc_glo_ind(ind_loc)), & 139 144 domglo_loc_ind(remote_ind_glo), domloc_glo_ind(ind_loc), dim3, dim4, request(1)%vector) 140 145 submessage_local%src_ind_loc = ind_loc … … 149 154 ! When data to send to remote_domain, add submessage in message%message_out 150 155 if( request(ind_loc)%points_HtoB(remote_ind_glo)%npoints > 0 ) then 151 submessage_out = make_submessage( request(ind_loc)%points_HtoB(remote_ind_glo), &156 submessage_out = make_submessage( field_type, request(ind_loc)%points_HtoB(remote_ind_glo), & 152 157 ind_loc, remote_ind_glo, dim3, dim4, request(1)%vector ) 153 158 call array_append_submessage( message_out_tmp, message_out_size, submessage_out ) 154 159 end if 155 160 if( request(ind_loc)%points_BtoH(remote_ind_glo)%npoints > 0 ) then 156 submessage_in = make_submessage( request(ind_loc)%points_BtoH(remote_ind_glo), &161 submessage_in = make_submessage( field_type, request(ind_loc)%points_BtoH(remote_ind_glo), & 157 162 ind_loc, remote_ind_glo, dim3, dim4, request(1)%vector ) 158 163 call array_append_submessage( message_in_tmp, message_in_size, submessage_in ) … … 185 190 contains 186 191 ! Generate submessage from points 187 function make_submessage(points, ind_loc, remote_ind_glo, dim3, dim4, vector) result(submessage) 188 use dimensions, only : swap_dimensions, iim, u_pos 192 function make_submessage(field_type, points, ind_loc, remote_ind_glo, dim3, dim4, vector) result(submessage) 193 use dimensions, only : swap_dimensions, iim, u_pos, z_pos 194 integer, intent(in) :: field_type 189 195 type(t_points), intent(in) :: points 190 196 integer, intent(in) :: ind_loc, remote_ind_glo, dim3, dim4 … … 199 205 allocate( submessage%displs( points%npoints ) ) 200 206 submessage%displs(:) = points%i + (points%j-1)*iim 201 if(allocated(points%edge)) submessage%displs = submessage%displs + u_pos( points%edge ) 207 if(field_type == field_U) submessage%displs = submessage%displs + u_pos( points%elt ) 208 if(field_type == field_Z) submessage%displs = submessage%displs + z_pos( points%elt ) 202 209 allocate(submessage%sign( points%npoints )) 203 if( vector ) then 204 submessage%sign(:) = (/( domain(ind_loc)%edge_assign_sign(points%e dge(k)-1, points%i(k), points%j(k)) ,k=1,points%npoints)/)210 if( vector ) then ! For U fields only 211 submessage%sign(:) = (/( domain(ind_loc)%edge_assign_sign(points%elt(k)-1, points%i(k), points%j(k)) ,k=1,points%npoints)/) 205 212 else 206 213 submessage%sign(:) = 1 … … 380 387 message%send_seq = message%send_seq + 1 381 388 !$omp end master 382 389 383 390 call enter_profile(profile_mpi_barrier) 384 391 !$omp barrier … … 411 418 do i = 1, size( message%message_local ) 412 419 if( assigned_domain( message%message_local(i)%dest_ind_loc ) ) then 413 !$acc loop collapse(2) 420 !$acc loop collapse(2) 414 421 do d4 = 1, dim4 415 422 do d3 = d3_begin, d3_end -
codes/icosagcm/trunk/src/parallel/transfert_requests.F90
r963 r965 2 2 module transfert_request_mod 3 3 use abort_mod 4 use field_mod, only : t_field, field_T, field_U 4 use field_mod, only : t_field, field_T, field_U, field_Z 5 5 use domain_mod, only : ndomain, ndomain_glo, domain, domain_glo, domloc_glo_ind, domglo_rank, domglo_loc_ind 6 6 implicit none … … 9 9 ! Points for t_request : list of points coordinates in a domain 10 10 type t_points 11 integer, allocatable :: i(:), j(:), edge(:) 11 integer, allocatable :: i(:), j(:) 12 integer, allocatable :: elt(:) ! Element : cell edge or cell point 12 13 integer :: npoints 13 14 end type … … 26 27 type(t_request),save,pointer :: req_e1_scal(:) ! Halos for U fields 27 28 type(t_request),save,pointer :: req_e1_vect(:) ! Halos for U fields (with sign change) 28 type(t_request),save,pointer :: req_z1_scal(:) => null() ! DO NOT USE : not initialized29 type(t_request),save,pointer :: req_z1_scal(:) ! Halos for Z fields 29 30 30 31 type(t_request),save,pointer :: req_i0(:) ! Duplicated cells for T fields … … 42 43 subroutine init_all_requests 43 44 use dimensions, only : swap_dimensions, ii_begin, ii_end, jj_begin, jj_end 44 use metric, only : right, rdown, ldown, left, lup, rup 45 use metric, only : right, rdown, ldown, left, lup, rup, vdown, vlup, vrdown, vrup, vup 45 46 integer :: ind, i, j 46 47 … … 131 132 req_e1_vect = req_e1_scal 132 133 req_e1_vect%vector = .true. 134 135 call request_init(req_z1_scal, field_z) 136 do ind=1,ndomain 137 call swap_dimensions(ind) 138 do i=ii_begin,ii_end 139 call request_add_point(req_z1_scal,ind,i,jj_begin-1,vrup) 140 call request_add_point(req_z1_scal,ind,i+1,jj_begin-1,vup) 141 enddo 142 do j=jj_begin,jj_end 143 call request_add_point(req_z1_scal,ind,ii_end+1,j,vlup) 144 enddo 145 do j=jj_begin,jj_end 146 call request_add_point(req_z1_scal,ind,ii_end+1,j-1,vup) 147 enddo 148 do i=ii_begin,ii_end 149 call request_add_point(req_z1_scal,ind,i,jj_end+1,vdown) 150 call request_add_point(req_z1_scal,ind,i-1,jj_end+1,vrdown) 151 enddo 152 do j=jj_begin,jj_end 153 call request_add_point(req_z1_scal,ind,ii_begin-1,j,vrup) 154 enddo 155 do j=jj_begin,jj_end 156 call request_add_point(req_z1_scal,ind,ii_begin-1,j,vrdown) 157 enddo 158 enddo 159 call request_exchange(req_z1_scal) 133 160 contains 134 161 135 162 ! ---- Points ---- 136 163 ! Initialize (allocate) points 137 subroutine points_init(points, has_e dges)164 subroutine points_init(points, has_elt) 138 165 type(t_points):: points 139 logical :: has_e dges166 logical :: has_elt 140 167 integer, parameter :: INITIAL_ALLOC_SIZE = 10 141 168 142 169 allocate(points%i(INITIAL_ALLOC_SIZE)) 143 170 allocate(points%j(INITIAL_ALLOC_SIZE)) 144 if(has_e dges) allocate(points%edge(INITIAL_ALLOC_SIZE))171 if(has_elt) allocate(points%elt(INITIAL_ALLOC_SIZE)) 145 172 points%npoints = 0 146 173 end subroutine 147 174 148 ! Append point (i, j [,e dge]) to point list149 subroutine points_append(points, i, j, e dge)175 ! Append point (i, j [,elt]) to point list 176 subroutine points_append(points, i, j, elt) 150 177 type(t_points), intent(inout):: points 151 178 integer, intent(in) :: i, j 152 integer, intent(in), optional :: e dge179 integer, intent(in), optional :: elt 153 180 integer :: begin_size 154 181 … … 157 184 begin_size=points%npoints 158 185 call array_append( points%j, begin_size, j ) 159 if(present(e dge)) then186 if(present(elt)) then 160 187 begin_size=points%npoints 161 call array_append( points%e dge, begin_size, edge)188 call array_append( points%elt, begin_size, elt ) 162 189 end if 163 190 164 191 points%npoints = points%npoints + 1 165 192 end subroutine 166 193 167 194 ! Add element to array, and reallocate if necessary 168 195 subroutine array_append( a, a_size, elt ) … … 172 199 integer, allocatable :: a_tmp(:) 173 200 integer, parameter :: GROW_FACTOR = 2 174 201 175 202 if( size( a ) <= a_size ) then 176 203 allocate( a_tmp ( a_size * GROW_FACTOR ) ) … … 192 219 193 220 displs = (points%i-1) + (points%j-1)*iim 194 if(allocated(points%e dge)) displs = displs + (points%edge-1)*iim*jjm221 if(allocated(points%elt)) displs = displs + (points%elt-1)*iim*jjm 195 222 196 223 last=0 … … 210 237 points%j(k) = mod((displs(k)-(points%i(k)-1))/iim, jjm)+1 211 238 end do 212 if(allocated(points%e dge)) then213 deallocate(points%e dge); allocate(points%edge(last))214 points%e dge(:) = displs(1:last)/(iim*jjm) + 1239 if(allocated(points%elt)) then 240 deallocate(points%elt); allocate(points%elt(last)) 241 points%elt(:) = displs(1:last)/(iim*jjm) + 1 215 242 end if 216 243 … … 231 258 allocate(request(ind)%points_HtoB(ndomain_glo)) 232 259 do remote_ind_glo = 1, ndomain_glo 233 call points_init(request(ind)%points_BtoH(remote_ind_glo), field_type == field_U)260 call points_init(request(ind)%points_BtoH(remote_ind_glo), field_type /= field_T) 234 261 !call points_init(request(ind)%points_HtoB(remote_ind_glo)) ! Initialized before MPI communication in request_exchange 235 262 end do … … 238 265 239 266 ! Append local point (ind_glo,i,j) to the point list of domain ind, where ind_glo is the domain owning the point 240 subroutine request_add_point(req, ind, i, j, e dge)267 subroutine request_add_point(req, ind, i, j, elt) 241 268 type(t_request), intent(inout) :: req(:) 242 269 integer :: ind, i, j 243 integer, optional :: edge 244 245 if( present(edge) ) then ! U field 246 if(domain(ind)%edge_assign_domain(edge-1,i,j) /= domloc_glo_ind(ind) ) & ! Add only if remote domain != local domain 247 call points_append(req(ind)%points_BtoH(domain(ind)%edge_assign_domain(edge-1,i,j)), i, j, edge ) 270 integer, optional :: elt 271 272 if( req(ind)%field_type == field_U ) then 273 if(domain(ind)%edge_assign_domain(elt-1,i,j) /= domloc_glo_ind(ind) ) & ! Add only if remote domain != local domain 274 call points_append(req(ind)%points_BtoH(domain(ind)%edge_assign_domain(elt-1,i,j)), i, j, elt ) 275 else if( req(ind)%field_type == field_Z ) then 276 if(domain(ind)%vertex_assign_domain(elt-1,i,j) /= domloc_glo_ind(ind) ) & ! Add only if remote domain != local domain 277 call points_append(req(ind)%points_BtoH(domain(ind)%vertex_assign_domain(elt-1,i,j)), i, j, elt ) 248 278 else ! T field 249 279 if(domain(ind)%assign_domain(i,j) /= domloc_glo_ind(ind) ) & … … 262 292 263 293 integer :: ind_loc, remote_ind_glo, k 264 integer :: i, j, e dge294 integer :: i, j, elt 265 295 integer :: reqs(ndomain, ndomain_glo, 2), reqs2(ndomain, ndomain_glo, 2), reqs3(ndomain, ndomain_glo, 3), ierr 266 296 … … 271 301 points => req(ind_loc)%points_BtoH(remote_ind_glo) 272 302 call points_sort(points) 273 call points_init( points_send(ind_loc,remote_ind_glo), req(ind_loc)%field_type == field_U)303 call points_init( points_send(ind_loc,remote_ind_glo), req(ind_loc)%field_type /= field_T ) 274 304 do k = 1, points%npoints 275 305 i = points%i(k) 276 306 j = points%j(k) 277 if(req(ind_loc)%field_type == field_T) then 307 if( req(ind_loc)%field_type == field_U ) then 308 elt = points%elt(k) - 1 ! WARNING : domain%edge_assign_* use index in 0:5 instead of 1:6 everywhere else 309 call points_append(points_send(ind_loc,remote_ind_glo), domain(ind_loc)%edge_assign_i(elt,i,j), & 310 domain(ind_loc)%edge_assign_j(elt,i,j), & 311 domain(ind_loc)%edge_assign_pos(elt,i,j)+1) 312 else if( req(ind_loc)%field_type == field_Z ) then 313 elt = points%elt(k) - 1 ! WARNING : domain%vertex_assign_* use index in 0:5 instead of 1:6 everywhere else 314 call points_append(points_send(ind_loc,remote_ind_glo), domain(ind_loc)%vertex_assign_i(elt,i,j), & 315 domain(ind_loc)%vertex_assign_j(elt,i,j), & 316 domain(ind_loc)%vertex_assign_pos(elt,i,j)+1) 317 else ! T field 278 318 call points_append(points_send(ind_loc,remote_ind_glo), domain(ind_loc)%assign_i(i,j), & 279 319 domain(ind_loc)%assign_j(i,j)) 280 else 281 edge = points%edge(k) - 1 ! WARNING : domain%edge_assign_* use index in 0:5 instead of 1:6 everywhere else 282 call points_append(points_send(ind_loc,remote_ind_glo), domain(ind_loc)%edge_assign_i(edge,i,j), & 283 domain(ind_loc)%edge_assign_j(edge,i,j), & 284 domain(ind_loc)%edge_assign_pos(edge,i,j)+1) 285 end if 320 endif 286 321 end do 287 322 end do … … 317 352 domloc_glo_ind(ind_loc)+ndomain_glo*remote_ind_glo, comm_icosa, reqs2(ind_loc,remote_ind_glo,2), ierr) 318 353 319 if(req(1)%field_type == field_U) then320 call MPI_Isend( points_send(ind_loc,remote_ind_glo)%e dge, points_send(ind_loc,remote_ind_glo)%npoints, MPI_INT, &354 if(req(1)%field_type /= field_T) then 355 call MPI_Isend( points_send(ind_loc,remote_ind_glo)%elt, points_send(ind_loc,remote_ind_glo)%npoints, MPI_INT, & 321 356 domglo_rank(remote_ind_glo), remote_ind_glo+ndomain_glo*domloc_glo_ind(ind_loc), & 322 357 comm_icosa, reqs3(ind_loc,remote_ind_glo,1), ierr ) 323 allocate( points%e dge( points%npoints ) )324 call MPI_Irecv( points%e dge, points%npoints, MPI_INT, &358 allocate( points%elt( points%npoints ) ) 359 call MPI_Irecv( points%elt, points%npoints, MPI_INT, & 325 360 domglo_rank(remote_ind_glo), domloc_glo_ind(ind_loc)+ndomain_glo*remote_ind_glo, & 326 361 comm_icosa, reqs3(ind_loc,remote_ind_glo,2), ierr ) … … 330 365 call waitall(reqs,ndomain*ndomain_glo*2) 331 366 call waitall(reqs2,ndomain*ndomain_glo*2) 332 if(req(1)%field_type == field_U) call waitall(reqs3,ndomain*ndomain_glo*2)367 if(req(1)%field_type /= field_T) call waitall(reqs3,ndomain*ndomain_glo*2) 333 368 end subroutine 334 369
Note: See TracChangeset
for help on using the changeset viewer.