Changeset 1203 for XIOS/dev/branch_yushan_merged/src

Timestamp:

07/10/17 15:18:16 (7 years ago)

Author:

yushan

Message:

prep to merge with trunk @1200

Location:

XIOS/dev/branch_yushan_merged/src

Files:

• client_client_dht_template_impl.hpp (modified) (4 diffs)
• filter/spatial_transform_filter.cpp (modified) (3 diffs)
• node/field.cpp (modified) (1 diff)
• test/test_remap_omp.f90 (modified) (4 diffs)
• test/test_unstruct_omp.f90 (modified) (1 diff)
• transformation/grid_transformation.cpp (modified) (6 diffs)

XIOS/dev/branch_yushan_merged/src/client_client_dht_template_impl.hpp

@@ -670 +670 @@
                                                        std::vector<ep_lib::MPI_Request>& requestSendIndex)
 {
+  printf("should not call this function sendIndexToClients");  
   ep_lib::MPI_Request request;
   requestSendIndex.push_back(request);
@@ -688 +689 @@
                                                          std::vector<ep_lib::MPI_Request>& requestRecvIndex)
 {
+  printf("should not call this function recvIndexFromClients");
   ep_lib::MPI_Request request;
   requestRecvIndex.push_back(request);
@@ -707 +709 @@
                                                       std::vector<ep_lib::MPI_Request>& requestSendInfo)
 {
+  printf("should not call this function sendInfoToClients");
   ep_lib::MPI_Request request;
   requestSendInfo.push_back(request);
@@ -726 +729 @@
                                                         std::vector<ep_lib::MPI_Request>& requestRecvInfo)
 {
+  printf("should not call this function recvInfoFromClients\n");
   ep_lib::MPI_Request request;
   requestRecvInfo.push_back(request);

XIOS/dev/branch_yushan_merged/src/filter/spatial_transform_filter.cpp

@@ -153 +153 @@
           sendBuff[idxSendBuff] = new double[itSend->second.numElements()];
       }
+      
+      const CGridTransformation::RecvIndexGridDestinationMap& localIndexToReceive = *itListRecv;
+      CGridTransformation::RecvIndexGridDestinationMap::const_iterator itbRecv = localIndexToReceive.begin(), itRecv,
+                                                                       iteRecv = localIndexToReceive.end();
 
       idxSendBuff = 0;
-      std::vector<ep_lib::MPI_Request> sendRecvRequest;
+      std::vector<ep_lib::MPI_Request> sendRecvRequest(localIndexToSend.size()+localIndexToReceive.size());
+      int position = 0;
       for (itSend = itbSend; itSend != iteSend; ++itSend, ++idxSendBuff)
       {
@@ -165 +170 @@
           sendBuff[idxSendBuff][idx] = dataCurrentSrc(localIndex_p(idx));
         }
-        sendRecvRequest.push_back(ep_lib::MPI_Request());
-        MPI_Isend(sendBuff[idxSendBuff], countSize, MPI_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest.back());
+        MPI_Isend(sendBuff[idxSendBuff], countSize, MPI_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest[position]);
+        position++;
       }
 
       // Receiving data on destination fields
-      const CGridTransformation::RecvIndexGridDestinationMap& localIndexToReceive = *itListRecv;
-      CGridTransformation::RecvIndexGridDestinationMap::const_iterator itbRecv = localIndexToReceive.begin(), itRecv,
-                                                                       iteRecv = localIndexToReceive.end();
+      
       int recvBuffSize = 0;
       for (itRecv = itbRecv; itRecv != iteRecv; ++itRecv) recvBuffSize += itRecv->second.size(); //(recvBuffSize < itRecv->second.size())
@@ -183 +186 @@
         int srcRank = itRecv->first;
         int countSize = itRecv->second.size();
-        sendRecvRequest.push_back(ep_lib::MPI_Request());
-        MPI_Irecv(recvBuff + currentBuff, countSize, MPI_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest.back());
+        
+        MPI_Irecv(recvBuff + currentBuff, countSize, MPI_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest[position]);
+        position++;
         currentBuff += countSize;
       }

XIOS/dev/branch_yushan_merged/src/node/field.cpp

@@ -715 +715 @@
      if (context->hasClient)
      {
+       printf("proc %d begein transformation\n", myRank);
        solveTransformedGrid();
+       printf("proc %d end transformation\n", myRank);
+       MPI_Barrier(context->client->intraComm);
      }
 

XIOS/dev/branch_yushan_merged/src/test/test_remap_omp.f90

@@ -183 +183 @@
   ierr=NF90_GET_VAR(ncid,varid, dst_boundslat, start=(/1,dst_ibegin+1/),count=(/dst_nvertex,dst_ni/))
 
-
+   !$omp barrier
+
+  !$omp master 
+  CALL MPI_barrier(comm, ierr)
+  !$omp end master
+
+  !$omp barrier
+
+  
   CALL xios_context_initialize("test",comm)
   CALL xios_get_handle("test",ctx_hdl)
@@ -214 +222 @@
   CALL xios_close_context_definition()
 
-  CALL xios_get_domain_attr("src_domain_regular_read", ni=src_tmp_ni, nj=src_tmp_nj)
-  ALLOCATE(tmp_field_0(src_tmp_ni*src_tmp_nj))
-
-  CALL xios_get_axis_attr("src_axis_curvilinear_read", n=src_tmp_n)
-  CALL xios_get_domain_attr("src_domain_curvilinear_read", ni=src_tmp_ni, nj=src_tmp_nj)
-  ALLOCATE(tmp_field_1(src_tmp_ni*src_tmp_nj*src_tmp_n))
-
-  CALL xios_get_domain_attr("src_domain_unstructured_read", ni=src_tmp_ni, nj=src_tmp_nj)
-  ALLOCATE(tmp_field_2(src_tmp_ni*src_tmp_nj))
-  
-  CALL xios_recv_field("src_field_regular", tmp_field_0)
-  CALL xios_recv_field("src_field_curvilinear", tmp_field_1)
-  CALL xios_recv_field("src_field_unstructured", tmp_field_2)
+!  CALL xios_get_domain_attr("src_domain_regular_read", ni=src_tmp_ni, nj=src_tmp_nj)
+!  ALLOCATE(tmp_field_0(src_tmp_ni*src_tmp_nj))
+
+!  CALL xios_get_axis_attr("src_axis_curvilinear_read", n=src_tmp_n)
+!  CALL xios_get_domain_attr("src_domain_curvilinear_read", ni=src_tmp_ni, nj=src_tmp_nj)
+!  ALLOCATE(tmp_field_1(src_tmp_ni*src_tmp_nj*src_tmp_n))
+
+!  CALL xios_get_domain_attr("src_domain_unstructured_read", ni=src_tmp_ni, nj=src_tmp_nj)
+!  ALLOCATE(tmp_field_2(src_tmp_ni*src_tmp_nj))
+  
+!  CALL xios_recv_field("src_field_regular", tmp_field_0)
+!  CALL xios_recv_field("src_field_curvilinear", tmp_field_1)
+!  CALL xios_recv_field("src_field_unstructured", tmp_field_2)
 
   DO ts=1,10
@@ -232 +240 @@
     CALL xios_send_field("src_field_2D",src_field_2D)
     
-    !DO i=1,src_ni
-    !  src_field_2D_clone(i) = src_field_2D(i)
-    !  IF ((23.5 * ts < src_lat(i)) .AND. (src_lat(i) < 65.5 *ts) .AND. (0 < src_lon(i)) .AND. (src_lon(i) < 30*ts)) THEN      
-    !    src_field_2D_clone(i) = missing_value    
-    !  ENDIF
-    !ENDDO
-
-    !CALL xios_send_field("src_field_2D_clone",src_field_2D_clone)
-    !CALL xios_send_field("src_field_3D",src_field_3D)
-    !CALL xios_send_field("src_field_3D_clone",src_field_3D)
-    !CALL xios_send_field("src_field_4D",src_field_4D)
-    !CALL xios_send_field("src_field_3D_pression",src_field_pression)
-    CALL xios_send_field("tmp_field_0",tmp_field_0)
-    CALL xios_send_field("tmp_field_1",tmp_field_1)
-    CALL xios_send_field("tmp_field_2",tmp_field_2)
+    DO i=1,src_ni
+      src_field_2D_clone(i) = src_field_2D(i)
+      IF ((23.5 * ts < src_lat(i)) .AND. (src_lat(i) < 65.5 *ts) .AND. (0 < src_lon(i)) .AND. (src_lon(i) < 30*ts)) THEN      
+        src_field_2D_clone(i) = missing_value    
+      ENDIF
+    ENDDO
+
+    CALL xios_send_field("src_field_2D_clone",src_field_2D_clone)
+    CALL xios_send_field("src_field_3D",src_field_3D)
+    CALL xios_send_field("src_field_3D_clone",src_field_3D)
+    CALL xios_send_field("src_field_4D",src_field_4D)
+    CALL xios_send_field("src_field_3D_pression",src_field_pression)
+ !   CALL xios_send_field("tmp_field_0",tmp_field_0)
+ !   CALL xios_send_field("tmp_field_1",tmp_field_1)
+ !   CALL xios_send_field("tmp_field_2",tmp_field_2)
     CALL wait_us(5000) ;
    ENDDO
@@ -254 +262 @@
   DEALLOCATE(src_lon, src_lat, src_boundslon,src_boundslat, src_field_2D)
   DEALLOCATE(dst_lon, dst_lat, dst_boundslon,dst_boundslat)
-  DEALLOCATE(tmp_field_0, tmp_field_1, tmp_field_2)
+  !DEALLOCATE(tmp_field_0, tmp_field_1, tmp_field_2)
   
   CALL xios_finalize()

XIOS/dev/branch_yushan_merged/src/test/test_unstruct_omp.f90

@@ -267 +267 @@
   CALL xios_finalize()
 
-print *, "Client : xios_finalize "
+print *, mpi_rank, "Client : xios_finalize "
 
   !$omp barrier

XIOS/dev/branch_yushan_merged/src/transformation/grid_transformation.cpp

@@ -473 +473 @@
 
   // Sending global index of grid source to corresponding process as well as the corresponding mask
-  std::vector<ep_lib::MPI_Request> requests;
-  requests.reserve(2*recvRankSizeMap.size()+2*globaIndexWeightFromSrcToDst.size());
+  std::vector<ep_lib::MPI_Request> requests(2*recvRankSizeMap.size()+2*globaIndexWeightFromSrcToDst.size());
   std::vector<ep_lib::MPI_Status> status;
   boost::unordered_map<int, unsigned char* > recvMaskDst;
   boost::unordered_map<int, unsigned long* > recvGlobalIndexSrc;
+  int position = 0;
   for (std::map<int,int>::const_iterator itRecv = recvRankSizeMap.begin(); itRecv != recvRankSizeMap.end(); ++itRecv)
   {
@@ -485 +485 @@
     recvGlobalIndexSrc[recvRank] = new unsigned long [recvSize];
 
-    requests.push_back(ep_lib::MPI_Request());
-    MPI_Irecv(recvGlobalIndexSrc[recvRank], recvSize, MPI_UNSIGNED_LONG, recvRank, 46, client->intraComm, &requests.back());
-    requests.push_back(ep_lib::MPI_Request());
-    MPI_Irecv(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 47, client->intraComm, &requests.back());
+
+    MPI_Irecv(recvGlobalIndexSrc[recvRank], recvSize, MPI_UNSIGNED_LONG, recvRank, 46, client->intraComm, &requests[position]);
+    position++;
+
+    MPI_Irecv(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 47, client->intraComm, &requests[position]);
+    position++;
   }
 
@@ -523 +525 @@
 
     // Send global index source and mask
-    requests.push_back(ep_lib::MPI_Request());
-    MPI_Isend(sendGlobalIndexSrc[sendRank], sendSize, MPI_UNSIGNED_LONG, sendRank, 46, client->intraComm, &requests.back());
-    requests.push_back(ep_lib::MPI_Request());
-    MPI_Isend(sendMaskDst[sendRank], sendSize, MPI_UNSIGNED_CHAR, sendRank, 47, client->intraComm, &requests.back());
+
+    MPI_Isend(sendGlobalIndexSrc[sendRank], sendSize, MPI_UNSIGNED_LONG, sendRank, 46, client->intraComm, &requests[position]);
+    position++;
+
+    MPI_Isend(sendMaskDst[sendRank], sendSize, MPI_UNSIGNED_CHAR, sendRank, 47, client->intraComm, &requests[position]);
+    position++;
   }
 
@@ -533 +537 @@
 
   // Okie, now use the mask to identify which index source we need to send, then also signal the destination which masked index we will return
-  std::vector<ep_lib::MPI_Request>().swap(requests);
-  std::vector<ep_lib::MPI_Status>().swap(status);
-  requests.reserve(sendRankSizeMap.size()+recvRankSizeMap.size());
+  //std::vector<ep_lib::MPI_Request>().swap(requests);
+  //std::vector<ep_lib::MPI_Status>().swap(status);
+  requests.resize(sendRankSizeMap.size()+recvRankSizeMap.size());
+  position = 0;
   // Okie, on destination side, we will wait for information of masked index of source
   for (std::map<int,int>::const_iterator itSend = sendRankSizeMap.begin(); itSend != sendRankSizeMap.end(); ++itSend)
@@ -542 +547 @@
     int recvSize = itSend->second;
 
-    requests.push_back(ep_lib::MPI_Request());
-    MPI_Irecv(sendMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests.back());
+    MPI_Irecv(sendMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests[position]);
+    position++;
   }
 
@@ -579 +584 @@
 
     // Okie, now inform the destination which source index are masked
-    requests.push_back(ep_lib::MPI_Request());
-    MPI_Isend(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests.back());
+    MPI_Isend(recvMaskDst[recvRank], recvSize, MPI_UNSIGNED_CHAR, recvRank, 48, client->intraComm, &requests[position]);
+    position++;
   }
   status.resize(requests.size());