Changeset 1328 for XIOS/dev/branch_openmp/src/filter

Timestamp:

11/15/17 12:14:34 (6 years ago)

Author:

yushan

Message:

dev_omp

Location:

XIOS/dev/branch_openmp/src/filter

Files:

• file_writer_filter.cpp (modified) (1 diff)
• source_filter.cpp (modified) (1 diff)
• spatial_transform_filter.cpp (modified) (10 diffs)
• spatial_transform_filter.hpp (modified) (2 diffs)
• store_filter.cpp (modified) (2 diffs)
• temporal_filter.cpp (modified) (1 diff)

XIOS/dev/branch_openmp/src/filter/file_writer_filter.cpp

@@ -17 +17 @@
   void CFileWriterFilter::onInputReady(std::vector<CDataPacketPtr> data)
   {
-    CDataPacketPtr packet = data[0];
-
     const bool detectMissingValue = (!field->detect_missing_value.isEmpty()
                                       && !field->default_value.isEmpty()
                                       && field->detect_missing_value == true);
+
+    CArray<double, 1> dataArray = (detectMissingValue) ? data[0]->data.copy() : data[0]->data;
+
     if (detectMissingValue)
     {
       const double missingValue = field->default_value;
-      const size_t nbData = packet->data.numElements();
+      const size_t nbData = dataArray.numElements();
       for (size_t idx = 0; idx < nbData; ++idx)
       {
-        if (NumTraits<double>::isnan(packet->data(idx)))
-          packet->data(idx) = missingValue;
+        if (NumTraits<double>::isnan(dataArray(idx)))
+          dataArray(idx) = missingValue;
       }
     }
 
-    field->sendUpdateData(packet->data);
+    field->sendUpdateData(dataArray);
   }
 

XIOS/dev/branch_openmp/src/filter/source_filter.cpp

@@ -99 +99 @@
   void CSourceFilter::signalEndOfStream(CDate date)
   {
+    date = date + offset; // this is a temporary solution, it should be part of a proper temporal filter
+
     CDataPacketPtr packet(new CDataPacket);
     packet->date = date;

XIOS/dev/branch_openmp/src/filter/spatial_transform_filter.cpp

@@ +1 @@
+#include "mpi.hpp"
 #include "spatial_transform_filter.hpp"
 #include "grid_transformation.hpp"
 #include "context.hpp"
 #include "context_client.hpp"
+using namespace ep_lib;
 
 namespace xios
@@ -65 +67 @@
   }
 
+  //std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > CSpatialTransformFilterEngine::engines;
   std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > *CSpatialTransformFilterEngine::engines_ptr = 0;
 
@@ -72 +75 @@
       ERROR("CSpatialTransformFilterEngine& CSpatialTransformFilterEngine::get(CGridTransformation* gridTransformation)",
             "Impossible to get the requested engine, the grid transformation is invalid.");
-
+    
     if(engines_ptr == NULL) engines_ptr = new std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> >;
 
+    //std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> >::iterator it = engines.find(gridTransformation);
     std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> >::iterator it = engines_ptr->find(gridTransformation);
+    //if (it == engines.end())
     if (it == engines_ptr->end())
     {
       boost::shared_ptr<CSpatialTransformFilterEngine> engine(new CSpatialTransformFilterEngine(gridTransformation));
+      //it = engines.insert(std::make_pair(gridTransformation, engine)).first;
       it = engines_ptr->insert(std::make_pair(gridTransformation, engine)).first;
     }
@@ -122 +128 @@
     double defaultValue = std::numeric_limits<double>::quiet_NaN();
     if (0 != dataDest.numElements()) ignoreMissingValue = NumTraits<double>::isnan(dataDest(0));
-    
-    const std::list<CGridTransformation::SendingIndexGridSourceMap> *listLocalIndexSend_ptr = & (gridTransformation->getLocalIndexToSendFromGridSource());
-    
+
     const std::list<CGridTransformation::SendingIndexGridSourceMap>& listLocalIndexSend = gridTransformation->getLocalIndexToSendFromGridSource();
     const std::list<CGridTransformation::RecvIndexGridDestinationMap>& listLocalIndexToReceive = gridTransformation->getLocalIndexToReceiveOnGridDest();
@@ -133 +137 @@
     CArray<double,1> dataCurrentDest(dataSrc.copy());
 
-    std::list<CGridTransformation::SendingIndexGridSourceMap>::const_iterator itListSend  = listLocalIndexSend_ptr->begin(),
-                                                                              iteListSend = listLocalIndexSend_ptr->end();
+    std::list<CGridTransformation::SendingIndexGridSourceMap>::const_iterator itListSend  = listLocalIndexSend.begin(),
+                                                                              iteListSend = listLocalIndexSend.end();
     std::list<CGridTransformation::RecvIndexGridDestinationMap>::const_iterator itListRecv = listLocalIndexToReceive.begin();
     std::list<size_t>::const_iterator itNbListRecv = listNbLocalIndexToReceive.begin();
@@ -155 +159 @@
           sendBuff[idxSendBuff] = new double[itSend->second.numElements()];
       }
-      
-      const CGridTransformation::RecvIndexGridDestinationMap& localIndexToReceive = *itListRecv;
-      CGridTransformation::RecvIndexGridDestinationMap::const_iterator itbRecv = localIndexToReceive.begin(), itRecv,
-                                                                       iteRecv = localIndexToReceive.end();
 
       idxSendBuff = 0;
-      std::vector<ep_lib::MPI_Request> sendRecvRequest(localIndexToSend.size()+localIndexToReceive.size());
+      std::vector<MPI_Request> sendRecvRequest(localIndexToSend.size() + itListRecv->size());
       int position = 0;
       for (itSend = itbSend; itSend != iteSend; ++itSend, ++idxSendBuff)
@@ -172 +172 @@
           sendBuff[idxSendBuff][idx] = dataCurrentSrc(localIndex_p(idx));
         }
-        MPI_Isend(sendBuff[idxSendBuff], countSize, MPI_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest[position]);
-        position++;
+        MPI_Isend(sendBuff[idxSendBuff], countSize, MPI_DOUBLE, destRank, 12, client->intraComm, &sendRecvRequest[position++]);
       }
 
       // Receiving data on destination fields
-      
+      const CGridTransformation::RecvIndexGridDestinationMap& localIndexToReceive = *itListRecv;
+      CGridTransformation::RecvIndexGridDestinationMap::const_iterator itbRecv = localIndexToReceive.begin(), itRecv,
+                                                                       iteRecv = localIndexToReceive.end();
       int recvBuffSize = 0;
       for (itRecv = itbRecv; itRecv != iteRecv; ++itRecv) recvBuffSize += itRecv->second.size(); //(recvBuffSize < itRecv->second.size())
@@ -188 +189 @@
         int srcRank = itRecv->first;
         int countSize = itRecv->second.size();
-        MPI_Irecv(recvBuff + currentBuff, countSize, MPI_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest[position]);
-        position++;
+        MPI_Irecv(recvBuff + currentBuff, countSize, MPI_DOUBLE, srcRank, 12, client->intraComm, &sendRecvRequest[position++]);
         currentBuff += countSize;
       }
-      std::vector<ep_lib::MPI_Status> status(sendRecvRequest.size());
+      std::vector<MPI_Status> status(sendRecvRequest.size());
       MPI_Waitall(sendRecvRequest.size(), &sendRecvRequest[0], &status[0]);
 
@@ -203 +203 @@
       std::vector<bool> localInitFlag(dataCurrentDest.numElements(), true);
       currentBuff = 0;
+      bool firstPass=true; 
       for (itRecv = itbRecv; itRecv != iteRecv; ++itRecv)
       {
@@ -211 +212 @@
                          dataCurrentDest,
                          localInitFlag,
-                         ignoreMissingValue);
+                         ignoreMissingValue,firstPass);
 
         currentBuff += countSize;
+        firstPass=false ;
       }
 

XIOS/dev/branch_openmp/src/filter/spatial_transform_filter.hpp

@@ -3 +3 @@
 
 #include "filter.hpp"
-
 namespace xios
 {
@@ -106 +105 @@
       //static std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > engines;
       static std::map<CGridTransformation*, boost::shared_ptr<CSpatialTransformFilterEngine> > *engines_ptr;
-      #pragma omp threadprivate(engines_ptr)
-      
   }; // class CSpatialTransformFilterEngine
 } // namespace xios

XIOS/dev/branch_openmp/src/filter/store_filter.cpp

@@ -78 +78 @@
   void CStoreFilter::onInputReady(std::vector<CDataPacketPtr> data)
   {
-    CDataPacketPtr packet = data[0];
+
+    CDataPacketPtr packet;
+    if (detectMissingValues)
+    {
+      const size_t nbData = data[0]->data.numElements();
+
+      packet = CDataPacketPtr(new CDataPacket);
+      packet->date = data[0]->date;
+      packet->timestamp = data[0]->timestamp;
+      packet->status = data[0]->status;
+      packet->data.resize(nbData);
+      packet->data = data[0]->data;
+
+      for (size_t idx = 0; idx < nbData; ++idx)
+      {
+        if (NumTraits<double>::isnan(packet->data(idx)))
+          packet->data(idx) = missingValue;
+      }
+
+    }
+
+    else
+    {
+      packet = data[0];
+    }
 
     packets.insert(std::make_pair(packet->timestamp, packet));
@@ -85 +109 @@
     gc.registerObject(this, packet->timestamp);
 
-    if (detectMissingValues)
-    {
-      const size_t nbData = packet->data.numElements();
-      for (size_t idx = 0; idx < nbData; ++idx)
-      {
-        if (NumTraits<double>::isnan(packet->data(idx)))
-          packet->data(idx) = missingValue;
-      }
-    }
   }
 

XIOS/dev/branch_openmp/src/filter/temporal_filter.cpp

@@ -14 +14 @@
     , isOnceOperation(functor->timeType() == func::CFunctor::once)
     , isInstantOperation(functor->timeType() == func::CFunctor::instant)
-    // If we can optimize the sampling when dealing with an instant functor we do it
-    , samplingFreq((isInstantOperation && samplingFreq == TimeStep && samplingOffset == NoneDu) ? opFreq : samplingFreq)
-    , samplingOffset((isInstantOperation && samplingFreq == TimeStep && samplingOffset == NoneDu) ? opFreq - initDate.getRelCalendar().getTimeStep() : samplingOffset)
+    , samplingFreq(samplingFreq)
+    , samplingOffset(samplingOffset)
     , opFreq(opFreq)
     , nextSamplingDate(initDate + this->samplingOffset + initDate.getRelCalendar().getTimeStep())