Changeset 569

Timestamp:

03/10/15 10:49:13 (9 years ago)

Author:

mhnguyen

Message:

Correct some bugs on discovering server index and do some code cleanings

+) Add some checking functions to make sure mpi_isend and mpi_irecv work correctly
+) Add comments to code
+) Remove some redundant code and comments

Test
+) On Curie
+) The new functions are tested in test_new_features.f90. Test_client and test_complete work like before
+) Test cases:

• 3 dimension grid with: 1 domain, 1 axis
• 3 dimension grid with: 3 axis
• Attached and connected

+) All pass and results are correct

TODO:
+) Fix zoom bug with grid composed of only one axis

Location:

XIOS/trunk

Files:

• inputs/Version2/iodef.xml (modified) (3 diffs)
• src/client_server_mapping.cpp (modified) (4 diffs)
• src/client_server_mapping.hpp (modified) (3 diffs)
• src/client_server_mapping_distributed.cpp (modified) (20 diffs)
• src/client_server_mapping_distributed.hpp (modified) (4 diffs)
• src/node/axis.cpp (modified) (1 diff)
• src/node/context.cpp (modified) (2 diffs)
• src/node/distribution_client.cpp (modified) (3 diffs)
• src/node/distribution_client.hpp (modified) (2 diffs)
• src/node/domain.cpp (modified) (3 diffs)
• src/node/field.cpp (modified) (2 diffs)
• src/node/field.hpp (modified) (1 diff)
• src/node/grid.cpp (modified) (3 diffs)
• src/server_distribution_description.cpp (modified) (5 diffs)
• src/test/test_new_features.f90 (modified) (3 diffs)

XIOS/trunk/inputs/Version2/iodef.xml

@@ -7 +7 @@
 
    <field_definition level="1" enabled=".TRUE.">
-<!--     <field id="field_A"  operation="average" freq_op="3600s" domain_ref="domain_A"  axis_ref="axis_A" />-->
+     <field id="field_AA"  operation="average" freq_op="3600s" domain_ref="domain_A"  axis_ref="axis_A" />
      <field id="field_A"  operation="average" freq_op="3600s" grid_ref="grid_A" />
      <field id="field_Axis"  operation="average" freq_op="3600s" grid_ref="grid_Axis" />
@@ -19 +19 @@
      </file>
      <file id="output_Axis" name="output_Axis">
-        <field field_ref="field_Axis" />
+<!--        <field field_ref="field_Axis" />-->
         <field field_ref="field_All_Axis" />
      </file>
@@ -28 +28 @@
      <axis id="axis_A" />
      <axis id="axis_B" />
-     <axis id="axis_C" zoom_size="2" zoom_end="2" />
+     <axis id="axis_C" zoom_size="2" zoom_end="2"/>
      <axis id="axis_D" zoom_size="2" zoom_end="3"/>
    </axis_definition>

XIOS/trunk/src/client_server_mapping.cpp

@@ -3 +3 @@
    \author Ha NGUYEN
    \since 04 Feb 2015
-   \date 06 Feb 2015
+   \date 09 Mars 2015
 
    \brief Mapping between index client and server.
@@ -20 +20 @@
 }
 
-void CClientServerMapping::computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient,
-                                                     const CArray<int,1>& localIndexOnClient)
-{
-//  defaultComputeServerIndexMapping(globalIndexOnClient, globalIndexServer);
-}
-
-
+/*!
+  Compute mapping global index of server which client sends to.
+  \param [in] globalIndexOnClient global index on client
+  \param [in] globalIndexServer global index of servers
+*/
 void CClientServerMapping::computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient,
                                                      const std::vector<CArray<size_t,1>* >& globalIndexServer)
@@ -33 +31 @@
 }
 
-//void CClientServerMapping::computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient,
-//                                                     const CArray<int,1>& localIndexOnClient,
-//                                                     const std::vector<CArray<size_t,1>* >& globalIndexOfServer)
-//{
-//  defaultComputeServerIndexMapping(globalIndexOnClient, globalIndexOfServer, &localIndexOnClient);
-//}
-
 /*!
    Compute index of data which are sent to server and index global on server side
    \param [in] globalIndexOnClient global index of data on client
    \param [in] globalIndexServer global index of server(s)
+   \param [in] localIndexOnClient local index of data on client which are sent to server
 */
 void CClientServerMapping::defaultComputeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient,
@@ -129 +121 @@
   for(int n=0;n<recvSize;n++) clientRes[recvBuff[n]]++ ;
 
-//  std::map<int,int> nbSenders;
   for(int n=0;n<nbConnectedServer;n++)
   {

XIOS/trunk/src/client_server_mapping.hpp

@@ -3 +3 @@
    \author Ha NGUYEN
    \since 04 Feb 2015
-   \date 06 Feb 2015
+   \date 09 Mars 2015
 
    \brief Mapping between index client and server.
@@ -13 +13 @@
 #include "array_new.hpp"
 #include "mpi.hpp"
-#include <boost/unordered_map.hpp>
 
 namespace xios {
@@ -31 +30 @@
     virtual ~CClientServerMapping();
 
+    // Only need global index on client to calculate mapping (supposed client has info of distribution)
+    virtual void computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient) = 0;
+
+    // In case of computing local index on client sent to server
+    virtual void computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient,
+                                           const CArray<int,1>& localIndexOnClient) = 0;
+
+    // Simple case, global index on client and index on servers
     virtual void computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient,
                                            const std::vector<CArray<size_t,1>* >& globalIndexOnServer);
-
-    virtual void computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient,
-                                           const CArray<int,1>& localIndexOnClient);
-
-//    virtual void computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient,
-//                                           const CArray<int,1>& localIndexOnClient,
-//                                           const std::vector<CArray<size_t,1>* >& globalIndexOnServer);
 
     std::map<int,int> computeConnectedClients(int nbServer, int nbClient,

XIOS/trunk/src/client_server_mapping_distributed.cpp

@@ +1 @@
+/*!
+   \file client_server_mapping.hpp
+   \author Ha NGUYEN
+   \since 27 Feb 2015
+   \date 09 Mars 2015
+
+   \brief Mapping between index client and server.
+   Clients pre-calculate all information of server distribution.
+ */
 #include "client_server_mapping_distributed.hpp"
 #include <limits>
@@ -7 +16 @@
 
 CClientServerMappingDistributed::CClientServerMappingDistributed(const boost::unordered_map<size_t,int>& globalIndexOfServer,
-                                                                 const MPI_Comm& clientIntraComm) : CClientServerMapping(), indexClientHash_()
+                                                                 const MPI_Comm& clientIntraComm)
+  : CClientServerMapping(), indexClientHash_(), countIndexGlobal_(0), countIndexServer_(0),
+    indexGlobalBuffBegin_(), indexServerBuffBegin_(), requestRecvIndexServer_()
 {
   clientIntraComm_ = clientIntraComm;
   MPI_Comm_size(clientIntraComm,&(nbClient_));
-  MPI_Comm_rank(clientIntraComm,&clientRank_) ;
+  MPI_Comm_rank(clientIntraComm,&clientRank_);
+  computeHashIndex();
   computeDistributedServerIndex(globalIndexOfServer, clientIntraComm);
 }
@@ -17 +29 @@
 CClientServerMappingDistributed::~CClientServerMappingDistributed()
 {
-
-}
-
+}
+
+/*!
+   Compute mapping global index of server which client sends to.
+   \param [in] globalIndexOnClient global index client has
+*/
+void CClientServerMappingDistributed::computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient)
+{
+  int ssize = globalIndexOnClient.numElements();
+  CArray<int,1>* localIndexOnClient = new CArray<int,1>(ssize);
+  for (int i = 0; i < ssize; ++i) (*localIndexOnClient)(i) = i;
+
+  this->computeServerIndexMapping(globalIndexOnClient, *localIndexOnClient);
+  delete localIndexOnClient;
+}
+
+/*!
+   Compute mapping global index of server which client sends to.
+   \param [in] globalIndexOnClient global index client has
+   \param [in] localIndexOnClient local index on client
+*/
 void CClientServerMappingDistributed::computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient,
                                                                 const CArray<int,1>& localIndexOnClient)
@@ -89 +119 @@
     recvBuffIndexServer = new int[nbIndexReceivedFromOthers];
 
-  resetRequestAndCount();
+  resetReceivingRequestAndCount();
   std::map<int, MPI_Request>::iterator itRequest;
   std::vector<int> demandAlreadyReceived, repondAlreadyReceived;
@@ -168 +198 @@
 }
 
+/*!
+  Compute the hash index distribution of whole size_t space then each client will have a range of this distribution
+*/
 void CClientServerMappingDistributed::computeHashIndex()
 {
@@ -184 +217 @@
 }
 
+/*!
+  Compute distribution of global index for servers
+  Each client already holds a piece of information about global index and the corresponding server.
+This information is redistributed into size_t sipace in which each client possesses a specific range of index.
+Afterh the redistribution, each client as long as its range of index contains all necessary information about server.
+  \param [in] globalIndexOfServer global index and the corresponding server
+  \param [in] clientIntraComm client joining distribution process.
+*/
 void CClientServerMappingDistributed::computeDistributedServerIndex(const boost::unordered_map<size_t,int>& globalIndexOfServer,
                                                                     const MPI_Comm& clientIntraComm)
 {
-  computeHashIndex();
-  int clientRank;
-  MPI_Comm_rank(clientIntraComm,&clientRank);
-
   int* sendBuff = new int[nbClient_];
   int* sendNbIndexBuff = new int[nbClient_];
@@ -214 +251 @@
     {
       int indexClient = std::distance(itbClientHash, itClientHash)-1;
-      if (clientRank == indexClient)
+      if (clientRank_ == indexClient)
       {
         globalIndexToServerMapping_.insert(std::make_pair<size_t,int>(it->first, it->second));
@@ -228 +265 @@
   }
 
-
-//    for (boost::unordered_map<size_t,int>::const_iterator it = globalIndexToServerMapping_.begin();
-//       it != globalIndexToServerMapping_.end(); ++it)
-//       std::cout << " " << it->first << ":" << it->second;
-//       std::cout << "First Number = " << globalIndexToServerMapping_.size() << std::endl;
-
-
+  // Calculate from how many clients each client receive message.
   int* recvBuff = new int[nbClient_];
   MPI_Allreduce(sendBuff, recvBuff, nbClient_, MPI_INT, MPI_SUM, clientIntraComm);
-
+  int recvNbClient = recvBuff[clientRank_];
+
+  // Calculate size of buffer for receiving message
   int* recvNbIndexBuff = new int[nbClient_];
   MPI_Allreduce(sendNbIndexBuff, recvNbIndexBuff, nbClient_, MPI_INT, MPI_SUM, clientIntraComm);
-
-  MPI_Status statusIndexGlobal, statusIndexServer;
-  int flag, countIndexGlobal_ = 0, countIndexServer_ = 0;
-
-
-  std::map<int, MPI_Request>::iterator itRequestIndexGlobal, itRequestIndexServer;
-  std::map<int, int> countBuffIndexServer, countBuffIndexGlobal;
-  std::vector<int> processedList;
-
-
-  bool isFinished=false;
-  int recvNbIndexCount = recvNbIndexBuff[clientRank];
-  int recvNbClient = recvBuff[clientRank];
+  int recvNbIndexCount = recvNbIndexBuff[clientRank_];
   unsigned long* recvIndexGlobalBuff = new unsigned long[recvNbIndexCount];
   int* recvIndexServerBuff = new int[recvNbIndexCount];
 
+  // If a client holds information about global index and servers which don't belong to it,
+  // it will send a message to the correct clients.
+  // Contents of the message are global index and its corresponding server index
   std::list<MPI_Request> sendRequest;
   std::map<int, std::vector<size_t> >::iterator itGlobal  = client2ClientIndexGlobal.begin(),
                                                 iteGlobal = client2ClientIndexGlobal.end();
-  for ( ; itGlobal != iteGlobal; ++itGlobal)
+  for ( ;itGlobal != iteGlobal; ++itGlobal)
     sendIndexGlobalToClients(itGlobal->first, itGlobal->second, clientIntraComm, sendRequest);
   std::map<int, std::vector<int> >::iterator itServer  = client2ClientIndexServer.begin(),
@@ -266 +290 @@
     sendIndexServerToClients(itServer->first, itServer->second, clientIntraComm, sendRequest);
 
-  resetRequestAndCount();
+  std::map<int, MPI_Request>::iterator itRequestIndexGlobal, itRequestIndexServer;
+  std::map<int, int> countBuffIndexServer, countBuffIndexGlobal;
+  std::vector<int> processedList;
+
+  bool isFinished = (0 == recvNbClient) ? true : false;
+
+  // Just to make sure before listening message, all counting index and receiving request have already beeen reset
+  resetReceivingRequestAndCount();
+
+  // Now each client trys to listen to demand from others.
+  // If they have message, it processes: pushing global index and corresponding server to its map
   while (!isFinished || (!sendRequest.empty()))
   {
@@ -304 +338 @@
         --recvNbClient;
       }
-
     }
 
@@ -324 +357 @@
   delete [] recvIndexGlobalBuff;
   delete [] recvIndexServerBuff;
-
-//    for (boost::unordered_map<size_t,int>::const_iterator it = globalIndexToServerMapping_.begin();
-//       it != globalIndexToServerMapping_.end(); ++it)
-//       std::cout << " " << it->first << ":" << it->second;
-//       std::cout << "Number = " << globalIndexToServerMapping_.size() << std::endl;
-
-}
-
+}
+
+/*!
+  Probe and receive message containg global index from other clients.
+  Each client can send a message of global index to other clients to fulfill their maps.
+Each client probes message from its queue then if the message is ready, it will be put into the receiving buffer
+  \param [in] recvIndexGlobalBuff buffer dedicated for receiving global index
+  \param [in] recvNbIndexCount size of the buffer
+*/
 void CClientServerMappingDistributed::probeIndexGlobalMessageFromClients(unsigned long* recvIndexGlobalBuff, int recvNbIndexCount)
 {
@@ -350 +384 @@
 }
 
+/*!
+  Probe and receive message containg server index from other clients.
+  Each client can send a message of server index to other clients to fulfill their maps.
+Each client probes message from its queue then if the message is ready, it will be put into the receiving buffer
+  \param [in] recvIndexServerBuff buffer dedicated for receiving server index
+  \param [in] recvNbIndexCount size of the buffer
+*/
 void CClientServerMappingDistributed::probeIndexServerMessageFromClients(int* recvIndexServerBuff, int recvNbIndexCount)
 {
@@ -369 +410 @@
 }
 
-
+/*!
+  Send message containing global index to clients
+  \param [in] clientDestRank rank of destination client
+  \param [in] indexGlobal global index to send
+  \param [in] clientIntraComm communication group of client
+  \param [in] requestSendIndexGlobal list of sending request
+*/
 void CClientServerMappingDistributed::sendIndexGlobalToClients(int clientDestRank, std::vector<size_t>& indexGlobal,
                                                                const MPI_Comm& clientIntraComm,
@@ -378 +425 @@
   MPI_Isend(&(indexGlobal)[0], (indexGlobal).size(), MPI_UNSIGNED_LONG,
             clientDestRank, 15, clientIntraComm, &(requestSendIndexGlobal.back()));
-
-//  int nbSendClient = indexGlobal.size();
-//  std::map<int, std::vector<size_t> >::iterator
-//                        itClient2ClientIndexGlobal  = indexGlobal.begin(),
-//                        iteClient2ClientIndexGlobal = indexGlobal.end();
-//
-//  for (; itClient2ClientIndexGlobal != iteClient2ClientIndexGlobal;
-//         ++itClient2ClientIndexGlobal)
-//  {
-//    MPI_Request request;
-//    requestSendIndexGlobal.push_back(request);
-//    MPI_Isend(&(itClient2ClientIndexGlobal->second)[0],
-//              (itClient2ClientIndexGlobal->second).size(),
-//              MPI_UNSIGNED_LONG,
-//              itClient2ClientIndexGlobal->first,
-//              15, clientIntraComm, &(requestSendIndexGlobal.back()));
-//  }
-
-}
-
+}
+
+/*!
+  Send message containing server index to clients
+  \param [in] clientDestRank rank of destination client
+  \param [in] indexServer server index to send
+  \param [in] clientIntraComm communication group of client
+  \param [in] requestSendIndexServer list of sending request
+*/
 void CClientServerMappingDistributed::sendIndexServerToClients(int clientDestRank, std::vector<int>& indexServer,
                                                                const MPI_Comm& clientIntraComm,
@@ -406 +442 @@
   MPI_Isend(&(indexServer)[0], (indexServer).size(), MPI_INT,
             clientDestRank, 12, clientIntraComm, &(requestSendIndexServer.back()));
-
-//  int nbSendClient = indexServer.size();
-//  std::map<int, std::vector<int> >::iterator
-//                        itClient2ClientIndexServer  = indexServer.begin(),
-//                        iteClient2ClientIndexServer = indexServer.end();
-
-//  for (; itClient2ClientIndexServer != iteClient2ClientIndexServer;
-//         ++itClient2ClientIndexServer)
-//  {
-//    MPI_Request request;
-//    requestSendIndexServer.push_back(request);
-//    MPI_Isend(&(itClient2ClientIndexServer->second)[0],
-//              (itClient2ClientIndexServer->second).size(),
-//              MPI_INT,
-//              itClient2ClientIndexServer->first,
-//              12, clientIntraComm, &(requestSendIndexServer.back()));
-//
-//  }
-}
-
+}
+
+/*!
+  Verify status of sending request
+  \param [in] sendRequest sending request to verify
+*/
 void CClientServerMappingDistributed::testSendRequest(std::list<MPI_Request>& sendRequest)
 {
@@ -441 +463 @@
       if (true == flag)
       {
-        --sizeListRequest;
         isErased = true;
         break;
@@ -451 +472 @@
 }
 
+/*!
+  Process the received request. Pushing global index and server index into map
+  \param[in] buffIndexGlobal pointer to the begining of buffer containing global index
+  \param[in] buffIndexServer pointer to the begining of buffer containing server index
+  \param[in] count size of received message
+*/
 void CClientServerMappingDistributed::processReceivedRequest(unsigned long* buffIndexGlobal, int* buffIndexServer, int count)
 {
@@ -457 +484 @@
 }
 
+/*!
+  Compute size of message containing global index
+  \param[in] requestRecv request of message
+*/
 int CClientServerMappingDistributed::computeBuffCountIndexGlobal(MPI_Request& requestRecv)
 {
@@ -471 +502 @@
 }
 
+/*!
+  Compute size of message containing server index
+  \param[in] requestRecv request of message
+*/
 int CClientServerMappingDistributed::computeBuffCountIndexServer(MPI_Request& requestRecv)
 {
@@ -485 +520 @@
 }
 
-void CClientServerMappingDistributed::resetRequestAndCount()
+/*!
+  Reset all receiving request map and counter
+*/
+void CClientServerMappingDistributed::resetReceivingRequestAndCount()
 {
   countIndexGlobal_ = countIndexServer_ = 0;

XIOS/trunk/src/client_server_mapping_distributed.hpp

@@ +1 @@
+/*!
+   \file client_server_mapping.hpp
+   \author Ha NGUYEN
+   \since 27 Feb 2015
+   \date 09 Mars 2015
+
+   \brief Mapping between index client and server.
+   Clients pre-calculate all information of server distribution.
+ */
+
 #ifndef __XIOS_CLIENT_SERVER_MAPPING_DISTRIBUTED_HPP__
 #define __XIOS_CLIENT_SERVER_MAPPING_DISTRIBUTED_HPP__
@@ -10 +20 @@
 namespace xios
 {
-
+/*!
+  \class CClientServerMappingDistributed
+  This class computes index of data which are sent to server as well as index of data
+on server side with a distributed alogrithm. Each client has a piece of information about the distribution
+of servers. To find out all these info, first of all, all client join a discovering process in which each client
+announces the others about the info they have as well as demand others info they are lacked of. After this process,
+each client has enough info to decide to which client it need to send a demand for corresponding server of a global index.
+The alogrithm depends on hashed index.
+*/
 class CClientServerMappingDistributed : public CClientServerMapping
 {
@@ -17 +35 @@
     CClientServerMappingDistributed(const boost::unordered_map<size_t,int>& globalIndexOfServer,
                                     const MPI_Comm& clientIntraComm);
+
+    virtual void computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient);
 
     virtual void computeServerIndexMapping(const CArray<size_t,1>& globalIndexOnClient,
@@ -25 +45 @@
 
   protected:
+    // Redistribute global index and server index among clients
     void computeDistributedServerIndex(const boost::unordered_map<size_t,int>& globalIndexOfServer,
                                        const MPI_Comm& clientIntraComm);
 
-    void processReceivedRequest(unsigned long* buffIndexGlobal, int* buffIndexServer, int count);
-
-    void testSendRequest(std::list<MPI_Request>& sendRequest);
-
-    int computeBuffCount(MPI_Request& requestRecvIndexGlobal, MPI_Request& requestRecvIndexServer);
-
-    void computeHashIndex();
-
-//    void sendIndexServerToClients(std::map<int, std::vector<int> >& indexServer,
-//                                  const MPI_Comm& clientIntraComm,
-//                                  std::list<MPI_Request>& requestSendIndexServer);
-
-//    void sendIndexGlobalToClients(std::map<int, std::vector<size_t> >& indexGlobal,
-//                                  const MPI_Comm& clientIntraComm,
-//                                  std::list<MPI_Request>& requestSendIndexGlobal);
-
+    // Send server index to clients
     void sendIndexServerToClients(int clientDestRank, std::vector<int>& indexServer,
                                   const MPI_Comm& clientIntraComm, std::list<MPI_Request>& requestSendIndexServer);
 
+    // Send global index to clients
     void sendIndexGlobalToClients(int clientDestRank, std::vector<size_t>& indexGlobal,
                                   const MPI_Comm& clientIntraComm, std::list<MPI_Request>& requestSendIndexGlobal);
 
-    void resetRequestAndCount();
+    // Verify sending request
+    void testSendRequest(std::list<MPI_Request>& sendRequest);
 
+    // Process request
+    void processReceivedRequest(unsigned long* buffIndexGlobal, int* buffIndexServer, int count);
+
+    // Probe and receive message of global index
     void probeIndexGlobalMessageFromClients(unsigned long* recvIndexGlobalBuff, int recvNbIndexCount);
 
+    // Probe and receive message of server index
     void probeIndexServerMessageFromClients(int* recvIndexServerBuff, int recvNbIndexCount);
 
+    // Compute range of hashing
+    void computeHashIndex();
+
+    // Compute size of receiving buffer for global index
     int computeBuffCountIndexGlobal(MPI_Request& requestRecv);
 
+    // Compute size of receiving buffer for server index
     int computeBuffCountIndexServer(MPI_Request& requestRecv);
+
+    // Reset request map
+    void resetReceivingRequestAndCount();
+
   private:
+    //! Mapping of global index to the corresponding server
     boost::unordered_map<size_t,int> globalIndexToServerMapping_;
 
+    //! Bounds of hash index
+    std::vector<size_t> indexClientHash_;
+
+    //! Number of client
+    int nbClient_;
+
+    //! Rank of client
+    int clientRank_;
+
+    //! Counting of buffer for receiving global index
+    int countIndexGlobal_;
+
+    //! Counting of buffer for receiving server index
+    int countIndexServer_;
+
+    //! intracommuntion of clients
+    MPI_Comm clientIntraComm_;
+
+    //! Request returned by MPI_IRecv function about global index
     std::map<int, MPI_Request> requestRecvIndexGlobal_;
 
+    //! Request returned by MPI_IRecv function about index of server
     std::map<int, MPI_Request> requestRecvIndexServer_;
 
-    std::vector<size_t> indexClientHash_;
-
-    int nbClient_;
-
-    int clientRank_;
-
-    int countIndexGlobal_;
-
-    int countIndexServer_;
-
-    MPI_Comm clientIntraComm_;
-
+    //! Mapping client rank and the beginning position of receiving buffer for message of global index from this client
     std::map<int, unsigned long*> indexGlobalBuffBegin_;
 
+    //! Mapping client rank and the begining position of receiving buffer for message of server index from this client
     std::map<int, int*> indexServerBuffBegin_;
-
 };
 

XIOS/trunk/src/node/axis.cpp

@@ -126 +126 @@
       this->zoom_end.setValue(zoom_end) ;
       this->zoom_size.setValue(zoom_size) ;
-
-      // compute client zoom indices
-//      zoom_begin_client = ibegin_client > zoom_begin ? begin_client : zoom_begin ;
-//      zoom_end_client   = iend_client < zoom_end ? iend_client : zoom_end ;
-//      zoom_size_client  = zoom_end_client-zoom_begin_client+1 ;
-//      if (zoom_ni_client<0) zoom_ni_client=0 ;
    }
 

XIOS/trunk/src/node/context.cpp

@@ -330 +330 @@
 
       // At last, we have all info of domain and axis, then send them
-//       sendRefDomainsAxis();
+       sendRefDomainsAxis();
 
       // After that, send all grid (if any)
@@ -799 +799 @@
 
    //! Client side: Send information of reference domain and axis of active fields
-//   void CContext::sendRefDomainsAxis()
-//   {
-//     std::set<StdString> domainIds;
-//     std::set<StdString> axisIds;
-//
-//     // Find all reference domain and axis of all active fields
-//     int numEnabledFiles = this->enabledFiles.size();
-//     for (int i = 0; i < numEnabledFiles; ++i)
-//     {
-//       std::vector<CField*> enabledFields = this->enabledFiles[i]->getEnabledFields();
-//       int numEnabledFields = enabledFields.size();
-//       for (int j = 0; j < numEnabledFields; ++j)
-//       {
-//         const std::pair<StdString, StdString>& prDomAxisId = enabledFields[j]->getDomainAxisIds();
-//         domainIds.insert(prDomAxisId.first);
-//         axisIds.insert(prDomAxisId.second);
-//       }
-//     }
-//
-//     // Create all reference axis on server side
-//     std::set<StdString>::iterator itDom, itAxis;
-//     std::set<StdString>::const_iterator itE;
-//
-//     StdString axiDefRoot("axis_definition");
-//     CAxisGroup* axisPtr = CAxisGroup::get(axiDefRoot);
-//     itE = axisIds.end();
-//     for (itAxis = axisIds.begin(); itAxis != itE; ++itAxis)
-//     {
-//       if (!itAxis->empty())
-//       {
-//         axisPtr->sendCreateChild(*itAxis);
-//         CAxis::get(*itAxis)->sendAllAttributesToServer();
-//       }
-//     }
-//
-//     // Create all reference domains on server side
-//     StdString domDefRoot("domain_definition");
-//     CDomainGroup* domPtr = CDomainGroup::get(domDefRoot);
-//     itE = domainIds.end();
-//     for (itDom = domainIds.begin(); itDom != itE; ++itDom)
-//     {
-//       if (!itDom->empty()) {
-//          domPtr->sendCreateChild(*itDom);
-//          CDomain::get(*itDom)->sendAllAttributesToServer();
-//       }
-//     }
-//   }
+   void CContext::sendRefDomainsAxis()
+   {
+     std::set<StdString> domainIds;
+     std::set<StdString> axisIds;
+
+     // Find all reference domain and axis of all active fields
+     int numEnabledFiles = this->enabledFiles.size();
+     for (int i = 0; i < numEnabledFiles; ++i)
+     {
+       std::vector<CField*> enabledFields = this->enabledFiles[i]->getEnabledFields();
+       int numEnabledFields = enabledFields.size();
+       for (int j = 0; j < numEnabledFields; ++j)
+       {
+         const std::pair<StdString, StdString>& prDomAxisId = enabledFields[j]->getRefDomainAxisIds();
+         domainIds.insert(prDomAxisId.first);
+         axisIds.insert(prDomAxisId.second);
+       }
+     }
+
+     // Create all reference axis on server side
+     std::set<StdString>::iterator itDom, itAxis;
+     std::set<StdString>::const_iterator itE;
+
+     StdString axiDefRoot("axis_definition");
+     CAxisGroup* axisPtr = CAxisGroup::get(axiDefRoot);
+     itE = axisIds.end();
+     for (itAxis = axisIds.begin(); itAxis != itE; ++itAxis)
+     {
+       if (!itAxis->empty())
+       {
+         axisPtr->sendCreateChild(*itAxis);
+         CAxis::get(*itAxis)->sendAllAttributesToServer();
+       }
+     }
+
+     // Create all reference domains on server side
+     StdString domDefRoot("domain_definition");
+     CDomainGroup* domPtr = CDomainGroup::get(domDefRoot);
+     itE = domainIds.end();
+     for (itDom = domainIds.begin(); itDom != itE; ++itDom)
+     {
+       if (!itDom->empty()) {
+          domPtr->sendCreateChild(*itDom);
+          CDomain::get(*itDom)->sendAllAttributesToServer();
+       }
+     }
+   }
 
    //! Update calendar in each time step

XIOS/trunk/src/node/distribution_client.cpp

@@ -3 +3 @@
    \author Ha NGUYEN
    \since 13 Jan 2015
-   \date 09 Feb 2015
+   \date 09 Mars 2015
 
    \brief Index distribution on client side.
@@ -56 +56 @@
   itbDom  = itDom  = domList.begin();  iteDom  = domList.end();
   itbAxis = itAxis = axisList.begin(); iteAxis = axisList.end();
-
-  // First of all, every attribute of domain and axis should be checked
-//  for (;itDom != iteDom; ++itDom) (*itDom)->checkAttributesOnClient();
-//  for (;itAxis != iteAxis; ++itAxis) (*itAxis)->checkAttributesOnClient();
 
   readDistributionInfo(domList, axisList, axisDomainOrder);
@@ -574 +570 @@
 }
 
+/*!
+  Return local data index on client which are sent to servers
+*/
 const CArray<int,1>& CDistributionClient::getLocalDataIndexSendToServerOnClient() const
 {

XIOS/trunk/src/node/distribution_client.hpp

@@ -3 +3 @@
    \author Ha NGUYEN
    \since 13 Jan 2015
-   \date 09 Feb 2015
+   \date 09 Mars 2015
 
    \brief Index distribution on client side.
@@ -112 +112 @@
 };
 
+/*!
+  A grid can have multiple dimension, so can its mask in the form of multi-dimension array.
+It's not a good idea to store all multi-dimension arrays corresponding to each mask.
+One of the ways is to convert this array into 1-dimension one and every process is taken place on it.
+  \param [in] multi-dimension array grid mask
+*/
 template<int N>
 void CDistributionClient::readGridMaskInfo(const CArray<bool,N>& gridMask)

XIOS/trunk/src/node/domain.cpp

@@ -15 +15 @@
 #include "array_new.hpp"
 #include "server_distribution_description.hpp"
+#include "client_server_mapping_distributed.hpp"
 
 namespace xios {
@@ -667 +668 @@
     int zoom_jend=zoom_jbegin+zoom_nj-1 ;
 
-    std::vector<int> nGlobDomain(2);
-    nGlobDomain[0] = ni_glo.getValue();
-    nGlobDomain[1] = nj_glo.getValue();
-    CServerDistributionDescription serverDescription(nGlobDomain);
-    serverDescription.computeServerDistribution(nbServer, doComputeGlobalIndexServer);
 
     // Precompute number of index
@@ -706 +702 @@
       }
 
-    CClientServerMapping clientServerMap;
-    clientServerMap.computeServerIndexMapping(globalIndexDomain, serverDescription.getGlobalIndex());
-    const std::map<int, std::vector<size_t> >& globalIndexDomainOnServer = clientServerMap.getGlobalIndexOnServer();
+     std::vector<int> nGlobDomain(2);
+     nGlobDomain[0] = ni_glo.getValue();
+     nGlobDomain[1] = nj_glo.getValue();
+     size_t globalSizeIndex = 1, indexBegin, indexEnd;
+     int range, clientSize = client->clientSize;
+     for (int i = 0; i < nGlobDomain.size(); ++i) globalSizeIndex *= nGlobDomain[i];
+     indexBegin = 0;
+     for (int i = 0; i < clientSize; ++i)
+     {
+       range = globalSizeIndex / clientSize;
+       if (i < (globalSizeIndex%clientSize)) ++range;
+       if (i == client->clientRank) break;
+       indexBegin += range;
+     }
+     indexEnd = indexBegin + range - 1;
+
+    CServerDistributionDescription serverDescription(nGlobDomain);
+    serverDescription.computeServerGlobalIndexInRange(nbServer, std::make_pair<size_t,size_t>(indexBegin, indexEnd));
+    CClientServerMapping* clientServerMap = new CClientServerMappingDistributed(serverDescription.getGlobalIndexRange(),
+                                                                                client->intraComm);
+    clientServerMap->computeServerIndexMapping(globalIndexDomain);
+    const std::map<int, std::vector<size_t> >& globalIndexDomainOnServer = clientServerMap->getGlobalIndexOnServer();
     std::vector<int> connectedServerRank;
     for (std::map<int, std::vector<size_t> >::const_iterator it = globalIndexDomainOnServer.begin(); it != globalIndexDomainOnServer.end(); ++it) {
       connectedServerRank.push_back(it->first);
     }
-    nbConnectedClients_ = clientServerMap.computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank);
+    nbConnectedClients_ = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank);
     indSrv_ = globalIndexDomainOnServer;
+
+    delete clientServerMap;
   }
 

XIOS/trunk/src/node/field.cpp

@@ -154 +154 @@
     if (!grid->doGridHaveDataDistributed())
     {
-       if (0 == client->getClientRank())
+       if (0 == client->clientRank)
        {
           for(it=grid->storeIndex_toSrv.begin();it!=grid->storeIndex_toSrv.end();it++)
@@ -799 +799 @@
    \return pair of Domain and Axis id
    */
-//   const std::pair<StdString,StdString>& CField::getDomainAxisIds()
-//   {
-//     CGrid* cgPtr = getRelGrid();
-//     if (NULL != cgPtr)
-//     {
-//       if (NULL != cgPtr->getRelDomain()) domAxisIds_.first = cgPtr->getRelDomain()->getId();
-//       if (NULL != cgPtr->getRelAxis()) domAxisIds_.second = cgPtr->getRelAxis()->getId();
-//     }
-//
-//     return (domAxisIds_);
-//   }
+   const std::pair<StdString,StdString>& CField::getRefDomainAxisIds()
+   {
+     CGrid* cgPtr = getRelGrid();
+     if (NULL != cgPtr)
+     {
+       std::vector<StdString>::iterator it;
+       if (!domain_ref.isEmpty())
+       {
+         std::vector<StdString> domainList = cgPtr->getDomainList();
+         it = std::find(domainList.begin(), domainList.end(), domain_ref.getValue());
+         if (domainList.end() != it) domAxisIds_.first = *it;
+       }
+
+       if (!axis_ref.isEmpty())
+       {
+         std::vector<StdString> axisList = cgPtr->getAxisList();
+         it = std::find(axisList.begin(), axisList.end(), axis_ref.getValue());
+         if (axisList.end() != it) domAxisIds_.second = *it;
+       }
+     }
+     return (domAxisIds_);
+   }
 
    CVariable* CField::addVariable(const string& id)

XIOS/trunk/src/node/field.hpp

@@ -157 +157 @@
 
 
-//        const std::pair<StdString, StdString>& getDomainAxisIds();
+        const std::pair<StdString, StdString>& getRefDomainAxisIds();
       public :
          /// Propriétés privées ///

XIOS/trunk/src/node/grid.cpp

@@ -340 +340 @@
                                                  clientDistribution_->getLocalDataIndexSendToServerOnClient());
 
-//     clientServerMap_->computeServerIndexMapping(clientDistribution_->getGlobalIndex(),
-//                                                 clientDistribution_->getLocalDataIndexSendToServerOnClient(),
-//                                                 serverDistributionDescription_->getGlobalIndex());
      const std::map<int, std::vector<size_t> >& globalIndexOnServer = clientServerMap_->getGlobalIndexOnServer();
      std::vector<int> connectedServerRank;
@@ -353 +350 @@
      storeIndex_client.resize(clientDistribution_->getLocalDataIndexOnClient().numElements());
      storeIndex_client = (clientDistribution_->getLocalDataIndexOnClient());
-
    }
 
@@ -457 +453 @@
     if (!doGridHaveDataDistributed())
     {
-      if (0 == client->getClientRank())
+      if (0 == client->clientRank)
       {
        for (int ns = 0; itGlobal != iteMap; ++itGlobal, ++itLocal, ++ns)

XIOS/trunk/src/server_distribution_description.cpp

@@ -3 +3 @@
    \author Ha NGUYEN
    \since 04 Jan 2015
-   \date 09 Feb 2015
+   \date 09 Mars 2015
 
    \brief Description of index distribution on server(s).
@@ -15 +15 @@
   : nGlobal_(globalDimensionSize), indexBegin_(), dimensionSizes_(), globalIndex_(), vecGlobalIndex_()
 {
-
 }
 
 CServerDistributionDescription::~CServerDistributionDescription()
 {
-//  if (0 != globalIndex_) delete globalIndex_;
   if (!vecGlobalIndex_.empty())
     for (int i = 0; i < vecGlobalIndex_.size(); ++i) delete vecGlobalIndex_[i];
@@ -92 +90 @@
 }
 
+/*!
+  Compute global index assigned to a server with a range.E.g: if a grid has 100 points and
+  there are 2 servers, the first one takes index from 0 to 49, the second has index from 50 to 99
+  \param [in] nServer number of server
+  \param [in] indexBeginEnd begining and ending index of range
+  \param [in] serType type of server distribution. For now, we can distribute server by band or plan
+*/
 void CServerDistributionDescription::computeServerGlobalIndexInRange(int nServer,
                                         const std::pair<size_t, size_t>& indexBeginEnd,
@@ -117 +122 @@
     size_t ssize = 1, idx = 0;
     for (int j = 0; j < dim; ++j) ssize *= dimensionSizes_[idxServer][j];
-    vecGlobalIndex_[idxServer] = new CArray<size_t,1>(ssize);
 
     std::vector<int> idxLoop(dim,0);
-
     int innerLoopSize = dimensionSizes_[idxServer][0];
 
@@ -243 +246 @@
 }
 
+/*!
+  Get global index calculated by computeServerGlobalIndexInRange
+*/
 const boost::unordered_map<size_t,int>& CServerDistributionDescription::getGlobalIndexRange() const
 {

XIOS/trunk/src/test/test_new_features.f90

@@ -17 +17 @@
   INTEGER,PARAMETER :: ni_glo=100
   INTEGER,PARAMETER :: nj_glo=100
-  INTEGER,PARAMETER :: llm=5
+  INTEGER,PARAMETER :: llm=10
   DOUBLE PRECISION  :: lval(llm)=1
   TYPE(xios_field) :: field_hdl
@@ -135 +135 @@
   CALL xios_get_domain_attr("domain_A",ni=ni,lonvalue=lonvalue)
 
-  print *,"ni",ni
-  print *,"lonvalue",lonvalue ;
+!  print *,"ni",ni
+!  print *,"lonvalue",lonvalue ;
 
   CALL xios_is_defined_field_attr("field_A",enabled=ok)
@@ -146 +146 @@
     CALL xios_update_calendar(ts)
     CALL xios_send_field("field_A",field_A)
-    CALL xios_send_field("field_Axis",field_Axis)
+!    CALL xios_send_field("field_Axis",field_Axis)
     CALL xios_send_field("field_All_Axis",field_All_Axis)
     CALL wait_us(5000) ;