Changeset 1576 for XIOS/dev/dev_olga/src/node/domain.cpp

Timestamp:

09/10/18 17:18:33 (6 years ago)

Author:

oabramkina

Message:

Adding a check on the client side if all servers have data to receive. If servers have no data to receive the master process will send empty data to such servers. This ensures that all servers participate in collective calls.

File:

• XIOS/dev/dev_olga/src/node/domain.cpp (modified) (1 diff)

XIOS/dev/dev_olga/src/node/domain.cpp

@@ -1848 +1848 @@
           // Even if a client has no index, it must connect to at least one server and
           // send an "empty" data to this server
-//          if (connectedServerRank_[nbServer].empty())
-//            connectedServerRank_[nbServer].push_back(client->clientRank % client->serverSize);
-
-          // Trying instead: send empty data to all servers
           if (connectedServerRank_[nbServer].empty())
-            for (int i = 0; i < client->serverSize; ++i)
-            connectedServerRank_[nbServer].push_back(i);
-
-
+            connectedServerRank_[nbServer].push_back(client->clientRank % client->serverSize);
+
+          // Now check if all servers have data to receive. If not, master client will send empty data.
+          // This ensures that all servers will participate in collective calls upon receiving even if they have no date to receive.
+          std::vector<int> counts (clientSize);
+          std::vector<int> displs (clientSize);
+          displs[0] = 0;
+          int localCount = connectedServerRank_[nbServer].size() ;
+          MPI_Gather(&localCount, 1, MPI_INT, &counts[0], 1, MPI_INT, 0, client->intraComm) ;
+          for (int i = 0; i < clientSize-1; ++i)
+          {
+            displs[i+1] = displs[i] + counts[i];
+          }
+          std::vector<int> allConnectedServers(displs[clientSize-1]+counts[clientSize-1]);
+          MPI_Gatherv(&(connectedServerRank_[nbServer])[0], localCount, MPI_INT, &allConnectedServers[0], &counts[0], &displs[0], MPI_INT, 0, client->intraComm);
+
+          if ((allConnectedServers.size() != nbServer) && (rank == 0))
+          {
+            std::vector<bool> isSrvConnected (nbServer, false);
+            for (int i = 0; i < allConnectedServers.size(); ++i) isSrvConnected[allConnectedServers[i]] = true;
+            for (int i = 0; i < nbServer; ++i)
+            {
+              if (!isSrvConnected[i]) connectedServerRank_[nbServer].push_back(i);
+            }
+          }
           nbSenders[nbServer] = clientServerMap->computeConnectedClients(client->serverSize, client->clientSize, client->intraComm, connectedServerRank_[nbServer]);
           delete clientServerMap;