Changeset 1856 for XIOS

Timestamp:

02/05/20 16:02:18 (4 years ago)

Author:

ymipsl

Message:

Impose reproducibilty when interpolation weights are read from file. MPI message must be received in a deterministic order.

YM

Location:

XIOS/trunk/src

Files:

• mpi_tag.hpp (modified) (1 diff)
• transformation/domain_algorithm_interpolate.cpp (modified) (8 diffs)

XIOS/trunk/src/mpi_tag.hpp

@@ -44 +44 @@
 #define MPI_DOMAIN_INTERPOLATION_WEIGHT 9
 
-
+/* Tag for mpi communication to send and receive info of DOMAIN in domain interpolation*/
+#define MPI_DOMAIN_INTERPOLATION_SOURCE_RANK 10
 
 #endif

XIOS/trunk/src/transformation/domain_algorithm_interpolate.cpp

@@ -609 +609 @@
   int* sendBuff = new int[nbClient];
   int* recvBuff = new int[nbClient];
+
+  int* sendParticipants = new int[nbClient];
+  int* recvParticipants = new int[nbClient];
+  
   for (int i = 0; i < nbClient; ++i)
   {
     sendBuff[i] = 0;
     recvBuff[i] = 0;
+    sendParticipants[i]=0 ;
+    recvParticipants[i]=0 ;
   }
   int sendBuffSize = 0;
@@ -627 +633 @@
     }
     sendBuff[itMap->first] = sizeIndex;
+    sendParticipants[itMap->first] = 1 ;
     sendBuffSize += sizeIndex;
   }
 
-
   MPI_Allreduce(sendBuff, recvBuff, nbClient, MPI_INT, MPI_SUM, client->intraComm);
+  MPI_Allreduce(sendParticipants, recvParticipants, nbClient, MPI_INT, MPI_SUM, client->intraComm);
 
   int* sendIndexDestBuff = new int [sendBuffSize];
@@ -657 +664 @@
       }
     }
+    
+    sendRequest.push_back(MPI_Request());
+    MPI_Isend(&clientRank,
+             1,
+             MPI_INT,
+             itMap->first,
+             MPI_DOMAIN_INTERPOLATION_SOURCE_RANK,
+             client->intraComm,
+             &sendRequest.back());
 
     sendRequest.push_back(MPI_Request());
@@ -686 +702 @@
 
   int recvBuffSize = recvBuff[clientRank];
+  int numberOfParticipants = recvParticipants[clientRank] ;
+
   int* recvIndexDestBuff = new int [recvBuffSize];
   int* recvIndexSrcBuff  = new int [recvBuffSize];
@@ -691 +709 @@
   int receivedSize = 0;
   int clientSrcRank;
-  while (receivedSize < recvBuffSize)
+ 
+ 
+ // this part is done to impose a specific order for reception to retrive reproducibility 
+  set<int> rankOrder ; 
+  for (int np=0 ; np < numberOfParticipants; ++np)
+  {
+    MPI_Status recvStatus;
+    int rank ;
+    MPI_Recv(&rank,
+             1,
+             MPI_INT,
+             MPI_ANY_SOURCE,
+             MPI_DOMAIN_INTERPOLATION_SOURCE_RANK,
+             client->intraComm,
+             &recvStatus);
+    rankOrder.insert(rank) ;
+  }
+  
+  for (auto  clientSrcRank : rankOrder)
   {
     MPI_Status recvStatus;
@@ -697 +733 @@
              recvBuffSize,
              MPI_INT,
-             MPI_ANY_SOURCE,
+             clientSrcRank,
              MPI_DOMAIN_INTERPOLATION_DEST_INDEX,
              client->intraComm,
@@ -704 +740 @@
     int countBuff = 0;
     MPI_Get_count(&recvStatus, MPI_INT, &countBuff);
-    clientSrcRank = recvStatus.MPI_SOURCE;
+//    clientSrcRank = recvStatus.MPI_SOURCE;
 
     MPI_Recv((recvIndexSrcBuff + receivedSize),
@@ -729 +765 @@
     receivedSize += countBuff;
   }
-
-  std::vector<MPI_Status> requestStatus(sendRequest.size());
+  
   MPI_Waitall(sendRequest.size(), &sendRequest[0], MPI_STATUS_IGNORE);
 
+  delete [] sendParticipants ;
+  delete [] recvParticipants ;
   delete [] sendIndexDestBuff;
   delete [] sendIndexSrcBuff;