Changeset 2666 for XIOS3/branches

Timestamp:

10/25/24 09:19:08 (3 months ago)

Author:

jderouillat

Message:

Backporting 1856 and 2663, related to reproductibilty of interpolation's order

Location:

XIOS3/branches/xios-3.0-beta/src

Files:

• mpi_tag.hpp (modified) (1 diff)
• transformation/domain_algorithm/domain_algorithm_interpolate.cpp (modified) (9 diffs)

XIOS3/branches/xios-3.0-beta/src/mpi_tag.hpp

@@ -44 +44 @@
 #define MPI_DOMAIN_INTERPOLATION_WEIGHT 9
 
-
+/* Tag for mpi communication to send and receive info of DOMAIN in domain interpolation*/ 
+#define MPI_DOMAIN_INTERPOLATION_SOURCE_RANK 10 
 
 #endif

XIOS3/branches/xios-3.0-beta/src/transformation/domain_algorithm/domain_algorithm_interpolate.cpp

@@ -548 +548 @@
       transWeight[it->first].push_back(tmp[i].second);
     }      
-  }      
+  }
+  
+  std::map<int, std::vector<double> > transWeightOrdered(transWeight.begin(), transWeight.end());
+  transWeight.clear();
+  std::map<int, std::vector<double> >::const_iterator itWeightO = transWeightOrdered.begin();
+  for(; itWeightO!=transWeightOrdered.end() ; ++itWeightO) {
+      transWeight[itWeightO->first] = itWeightO->second;
+  }
+  
+  std::map<int, std::vector<int> > transMapOrdered(transMap.begin(), transMap.end());
+  transMap.clear();
+  std::map<int, std::vector<int> >::const_iterator itMapO = transMapOrdered.begin();
+  for(; itMapO!=transMapOrdered.end() ; ++itMapO) {
+      transMap[itMapO->first] = itMapO->second;
+  }
 }
 CATCH
@@ -599 +613 @@
   int* sendBuff = new int[nbClient];
   int* recvBuff = new int[nbClient];
+
+  int* sendParticipants = new int[nbClient];
+  int* recvParticipants = new int[nbClient];
+  
   for (int i = 0; i < nbClient; ++i)
   {
     sendBuff[i] = 0;
     recvBuff[i] = 0;
+    sendParticipants[i]=0 ;
+    recvParticipants[i]=0 ;
   }
   int sendBuffSize = 0;
@@ -617 +637 @@
     }
     sendBuff[itMap->first] = sizeIndex;
+    sendParticipants[itMap->first] = 1 ;
     sendBuffSize += sizeIndex;
   }
 
-
   MPI_Allreduce(sendBuff, recvBuff, nbClient, MPI_INT, MPI_SUM, context->intraComm_);
+  MPI_Allreduce(sendParticipants, recvParticipants, nbClient, MPI_INT, MPI_SUM, context->intraComm_);
 
   int* sendIndexDestBuff = new int [sendBuffSize];
@@ -647 +668 @@
       }
     }
+    
+    sendRequest.push_back(MPI_Request());
+    MPI_Isend(&clientRank,
+             1,
+             MPI_INT,
+             itMap->first,
+             MPI_DOMAIN_INTERPOLATION_SOURCE_RANK,
+             context->intraComm_,
+             &sendRequest.back());
 
     sendRequest.push_back(MPI_Request());
@@ -676 +706 @@
 
   int recvBuffSize = recvBuff[clientRank];
+  int numberOfParticipants = recvParticipants[clientRank] ;
+
   int* recvIndexDestBuff = new int [recvBuffSize];
   int* recvIndexSrcBuff  = new int [recvBuffSize];
@@ -681 +713 @@
   int receivedSize = 0;
   int clientSrcRank;
-  while (receivedSize < recvBuffSize)
+ 
+ 
+  // this part is done to impose a specific order for reception to retrive reproducibility 
+  set<int> rankOrder ; 
+  for (int np=0 ; np < numberOfParticipants; ++np)
+  {
+    MPI_Status recvStatus;
+    int rank ;
+    MPI_Recv(&rank,
+             1,
+             MPI_INT,
+             MPI_ANY_SOURCE,
+             MPI_DOMAIN_INTERPOLATION_SOURCE_RANK,
+             context->intraComm_,
+             &recvStatus);
+    rankOrder.insert(rank) ;
+  }
+  
+  for (auto  clientSrcRank : rankOrder)
   {
     MPI_Status recvStatus;
@@ -687 +737 @@
              recvBuffSize,
              MPI_INT,
-             MPI_ANY_SOURCE,
+             clientSrcRank,
              MPI_DOMAIN_INTERPOLATION_DEST_INDEX,
              context->intraComm_,
@@ -694 +744 @@
     int countBuff = 0;
     MPI_Get_count(&recvStatus, MPI_INT, &countBuff);
-    clientSrcRank = recvStatus.MPI_SOURCE;
+//    clientSrcRank = recvStatus.MPI_SOURCE;
 
     MPI_Recv((recvIndexSrcBuff + receivedSize),
@@ -720 +770 @@
   }
 
+  std::map<int, std::vector<double> > transWeightOrdered(transWeight.begin(), transWeight.end());
+  transWeight.clear();
+  std::map<int, std::vector<double> >::const_iterator itWeightO = transWeightOrdered.begin();
+  for(; itWeightO!=transWeightOrdered.end() ; ++itWeightO) {
+      transWeight[itWeightO->first] = itWeightO->second;
+  }
+  
+  std::map<int, std::vector<int> > transMapOrdered(transMap.begin(), transMap.end());
+  transMap.clear();
+  std::map<int, std::vector<int> >::const_iterator itMapO = transMapOrdered.begin();
+  for(; itMapO!=transMapOrdered.end() ; ++itMapO) {
+      transMap[itMapO->first] = itMapO->second;
+  }
+  
   std::vector<MPI_Status> requestStatus(sendRequest.size());
   MPI_Waitall(sendRequest.size(), &sendRequest[0], MPI_STATUS_IGNORE);
 
+  delete [] sendParticipants ;                                                                                                                              
+  delete [] recvParticipants ;
   delete [] sendIndexDestBuff;
   delete [] sendIndexSrcBuff;