Changeset 1149 for XIOS/dev

Timestamp:

05/31/17 10:51:06 (7 years ago)

Author:

yushan

Message:

bug fixed in mpi_comm_split. Key needs to be specifify.

Location:

XIOS/dev/branch_yushan_merged

Files:

• extern/remap/src/intersect.cpp (modified) (1 diff)
• extern/remap/src/mapper.cpp (modified) (11 diffs)
• extern/src_ep_dev/ep_test.cpp (modified) (4 diffs)
• extern/src_ep_dev/ep_wait.cpp (modified) (1 diff)
• src/io/netCdfInterface.cpp (modified) (2 diffs)
• src/node/field.cpp (modified) (3 diffs)
• src/node/file.cpp (modified) (1 diff)
• src/transformation/domain_algorithm_interpolate.cpp (modified) (13 diffs)

XIOS/dev/branch_yushan_merged/extern/remap/src/intersect.cpp

@@ -56 +56 @@
     for (int j = 0; j < b.n; j++)
     {
-// share a full edge ? be carefull at the orientation
-      assert(squaredist(a.vertex[ i       ], b.vertex[ j       ]) > 1e-10*1e-10 ||
-             squaredist(a.vertex[(i+1)%a.n], b.vertex[(j+1)%b.n]) > 1e-10*1e-10);
+      // share a full edge ? be carefull at the orientation
+      /*
+      if(squaredist(a.vertex[i], b.vertex[j]) > 1e-10*1e-10 ||
+         squaredist(a.vertex[(i+1)%a.n], b.vertex[(j+1)%b.n]) > 1e-10*1e-10 )
+      {      
+        printf("A : squaredist(a.vertex[%d], b.vertex[%d]) = %.10e  %d\n", 
+                i, j, squaredist(a.vertex[i], b.vertex[j]), 
+                squaredist(a.vertex[i], b.vertex[j]) > 1e-10*1e-10 ? true : false);
+        printf("B : squaredist(a.vertex[%d], b.vertex[%d]) = %.10e  %d\n", 
+                (i+1)%a.n, (j+1)%b.n, squaredist(a.vertex[(i+1)%a.n], b.vertex[(j+1)%b.n]),
+                squaredist(a.vertex[(i+1)%a.n], b.vertex[(j+1)%b.n]) > 1e-10*1e-10 ? true : false);
+
+        assert(squaredist(a.vertex[ i       ], b.vertex[ j       ]) > 1e-10*1e-10 ||
+               squaredist(a.vertex[(i+1)%a.n], b.vertex[(j+1)%b.n]) > 1e-10*1e-10);
+        exit(0);
+      }*/
       if (   squaredist(a.vertex[ i       ], b.vertex[ j           ]) < 1e-10*1e-10 &&
              squaredist(a.vertex[(i+1)%a.n], b.vertex[(j+b.n-1)%b.n]) < 1e-10*1e-10)

XIOS/dev/branch_yushan_merged/extern/remap/src/mapper.cpp

@@ -270 +270 @@
     MPI_Status *status = new MPI_Status[4*mpiSize];
     
+    MPI_Waitall(nbRecvRequest, recvRequest, status);
     MPI_Waitall(nbSendRequest, sendRequest, status);
-    MPI_Waitall(nbRecvRequest, recvRequest, status);
+    
 
     /* for all indices that have been received from requesting ranks: pack values and gradients, then send */
@@ -300 +301 @@
                     sendNeighIds[rank][j] = sstree.localElements[recvElement[rank][j]].src_id;
             }
-            MPI_Issend(sendValue[rank],  nbRecvElement[rank], MPI_DOUBLE, rank, 0, communicator, &sendRequest[nbSendRequest]);
+            MPI_Issend(sendValue[rank], nbRecvElement[rank], MPI_DOUBLE, rank, 0, communicator, &sendRequest[nbSendRequest]);
             nbSendRequest++;
-            MPI_Issend(sendArea[rank],  nbRecvElement[rank], MPI_DOUBLE, rank, 0, communicator, &sendRequest[nbSendRequest]);
+            MPI_Issend(sendArea[rank], nbRecvElement[rank], MPI_DOUBLE, rank, 0, communicator, &sendRequest[nbSendRequest]);
             nbSendRequest++;
             if (order == 2)
@@ -317 +318 @@
                 MPI_Issend(sendNeighIds[rank], 4*nbRecvElement[rank], MPI_INT, rank, 0, communicator, &sendRequest[nbSendRequest]);
                 //ym  --> attention taille GloId
-                nbSendRequest++;
-                
+                nbSendRequest++;                
             }
         }
@@ -345 +345 @@
     }
     
-    MPI_Waitall(nbSendRequest, sendRequest, status);
     MPI_Waitall(nbRecvRequest, recvRequest, status);
+    MPI_Waitall(nbSendRequest, sendRequest, status); 
+    
     
 
@@ -407 +408 @@
         }
     }
-
+    
     /* free all memory allocated in this function */
-    for (int rank = 0; rank < mpiSize; rank++)
+    /*for (int rank = 0; rank < mpiSize; rank++)
     {
         if (nbSendElement[rank] > 0)
@@ -446 +447 @@
     delete[] sendNeighIds;
     delete[] recvNeighIds;
+    */
     return i;
 }
@@ -546 +548 @@
         }
     }
-
+    
+    MPI_Waitall(nbRecvRequest, recvRequest, status);
     MPI_Waitall(nbSendRequest, sendRequest, status);
-    MPI_Waitall(nbRecvRequest, recvRequest, status);
 
     for (int rank = 0; rank < mpiSize; rank++)
@@ -619 +621 @@
     }
 
+    MPI_Waitall(nbRecvRequest, recvRequest, status);
     MPI_Waitall(nbSendRequest, sendRequest, status);
-    MPI_Waitall(nbRecvRequest, recvRequest, status);
  
     int nbNeighbourNodes = 0;
@@ -803 +805 @@
     }
  
+    MPI_Waitall(nbRecvRequest, recvRequest, status);
     MPI_Waitall(nbSendRequest, sendRequest, status);
-    MPI_Waitall(nbRecvRequest, recvRequest, status);
+    
     
     char **sendBuffer2 = new char*[mpiSize];
@@ -878 +881 @@
         {
             MPI_Issend(sendBuffer2[rank], sentMessageSize[rank], MPI_CHAR, rank, 0, communicator, &sendRequest[nbSendRequest]);
-            printf("proc %d send %d elements to proc %d\n", mpiRank, sentMessageSize[rank], rank);
             nbSendRequest++;
         }
@@ -884 +886 @@
         {
             MPI_Irecv(recvBuffer2[rank], recvMessageSize[rank], MPI_CHAR, rank, 0, communicator, &recvRequest[nbRecvRequest]);
-            printf("proc %d recv %d elements from proc %d\n", mpiRank, recvMessageSize[rank], rank);
             nbRecvRequest++;
         }
     }
     
+    MPI_Waitall(nbRecvRequest, recvRequest, status);
     MPI_Waitall(nbSendRequest, sendRequest, status);
-    MPI_Waitall(nbRecvRequest, recvRequest, status);
+   
    
 

XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_test.cpp

@@ -15 +15 @@
 namespace ep_lib {
 
-        int MPI_Test(MPI_Request *request, int *flag, MPI_Status *status)
-        {
-                Debug("MPI_Test with EP");
+  int MPI_Test(MPI_Request *request, int *flag, MPI_Status *status)
+  {
+    Debug("MPI_Test with EP");
 
     *flag = false;
@@ -56 +56 @@
     if(request->type == 3)  // imrecv
     {
-      ::MPI_Request mpi_request = static_cast< ::MPI_Request >(request->mpi_request);
+      ::MPI_Request *mpi_request = static_cast< ::MPI_Request* >(&(request->mpi_request));
       ::MPI_Status mpi_status;
-      ::MPI_Test(&mpi_request, flag, &mpi_status);
+      
+      ::MPI_Errhandler_set(MPI_COMM_WORLD_STD, MPI_ERRORS_RETURN);
+      int error_code = ::MPI_Test(mpi_request, flag, &mpi_status);
+      if (error_code != MPI_SUCCESS) {
+      
+         char error_string[BUFSIZ];
+         int length_of_error_string, error_class;
+      
+         ::MPI_Error_class(error_code, &error_class);
+         ::MPI_Error_string(error_class, error_string, &length_of_error_string);
+         printf("%s\n", error_string);
+      }
+      
       if(*flag)
       {
@@ -79 +91 @@
     }
                 
-        }
+  }
 
 
-        int MPI_Testall(int count, MPI_Request *array_of_requests, int *flag, MPI_Status *array_of_statuses)
-        {
-          Debug("MPI_Testall with EP");
+  int MPI_Testall(int count, MPI_Request *array_of_requests, int *flag, MPI_Status *array_of_statuses)
+  {
+    Debug("MPI_Testall with EP");
     *flag = true;
     int i=0;
@@ -92 +104 @@
       i++;
     }
-        }
+  }
 
 

XIOS/dev/branch_yushan_merged/extern/src_ep_dev/ep_wait.cpp

@@ -100 +100 @@
           if(array_of_requests[i].type != 2) // isend or imrecv
           {      
-            MPI_Wait(&array_of_requests[i], &array_of_statuses[i]);
+            //MPI_Wait(&array_of_requests[i], &array_of_statuses[i]);
+            int tested=false;
+            while(!tested) MPI_Test(&array_of_requests[i], &tested, &array_of_statuses[i]);
             finished++;
             finished_index[i] = true;

XIOS/dev/branch_yushan_merged/src/io/netCdfInterface.cpp

@@ -128 +128 @@
 int CNetCdfInterface::close(int ncId)
 {
-  int status = nc_close(ncId);
+  int status = NC_NOERR;
+  //#pragma omp critical (_netcdf)
+  #pragma omp master
+  {
+  status = nc_close(ncId);
   if (NC_NOERR != status)
   {
@@ -139 +143 @@
     throw CNetCdfException(e);
   }
-
+  }
   return status;
 }

XIOS/dev/branch_yushan_merged/src/node/field.cpp

@@ -26 +26 @@
 namespace xios{
 
-   /// ////////////////////// Définitions ////////////////////// ///
+   /// ////////////////////// Dfinitions ////////////////////// ///
 
    CField::CField(void)
@@ -691 +691 @@
      CContext* context = CContext::getCurrent();
      solveOnlyReferenceEnabledField(doSending2Server);
+     int myRank;
+     MPI_Comm_rank(context->client->intraComm, &myRank);
 
      if (!areAllReferenceSolved)
@@ -711 +713 @@
      if (context->hasClient)
      {
+       MPI_Barrier(context->client->intraComm);
+       printf("Proc %d enters function\n", myRank);
        solveTransformedGrid();
+       MPI_Barrier(context->client->intraComm);
+       printf("Proc %d exits function\n", myRank);
      }
 

XIOS/dev/branch_yushan_merged/src/node/file.cpp

@@ -634 +634 @@
 
      // Now everything is ok, close it
-     close();
+     //close();
    }
 

XIOS/dev/branch_yushan_merged/src/transformation/domain_algorithm_interpolate.cpp

@@ -404 +404 @@
   CContext* context = CContext::getCurrent();
   CContextClient* client=context->client;
+  int split_key;
+  ep_lib::MPI_Comm_rank(client->intraComm, &split_key);
 
   ep_lib::MPI_Comm poleComme(MPI_COMM_NULL);
-  ep_lib::MPI_Comm_split(client->intraComm, interMapValuePole.empty() ? MPI_UNDEFINED : 1, 0, &poleComme);
+  ep_lib::MPI_Comm_split(client->intraComm, interMapValuePole.empty() ? 9 : 1, split_key, &poleComme);
   if (MPI_COMM_NULL != poleComme)
   {
@@ -421 +423 @@
     std::vector<int> recvCount(nbClientPole,0);
     std::vector<int> displ(nbClientPole,0);
-    MPI_Allgather(&nbWeight,1,MPI_INT,&recvCount[0],1,MPI_INT,poleComme) ;
+    ep_lib::MPI_Allgather(&nbWeight,1,MPI_INT,&recvCount[0],1,MPI_INT,poleComme) ;
 
     displ[0]=0;
     for(int n=1;n<nbClientPole;++n) displ[n]=displ[n-1]+recvCount[n-1] ;
     int recvSize=displ[nbClientPole-1]+recvCount[nbClientPole-1] ;
+    
 
     std::vector<int> sendSourceIndexBuff(nbWeight);
@@ -444 +447 @@
 
     // Gather all index and weight for pole
-    MPI_Allgatherv(&sendSourceIndexBuff[0],nbWeight,MPI_INT,&recvSourceIndexBuff[0],&recvCount[0],&displ[0],MPI_INT,poleComme);
-    MPI_Allgatherv(&sendSourceWeightBuff[0],nbWeight,MPI_DOUBLE,&recvSourceWeightBuff[0],&recvCount[0],&displ[0],MPI_DOUBLE,poleComme);
+    ep_lib::MPI_Allgatherv(&sendSourceIndexBuff[0],nbWeight,MPI_INT,&recvSourceIndexBuff[0],&recvCount[0],&displ[0],MPI_INT,poleComme);
+    ep_lib::MPI_Allgatherv(&sendSourceWeightBuff[0],nbWeight,MPI_DOUBLE,&recvSourceWeightBuff[0],&recvCount[0],&displ[0],MPI_DOUBLE,poleComme);
 
     std::map<int,double> recvTemp;
@@ -567 +570 @@
 
 
-  MPI_Allreduce(sendBuff, recvBuff, nbClient, MPI_INT, MPI_SUM, client->intraComm);
+  ep_lib::MPI_Allreduce(sendBuff, recvBuff, nbClient, MPI_INT, MPI_SUM, client->intraComm);
 
   int* sendIndexDestBuff = new int [sendBuffSize];
@@ -595 +598 @@
 
     sendRequest.push_back(ep_lib::MPI_Request());
-    MPI_Isend(sendIndexDestBuff + sendOffSet,
+    ep_lib::MPI_Isend(sendIndexDestBuff + sendOffSet,
              k,
              MPI_INT,
@@ -603 +606 @@
              &sendRequest.back());
     sendRequest.push_back(ep_lib::MPI_Request());
-    MPI_Isend(sendIndexSrcBuff + sendOffSet,
+    ep_lib::MPI_Isend(sendIndexSrcBuff + sendOffSet,
              k,
              MPI_INT,
@@ -611 +614 @@
              &sendRequest.back());
     sendRequest.push_back(ep_lib::MPI_Request());
-    MPI_Isend(sendWeightBuff + sendOffSet,
+    ep_lib::MPI_Isend(sendWeightBuff + sendOffSet,
              k,
              MPI_DOUBLE,
@@ -630 +633 @@
   {
     ep_lib::MPI_Status recvStatus;
-    MPI_Recv((recvIndexDestBuff + receivedSize),
+    ep_lib::MPI_Recv((recvIndexDestBuff + receivedSize),
              recvBuffSize,
              MPI_INT,
@@ -639 +642 @@
 
     int countBuff = 0;
-    MPI_Get_count(&recvStatus, MPI_INT, &countBuff);
+    ep_lib::MPI_Get_count(&recvStatus, MPI_INT, &countBuff);
     #ifdef _usingMPI
     clientSrcRank = recvStatus.MPI_SOURCE;
@@ -645 +648 @@
     clientSrcRank = recvStatus.ep_src;
     #endif
-    MPI_Recv((recvIndexSrcBuff + receivedSize),
+    ep_lib::MPI_Recv((recvIndexSrcBuff + receivedSize),
              recvBuffSize,
              MPI_INT,
@@ -653 +656 @@
              &recvStatus);
 
-    MPI_Recv((recvWeightBuff + receivedSize),
+    ep_lib::MPI_Recv((recvWeightBuff + receivedSize),
              recvBuffSize,
              MPI_DOUBLE,
@@ -671 +674 @@
   std::vector<ep_lib::MPI_Status> requestStatus(sendRequest.size());
   ep_lib::MPI_Status stat_ignore;
-  MPI_Waitall(sendRequest.size(), &sendRequest[0], &stat_ignore);
+  ep_lib::MPI_Waitall(sendRequest.size(), &sendRequest[0], &stat_ignore);
 
   delete [] sendIndexDestBuff;
@@ -761 +764 @@
   }
 
-  MPI_Allreduce(&localNbWeight, &globalNbWeight, 1, MPI_LONG, MPI_SUM, client->intraComm);
+  ep_lib::MPI_Allreduce(&localNbWeight, &globalNbWeight, 1, MPI_LONG, MPI_SUM, client->intraComm);
   ep_lib::MPI_Scan(&localNbWeight, &startIndex, 1, MPI_LONG, MPI_SUM, client->intraComm);