Changeset 1518 for XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

Timestamp:

06/01/18 19:36:12 (6 years ago)

Author:

yushan

Message:

optimisation partially finished. To do : special case for intercomm_create and intercomm_merge

File:

• XIOS/dev/branch_openmp/extern/ep_dev/main.cpp (modified) (3 diffs)

XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

@@ -826 +826 @@
       MPI_Barrier(comm);
 
-      if(color==2 && split_rank==1)
+      if(color==2 && split_rank==0)
       {
         double sendbuf[9]={1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9};
         MPI_Request send_request;
         MPI_Status send_status;
-        MPI_Isend(sendbuf, 9, MPI_DOUBLE, 5, 10, inter_comm, &send_request);
+        MPI_Isend(sendbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &send_request);
         MPI_Wait(&send_request, &send_status);
       }
 
-      if(color==1 && split_rank==5)
+      if(color==1 && split_rank==0)
       {
         double recvbuf[9];
         MPI_Request recv_request;
         MPI_Status recv_status;
-        MPI_Irecv(recvbuf, 9, MPI_DOUBLE, 1, 10, inter_comm, &recv_request);
+        MPI_Irecv(recvbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &recv_request);
         MPI_Wait(&recv_request, &recv_status);
         for(int i=0; i<9; i++)
@@ -867 +867 @@
       MPI_Comm_dup(inter_comm, &inter_comm_dup);
 
+      int inter_comm_dup_size;
+      int inter_comm_dup_remote_size;
+      MPI_Comm_size(inter_comm_dup, &inter_comm_dup_size);
+
+      MPI_Comm_remote_size(inter_comm_dup, &inter_comm_dup_remote_size);
+
+      bool high = inter_comm_dup_size>inter_comm_dup_remote_size;
+
+
+      //printf("rank = %d, high = %d, inter_comm_dup_size = %d, inter_comm_dup_remote_size = %d\n", rank, high, inter_comm_dup_size, inter_comm_dup_remote_size);
+
+      MPI_Comm inter_comm_dup_merged;
+
+      MPI_Intercomm_merge(inter_comm_dup, high, &inter_comm_dup_merged);
+      
+      int inter_comm_dup_merged_rank;
+      MPI_Comm_rank(inter_comm_dup_merged, &inter_comm_dup_merged_rank);
+
+      int inter_comm_dup_merged_size;
+      MPI_Comm_size(inter_comm_dup_merged, &inter_comm_dup_merged_size);
+
+      printf("rank = %d, inter_comm_dup_merged_rank = %d, inter_comm_dup_merged_size = %d\n", rank, inter_comm_dup_merged_rank, inter_comm_dup_merged_size);
+
+      // TEST OF GATHER
+      {
+        int gather_root = 99;
+  
+        if(inter_comm_dup_merged_rank == 0) 
+        {
+          gather_root = rand() % inter_comm_dup_merged_size;
+        }
+  
+        MPI_Bcast(&gather_root, 1, MPI_INT, 0, inter_comm_dup_merged);
+
+        double sendbuf[2];
+        sendbuf[0] = inter_comm_dup_merged_rank * 1.0;
+        sendbuf[1] = inter_comm_dup_merged_size * (-1.0);
+
+        std::vector<double>recvbuf(2*inter_comm_dup_merged_size, 0);
+
+        MPI_Gather(sendbuf, 2, MPI_DOUBLE, recvbuf.data(), 2, MPI_DOUBLE, gather_root, inter_comm_dup_merged);
+
+        bool gather_result = true;
+
+        if(inter_comm_dup_merged_rank == gather_root)
+        {
+          for(int i=0; i<inter_comm_dup_merged_size; i++)
+          {
+            if(abs(recvbuf[2*i] - i) > 1.e-10 || abs(recvbuf[2*i+1] + inter_comm_dup_merged_size) > 1.e-10)
+            {
+              gather_result = false;
+              break;
+            }  
+          }
+
+          if(gather_result) printf("root = %d : \t test MPI_Gather for merged comm\t OK \n", gather_root);
+          else              printf("root = %d : \t test MPI_Gather for merged comm\t FAILED\n", gather_root);
+        }
+      }
+
+      // TEST OF ALLREDUCE
+      {
+ 
+        int sendbuf[2];
+        sendbuf[0] = inter_comm_dup_merged_rank;
+        sendbuf[1] = -inter_comm_dup_merged_size;
+
+        std::vector<int>recvbuf(2, 0);
+
+        MPI_Op op = MPI_MIN;
+
+        MPI_Allreduce(sendbuf, recvbuf.data(), 2, MPI_INT, op, inter_comm_dup_merged);
+
+
+        int allreduce_test = 1;
+
+      
+        if((op == MPI_SUM && (abs(recvbuf[0]-(inter_comm_dup_merged_size-1)*inter_comm_dup_merged_size/2) > 1.e-10 || abs(recvbuf[1] + inter_comm_dup_merged_size * inter_comm_dup_merged_size) > 1.e-10) ) ||
+           (op == MPI_MAX && (abs(recvbuf[0]-(inter_comm_dup_merged_size-1)) > 1.e-10 || abs(recvbuf[1] + inter_comm_dup_merged_size) > 1.e-10) )               ||
+           (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] + inter_comm_dup_merged_size) > 1.e-10) ) )
+        {
+          allreduce_test = 0; printf("%d %d\n", recvbuf[0], recvbuf[1]);
+        }  
+      
+
+        int allreduce_result;
+        MPI_Reduce(&allreduce_test, &allreduce_result, 1, MPI_INT, MPI_MIN, 0, inter_comm_dup_merged);
+
+        if(inter_comm_dup_merged_rank == 0 && allreduce_result)  printf("            \t test MPI_Allreduce for merged comm \t OK\n");
+        if(inter_comm_dup_merged_rank == 0 && !allreduce_result) printf("            \t test MPI_Allreduce for merged comm \t FAILED\n");
+    
+      }
+
+      // TEST OF EXSCAN
+      {
+ 
+        std::vector<int>sendbuf(2, inter_comm_dup_merged_rank);
+        std::vector<int>recvbuf(2, -1);
+
+        MPI_Op op = MPI_SUM;
+           
+
+        MPI_Exscan(sendbuf.data(), recvbuf.data(), 2, MPI_INT, op, inter_comm_dup_merged);
+
+        int exscan_test = 1;
+
+        if(inter_comm_dup_merged_rank >0)
+        if((op == MPI_SUM && (abs(recvbuf[0]-inter_comm_dup_merged_rank*(inter_comm_dup_merged_rank-1)/2) > 1.e-10 || abs(recvbuf[1]-inter_comm_dup_merged_rank*(inter_comm_dup_merged_rank-1)/2) > 1.e-10) ) ||
+           (op == MPI_MIN && (abs(recvbuf[0] ) > 1.e-10 || abs(recvbuf[1]) > 1.e-10) )  || 
+           (op == MPI_MAX && (abs(recvbuf[0] - inter_comm_dup_merged_rank+1) > 1.e-10 || abs(recvbuf[1] - inter_comm_dup_merged_rank+1) > 1.e-10) ) )
+        {
+          exscan_test = 0; 
+        }
+
+        //printf(" ID=%d : sendbuf = (%d, %d), recvbuf = (%d, %d), exscan_test = %d \n", inter_comm_dup_merged_rank, sendbuf[0], sendbuf[1], recvbuf[0], recvbuf[1], exscan_test); 
+        
+        int exscan_result;
+        MPI_Reduce(&exscan_test, &exscan_result, 1, MPI_INT, MPI_MIN, 0, inter_comm_dup_merged);
+    
+        if(inter_comm_dup_merged_rank == 0 && exscan_result)  printf("            \t test MPI_Exscan for merged comm \t OK\n");
+        if(inter_comm_dup_merged_rank == 0 && !exscan_result) printf("            \t test MPI_Exscan for merged comm \t FAILED %d\n", exscan_result);
+      }
+
+      // TEST OF SCATTERV
+      {
+
+        int scatterv_root;
+  
+        if(inter_comm_dup_merged_rank == 0) scatterv_root = rand() % inter_comm_dup_merged_size;
+  
+        MPI_Bcast(&scatterv_root, 1, MPI_INT, 0, inter_comm_dup_merged);
+ 
+        std::vector<int>sendbuf(2*inter_comm_dup_merged_size, inter_comm_dup_merged_rank);
+        std::vector<int>recvbuf(2, -1);
+        std::vector<int>sendcounts(inter_comm_dup_merged_size, 2);
+        std::vector<int>displs(inter_comm_dup_merged_size, 0);
+
+        for(int i=0; i<inter_comm_dup_merged_size; i++) displs[i] = 2*(inter_comm_dup_merged_size-1-i);
+
+        if(inter_comm_dup_merged_rank == scatterv_root) 
+        {
+          for(int i=0; i<inter_comm_dup_merged_size; i++) 
+          {
+            sendbuf[2*i] = i;
+            sendbuf[2*i+1] = inter_comm_dup_merged_size;
+          }
+        }
+
+
+        MPI_Scatterv(sendbuf.data(), sendcounts.data(), displs.data(), MPI_INT, recvbuf.data(), 2, MPI_INT, scatterv_root, inter_comm_dup_merged);
+
+        //printf("ID = %d : %d %d\n", inter_comm_dup_merged_rank, recvbuf[0], recvbuf[1]);
+
+        int scatterv_test = 1;
+
+     
+        if( abs(recvbuf[0]-(inter_comm_dup_merged_size-1-inter_comm_dup_merged_rank)) > 1.e-10 || abs(recvbuf[1]-inter_comm_dup_merged_size) > 1.e-10 )
+        {
+          scatterv_test = 0; printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], inter_comm_dup_merged_rank);
+        }  
+    
+
+        int scatterv_result;
+        MPI_Reduce(&scatterv_test, &scatterv_result, 1, MPI_INT, MPI_MIN, scatterv_root, inter_comm_dup_merged);
+
+        if(inter_comm_dup_merged_rank == scatterv_root && scatterv_result)  printf("root = %d : \t test MPI_Scatterv for merged comm \t OK\n", scatterv_root);
+        if(inter_comm_dup_merged_rank == scatterv_root && !scatterv_result) printf("root = %d : \t test MPI_Scatterv for merged comm \t FAILED\n", scatterv_root);
+      }
+
+
+
+
       MPI_Barrier(comm);
       MPI_Barrier(comm);
@@ -872 +1044 @@
 
       MPI_Comm_free(&inter_comm_dup);
+
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+
+
+      MPI_Comm_free(&inter_comm_dup_merged);
+