Changeset 1382 for XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

Timestamp:

12/19/17 14:44:41 (7 years ago)

Author:

yushan

Message:

dev EP_RMA

File:

• XIOS/dev/branch_openmp/extern/ep_dev/main.cpp (modified) (2 diffs)

XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

@@ -30 +30 @@
   MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
  
+  
+
+
+
+
+
+  //omp_set_num_threads(4);
+
+  
+
+  #pragma omp parallel default(shared)
+  {
+    MPI_Comm_rank(MPI_COMM_WORLD , &mpi_rank);
+
+    int num_ep = omp_get_num_threads();
+    MPI_Info info;
+
+    //printf("mpi_rank = %d, thread_num = %d\n", mpi_rank, omp_get_thread_num());
+
+    MPI_Comm *ep_comm;
+    #pragma omp master
+    {
+      MPI_Comm *ep_comm;
+      MPI_Comm_create_endpoints(MPI_COMM_WORLD.mpi_comm, num_ep, info, ep_comm);
+      passage = ep_comm;  
+    }
+
+    #pragma omp barrier
+
+
+    MPI_Comm comm; // this should act as EP_COMM_WORLD
+    comm = passage[omp_get_thread_num()];
+
+    int rank, size;
+    MPI_Comm_rank(comm, &rank);
+    MPI_Comm_size(comm, &size);
+
+    // TIMING SYCHRONIZATION
+    {
+      int n=100000;
+
+      MPI_Barrier(comm);
+      
+      high_resolution_clock::time_point t1 = high_resolution_clock::now();
+
+      for(int i=0; i<n; i++)
+        MPI_Barrier_local(comm);
+
+      high_resolution_clock::time_point t2 = high_resolution_clock::now();
+      duration<double> time_span = duration_cast<duration<double>>(t2 - t1);
+      #pragma omp master
+      std::cout << "proc "<< mpi_rank <<" ep_barrier "<< time_span.count() << " seconds."<<std::endl;
+
+      t1 = high_resolution_clock::now();
+
+      for(int i=0; i<n; i++)
+      {
+        #pragma omp barrier
+      }
+
+      t2 = high_resolution_clock::now();
+      time_span = duration_cast<duration<double>>(t2 - t1);
+      
+      #pragma omp master
+      std::cout << "proc "<< mpi_rank <<" omp_barrier "<< time_span.count() << " seconds."<<std::endl;
+
+      t1 = high_resolution_clock::now();
+
+      for(int i=0; i<n; i++)
+      {
+        //#pragma omp barrier
+      }
+
+      t2 = high_resolution_clock::now();
+      time_span = duration_cast<duration<double>>(t2 - t1);
+      
+      MPI_Barrier(comm);
+
+      #pragma omp master
+      std::cout << "proc "<< mpi_rank <<" for_loop "<< time_span.count() << " seconds."<<std::endl;
+    }
+    
+
+    // TEST OF BCAST FROM A RANDOM ROOT
+    {
+      int bcast_root;
+  
+      if(rank == 0) bcast_root = rand() % size;
+  
+      MPI_Bcast(&bcast_root, 1, MPI_INT, 0, comm);
+  
+      int sendbuf[2];
+
+      sendbuf[0] = rank;
+      sendbuf[1] = size;
+  
+      MPI_Bcast(sendbuf, 2, MPI_INT, bcast_root, comm);
+  
+      int bcast_test = 0;
+      if(sendbuf[0] ==  bcast_root && sendbuf[1] == size) bcast_test = 1;
+  
+      int bcast_result;
+  
+      MPI_Reduce(&bcast_test, &bcast_result, 1, MPI_INT, MPI_MIN, bcast_root, comm);
+  
+      if(bcast_result && rank == bcast_root)  printf("root = %d : \t test MPI_Bcast \t OK\n", bcast_root);
+      if(!bcast_result && rank == bcast_root) printf("root = %d : \t test MPI_Bcast \t FAILED %d\n", bcast_root, bcast_result);
+    }
+
+    MPI_Barrier(comm);
+
+    // TEST OF GATHER FROM A RAMDOM ROOT
+    {
+      int gather_root;
+  
+      if(rank == 0) gather_root = rand() % size;
+  
+      MPI_Bcast(&gather_root, 1, MPI_INT, 0, comm);
+
+      double sendbuf[2];
+      sendbuf[0] = rank * 1.0;
+      sendbuf[1] = size * (-1.0);
+
+      std::vector<double>recvbuf(2*size, 0);
+
+      MPI_Gather(sendbuf, 2, MPI_DOUBLE, recvbuf.data(), 2, MPI_DOUBLE, gather_root, comm);
+
+      bool gather_result = true;
+
+      if(rank == gather_root)
+      {
+        for(int i=0; i<size; i++)
+        {
+          if(abs(recvbuf[2*i] - i) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
+          {
+            gather_result = false;
+            break;
+          }  
+        }
+
+        if(gather_result) printf("root = %d : \t test MPI_Gather \t OK \n", gather_root);
+        else              printf("root = %d : \t test MPI_Gather \t FAILED\n", gather_root);
+      }
+    }
+
+    MPI_Barrier(comm);
+
+    // TEST OF GATHERV FROM A RAMDOM ROOT
+    {
+      int gatherv_root;
+  
+      if(rank == 0) gatherv_root = rand() % size;
+  
+      MPI_Bcast(&gatherv_root, 1, MPI_INT, 0, comm);
+
+      int sendbuf[2];
+      sendbuf[0] = rank;
+      sendbuf[1] = -size;
+
+      std::vector<int>recvbuf(2*size, 0);
+
+      std::vector<int>recvcounts(size, 2);
+      std::vector<int>displs(size, 0);
+
+      for(int i=0; i<size; i++) displs[i] = 2*(size-1-i);
+
+      MPI_Gatherv(sendbuf, 2, MPI_INT, recvbuf.data(), recvcounts.data(), displs.data(), MPI_INT, gatherv_root, comm);
+
+      bool gatherv_result = true;
+
+      if(rank == gatherv_root)
+      {
+        for(int i=0; i<size; i++)
+        {
+          if(abs(recvbuf[2*i] - (size-1-i)) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
+          {
+            gatherv_result = false; printf("%lf %lf root = %d, i = %d\n", recvbuf[2*i], recvbuf[2*i+1], gatherv_root, i);
+            break;
+          }  
+        }
+        
+        //for(int i=0; i<size*2; i++) printf("%lf\t", recvbuf[i]);
+        //printf("\n");
+
+        if(gatherv_result) printf("root = %d : \t test MPI_Gatherv \t OK\n", gatherv_root);
+        else               printf("root = %d : \t test MPI_Gatherv \t FAILED\n", gatherv_root);
+      }
+    }
+
+    MPI_Barrier(comm);
+
+    // TEST OF ALLGATHER
+    {
+      double sendbuf[2];
+      sendbuf[0] = rank * 1.0;
+      sendbuf[1] = size * (-1.0);
+
+      std::vector<double>recvbuf(2*size, 0);
+
+      MPI_Allgather(sendbuf, 2, MPI_DOUBLE, recvbuf.data(), 2, MPI_DOUBLE, comm);
+
+      int allgather_test = 1;
+
+      for(int i=0; i<size; i++)
+      {
+        if(abs(recvbuf[2*i] - i) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
+        {
+          allgather_test = 0;
+          break;
+        }  
+      }
+
+      int allgather_result;
+      MPI_Reduce(&allgather_test, &allgather_result, 1, MPI_INT, MPI_MIN, 0, comm);
+
+      if(rank == 0 && allgather_result)  printf("           \t test MPI_Allgather \t OK \n");
+      if(rank == 0 && !allgather_result) printf("           \t test MPI_Allgather \t OK \n");
+      
+    }
+
+    MPI_Barrier(comm);
+
+    // TEST OF ALLGATHERV
+    {
+      int sendbuf[2];
+      sendbuf[0] = rank;
+      sendbuf[1] = -size;
+
+      std::vector<int>recvbuf(2*size, 0);
+
+      std::vector<int>recvcounts(size, 2);
+      std::vector<int>displs(size, 0);
+
+      for(int i=0; i<size; i++) displs[i] = 2*(size-1-i);
+
+      MPI_Allgatherv(sendbuf, 2, MPI_INT, recvbuf.data(), recvcounts.data(), displs.data(), MPI_INT, comm);
+
+      int allgatherv_test = 1;
+
+      
+      
+      for(int i=0; i<size; i++)
+      {
+        if(abs(recvbuf[2*i] - (size-1-i)) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
+        {
+          allgatherv_test = 0; printf("ID : %d %d %d %d %d\n", rank, recvbuf[2*i], recvbuf[2*i+1] , recvbuf[2*i] - (size-1-i), recvbuf[2*i+1] + size);
+          break;
+        }  
+      }
+        
+      
+      int allgatherv_result;
+      MPI_Reduce(&allgatherv_test, &allgatherv_result, 1, MPI_INT, MPI_MIN, 0, comm);
+
+      if(rank == 0 && allgatherv_result)  printf("           \t test MPI_Allgatherv \t OK \n");
+      if(rank == 0 && !allgatherv_result) printf("           \t test MPI_Allgatherv \t FAILED %d\n", allgatherv_result);
+      
+    }
+
+    MPI_Barrier(comm);
+
+    // TEST OF REDUCE
+    {
+      int reduce_root;
+  
+      if(rank == 0) reduce_root = rand() % size;
+  
+      MPI_Bcast(&reduce_root, 1, MPI_INT, 0, comm);
+
+      int sendbuf[2];
+      sendbuf[0] = rank;
+      sendbuf[1] = -size;
+
+      std::vector<int>recvbuf(2, 0);
+
+      MPI_Op op = MPI_MIN;
+
+      MPI_Reduce(sendbuf, recvbuf.data(), 2, MPI_INT, op, reduce_root, comm);
+
+
+      bool reduce_result = true;
+
+      if(rank == reduce_root)
+      {
+        for(int i=0; i<2; i++)
+        {
+          if((op == MPI_SUM && (abs(recvbuf[0]-(size-1)*size/2) > 1.e-10 || abs(recvbuf[1] + size * size) > 1.e-10) ) ||
+             (op == MPI_MAX && (abs(recvbuf[0]-(size-1)) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) )               ||
+             (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) ) )
+          {
+            reduce_result = false; printf("%d %d root = %d, i = %d\n", recvbuf[0], recvbuf[1], reduce_root, i);
+            break;
+          }  
+        }
+      }
+
+      if(rank == reduce_root && reduce_result)  printf("root = %d : \t test MPI_Reduce \t OK\n", reduce_root);
+      if(rank == reduce_root && !reduce_result) printf("root = %d : \t test MPI_Reduce \t FAILED\n", reduce_root);
+    }
+    
+
+    MPI_Barrier(comm);
+
+    // TEST OF ALLREDUCE
+    {
+ 
+      int sendbuf[2];
+      sendbuf[0] = rank;
+      sendbuf[1] = -size;
+
+      std::vector<int>recvbuf(2, 0);
+
+      MPI_Op op = MPI_MIN;
+
+      MPI_Allreduce(sendbuf, recvbuf.data(), 2, MPI_INT, op, comm);
+
+
+      int allreduce_test = 1;
+
+      
+      if((op == MPI_SUM && (abs(recvbuf[0]-(size-1)*size/2) > 1.e-10 || abs(recvbuf[1] + size * size) > 1.e-10) ) ||
+         (op == MPI_MAX && (abs(recvbuf[0]-(size-1)) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) )               ||
+         (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) ) )
+      {
+        allreduce_test = 0; printf("%d %d\n", recvbuf[0], recvbuf[1]);
+      }  
+      
+
+      int allreduce_result;
+      MPI_Reduce(&allreduce_test, &allreduce_result, 1, MPI_INT, MPI_MIN, 0, comm);
+
+      if(rank == 0 && allreduce_result)  printf("            \t test MPI_Allreduce \t OK\n");
+      if(rank == 0 && !allreduce_result) printf("            \t test MPI_Allreduce \t FAILED\n");
+    }
+
+
+    MPI_Barrier(comm);
+
+    // TEST OF REDUCE_SCATTER
+    {
+ 
+      std::vector<int>sendbuf(2*size, rank);
+      std::vector<int>recvbuf(2, -1);
+      std::vector<int>recvcounts(size, 2);
+
+      MPI_Op op = MPI_MIN;
+
+      MPI_Reduce_scatter(sendbuf.data(), recvbuf.data(), recvcounts.data(), MPI_INT, op, comm);
+
+
+      int reduce_scatter_test = 1;
+
+     
+      if((op == MPI_SUM && (abs(recvbuf[0]-(size-1)*size/2) > 1.e-10 || abs(recvbuf[0]-(size-1)*size/2) > 1.e-10) ) ||
+         (op == MPI_MAX && (abs(recvbuf[0]-(size-1)) > 1.e-10 || abs(recvbuf[1]-(size-1)) > 1.e-10) )               ||
+         (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] ) > 1.e-10) ) )
+      {
+        reduce_scatter_test = 0; //printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], rank);
+      }  
+    
+
+      int reduce_scatter_result;
+      MPI_Reduce(&reduce_scatter_test, &reduce_scatter_result, 1, MPI_INT, MPI_MIN, 0, comm);
+
+      if(rank == 0 && reduce_scatter_result)  printf("            \t test MPI_Reduce_scatter OK\n");
+      if(rank == 0 && !reduce_scatter_result) printf("            \t test MPI_Reduce_scatter FAILED\n");
+    }
+
+    MPI_Barrier(comm);
+
+    // TEST OF SCATTER
+    {
+
+      int scatter_root;
+  
+      if(rank == 0) scatter_root = rand() % size;
+  
+      MPI_Bcast(&scatter_root, 1, MPI_INT, 0, comm);
+ 
+      std::vector<int>sendbuf(2*size, rank);
+      std::vector<int>recvbuf(2, -1);
+      std::vector<int>recvcounts(size, 2);
+
+      if(rank == scatter_root) 
+      {
+        for(int i=0; i<size; i++) 
+        {
+          sendbuf[2*i] = i;
+          sendbuf[2*i+1] = size;
+        }
+        //for(int i=0; i<size*2; i++) printf("%d\t", sendbuf[i]);
+      }
+
+
+      MPI_Scatter(sendbuf.data(), 2, MPI_INT, recvbuf.data(), 2, MPI_INT, scatter_root, comm);
+
+      //printf("ID = %d : %d %d\n", rank, recvbuf[0], recvbuf[1]);
+
+      int scatter_test = 1;
+
+     
+      if( abs(recvbuf[0]-rank) > 1.e-10 || abs(recvbuf[1]-size) > 1.e-10 )
+      {
+        scatter_test = 0; //printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], rank);
+      }  
+    
+
+      int scatter_result;
+      MPI_Reduce(&scatter_test, &scatter_result, 1, MPI_INT, MPI_MIN, scatter_root, comm);
+
+      if(rank == scatter_root && scatter_result)  printf("root = %d : \t test MPI_Scatter \t OK\n", scatter_root);
+      if(rank == scatter_root && !scatter_result) printf("root = %d : \t test MPI_Scatter \t FAILED\n", scatter_root);
+    }
+   
+    MPI_Barrier(comm);
+
+    // TEST OF SCATTERV
+    {
+
+      int scatterv_root;
+  
+      if(rank == 0) scatterv_root = rand() % size;
+  
+      MPI_Bcast(&scatterv_root, 1, MPI_INT, 0, comm);
+ 
+      std::vector<int>sendbuf(2*size, rank);
+      std::vector<int>recvbuf(2, -1);
+      std::vector<int>sendcounts(size, 2);
+      std::vector<int>displs(size, 0);
+
+      for(int i=0; i<size; i++) displs[i] = 2*(size-1-i);
+
+      if(rank == scatterv_root) 
+      {
+        for(int i=0; i<size; i++) 
+        {
+          sendbuf[2*i] = i;
+          sendbuf[2*i+1] = size;
+        }
+      }
+
+
+      MPI_Scatterv(sendbuf.data(), sendcounts.data(), displs.data(), MPI_INT, recvbuf.data(), 2, MPI_INT, scatterv_root, comm);
+
+      //printf("ID = %d : %d %d\n", rank, recvbuf[0], recvbuf[1]);
+
+      int scatterv_test = 1;
+
+     
+      if( abs(recvbuf[0]-(size-1-rank)) > 1.e-10 || abs(recvbuf[1]-size) > 1.e-10 )
+      {
+        scatterv_test = 0; printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], rank);
+      }  
+    
+
+      int scatterv_result;
+      MPI_Reduce(&scatterv_test, &scatterv_result, 1, MPI_INT, MPI_MIN, scatterv_root, comm);
+
+      if(rank == scatterv_root && scatterv_result)  printf("root = %d : \t test MPI_Scatterv \t OK\n", scatterv_root);
+      if(rank == scatterv_root && !scatterv_result) printf("root = %d : \t test MPI_Scatterv \t FAILED\n", scatterv_root);
+    }
+
+    MPI_Barrier(comm);
+
+    // TEST OF ALLTOALL
+    {
+
+      std::vector<int>sendbuf(size, rank);
+      std::vector<int>recvbuf(size, -1);
+           
+
+      MPI_Alltoall(sendbuf.data(), 1, MPI_INT, recvbuf.data(), 1, MPI_INT, comm);
+
+      int alltoall_result = 1;
+
+     
+      for(int i=0; i<size; i++)
+      if( abs(recvbuf[i]-i) > 1.e-10 )
+      {
+        alltoall_result = 0; printf("%d  id = %d\n", recvbuf[i], rank);
+      }  
+    
+      if(rank == 0 && alltoall_result)  printf("            \t test MPI_Alltoall \t OK\n");
+      if(rank == 0 && !alltoall_result) printf("            \t test MPI_Alltoall \t FAILED\n");
+    }
+
+    // TEST OF SCAN
+    {
+ 
+      std::vector<int>sendbuf(2, rank);
+      std::vector<int>recvbuf(2, -1);
+
+      MPI_Op op = MPI_SUM;
+           
+
+      MPI_Scan(sendbuf.data(), recvbuf.data(), 2, MPI_INT, op, comm);
+
+      int scan_test = 1;
+
+      // printf(" ID=%d : %d  %d \n", rank, recvbuf[0], recvbuf[1]);
+     
+      if((op == MPI_SUM && (abs(recvbuf[0]-rank*(rank+1)/2) > 1.e-10 || abs(recvbuf[1]-rank*(rank+1)/2) > 1.e-10) ) ||
+         (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1]) > 1.e-10) )  || 
+         (op == MPI_MAX && (abs(recvbuf[0] - rank) > 1.e-10 || abs(recvbuf[1] - rank) > 1.e-10) ) )
+      {
+        scan_test = 0; //printf("%d  id = %d\n", recvbuf[i], rank);
+      }  
+
+      int scan_result;
+      MPI_Reduce(&scan_test, &scan_result, 1, MPI_INT, MPI_MIN, 0, comm);
+    
+      if(rank == 0 && scan_result)  printf("            \t test MPI_Scan \t\t OK\n");
+      if(rank == 0 && !scan_result) printf("            \t test MPI_Scan \t\t FAILED\n");
+    }
+
+
+    // TEST OF EXSCAN
+    {
+ 
+      std::vector<int>sendbuf(2, rank);
+      std::vector<int>recvbuf(2, -1);
+
+      MPI_Op op = MPI_SUM;
+           
+
+      MPI_Exscan(sendbuf.data(), recvbuf.data(), 2, MPI_INT, op, comm);
+
+      int exscan_test = 1;
+
+      // printf(" ID=%d : %d  %d \n", rank, recvbuf[0], recvbuf[1]);
+     
+      if(rank >0)
+      if((op == MPI_SUM && (abs(recvbuf[0]-rank*(rank-1)/2) > 1.e-10 || abs(recvbuf[1]-rank*(rank-1)/2) > 1.e-10) ) ||
+         (op == MPI_MIN && (abs(recvbuf[0] ) > 1.e-10 || abs(recvbuf[1]) > 1.e-10) )  || 
+         (op == MPI_MAX && (abs(recvbuf[0] - rank+1) > 1.e-10 || abs(recvbuf[1] - rank+1) > 1.e-10) ) )
+      {
+        exscan_test = 0; //printf("%d  id = %d\n", recvbuf[i], rank);
+      }  
+
+      int exscan_result;
+      MPI_Reduce(&exscan_test, &exscan_result, 1, MPI_INT, MPI_MIN, 0, comm);
+    
+      if(rank == 0 && exscan_result)  printf("            \t test MPI_Exscan \t OK\n");
+      if(rank == 0 && !exscan_result) printf("            \t test MPI_Exscan \t FAILED\n");
+    }
+
+
+
+
+/*
+    MPI_Barrier(comm);
+    {
+    int rank, size;
+    MPI_Comm_rank(comm, &rank);
+    MPI_Comm_size(comm, &size);
+
+    //int color = rank%2;
+    int color, remote_leader;
+    if(rank<size-2) {color = 1; remote_leader = size-2;}
+    else {color = 0; remote_leader = 0;}
+    
+    printf("rank = %d, color = %d, remote_leader = %d\n", rank, color, remote_leader);
+  
+    MPI_Comm sub_comm;
+    MPI_Comm_split(comm, color, rank, &sub_comm);
+
+
+    
+    int sub_rank;
+    MPI_Comm_rank(sub_comm, &sub_rank);
+
+
+    MPI_Barrier(comm);
+    if(rank == 0) printf("\tMPI_Comm_split OK\n");
+    MPI_Barrier(comm);
+
+    MPI_Comm inter_comm;
+    //MPI_Intercomm_create(sub_comm, 0, comm, (color+1)%2, 99, &inter_comm);
+    MPI_Intercomm_create(sub_comm, 0, comm, remote_leader, 99, &inter_comm);
+
+    MPI_Barrier(comm);
+    if(rank == 0) printf("\tMPI_Intercomm_create OK\n");
+    MPI_Barrier(comm);
+
+    
+
+    int high=color;
+    MPI_Comm intra_comm;
+    MPI_Intercomm_merge(inter_comm, high, &intra_comm);
+    
+    int intra_rank, intra_size;
+    MPI_Comm_rank(intra_comm, &intra_rank);
+    MPI_Comm_size(intra_comm, &intra_size);
+
+    MPI_Barrier(comm);
+    if(rank == 0) printf("\tMPI_Intercomm_merge OK\n");
+    MPI_Barrier(comm);
+    }
+
+    //check_test_gatherv(comm);
+
+    // MPI_Barrier(comm);
+    // MPI_Comm_free(&sub_comm);
+
+
+    // MPI_Barrier(comm);
+    // MPI_Comm_free(&inter_comm);
+*/
+
+    MPI_Barrier(comm);
+    MPI_Comm_free(&comm);
+  }
+
   int num_threads;
   if(mpi_rank < mpi_size-2)
@@ -102 +715 @@
   }
 
-
-
-
-
-  omp_set_num_threads(4);
-
-  
-
-  #pragma omp parallel default(shared)
-  {
-    MPI_Comm_rank(MPI_COMM_WORLD , &mpi_rank);
-
-    int num_ep = omp_get_num_threads();
-    MPI_Info info;
-
-    //printf("mpi_rank = %d, thread_num = %d\n", mpi_rank, omp_get_thread_num());
-
-    MPI_Comm *ep_comm;
-    #pragma omp master
-    {
-      MPI_Comm *ep_comm;
-      MPI_Comm_create_endpoints(MPI_COMM_WORLD.mpi_comm, num_ep, info, ep_comm);
-      passage = ep_comm;  
-    }
-
-    #pragma omp barrier
-
-
-    MPI_Comm comm; // this should act as EP_COMM_WORLD
-    comm = passage[omp_get_thread_num()];
-
-    int rank, size;
-    MPI_Comm_rank(comm, &rank);
-    MPI_Comm_size(comm, &size);
-
-    // TIMING SYCHRONIZATION
-    {
-      int n=100000;
-      
-      high_resolution_clock::time_point t1 = high_resolution_clock::now();
-
-      for(int i=0; i<n; i++)
-        MPI_Barrier_local(comm);
-
-      high_resolution_clock::time_point t2 = high_resolution_clock::now();
-      duration<double> time_span = duration_cast<duration<double>>(t2 - t1);
-      #pragma omp master
-      std::cout << "proc "<< mpi_rank <<" ep_barrier "<< time_span.count() << " seconds."<<std::endl;
-
-      t1 = high_resolution_clock::now();
-
-      for(int i=0; i<n; i++)
-      {
-        #pragma omp barrier
-      }
-
-      t2 = high_resolution_clock::now();
-      time_span = duration_cast<duration<double>>(t2 - t1);
-      
-      #pragma omp master
-      std::cout << "proc "<< mpi_rank <<" omp_barrier "<< time_span.count() << " seconds."<<std::endl;
-
-      t1 = high_resolution_clock::now();
-
-      for(int i=0; i<n; i++)
-      {
-        //#pragma omp barrier
-      }
-
-      t2 = high_resolution_clock::now();
-      time_span = duration_cast<duration<double>>(t2 - t1);
-      
-      MPI_Barrier(comm);
-      
-      #pragma omp master
-      std::cout << "proc "<< mpi_rank <<" for_loop "<< time_span.count() << " seconds."<<std::endl;
-    }
-    
-
-    // TEST OF BCAST FROM A RANDOM ROOT
-    {
-      int bcast_root;
-  
-      if(rank == 0) bcast_root = rand() % size;
-  
-      MPI_Bcast(&bcast_root, 1, MPI_INT, 0, comm);
-  
-      int sendbuf[2];
-
-      sendbuf[0] = rank;
-      sendbuf[1] = size;
-  
-      MPI_Bcast(sendbuf, 2, MPI_INT, bcast_root, comm);
-  
-      int bcast_test = 0;
-      if(sendbuf[0] ==  bcast_root && sendbuf[1] == size) bcast_test = 1;
-  
-      int bcast_result;
-  
-      MPI_Reduce(&bcast_test, &bcast_result, 1, MPI_INT, MPI_MIN, bcast_root, comm);
-  
-      if(bcast_result && rank == bcast_root)  printf("root = %d : \t test MPI_Bcast \t OK\n", bcast_root);
-      if(!bcast_result && rank == bcast_root) printf("root = %d : \t test MPI_Bcast \t FAILED %d\n", bcast_root, bcast_result);
-    }
-
-    MPI_Barrier(comm);
-
-    // TEST OF GATHER FROM A RAMDOM ROOT
-    {
-      int gather_root;
-  
-      if(rank == 0) gather_root = rand() % size;
-  
-      MPI_Bcast(&gather_root, 1, MPI_INT, 0, comm);
-
-      double sendbuf[2];
-      sendbuf[0] = rank * 1.0;
-      sendbuf[1] = size * (-1.0);
-
-      std::vector<double>recvbuf(2*size, 0);
-
-      MPI_Gather(sendbuf, 2, MPI_DOUBLE, recvbuf.data(), 2, MPI_DOUBLE, gather_root, comm);
-
-      bool gather_result = true;
-
-      if(rank == gather_root)
-      {
-        for(int i=0; i<size; i++)
-        {
-          if(abs(recvbuf[2*i] - i) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
-          {
-            gather_result = false;
-            break;
-          }  
-        }
-
-        if(gather_result) printf("root = %d : \t test MPI_Gather \t OK \n", gather_root);
-        else              printf("root = %d : \t test MPI_Gather \t FAILED\n", gather_root);
-      }
-    }
-
-    MPI_Barrier(comm);
-
-    // TEST OF GATHERV FROM A RAMDOM ROOT
-    {
-      int gatherv_root;
-  
-      if(rank == 0) gatherv_root = rand() % size;
-  
-      MPI_Bcast(&gatherv_root, 1, MPI_INT, 0, comm);
-
-      int sendbuf[2];
-      sendbuf[0] = rank;
-      sendbuf[1] = -size;
-
-      std::vector<int>recvbuf(2*size, 0);
-
-      std::vector<int>recvcounts(size, 2);
-      std::vector<int>displs(size, 0);
-
-      for(int i=0; i<size; i++) displs[i] = 2*(size-1-i);
-
-      MPI_Gatherv(sendbuf, 2, MPI_INT, recvbuf.data(), recvcounts.data(), displs.data(), MPI_INT, gatherv_root, comm);
-
-      bool gatherv_result = true;
-
-      if(rank == gatherv_root)
-      {
-        for(int i=0; i<size; i++)
-        {
-          if(abs(recvbuf[2*i] - (size-1-i)) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
-          {
-            gatherv_result = false; printf("%lf %lf root = %d, i = %d\n", recvbuf[2*i], recvbuf[2*i+1], gatherv_root, i);
-            break;
-          }  
-        }
-        
-        //for(int i=0; i<size*2; i++) printf("%lf\t", recvbuf[i]);
-        //printf("\n");
-
-        if(gatherv_result) printf("root = %d : \t test MPI_Gatherv \t OK\n", gatherv_root);
-        else               printf("root = %d : \t test MPI_Gatherv \t FAILED\n", gatherv_root);
-      }
-    }
-
-    MPI_Barrier(comm);
-
-    // TEST OF ALLGATHER
-    {
-      double sendbuf[2];
-      sendbuf[0] = rank * 1.0;
-      sendbuf[1] = size * (-1.0);
-
-      std::vector<double>recvbuf(2*size, 0);
-
-      MPI_Allgather(sendbuf, 2, MPI_DOUBLE, recvbuf.data(), 2, MPI_DOUBLE, comm);
-
-      int allgather_test = 1;
-
-      for(int i=0; i<size; i++)
-      {
-        if(abs(recvbuf[2*i] - i) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
-        {
-          allgather_test = 0;
-          break;
-        }  
-      }
-
-      int allgather_result;
-      MPI_Reduce(&allgather_test, &allgather_result, 1, MPI_INT, MPI_MIN, 0, comm);
-
-      if(rank == 0 && allgather_result)  printf("           \t test MPI_Allgather \t OK \n");
-      if(rank == 0 && !allgather_result) printf("           \t test MPI_Allgather \t OK \n");
-      
-    }
-
-    MPI_Barrier(comm);
-
-    // TEST OF ALLGATHERV
-    {
-      int sendbuf[2];
-      sendbuf[0] = rank;
-      sendbuf[1] = -size;
-
-      std::vector<int>recvbuf(2*size, 0);
-
-      std::vector<int>recvcounts(size, 2);
-      std::vector<int>displs(size, 0);
-
-      for(int i=0; i<size; i++) displs[i] = 2*(size-1-i);
-
-      MPI_Allgatherv(sendbuf, 2, MPI_INT, recvbuf.data(), recvcounts.data(), displs.data(), MPI_INT, comm);
-
-      int allgatherv_test = 1;
-
-      
-      
-      for(int i=0; i<size; i++)
-      {
-        if(abs(recvbuf[2*i] - (size-1-i)) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
-        {
-          allgatherv_test = 0; printf("ID : %d %d %d %d %d\n", rank, recvbuf[2*i], recvbuf[2*i+1] , recvbuf[2*i] - (size-1-i), recvbuf[2*i+1] + size);
-          break;
-        }  
-      }
-        
-      
-      int allgatherv_result;
-      MPI_Reduce(&allgatherv_test, &allgatherv_result, 1, MPI_INT, MPI_MIN, 0, comm);
-
-      if(rank == 0 && allgatherv_result)  printf("           \t test MPI_Allgatherv \t OK \n");
-      if(rank == 0 && !allgatherv_result) printf("           \t test MPI_Allgatherv \t FAILED %d\n", allgatherv_result);
-      
-    }
-
-    MPI_Barrier(comm);
-
-    // TEST OF REDUCE
-    {
-      int reduce_root;
-  
-      if(rank == 0) reduce_root = rand() % size;
-  
-      MPI_Bcast(&reduce_root, 1, MPI_INT, 0, comm);
-
-      int sendbuf[2];
-      sendbuf[0] = rank;
-      sendbuf[1] = -size;
-
-      std::vector<int>recvbuf(2, 0);
-
-      MPI_Op op = MPI_MIN;
-
-      MPI_Reduce(sendbuf, recvbuf.data(), 2, MPI_INT, op, reduce_root, comm);
-
-
-      bool reduce_result = true;
-
-      if(rank == reduce_root)
-      {
-        for(int i=0; i<2; i++)
-        {
-          if((op == MPI_SUM && (abs(recvbuf[0]-(size-1)*size/2) > 1.e-10 || abs(recvbuf[1] + size * size) > 1.e-10) ) ||
-             (op == MPI_MAX && (abs(recvbuf[0]-(size-1)) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) )               ||
-             (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) ) )
-          {
-            reduce_result = false; printf("%d %d root = %d, i = %d\n", recvbuf[0], recvbuf[1], reduce_root, i);
-            break;
-          }  
-        }
-      }
-
-      if(rank == reduce_root && reduce_result)  printf("root = %d : \t test MPI_Reduce \t OK\n", reduce_root);
-      if(rank == reduce_root && !reduce_result) printf("root = %d : \t test MPI_Reduce \t FAILED\n", reduce_root);
-    }
-    
-
-    MPI_Barrier(comm);
-
-    // TEST OF ALLREDUCE
-    {
- 
-      int sendbuf[2];
-      sendbuf[0] = rank;
-      sendbuf[1] = -size;
-
-      std::vector<int>recvbuf(2, 0);
-
-      MPI_Op op = MPI_MIN;
-
-      MPI_Allreduce(sendbuf, recvbuf.data(), 2, MPI_INT, op, comm);
-
-
-      int allreduce_test = 1;
-
-      
-      if((op == MPI_SUM && (abs(recvbuf[0]-(size-1)*size/2) > 1.e-10 || abs(recvbuf[1] + size * size) > 1.e-10) ) ||
-         (op == MPI_MAX && (abs(recvbuf[0]-(size-1)) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) )               ||
-         (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) ) )
-      {
-        allreduce_test = 0; printf("%d %d\n", recvbuf[0], recvbuf[1]);
-      }  
-      
-
-      int allreduce_result;
-      MPI_Reduce(&allreduce_test, &allreduce_result, 1, MPI_INT, MPI_MIN, 0, comm);
-
-      if(rank == 0 && allreduce_result)  printf("            \t test MPI_Allreduce \t OK\n");
-      if(rank == 0 && !allreduce_result) printf("            \t test MPI_Allreduce \t FAILED\n");
-    }
-
-
-    MPI_Barrier(comm);
-
-    // TEST OF REDUCE_SCATTER
-    {
- 
-      std::vector<int>sendbuf(2*size, rank);
-      std::vector<int>recvbuf(2, -1);
-      std::vector<int>recvcounts(size, 2);
-
-      MPI_Op op = MPI_MIN;
-
-      MPI_Reduce_scatter(sendbuf.data(), recvbuf.data(), recvcounts.data(), MPI_INT, op, comm);
-
-
-      int reduce_scatter_test = 1;
-
-     
-      if((op == MPI_SUM && (abs(recvbuf[0]-(size-1)*size/2) > 1.e-10 || abs(recvbuf[0]-(size-1)*size/2) > 1.e-10) ) ||
-         (op == MPI_MAX && (abs(recvbuf[0]-(size-1)) > 1.e-10 || abs(recvbuf[1]-(size-1)) > 1.e-10) )               ||
-         (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] ) > 1.e-10) ) )
-      {
-        reduce_scatter_test = 0; //printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], rank);
-      }  
-    
-
-      int reduce_scatter_result;
-      MPI_Reduce(&reduce_scatter_test, &reduce_scatter_result, 1, MPI_INT, MPI_MIN, 0, comm);
-
-      if(rank == 0 && reduce_scatter_result)  printf("            \t test MPI_Reduce_scatter OK\n");
-      if(rank == 0 && !reduce_scatter_result) printf("            \t test MPI_Reduce_scatter FAILED\n");
-    }
-
-    MPI_Barrier(comm);
-
-    // TEST OF SCATTER
-    {
-
-      int scatter_root;
-  
-      if(rank == 0) scatter_root = rand() % size;
-  
-      MPI_Bcast(&scatter_root, 1, MPI_INT, 0, comm);
- 
-      std::vector<int>sendbuf(2*size, rank);
-      std::vector<int>recvbuf(2, -1);
-      std::vector<int>recvcounts(size, 2);
-
-      if(rank == scatter_root) 
-      {
-        for(int i=0; i<size; i++) 
-        {
-          sendbuf[2*i] = i;
-          sendbuf[2*i+1] = size;
-        }
-        //for(int i=0; i<size*2; i++) printf("%d\t", sendbuf[i]);
-      }
-
-
-      MPI_Scatter(sendbuf.data(), 2, MPI_INT, recvbuf.data(), 2, MPI_INT, scatter_root, comm);
-
-      //printf("ID = %d : %d %d\n", rank, recvbuf[0], recvbuf[1]);
-
-      int scatter_test = 1;
-
-     
-      if( abs(recvbuf[0]-rank) > 1.e-10 || abs(recvbuf[1]-size) > 1.e-10 )
-      {
-        scatter_test = 0; //printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], rank);
-      }  
-    
-
-      int scatter_result;
-      MPI_Reduce(&scatter_test, &scatter_result, 1, MPI_INT, MPI_MIN, scatter_root, comm);
-
-      if(rank == scatter_root && scatter_result)  printf("root = %d : \t test MPI_Scatter \t OK\n", scatter_root);
-      if(rank == scatter_root && !scatter_result) printf("root = %d : \t test MPI_Scatter \t FAILED\n", scatter_root);
-    }
-   
-    MPI_Barrier(comm);
-
-    // TEST OF SCATTERV
-    {
-
-      int scatterv_root;
-  
-      if(rank == 0) scatterv_root = rand() % size;
-  
-      MPI_Bcast(&scatterv_root, 1, MPI_INT, 0, comm);
- 
-      std::vector<int>sendbuf(2*size, rank);
-      std::vector<int>recvbuf(2, -1);
-      std::vector<int>sendcounts(size, 2);
-      std::vector<int>displs(size, 0);
-
-      for(int i=0; i<size; i++) displs[i] = 2*(size-1-i);
-
-      if(rank == scatterv_root) 
-      {
-        for(int i=0; i<size; i++) 
-        {
-          sendbuf[2*i] = i;
-          sendbuf[2*i+1] = size;
-        }
-      }
-
-
-      MPI_Scatterv(sendbuf.data(), sendcounts.data(), displs.data(), MPI_INT, recvbuf.data(), 2, MPI_INT, scatterv_root, comm);
-
-      //printf("ID = %d : %d %d\n", rank, recvbuf[0], recvbuf[1]);
-
-      int scatterv_test = 1;
-
-     
-      if( abs(recvbuf[0]-(size-1-rank)) > 1.e-10 || abs(recvbuf[1]-size) > 1.e-10 )
-      {
-        scatterv_test = 0; printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], rank);
-      }  
-    
-
-      int scatterv_result;
-      MPI_Reduce(&scatterv_test, &scatterv_result, 1, MPI_INT, MPI_MIN, scatterv_root, comm);
-
-      if(rank == scatterv_root && scatterv_result)  printf("root = %d : \t test MPI_Scatterv \t OK\n", scatterv_root);
-      if(rank == scatterv_root && !scatterv_result) printf("root = %d : \t test MPI_Scatterv \t FAILED\n", scatterv_root);
-    }
-
-    MPI_Barrier(comm);
-
-    // TEST OF ALLTOALL
-    {
-
-      std::vector<int>sendbuf(size, rank);
-      std::vector<int>recvbuf(size, -1);
-           
-
-      MPI_Alltoall(sendbuf.data(), 1, MPI_INT, recvbuf.data(), 1, MPI_INT, comm);
-
-      int alltoall_result = 1;
-
-     
-      for(int i=0; i<size; i++)
-      if( abs(recvbuf[i]-i) > 1.e-10 )
-      {
-        alltoall_result = 0; printf("%d  id = %d\n", recvbuf[i], rank);
-      }  
-    
-      if(rank == 0 && alltoall_result)  printf("            \t test MPI_Alltoall \t OK\n");
-      if(rank == 0 && !alltoall_result) printf("            \t test MPI_Alltoall \t FAILED\n");
-    }
-
-    // TEST OF SCAN
-    {
- 
-      std::vector<int>sendbuf(2, rank);
-      std::vector<int>recvbuf(2, -1);
-
-      MPI_Op op = MPI_SUM;
-           
-
-      MPI_Scan(sendbuf.data(), recvbuf.data(), 2, MPI_INT, op, comm);
-
-      int scan_test = 1;
-
-      // printf(" ID=%d : %d  %d \n", rank, recvbuf[0], recvbuf[1]);
-     
-      if((op == MPI_SUM && (abs(recvbuf[0]-rank*(rank+1)/2) > 1.e-10 || abs(recvbuf[1]-rank*(rank+1)/2) > 1.e-10) ) ||
-         (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1]) > 1.e-10) )  || 
-         (op == MPI_MAX && (abs(recvbuf[0] - rank) > 1.e-10 || abs(recvbuf[1] - rank) > 1.e-10) ) )
-      {
-        scan_test = 0; //printf("%d  id = %d\n", recvbuf[i], rank);
-      }  
-
-      int scan_result;
-      MPI_Reduce(&scan_test, &scan_result, 1, MPI_INT, MPI_MIN, 0, comm);
-    
-      if(rank == 0 && scan_result)  printf("            \t test MPI_Scan \t\t OK\n");
-      if(rank == 0 && !scan_result) printf("            \t test MPI_Scan \t\t FAILED\n");
-    }
-
-
-    // TEST OF EXSCAN
-    {
- 
-      std::vector<int>sendbuf(2, rank);
-      std::vector<int>recvbuf(2, -1);
-
-      MPI_Op op = MPI_SUM;
-           
-
-      MPI_Exscan(sendbuf.data(), recvbuf.data(), 2, MPI_INT, op, comm);
-
-      int exscan_test = 1;
-
-      // printf(" ID=%d : %d  %d \n", rank, recvbuf[0], recvbuf[1]);
-     
-      if(rank >0)
-      if((op == MPI_SUM && (abs(recvbuf[0]-rank*(rank-1)/2) > 1.e-10 || abs(recvbuf[1]-rank*(rank-1)/2) > 1.e-10) ) ||
-         (op == MPI_MIN && (abs(recvbuf[0] ) > 1.e-10 || abs(recvbuf[1]) > 1.e-10) )  || 
-         (op == MPI_MAX && (abs(recvbuf[0] - rank+1) > 1.e-10 || abs(recvbuf[1] - rank+1) > 1.e-10) ) )
-      {
-        exscan_test = 0; //printf("%d  id = %d\n", recvbuf[i], rank);
-      }  
-
-      int exscan_result;
-      MPI_Reduce(&exscan_test, &exscan_result, 1, MPI_INT, MPI_MIN, 0, comm);
-    
-      if(rank == 0 && exscan_result)  printf("            \t test MPI_Exscan \t OK\n");
-      if(rank == 0 && !exscan_result) printf("            \t test MPI_Exscan \t FAILED\n");
-    }
-
-
-
-
-/*
-    MPI_Barrier(comm);
-    {
-    int rank, size;
-    MPI_Comm_rank(comm, &rank);
-    MPI_Comm_size(comm, &size);
-
-    //int color = rank%2;
-    int color, remote_leader;
-    if(rank<size-2) {color = 1; remote_leader = size-2;}
-    else {color = 0; remote_leader = 0;}
-    
-    printf("rank = %d, color = %d, remote_leader = %d\n", rank, color, remote_leader);
-  
-    MPI_Comm sub_comm;
-    MPI_Comm_split(comm, color, rank, &sub_comm);
-
-
-    
-    int sub_rank;
-    MPI_Comm_rank(sub_comm, &sub_rank);
-
-
-    MPI_Barrier(comm);
-    if(rank == 0) printf("\tMPI_Comm_split OK\n");
-    MPI_Barrier(comm);
-
-    MPI_Comm inter_comm;
-    //MPI_Intercomm_create(sub_comm, 0, comm, (color+1)%2, 99, &inter_comm);
-    MPI_Intercomm_create(sub_comm, 0, comm, remote_leader, 99, &inter_comm);
-
-    MPI_Barrier(comm);
-    if(rank == 0) printf("\tMPI_Intercomm_create OK\n");
-    MPI_Barrier(comm);
-
-    
-
-    int high=color;
-    MPI_Comm intra_comm;
-    MPI_Intercomm_merge(inter_comm, high, &intra_comm);
-    
-    int intra_rank, intra_size;
-    MPI_Comm_rank(intra_comm, &intra_rank);
-    MPI_Comm_size(intra_comm, &intra_size);
-
-    MPI_Barrier(comm);
-    if(rank == 0) printf("\tMPI_Intercomm_merge OK\n");
-    MPI_Barrier(comm);
-    }
-
-    //check_test_gatherv(comm);
-
-    // MPI_Barrier(comm);
-    // MPI_Comm_free(&sub_comm);
-
-
-    // MPI_Barrier(comm);
-    // MPI_Comm_free(&inter_comm);
-*/
-
-    MPI_Barrier(comm);
-    MPI_Comm_free(&comm);
-  }
-
   MPI_Finalize();