source: XIOS/dev/branch_openmp/extern/ep_dev/main.cpp @ 1527

#include "ep_lib.hpp"

#include <stdio.h>
#include <assert.h>
#include "ep_declaration.hpp"
#include <omp.h>
#include <time.h>       /* time */
#include <ctime>
#include <ratio>
#include <chrono>

using namespace ep_lib;
using namespace std::chrono;



int main(int argc, char **argv)
{
  srand (time(NULL));

  //printf("Testing ep_lib\n");
  int required=3, provided;

  MPI_Init_thread(&argc, &argv, required, &provided);

  assert(required==provided);

  int mpi_rank;
  int mpi_size;

  MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
  MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);

  #pragma omp parallel default(shared)
  {
    MPI_Comm_rank(MPI_COMM_WORLD , &mpi_rank);

    int num_ep = omp_get_num_threads();
    MPI_Info info;

    //printf("mpi_rank = %d, thread_num = %d\n", mpi_rank, omp_get_thread_num());

    MPI_Comm *ep_comm;
    #pragma omp master
    {
      MPI_Comm *ep_comm;
      MPI_Comm_create_endpoints(MPI_COMM_WORLD->mpi_comm, num_ep, info, ep_comm);
      passage = ep_comm;  
    }

    #pragma omp barrier

    MPI_Comm comm; // this should act as EP_COMM_WORLD
        
    comm = passage[omp_get_thread_num()];

    // TEST OF COMM_SPLIT
    {

      MPI_Barrier(comm);
      int rank, size;
      MPI_Comm_rank(comm, &rank);
      MPI_Comm_size(comm, &size);
    
      int tab_color[16] = {2, 2, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1}; // used for config 1
      int tab_key[16]   = {2, 1, 4, 1, 0, 4, 3, 3, 4, 3, 4, 4, 4, 0, 0, 2}; // used for config 1

      int color = tab_color[rank];
      int key = tab_key[rank];
      
  
      MPI_Comm split_comm;
      MPI_Comm_split(comm, color, key, &split_comm);


    
      int split_rank, split_size;
      MPI_Comm_rank(split_comm, &split_rank);
      MPI_Comm_size(split_comm, &split_size);
    
#ifdef _Memory_check    
      printf("rank = %d, color = %d, key = %d, split_rank = %d, local_rank=%d\n", rank, color, key, split_rank, split_comm->ep_comm_ptr->size_rank_info[1].first);
#endif

      MPI_Barrier(comm);
      MPI_Barrier(comm);
      MPI_Barrier(comm);
      
      if(rank == 0) printf("            \t MPI_Comm_split \t OK\n");


      int local_leader = 0;
      int remote_leader = color==2? 13: 4; // used for config 1

      MPI_Comm peer_comm = comm;

      MPI_Comm inter_comm;
      MPI_Intercomm_create(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm);

      int inter_rank, inter_size, remote_size;
      MPI_Comm_rank(inter_comm, &inter_rank);
      MPI_Comm_size(inter_comm, &inter_size);
      MPI_Comm_remote_size(inter_comm, &remote_size);
      
      
      MPI_Barrier(comm);
      MPI_Barrier(comm);

      if(rank == 0) printf("            \t MPI_Intercomm_create \t OK\n");

      //printf("rank = %d, split_rank = %d, split_size = %d, inter_rank=%d, inter_size=%d, remote_size=%d\n", rank, split_rank, split_size, inter_rank, inter_size, remote_size);


      MPI_Comm_free(&split_comm);

      
      MPI_Barrier(comm);


      if(color==2 && split_rank==0)
      {
        double sendbuf[9]={1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9};
        MPI_Request send_request;
        MPI_Status send_status;
        MPI_Isend(sendbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &send_request);
        MPI_Wait(&send_request, &send_status);
      }

      if(color==1 && split_rank==0)
      {
        double recvbuf[9];
        MPI_Request recv_request;
        MPI_Status recv_status;
        MPI_Irecv(recvbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &recv_request);
        MPI_Wait(&recv_request, &recv_status);
        for(int i=0; i<9; i++)
        {
          printf("recvbuf[%d] = %lf\n", i, recvbuf[i]);
        }
      }

      MPI_Barrier(comm);
      MPI_Barrier(comm);

      if(rank == 0) printf("            \t Test iP2P for intercomm \t OK\n");


      MPI_Comm inter_comm_dup;
      MPI_Comm_dup(inter_comm, &inter_comm_dup);
      int inter_dup_rank, inter_dup_size;
      MPI_Comm_rank(inter_comm_dup, &inter_dup_rank);
      MPI_Comm_size(inter_comm_dup, &inter_dup_size);

      //printf("rank = %d, split_rank = %d, split_size = %d, inter_dup_rank=%d, inter_dup_size=%d\n", rank, split_rank, split_size, inter_dup_rank, inter_dup_size);



      MPI_Comm inter_comm_dup_merged;
      bool high;
      if(color==2) high=true;
      else high = false;

      MPI_Intercomm_merge(inter_comm_dup, high, &inter_comm_dup_merged);
      int merged_rank, merged_size;
      MPI_Comm_rank(inter_comm_dup_merged, &merged_rank);
      MPI_Comm_size(inter_comm_dup_merged, &merged_size);

      //printf("merged_rank=%d, merged_rank_loc=%d, root_ep_loc = %d\n", merged_rank, inter_comm_dup_merged->ep_comm_ptr->size_rank_info[1].first, inter_comm_dup_merged->ep_rank_map->at(0).first);

      MPI_Barrier(comm);
      MPI_Barrier(comm);
            
      int sendbuf=0;
      if(merged_rank == 0) sendbuf = 99;
      MPI_Bcast(&sendbuf, 1, MPI_INT, 0, inter_comm_dup_merged);
      printf("merged_rank = %d, sendbuf = %d\n", merged_rank, sendbuf);



      MPI_Barrier(comm);

      MPI_Comm_free(&inter_comm_dup_merged);
      MPI_Comm_free(&inter_comm_dup);




      MPI_Comm_free(&inter_comm);
           
    }
    
    

    MPI_Barrier(comm);
    MPI_Comm_free(&comm);
  }

  //MPI_Finalize();
  //return 0;



 
  

  #pragma omp parallel default(shared)
  {
    MPI_Comm_rank(MPI_COMM_WORLD , &mpi_rank);

    int num_ep = omp_get_num_threads();
    MPI_Info info;

    //printf("mpi_rank = %d, thread_num = %d\n", mpi_rank, omp_get_thread_num());

    MPI_Comm *ep_comm;
    #pragma omp master
    {
      MPI_Comm *ep_comm;
      MPI_Comm_create_endpoints(MPI_COMM_WORLD->mpi_comm, num_ep, info, ep_comm);
      passage = ep_comm;  
    }

    #pragma omp barrier


    MPI_Comm comm_for_dup; // this should act as EP_COMM_WORLD
    MPI_Comm comm; // this should act as EP_COMM_WORLD
        
    comm_for_dup = passage[omp_get_thread_num()];
    MPI_Comm_dup(comm_for_dup, &comm);
    
    MPI_Comm_free(&comm_for_dup);
    
    MPI_Barrier(comm);

    int rank, size;
    MPI_Comm_rank(comm, &rank);
    MPI_Comm_size(comm, &size);
    
    if(rank == 0) printf("           \t test MPI_Comm_dup \t OK \n");

    // TIMING SYCHRONIZATION
    {
      int n=100000;

      MPI_Barrier(comm);
      
      high_resolution_clock::time_point t1 = high_resolution_clock::now();

      for(int i=0; i<n; i++)
        MPI_Barrier_local(comm);

      high_resolution_clock::time_point t2 = high_resolution_clock::now();
      duration<double> time_span = duration_cast<duration<double>>(t2 - t1);
      #pragma omp master
      std::cout << "proc "<< mpi_rank <<" ep_barrier "<< time_span.count() << " seconds."<<std::endl;

      t1 = high_resolution_clock::now();

      for(int i=0; i<n; i++)
      {
        #pragma omp barrier
      }

      t2 = high_resolution_clock::now();
      time_span = duration_cast<duration<double>>(t2 - t1);
      
      #pragma omp master
      std::cout << "proc "<< mpi_rank <<" omp_barrier "<< time_span.count() << " seconds."<<std::endl;

      t1 = high_resolution_clock::now();

      for(int i=0; i<n; i++)
      {
        //#pragma omp barrier
      }

      t2 = high_resolution_clock::now();
      time_span = duration_cast<duration<double>>(t2 - t1);
      
      MPI_Barrier(comm);

      #pragma omp master
      std::cout << "proc "<< mpi_rank <<" for_loop "<< time_span.count() << " seconds."<<std::endl;
    }// END TIMING SYCHRONIZATION
    
    // TEST of p2p blocking communication
    {
      MPI_Barrier(comm);
      MPI_Barrier(comm);
      
      MPI_Comm equal_comm = comm;
      
      
      
      
      double sendbuf[10];
      double recvbuf[20];
      
      int sender;
      if(rank == 0) sender = rand() % size;
      MPI_Bcast(&sender, 1, MPI_INT, 0, comm);
      
      int receiver = sender;
      if(rank == 0) while(sender == receiver) {receiver = rand() % size;}
      MPI_Bcast(&receiver, 1, MPI_INT, 0, comm);
      
      
      
      
      if(rank == sender)
      {
        for(int i=0; i<10; i++) sendbuf[i] = 99.99;
        MPI_Send(sendbuf, 10, MPI_DOUBLE, receiver, 99, equal_comm);
        for(int i=0; i<10; i++) sendbuf[i] = -99.99;
        MPI_Send(sendbuf, 10, MPI_DOUBLE, receiver, 11, equal_comm);
      }
      
      if(rank == receiver)
      {
        MPI_Status status;
        for(int i=0; i<20; i++) recvbuf[i] = 0.0;
        MPI_Recv(&recvbuf[10], 10, MPI_DOUBLE, sender, 99, comm, &status);
        MPI_Recv(recvbuf, 10, MPI_DOUBLE, sender, 11, comm, &status);
        
        for(int i=0; i<20; i++) std::cout << "recvbuf["<< i <<"] = "<< recvbuf[i] << std::endl;
        printf("sender = %d\nreceiver = %d \tTEST of p2p blocking communication\tOK\n", sender, receiver);
      }
      
      MPI_Barrier(comm);
    
    }//TEST of p2p blocking communication
    
    // TEST of p2p non-blocking communication
    {
      MPI_Barrier(comm);
      MPI_Barrier(comm);
      
      double sendbuf[10];
      double recvbuf[20];
      
      int sender;
      if(rank == 0) sender = rand() % size;
      MPI_Bcast(&sender, 1, MPI_INT, 0, comm);
      
      int receiver = sender;
      if(rank == 0) receiver = rand() % size;
      MPI_Bcast(&receiver, 1, MPI_INT, 0, comm);


            
      MPI_Request request[2];
      
      if(rank == sender)
      {
        
        for(int i=0; i<10; i++) sendbuf[i] = 99.99;
        MPI_Isend(sendbuf, 10, MPI_DOUBLE, receiver, 99, comm, &request[0]);
        for(int i=0; i<10; i++) sendbuf[i] = -99.99;
        MPI_Isend(sendbuf, 10, MPI_DOUBLE, receiver, 11, comm, &request[1]);
      }
      
      if(rank == receiver)
      {
        for(int i=0; i<20; i++) recvbuf[i] = 0.0;
        MPI_Irecv(&recvbuf[10], 10, MPI_DOUBLE, sender, 11, comm, &request[0]);
        MPI_Irecv(recvbuf, 10, MPI_DOUBLE, sender, 99, comm, &request[1]);
      }
      
      MPI_Barrier(comm);
      
      if(rank == receiver || rank == sender)
      {
        MPI_Status status[2];
        MPI_Waitall(2, request, status);
      }
      
      MPI_Barrier(comm);
      
      if(rank == receiver)
      {
        for(int i=0; i<20; i++) std::cout << "recvbuf["<< i <<"] = "<< recvbuf[i] << std::endl;
        printf("sender = %d\nreceiver = %d \tTEST of p2p non-blocking communication\tOK\n", sender, receiver);
      }
    
    }//TEST of p2p blocking communication
    

    // TEST OF BCAST FROM A RANDOM ROOT
    {
      int bcast_root;
  
      if(rank == 0) bcast_root = rand() % size;
  
      MPI_Bcast(&bcast_root, 1, MPI_INT, 0, comm);
  
      int sendbuf[2];

      sendbuf[0] = rank;
      sendbuf[1] = size;
  
      MPI_Bcast(sendbuf, 2, MPI_INT, bcast_root, comm);
  
      int bcast_test = 0;
      if(sendbuf[0] ==  bcast_root && sendbuf[1] == size) bcast_test = 1;
  
      int bcast_result;
  
      MPI_Reduce(&bcast_test, &bcast_result, 1, MPI_INT, MPI_MIN, bcast_root, comm);
  
      if(bcast_result && rank == bcast_root)  printf("root = %d : \t test MPI_Bcast \t OK\n", bcast_root);
      if(!bcast_result && rank == bcast_root) printf("root = %d : \t test MPI_Bcast \t FAILED %d\n", bcast_root, bcast_result);
    }

    MPI_Barrier(comm);

    // TEST OF GATHER FROM A RAMDOM ROOT
    {
      int gather_root;
  
      if(rank == 0) gather_root = rand() % size;
  
      MPI_Bcast(&gather_root, 1, MPI_INT, 0, comm);

      double sendbuf[2];
      sendbuf[0] = rank * 1.0;
      sendbuf[1] = size * (-1.0);

      std::vector<double>recvbuf(2*size, 0);

      MPI_Gather(sendbuf, 2, MPI_DOUBLE, recvbuf.data(), 2, MPI_DOUBLE, gather_root, comm);

      bool gather_result = true;

      if(rank == gather_root)
      {
        for(int i=0; i<size; i++)
        {
          if(abs(recvbuf[2*i] - i) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
          {
            gather_result = false;
            break;
          }  
        }

        if(gather_result) printf("root = %d : \t test MPI_Gather \t OK \n", gather_root);
        else              printf("root = %d : \t test MPI_Gather \t FAILED\n", gather_root);
      }
    }

    MPI_Barrier(comm);

    // TEST OF GATHERV FROM A RAMDOM ROOT
    {
      int gatherv_root;
  
      if(rank == 0) gatherv_root = rand() % size;
  
      MPI_Bcast(&gatherv_root, 1, MPI_INT, 0, comm);

      int sendbuf[2];
      sendbuf[0] = rank;
      sendbuf[1] = -size;

      std::vector<int>recvbuf(2*size, 0);

      std::vector<int>recvcounts(size, 2);
      std::vector<int>displs(size, 0);

      for(int i=0; i<size; i++) displs[i] = 2*(size-1-i);

      MPI_Gatherv(sendbuf, 2, MPI_INT, recvbuf.data(), recvcounts.data(), displs.data(), MPI_INT, gatherv_root, comm);

      bool gatherv_result = true;

      if(rank == gatherv_root)
      {
        for(int i=0; i<size; i++)
        {
          if(abs(recvbuf[2*i] - (size-1-i)) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
          {
            gatherv_result = false; printf("%lf %lf root = %d, i = %d\n", recvbuf[2*i], recvbuf[2*i+1], gatherv_root, i);
            break;
          }  
        }
        
        //for(int i=0; i<size*2; i++) printf("%lf\t", recvbuf[i]);
        //printf("\n");

        if(gatherv_result) printf("root = %d : \t test MPI_Gatherv \t OK\n", gatherv_root);
        else               printf("root = %d : \t test MPI_Gatherv \t FAILED\n", gatherv_root);
      }
    }

    MPI_Barrier(comm);

    // TEST OF ALLGATHER
    {
      double sendbuf[2];
      sendbuf[0] = rank * 1.0;
      sendbuf[1] = size * (-1.0);

      std::vector<double>recvbuf(2*size, 0);

      MPI_Allgather(sendbuf, 2, MPI_DOUBLE, recvbuf.data(), 2, MPI_DOUBLE, comm);

      int allgather_test = 1;

      for(int i=0; i<size; i++)
      {
        if(abs(recvbuf[2*i] - i) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
        {
          allgather_test = 0;
          break;
        }  
      }

      int allgather_result;
      MPI_Reduce(&allgather_test, &allgather_result, 1, MPI_INT, MPI_MIN, 0, comm);

      if(rank == 0 && allgather_result)  printf("           \t test MPI_Allgather \t OK \n");
      if(rank == 0 && !allgather_result) printf("           \t test MPI_Allgather \t OK \n");
      
    }

    MPI_Barrier(comm);

    // TEST OF ALLGATHERV
    {
      int sendbuf[2];
      sendbuf[0] = rank;
      sendbuf[1] = -size;

      std::vector<int>recvbuf(2*size, 0);

      std::vector<int>recvcounts(size, 2);
      std::vector<int>displs(size, 0);

      for(int i=0; i<size; i++) displs[i] = 2*(size-1-i);

      MPI_Allgatherv(sendbuf, 2, MPI_INT, recvbuf.data(), recvcounts.data(), displs.data(), MPI_INT, comm);

      int allgatherv_test = 1;

      
      
      for(int i=0; i<size; i++)
      {
        if(abs(recvbuf[2*i] - (size-1-i)) > 1.e-10 || abs(recvbuf[2*i+1] + size) > 1.e-10)
        {
          allgatherv_test = 0; printf("ID : %d %d %d %d %d\n", rank, recvbuf[2*i], recvbuf[2*i+1] , recvbuf[2*i] - (size-1-i), recvbuf[2*i+1] + size);
          break;
        }  
      }
        
      
      int allgatherv_result;
      MPI_Reduce(&allgatherv_test, &allgatherv_result, 1, MPI_INT, MPI_MIN, 0, comm);

      if(rank == 0 && allgatherv_result)  printf("           \t test MPI_Allgatherv \t OK \n");
      if(rank == 0 && !allgatherv_result) printf("           \t test MPI_Allgatherv \t FAILED %d\n", allgatherv_result);
      
    }

    MPI_Barrier(comm);

    // TEST OF REDUCE
    {
      int reduce_root;
  
      if(rank == 0) reduce_root = rand() % size;
  
      MPI_Bcast(&reduce_root, 1, MPI_INT, 0, comm);

      int sendbuf[2];
      sendbuf[0] = rank;
      sendbuf[1] = -size;

      std::vector<int>recvbuf(2, 0);

      MPI_Op op = MPI_MIN;

      MPI_Reduce(sendbuf, recvbuf.data(), 2, MPI_INT, op, reduce_root, comm);


      bool reduce_result = true;

      if(rank == reduce_root)
      {
        for(int i=0; i<2; i++)
        {
          if((op == MPI_SUM && (abs(recvbuf[0]-(size-1)*size/2) > 1.e-10 || abs(recvbuf[1] + size * size) > 1.e-10) ) ||
             (op == MPI_MAX && (abs(recvbuf[0]-(size-1)) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) )               ||
             (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) ) )
          {
            reduce_result = false; printf("%d %d root = %d, i = %d\n", recvbuf[0], recvbuf[1], reduce_root, i);
            break;
          }  
        }
      }

      if(rank == reduce_root && reduce_result)  printf("root = %d : \t test MPI_Reduce \t OK\n", reduce_root);
      if(rank == reduce_root && !reduce_result) printf("root = %d : \t test MPI_Reduce \t FAILED\n", reduce_root);
    }
    

    MPI_Barrier(comm);

    // TEST OF ALLREDUCE
    {
 
      int sendbuf[2];
      sendbuf[0] = rank;
      sendbuf[1] = -size;

      std::vector<int>recvbuf(2, 0);

      MPI_Op op = MPI_MIN;

      MPI_Allreduce(sendbuf, recvbuf.data(), 2, MPI_INT, op, comm);


      int allreduce_test = 1;

      
      if((op == MPI_SUM && (abs(recvbuf[0]-(size-1)*size/2) > 1.e-10 || abs(recvbuf[1] + size * size) > 1.e-10) ) ||
         (op == MPI_MAX && (abs(recvbuf[0]-(size-1)) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) )               ||
         (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] + size) > 1.e-10) ) )
      {
        allreduce_test = 0; printf("%d %d\n", recvbuf[0], recvbuf[1]);
      }  
      

      int allreduce_result;
      MPI_Reduce(&allreduce_test, &allreduce_result, 1, MPI_INT, MPI_MIN, 0, comm);

      if(rank == 0 && allreduce_result)  printf("            \t test MPI_Allreduce \t OK\n");
      if(rank == 0 && !allreduce_result) printf("            \t test MPI_Allreduce \t FAILED\n");
    }


    MPI_Barrier(comm);

    // TEST OF REDUCE_SCATTER
    {
 
      std::vector<int>sendbuf(2*size, rank);
      std::vector<int>recvbuf(2, -1);
      std::vector<int>recvcounts(size, 2);

      MPI_Op op = MPI_MIN;

      MPI_Reduce_scatter(sendbuf.data(), recvbuf.data(), recvcounts.data(), MPI_INT, op, comm);


      int reduce_scatter_test = 1;

     
      if((op == MPI_SUM && (abs(recvbuf[0]-(size-1)*size/2) > 1.e-10 || abs(recvbuf[0]-(size-1)*size/2) > 1.e-10) ) ||
         (op == MPI_MAX && (abs(recvbuf[0]-(size-1)) > 1.e-10 || abs(recvbuf[1]-(size-1)) > 1.e-10) )               ||
         (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] ) > 1.e-10) ) )
      {
        reduce_scatter_test = 0; //printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], rank);
      }  
    

      int reduce_scatter_result;
      MPI_Reduce(&reduce_scatter_test, &reduce_scatter_result, 1, MPI_INT, MPI_MIN, 0, comm);

      if(rank == 0 && reduce_scatter_result)  printf("            \t test MPI_Reduce_scatter OK\n");
      if(rank == 0 && !reduce_scatter_result) printf("            \t test MPI_Reduce_scatter FAILED\n");
    }

    MPI_Barrier(comm);

    // TEST OF SCATTER
    {

      int scatter_root;
  
      if(rank == 0) scatter_root = rand() % size;
  
      MPI_Bcast(&scatter_root, 1, MPI_INT, 0, comm);
 
      std::vector<int>sendbuf(2*size, rank);
      std::vector<int>recvbuf(2, -1);
      std::vector<int>recvcounts(size, 2);

      if(rank == scatter_root) 
      {
        for(int i=0; i<size; i++) 
        {
          sendbuf[2*i] = i;
          sendbuf[2*i+1] = size;
        }
        //for(int i=0; i<size*2; i++) printf("%d\t", sendbuf[i]);
      }


      MPI_Scatter(sendbuf.data(), 2, MPI_INT, recvbuf.data(), 2, MPI_INT, scatter_root, comm);

      //printf("ID = %d : %d %d\n", rank, recvbuf[0], recvbuf[1]);

      int scatter_test = 1;

     
      if( abs(recvbuf[0]-rank) > 1.e-10 || abs(recvbuf[1]-size) > 1.e-10 )
      {
        scatter_test = 0; //printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], rank);
      }  
    

      int scatter_result;
      MPI_Reduce(&scatter_test, &scatter_result, 1, MPI_INT, MPI_MIN, scatter_root, comm);

      if(rank == scatter_root && scatter_result)  printf("root = %d : \t test MPI_Scatter \t OK\n", scatter_root);
      if(rank == scatter_root && !scatter_result) printf("root = %d : \t test MPI_Scatter \t FAILED\n", scatter_root);
    }
   
    MPI_Barrier(comm);

    // TEST OF SCATTERV
    {

      int scatterv_root;
  
      if(rank == 0) scatterv_root = rand() % size;
  
      MPI_Bcast(&scatterv_root, 1, MPI_INT, 0, comm);
 
      std::vector<int>sendbuf(2*size, rank);
      std::vector<int>recvbuf(2, -1);
      std::vector<int>sendcounts(size, 2);
      std::vector<int>displs(size, 0);

      for(int i=0; i<size; i++) displs[i] = 2*(size-1-i);

      if(rank == scatterv_root) 
      {
        for(int i=0; i<size; i++) 
        {
          sendbuf[2*i] = i;
          sendbuf[2*i+1] = size;
        }
      }


      MPI_Scatterv(sendbuf.data(), sendcounts.data(), displs.data(), MPI_INT, recvbuf.data(), 2, MPI_INT, scatterv_root, comm);

      //printf("ID = %d : %d %d\n", rank, recvbuf[0], recvbuf[1]);

      int scatterv_test = 1;

     
      if( abs(recvbuf[0]-(size-1-rank)) > 1.e-10 || abs(recvbuf[1]-size) > 1.e-10 )
      {
        scatterv_test = 0; printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], rank);
      }  
    

      int scatterv_result;
      MPI_Reduce(&scatterv_test, &scatterv_result, 1, MPI_INT, MPI_MIN, scatterv_root, comm);

      if(rank == scatterv_root && scatterv_result)  printf("root = %d : \t test MPI_Scatterv \t OK\n", scatterv_root);
      if(rank == scatterv_root && !scatterv_result) printf("root = %d : \t test MPI_Scatterv \t FAILED\n", scatterv_root);
    }

    MPI_Barrier(comm);

    // TEST OF ALLTOALL
    {

      std::vector<int>sendbuf(size, rank);
      std::vector<int>recvbuf(size, -1);
           

      MPI_Alltoall(sendbuf.data(), 1, MPI_INT, recvbuf.data(), 1, MPI_INT, comm);

      int alltoall_result = 1;

     
      for(int i=0; i<size; i++)
      if( abs(recvbuf[i]-i) > 1.e-10 )
      {
        alltoall_result = 0; printf("%d  id = %d\n", recvbuf[i], rank);
      }  
    
      if(rank == 0 && alltoall_result)  printf("            \t test MPI_Alltoall \t OK\n");
      if(rank == 0 && !alltoall_result) printf("            \t test MPI_Alltoall \t FAILED\n");
    }

    // TEST OF SCAN
    {
 
      std::vector<int>sendbuf(2, rank);
      std::vector<int>recvbuf(2, -1);

      MPI_Op op = MPI_SUM;
           

      MPI_Scan(sendbuf.data(), recvbuf.data(), 2, MPI_INT, op, comm);

      int scan_test = 1;

      // printf(" ID=%d : %d  %d \n", rank, recvbuf[0], recvbuf[1]);
     
      if((op == MPI_SUM && (abs(recvbuf[0]-rank*(rank+1)/2) > 1.e-10 || abs(recvbuf[1]-rank*(rank+1)/2) > 1.e-10) ) ||
         (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1]) > 1.e-10) )  || 
         (op == MPI_MAX && (abs(recvbuf[0] - rank) > 1.e-10 || abs(recvbuf[1] - rank) > 1.e-10) ) )
      {
        scan_test = 0; //printf("%d  id = %d\n", recvbuf[i], rank);
      }  

      int scan_result;
      MPI_Reduce(&scan_test, &scan_result, 1, MPI_INT, MPI_MIN, 0, comm);
    
      if(rank == 0 && scan_result)  printf("            \t test MPI_Scan \t\t OK\n");
      if(rank == 0 && !scan_result) printf("            \t test MPI_Scan \t\t FAILED\n");
    }


    // TEST OF EXSCAN
    {
 
      std::vector<int>sendbuf(2, rank);
      std::vector<int>recvbuf(2, -1);

      MPI_Op op = MPI_SUM;
           

      MPI_Exscan(sendbuf.data(), recvbuf.data(), 2, MPI_INT, op, comm);

      int exscan_test = 1;

      // printf(" ID=%d : %d  %d \n", rank, recvbuf[0], recvbuf[1]);
     
      if(rank >0)
      if((op == MPI_SUM && (abs(recvbuf[0]-rank*(rank-1)/2) > 1.e-10 || abs(recvbuf[1]-rank*(rank-1)/2) > 1.e-10) ) ||
         (op == MPI_MIN && (abs(recvbuf[0] ) > 1.e-10 || abs(recvbuf[1]) > 1.e-10) )  || 
         (op == MPI_MAX && (abs(recvbuf[0] - rank+1) > 1.e-10 || abs(recvbuf[1] - rank+1) > 1.e-10) ) )
      {
        exscan_test = 0; //printf("%d  id = %d\n", recvbuf[i], rank);
      }  

      int exscan_result;
      MPI_Reduce(&exscan_test, &exscan_result, 1, MPI_INT, MPI_MIN, 0, comm);
    
      if(rank == 0 && exscan_result)  printf("            \t test MPI_Exscan \t OK\n");
      if(rank == 0 && !exscan_result) printf("            \t test MPI_Exscan \t FAILED\n");
    }



    // TEST OF COMM_SPLIT
    {

      MPI_Barrier(comm);
      int rank, size;
      MPI_Comm_rank(comm, &rank);
      MPI_Comm_size(comm, &size);
    
      int color = rand()%3;
      int key = rand()%5;
      
      int color2 = rand()%3;
      int key2 = rand()%5;
  
      MPI_Comm split_comm;
      MPI_Comm_split(comm, color, key, &split_comm);

      
      MPI_Comm split_comm2;
      MPI_Comm_split(comm, color2, key2, &split_comm2);


    
      int split_rank, split_size;
      MPI_Comm_rank(split_comm, &split_rank);
      MPI_Comm_size(split_comm, &split_size);
    
#ifdef _Memory_check    
      printf("rank = %d, color = %d, key = %d, split_rank = %d, local_rank=%d\n", rank, color, key, split_rank, split_comm->ep_comm_ptr->size_rank_info[1].first);
#endif

      MPI_Barrier(comm);
      MPI_Barrier(comm);
      MPI_Barrier(comm);
      
      if(rank == 0) printf("            \t MPI_Comm_split \t OK\n");
      
      MPI_Barrier(comm);
      MPI_Barrier(comm);
      
      int bcast_buf_size=100;
      
      std::vector<int> bcast_buf(bcast_buf_size, 0);
      
      if(split_rank==0) bcast_buf.assign(bcast_buf_size, (color+1)*split_size);
      
      MPI_Bcast(bcast_buf.data(), bcast_buf_size, MPI_INT, 0, split_comm);
      
      int bcast_test = 0;
      for(int i=0; i<bcast_buf_size; i++)
      {
        if(bcast_buf[i] != (color+1)*split_size)
          bcast_test = 1;
      }
  
      int bcast_result;
  
      MPI_Reduce(&bcast_test, &bcast_result, 1, MPI_INT, MPI_SUM, 0, comm);
  
      MPI_Barrier(split_comm);
      MPI_Comm_free(&split_comm);
      
      MPI_Barrier(split_comm2);
      MPI_Comm_free(&split_comm2);

      
      
      if(bcast_result == 0 && rank == 0)  printf("            \t test MPI_Bcast for split comm\t OK\n");
      if(bcast_result != 0 && rank == 0)  printf("            \t test MPI_Bcast for split comm\t FAILED %d\n", bcast_result);
      
    }
    
    MPI_Barrier(comm);
    MPI_Barrier(comm);
    MPI_Barrier(comm);
    
    // TESE OF INTERCOMM_CREATE
    {
      MPI_Barrier(comm);
      
      int rank, size;
      MPI_Comm_rank(comm, &rank);
      MPI_Comm_size(comm, &size);

      int config = 1;
      if(config=1)
      {
        assert(size == 16 && omp_get_num_threads()==4);
      }
      
      int tab_color[16] = {2, 2, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1}; // used for config 1
      int tab_key[16]   = {2, 1, 4, 1, 0, 4, 3, 3, 4, 3, 4, 4, 4, 0, 0, 2}; // used for config 1

      int color = tab_color[rank];
      int key = tab_key[rank];
      
  
      MPI_Comm split_comm;
      MPI_Comm_split(comm, color, key, &split_comm);

    
      int split_rank, split_size;
      MPI_Comm_rank(split_comm, &split_rank);
      MPI_Comm_size(split_comm, &split_size);


      

      MPI_Barrier(comm);
      MPI_Barrier(comm);
      
      if(rank == 0) printf("            \t MPI_Comm_split \t OK\n");
      
      MPI_Barrier(comm);
      MPI_Barrier(comm);
      
      
      
      
      
      int local_leader = 0;
      int remote_leader = color==2? 13: 4; // used for config 1

      MPI_Comm peer_comm = comm;

      MPI_Comm inter_comm;
      MPI_Intercomm_create(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm);

      int inter_rank, inter_size, remote_size;
      MPI_Comm_rank(inter_comm, &inter_rank);
      MPI_Comm_size(inter_comm, &inter_size);
      MPI_Comm_remote_size(inter_comm, &remote_size);
      
      
      MPI_Barrier(comm);
      MPI_Barrier(comm);

      if(rank == 0) printf("            \t MPI_Intercomm_create \t OK\n");

      printf("rank = %d, split_rank = %d, split_size = %d, inter_rank=%d, inter_size=%d, remote_size=%d\n", rank, split_rank, split_size, inter_rank, inter_size, remote_size);

      
      MPI_Barrier(comm);
      MPI_Barrier(comm);


      if(color==2 && split_rank==0)
      {
        double sendbuf[9]={1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9};
        MPI_Request send_request;
        MPI_Status send_status;
        MPI_Isend(sendbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &send_request);
        MPI_Wait(&send_request, &send_status);
      }

      if(color==1 && split_rank==0)
      {
        double recvbuf[9];
        MPI_Request recv_request;
        MPI_Status recv_status;
        MPI_Irecv(recvbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &recv_request);
        MPI_Wait(&recv_request, &recv_status);
        for(int i=0; i<9; i++)
        {
          printf("recvbuf[%d] = %lf\n", i, recvbuf[i]);
        }
      }

      MPI_Barrier(comm);
      MPI_Barrier(comm);

      if(rank == 0) printf("            \t Test iP2P for intercomm \t OK\n");
      
      


      MPI_Barrier(comm);
      MPI_Barrier(comm);

      MPI_Barrier(split_comm);
      MPI_Comm_free(&split_comm);


      MPI_Barrier(comm);
      MPI_Barrier(comm);

      MPI_Comm inter_comm_dup;
      MPI_Comm_dup(inter_comm, &inter_comm_dup);

      int inter_comm_dup_size;
      int inter_comm_dup_remote_size;
      MPI_Comm_size(inter_comm_dup, &inter_comm_dup_size);

      MPI_Comm_remote_size(inter_comm_dup, &inter_comm_dup_remote_size);

      bool high = inter_comm_dup_size>inter_comm_dup_remote_size;


      printf("rank = %d, high = %d, inter_comm_dup_size = %d, inter_comm_dup_remote_size = %d\n", rank, high, inter_comm_dup_size, inter_comm_dup_remote_size);

      MPI_Comm inter_comm_dup_merged;

      MPI_Intercomm_merge(inter_comm_dup, high, &inter_comm_dup_merged);
      
      int inter_comm_dup_merged_rank;
      MPI_Comm_rank(inter_comm_dup_merged, &inter_comm_dup_merged_rank);

      int inter_comm_dup_merged_size;
      MPI_Comm_size(inter_comm_dup_merged, &inter_comm_dup_merged_size);

      //printf("rank = %d, inter_comm_dup_merged_rank = %d, inter_comm_dup_merged_size = %d\n", rank, inter_comm_dup_merged_rank, inter_comm_dup_merged_size);
      
      
      MPI_Barrier(inter_comm_dup_merged);

      // TEST OF GATHER
      {
        int inter_comm_dup_merged_rank;
        MPI_Comm_rank(inter_comm_dup_merged, &inter_comm_dup_merged_rank);

        int inter_comm_dup_merged_size;
        MPI_Comm_size(inter_comm_dup_merged, &inter_comm_dup_merged_size);
        
        

        int gather_root = 99;
  
        if(inter_comm_dup_merged_rank == 0) 
        {
          gather_root = rand() % inter_comm_dup_merged_size;
        }


  
        MPI_Bcast(&gather_root, 1, MPI_INT, 0, inter_comm_dup_merged);

        printf("rank = %d, inter_comm_dup_merged_rank = %d, inter_comm_dup_merged_size = %d, gather_root = %d\n", rank, inter_comm_dup_merged_rank, inter_comm_dup_merged_size, gather_root);


        double sendbuf[2];
        sendbuf[0] = inter_comm_dup_merged_rank * 1.0;
        sendbuf[1] = inter_comm_dup_merged_size * (-1.0);

        std::vector<double>recvbuf(2*inter_comm_dup_merged_size, 0);

        MPI_Gather(sendbuf, 2, MPI_DOUBLE, recvbuf.data(), 2, MPI_DOUBLE, gather_root, inter_comm_dup_merged);

        bool gather_result = true;

        if(inter_comm_dup_merged_rank == gather_root)
        {
          for(int i=0; i<inter_comm_dup_merged_size; i++)
          {
            if(abs(recvbuf[2*i] - i) > 1.e-10 || abs(recvbuf[2*i+1] + inter_comm_dup_merged_size) > 1.e-10)
            {
              gather_result = false;
              break;
            }  
          }

          if(gather_result) printf("root = %d : \t test MPI_Gather for merged comm\t OK \n", gather_root);
          else              printf("root = %d : \t test MPI_Gather for merged comm\t FAILED\n", gather_root);
        }
      }

      MPI_Barrier(inter_comm_dup_merged);

      // TEST OF ALLREDUCE
      {
 
        int sendbuf[2];
        sendbuf[0] = inter_comm_dup_merged_rank;
        sendbuf[1] = -inter_comm_dup_merged_size;

        std::vector<int>recvbuf(2, 0);

        MPI_Op op = MPI_MIN;

        MPI_Allreduce(sendbuf, recvbuf.data(), 2, MPI_INT, op, inter_comm_dup_merged);


        int allreduce_test = 1;

      
        if((op == MPI_SUM && (abs(recvbuf[0]-(inter_comm_dup_merged_size-1)*inter_comm_dup_merged_size/2) > 1.e-10 || abs(recvbuf[1] + inter_comm_dup_merged_size * inter_comm_dup_merged_size) > 1.e-10) ) ||
           (op == MPI_MAX && (abs(recvbuf[0]-(inter_comm_dup_merged_size-1)) > 1.e-10 || abs(recvbuf[1] + inter_comm_dup_merged_size) > 1.e-10) )               ||
           (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] + inter_comm_dup_merged_size) > 1.e-10) ) )
        {
          allreduce_test = 0; printf("%d %d\n", recvbuf[0], recvbuf[1]);
        }  
      

        int allreduce_result;
        MPI_Reduce(&allreduce_test, &allreduce_result, 1, MPI_INT, MPI_MIN, 0, inter_comm_dup_merged);

        if(inter_comm_dup_merged_rank == 0 && allreduce_result)  printf("            \t test MPI_Allreduce for merged comm \t OK\n");
        if(inter_comm_dup_merged_rank == 0 && !allreduce_result) printf("            \t test MPI_Allreduce for merged comm \t FAILED\n");
    
      }

      MPI_Barrier(inter_comm_dup_merged);

      // TEST OF EXSCAN
      {
 
        std::vector<int>sendbuf(2, inter_comm_dup_merged_rank);
        std::vector<int>recvbuf(2, -1);

        MPI_Op op = MPI_SUM;
           

        MPI_Exscan(sendbuf.data(), recvbuf.data(), 2, MPI_INT, op, inter_comm_dup_merged);

        int exscan_test = 1;

        if(inter_comm_dup_merged_rank >0)
        if((op == MPI_SUM && (abs(recvbuf[0]-inter_comm_dup_merged_rank*(inter_comm_dup_merged_rank-1)/2) > 1.e-10 || abs(recvbuf[1]-inter_comm_dup_merged_rank*(inter_comm_dup_merged_rank-1)/2) > 1.e-10) ) ||
           (op == MPI_MIN && (abs(recvbuf[0] ) > 1.e-10 || abs(recvbuf[1]) > 1.e-10) )  || 
           (op == MPI_MAX && (abs(recvbuf[0] - inter_comm_dup_merged_rank+1) > 1.e-10 || abs(recvbuf[1] - inter_comm_dup_merged_rank+1) > 1.e-10) ) )
        {
          exscan_test = 0; 
        }
        
        int exscan_result;
        MPI_Reduce(&exscan_test, &exscan_result, 1, MPI_INT, MPI_MIN, 0, inter_comm_dup_merged);
    
        if(inter_comm_dup_merged_rank == 0 && exscan_result)  printf("            \t test MPI_Exscan for merged comm \t OK\n");
        if(inter_comm_dup_merged_rank == 0 && !exscan_result) printf("            \t test MPI_Exscan for merged comm \t FAILED %d\n", exscan_result);
      }

      MPI_Barrier(inter_comm_dup_merged);

      // TEST OF SCATTERV
      {

        int scatterv_root;
  
        if(inter_comm_dup_merged_rank == 0) scatterv_root = rand() % inter_comm_dup_merged_size;
  
        MPI_Bcast(&scatterv_root, 1, MPI_INT, 0, inter_comm_dup_merged);
 
        std::vector<int>sendbuf(2*inter_comm_dup_merged_size, inter_comm_dup_merged_rank);
        std::vector<int>recvbuf(2, -1);
        std::vector<int>sendcounts(inter_comm_dup_merged_size, 2);
        std::vector<int>displs(inter_comm_dup_merged_size, 0);

        for(int i=0; i<inter_comm_dup_merged_size; i++) displs[i] = 2*(inter_comm_dup_merged_size-1-i);

        if(inter_comm_dup_merged_rank == scatterv_root) 
        {
          for(int i=0; i<inter_comm_dup_merged_size; i++) 
          {
            sendbuf[2*i] = i;
            sendbuf[2*i+1] = inter_comm_dup_merged_size;
          }
        }


        MPI_Scatterv(sendbuf.data(), sendcounts.data(), displs.data(), MPI_INT, recvbuf.data(), 2, MPI_INT, scatterv_root, inter_comm_dup_merged);

        int scatterv_test = 1;

     
        if( abs(recvbuf[0]-(inter_comm_dup_merged_size-1-inter_comm_dup_merged_rank)) > 1.e-10 || abs(recvbuf[1]-inter_comm_dup_merged_size) > 1.e-10 )
        {
          scatterv_test = 0; printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], inter_comm_dup_merged_rank);
        }  
    

        int scatterv_result;
        MPI_Reduce(&scatterv_test, &scatterv_result, 1, MPI_INT, MPI_MIN, scatterv_root, inter_comm_dup_merged);

        if(inter_comm_dup_merged_rank == scatterv_root && scatterv_result)  printf("root = %d : \t test MPI_Scatterv for merged comm \t OK\n", scatterv_root);
        if(inter_comm_dup_merged_rank == scatterv_root && !scatterv_result) printf("root = %d : \t test MPI_Scatterv for merged comm \t FAILED\n", scatterv_root);
      }

      MPI_Barrier(inter_comm_dup_merged);

      // TEST OF TESTALL
      {
        int merged_size, merged_rank;
        MPI_Comm_rank(inter_comm_dup_merged, &merged_rank);
        MPI_Comm_size(inter_comm_dup_merged, &merged_size);
        
        int left = merged_rank>0? merged_rank-1 : merged_size-1;
        int right = (merged_rank+1) % merged_size;
                
        int N=10000;
        int *left_buf = new int[N];
        int *right_buf = new int[N];
                
        for(int i=0; i<N; i++)
        {
          right_buf[i] = merged_rank*100000+i;
        }
        
        MPI_Request request[2];
        MPI_Status status[2];
        
        MPI_Irecv(left_buf, N, MPI_INT, left, 0, inter_comm_dup_merged, &request[0]);
        MPI_Isend(right_buf, N, MPI_INT, right, 0, inter_comm_dup_merged, &request[1]);
        
        
        int flag;
        MPI_Testall(2, request, &flag, status);
      
        printf("rank = %d, flag = %d\n", merged_rank, flag);
        
        while(!flag) MPI_Testall(2, request, &flag, status);
        
        int recv_count;
        MPI_Get_count(&status[0], MPI_INT, &recv_count);
        
        MPI_Barrier(inter_comm_dup_merged);
        
        printf("rank = %d, recv_count = %d, left_buf[5-10] = %d\t%d\t%d\t%d\t%d\t%d\n", merged_rank, recv_count, left_buf[5], left_buf[6], left_buf[7], left_buf[8], left_buf[9], left_buf[10]);
                        
        delete[] left_buf;
        delete[] right_buf;
      }
      
      MPI_Barrier(inter_comm_dup_merged);

      // TEST WAITALL
      {
        int merged_rank;
        int merged_size;

        MPI_Comm_rank(inter_comm_dup_merged, &merged_rank);
        MPI_Comm_size(inter_comm_dup_merged, &merged_size);

        int left = merged_rank>0? merged_rank-1 : merged_size-1;
        int right = (merged_rank+1)%merged_size;

        printf("merged_rank = %d, left = %d, right = %d\n", merged_rank, left, right);

        int NN=10000;

        double *left_buf = new double[NN];
        double *right_buf = new double[NN];

        for(int i=0; i<NN; i++)
        {
          right_buf[i] = merged_rank*1000000 + i;
        }

        MPI_Request request[2];
        MPI_Status  status[2];

        MPI_Irecv(left_buf,  NN, MPI_DOUBLE, left,  0, inter_comm_dup_merged, &request[0]);
        MPI_Isend(right_buf, NN, MPI_DOUBLE, right, 0, inter_comm_dup_merged, &request[1]);
        

        MPI_Waitall(2, request, status);

        printf("merged_rank = %d, left_buf[0-4] = %lf\t%lf\t%lf\t%lf\t%lf\n", merged_rank, left_buf[0], left_buf[1], left_buf[2], left_buf[3], left_buf[4]);


        delete left_buf;
        delete right_buf;
      }


      MPI_Barrier(comm);
      MPI_Barrier(comm);


      MPI_Comm_free(&inter_comm_dup);

      MPI_Barrier(comm);
      MPI_Barrier(comm);


      MPI_Comm_free(&inter_comm_dup_merged);



      MPI_Barrier(comm);
      MPI_Barrier(comm);

      MPI_Barrier(inter_comm);
      MPI_Comm_free(&inter_comm);
      
    }

    // TESE OF INTERCOMM_CREATE
    {
      MPI_Barrier(comm);
      
      int rank, size;
      MPI_Comm_rank(comm, &rank);
      MPI_Comm_size(comm, &size);

      int config = 1;
      if(config=1)
      {
        assert(size == 16 && omp_get_num_threads()==4);
      }
      
      int tab_color[16] = {2, 2, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1}; // used for config 1
      int tab_key[16]   = {2, 1, 4, 1, 0, 4, 3, 3, 4, 3, 4, 4, 4, 0, 0, 2}; // used for config 1

      int color = tab_color[rank];
      int key = tab_key[rank];
      
  
      MPI_Comm split_comm;
      MPI_Comm_split(comm, color, key, &split_comm);

    
      int split_rank, split_size;
      MPI_Comm_rank(split_comm, &split_rank);
      MPI_Comm_size(split_comm, &split_size);


      

      MPI_Barrier(comm);
      MPI_Barrier(comm);
      
      if(rank == 0) printf("            \t MPI_Comm_split \t OK\n");
      
      MPI_Barrier(comm);
      MPI_Barrier(comm);
      
            
      int local_leader = 5;
      int remote_leader = color==2? 5: 2; // used for config 1

      MPI_Comm peer_comm = comm;

      MPI_Comm inter_comm;
      MPI_Intercomm_create(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm);

      int inter_rank;
      int inter_size;
      MPI_Comm_rank(inter_comm, &inter_rank);
      MPI_Comm_size(inter_comm, &inter_size);

      printf("split_rank = %d, inter_rank = %d / %d\n", split_rank, inter_rank, inter_size);

      MPI_Barrier(split_comm);
      MPI_Comm_free(&split_comm);

      MPI_Barrier(inter_comm);
      MPI_Comm_free(&inter_comm);
      
    }
  
    MPI_Barrier(comm);
    MPI_Comm_free(&comm);
  }

/*
  int num_threads;
  if(mpi_rank < mpi_size-2)
  {
    printf("Proc %d is client\n", mpi_rank);
    num_threads = 2;//+mpi_rank;
  }
  else
  {
    printf("Proc %d is server\n", mpi_rank); 
    num_threads = 1;
  }
  
  omp_set_num_threads(num_threads);

  #pragma omp parallel default(shared) firstprivate(num_threads)
  {
    int num_ep = num_threads;
    MPI_Info info;

    //printf("omp_get_thread_num() = %d, omp_get_num_threads() = %d, num_threads = %d\n", omp_get_thread_num(), omp_get_num_threads(), num_threads);
    MPI_Comm *ep_comm;
    #pragma omp master
    {
      MPI_Comm *ep_comm;
      MPI_Comm_create_endpoints(MPI_COMM_WORLD->mpi_comm, num_ep, info, ep_comm);
      passage = ep_comm;  
    }

    #pragma omp barrier


    MPI_Comm comm; // this should act as EP_COMM_WORLD
    comm = passage[omp_get_thread_num()];
    
    int rank, size;
    MPI_Comm_rank(comm, &rank);
    MPI_Comm_size(comm, &size);
    
    

    bool isClient = false;
    bool isServer = false;

    if(omp_get_num_threads()>1) isClient = true;
    else isServer = true;

    printf("mpi_rank = %d, ep_rank = %d, isClient = %d\n", mpi_rank, rank, isClient);

    MPI_Win ep_win;
    MPI_Aint buf_size=1;
    int buf = rank;
    int local_buf = rank;
    int result_buf = -1;
    MPI_Win_create(&buf, buf_size, sizeof(int), info, comm, &ep_win);
    MPI_Barrier(comm);
    
    // MPI_Win_fence(MPI_MODE_NOPRECEDE, ep_win);

    MPI_Barrier(comm);
    sleep(0.2);
    MPI_Barrier(comm);

    MPI_Win_fence(0, ep_win);

    if(rank == 0)
    {
      local_buf = 99;
      MPI_Aint displs=0;
      MPI_Put(&local_buf, 1, MPI_INT, size-1, displs, 1, MPI_INT, ep_win);
    }

    if(rank == size-2)
    {
      MPI_Aint displs(0);
      MPI_Get(&local_buf, 1, MPI_INT, 2, displs, 1, MPI_INT, ep_win);
    }

    MPI_Win_fence(0, ep_win);

    if(rank == 1)
    {
      MPI_Aint displs=0;
      MPI_Accumulate(&local_buf, 1, MPI_INT, size-1, displs, 1, MPI_INT, MPI_REPLACE, ep_win);
    }


    MPI_Barrier(comm);

    MPI_Win_fence(0, ep_win);

    if(rank == 2)
    {
      MPI_Aint displs = 0;
      MPI_Get_accumulate(&local_buf, 1, MPI_INT, &result_buf, 
                         1, MPI_INT, size-2, displs,
                         1, MPI_INT, MPI_SUM, ep_win);
    }
    
    MPI_Win_fence(0, ep_win);
    
    if(rank == 6)
    {
      MPI_Aint displs = 0;
      MPI_Fetch_and_op(&local_buf, &result_buf, MPI_INT, size-1, displs,
                       MPI_SUM, ep_win);
    }

    MPI_Win_fence(0, ep_win);

    if(rank == 7)
    {
      MPI_Aint displs = 0;
      MPI_Compare_and_swap(&local_buf, &buf, &result_buf, MPI_INT, size-1, displs, ep_win);
    }

    MPI_Win_fence(0, ep_win);

    //::MPI_Compare_and_swap(origin_addr, compare_addr, result_addr, to_mpi_type(datatype), target_mpi_rank, to_mpi_aint(target_disp), to_mpi_win(win.server_win[target_local_rank]));

    MPI_Win ep_win_allocated;
    int* baseptr = new int[10];
    MPI_Aint base_size = 4;

    MPI_Win_allocate (base_size, sizeof(int), info, comm, baseptr, &ep_win_allocated);

    MPI_Win_fence(0, ep_win_allocated);

    MPI_Win_free(&ep_win_allocated);
    delete[] baseptr;

    MPI_Win_free(&ep_win);
    printf("rank = %d, buf = %d, local_buf = %d, result_buf = %d\n", rank, buf, local_buf, result_buf);
    
    MPI_Comm_free(&comm);

  }
*/
  MPI_Finalize();

}