Changeset 1511 for XIOS

Timestamp:

05/30/18 20:08:15 (6 years ago)

Author:

yushan

Message:

save dev

Location:

XIOS/dev/branch_openmp/extern/ep_dev

Files:

• ep_allgather.cpp (modified) (1 diff)
• ep_comm.hpp (modified) (1 diff)
• ep_create.cpp (modified) (1 diff)
• ep_intercomm.cpp (modified) (3 diffs)
• ep_intercomm.hpp (modified) (2 diffs)
• ep_intercomm_kernel.cpp (modified) (4 diffs)
• ep_split.cpp (modified) (1 diff)
• ep_type.hpp (modified) (5 diffs)
• main.cpp (modified) (1 diff)
• makefile (modified) (3 diffs)

XIOS/dev/branch_openmp/extern/ep_dev/ep_allgather.cpp

@@ -37 +37 @@
         memcpy(recvbuf + datasize * i * count, comm->my_buffer->void_buffer[i], datasize * count);
     }
+    MPI_Barrier_local(comm);
   }
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_comm.hpp

@@ -52 +52 @@
     BUFFER     *my_buffer;
     ep_barrier *ep_barrier;
-    //RANK_MAP   *rank_map;
     EP_RANK_MAP   *ep_rank_map;
     void* mpi_comm;

XIOS/dev/branch_openmp/extern/ep_dev/ep_create.cpp

@@ -101 +101 @@
   } //MPI_Comm_create_endpoints
 
+  
 
 } //namespace ep_lib

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.cpp

@@ -61 +61 @@
       if( leader_ranks[1] * leader_ranks[4] == 1)
       {
-        if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_unique_leader\n");
+        if(ep_rank == local_leader) printf("calling MPI_Intercomm_create_unique_leader\n");
         local_comm->ep_comm_ptr->comm_label = -99;
 
@@ -167 +167 @@
         local_comm->ep_comm_ptr->comm_label = tag;
 
-        if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_from_world\n");
+        if(ep_rank == local_leader) printf("calling MPI_Intercomm_create_from_world\n");
 
         return MPI_Intercomm_create_from_world(local_comm, new_local_leader, MPI_COMM_WORLD->mpi_comm, leader_in_world[1], new_tag_in_world, newintercomm);
@@ -174 +174 @@
     }
 
-    if(ep_rank == local_leader) Debug("calling MPI_Intercomm_create_kernel\n");
+    if(ep_rank == local_leader) printf("calling MPI_Intercomm_create_kernel\n");
 
     return MPI_Intercomm_create_kernel(local_comm, local_leader, peer_comm, remote_leader, tag, newintercomm);

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm.hpp

@@ -5 +5 @@
 typedef std::vector< std::pair<int, int> > RANK_MAP;  // at(ep_rank) = <ep_rank_local, mpi_rank>
 typedef std::map<int, std::pair<int, int> > EP_RANK_MAP;  // key(ep_rank) = <ep_rank_local, mpi_rank>
-//typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP;
+typedef std::map<int, std::pair< int, std::pair<int, int> > > INTERCOMM_RANK_MAP;
 
 namespace ep_lib
@@ -18 +18 @@
     void *mpi_inter_comm;
 
-    RANK_MAP *intercomm_rank_map;
+    INTERCOMM_RANK_MAP *intercomm_rank_map;
     RANK_MAP *local_rank_map;
     RANK_MAP *remote_rank_map;

XIOS/dev/branch_openmp/extern/ep_dev/ep_intercomm_kernel.cpp

@@ -20 +20 @@
     num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second;
     mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second;
+
+
+    // step 1 : local leaders exchange ep_size then bcast to all ep in local_comm
+
+    int remote_ep_size;
+    bool is_local_leader = ep_rank==local_leader? true: false;
+
+    if(is_local_leader)
+    {
+      MPI_Request request[2];
+      MPI_Status status[2];
+      MPI_Isend(&ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request[0]);
+      MPI_Irecv(&remote_ep_size, 1, MPI_INT, remote_leader, tag, peer_comm, &request[1]);
+      MPI_Waitall(2, request, status);
+    }
+
+    MPI_Bcast(&remote_ep_size, 1, MPI_INT, local_leader, local_comm);
+
+#ifdef _showinfo
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    #pragma omp critical(stdoutput)
+    printf("peer_rank = %d, ep_size = %d, remote_ep_size = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_size, remote_ep_size);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm); 
+#endif
+
+    // step 2 : gather ranks in world for both local and remote comm
+
+    int rank_in_world;
+    ::MPI_Comm_rank(to_mpi_comm(MPI_COMM_WORLD->mpi_comm), &rank_in_world);
+
+    int *ranks_in_world_local  = new int[ep_size];
+    int *ranks_in_world_remote = new int[remote_ep_size];
+
+    MPI_Allgather(&rank_in_world, 1, MPI_INT, ranks_in_world_local, 1, MPI_INT, local_comm);
+
+    if(is_local_leader)
+    {
+      MPI_Request request[2];
+      MPI_Status status[2];
+      MPI_Isend(ranks_in_world_local,  ep_size,        MPI_INT, remote_leader, tag, peer_comm, &request[0]);
+      MPI_Irecv(ranks_in_world_remote, remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request[1]);
+      MPI_Waitall(2, request, status);
+    }
+
+    MPI_Bcast(ranks_in_world_remote, remote_ep_size, MPI_INT, local_leader, local_comm);
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+    if(remote_leader == 4)
+    {
+      for(int i=0; i<ep_size; i++)
+      {
+        if(ep_rank == i)
+        {
+          printf("peer_rank = %d, ranks_in_world_local = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_local[i]);
+          }
+    
+          printf("\npeer_rank = %d, ranks_in_world_remote = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<remote_ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_remote[i]);
+          }
+          printf("\n");
+         
+        }
+        
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+      }
+    }
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    
+    if(remote_leader == 13)
+    {
+      for(int i=0; i<ep_size; i++)
+      {
+        if(ep_rank == i)
+        {
+          printf("peer_rank = %d, ranks_in_world_local = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_local[i]);
+          }
+    
+          printf("\npeer_rank = %d, ranks_in_world_remote = \n", peer_comm->ep_comm_ptr->size_rank_info[0].first);
+          for(int i=0; i<remote_ep_size; i++)
+          {
+            printf("%d\t", ranks_in_world_remote[i]);
+          }
+          printf("\n");
+         
+        }
+        
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+        MPI_Barrier(local_comm);
+      }
+    }
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+
+    // step 3 : determine the priority and ownership of each ep
+
+    int local_leader_rank_in_peer;
+    int remote_leader_rank_in_peer;
+
+    if(is_local_leader) MPI_Comm_rank(peer_comm, &local_leader_rank_in_peer);
+
+    if(is_local_leader)
+    {
+      MPI_Request request[2];
+      MPI_Status status[2];
+      MPI_Isend(&local_leader_rank_in_peer,  1, MPI_INT, remote_leader, tag, peer_comm, &request[0]);
+      MPI_Irecv(&remote_leader_rank_in_peer, 1, MPI_INT, remote_leader, tag, peer_comm, &request[1]);
+      MPI_Waitall(2, request, status);
+    }
+
+    MPI_Bcast(&local_leader_rank_in_peer,  1, MPI_INT, local_leader, local_comm);
+    MPI_Bcast(&remote_leader_rank_in_peer, 1, MPI_INT, local_leader, local_comm);
+
+    bool priority = local_leader_rank_in_peer > remote_leader_rank_in_peer? true : false;
+
+#ifdef _showinfo
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, priority = %d, local_leader_rank_in_peer = %d, remote_leader_rank_in_peer = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, priority, local_leader_rank_in_peer, remote_leader_rank_in_peer);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+#endif
+
+    int local_leader_rank_in_world = ranks_in_world_local[local_leader];
+    int remote_leader_rank_in_world;
+
+    if(is_local_leader)
+    {
+      MPI_Request request[2];
+      MPI_Status status[2];
+      MPI_Isend(&local_leader_rank_in_world,  1, MPI_INT, remote_leader, tag, peer_comm, &request[0]);
+      MPI_Irecv(&remote_leader_rank_in_world, 1, MPI_INT, remote_leader, tag, peer_comm, &request[1]);
+      MPI_Waitall(2, request, status);
+    }
+
+    MPI_Bcast(&remote_leader_rank_in_world, 1, MPI_INT, local_leader, local_comm);
+
+    int ownership;
+
+    if(rank_in_world == ranks_in_world_local[local_leader]) ownership = 1;
+    else if(rank_in_world == remote_leader_rank_in_world)   ownership = 0;
+    else
+    {
+      ownership = 1;
+      for(int i=0; i<remote_ep_size; i++)
+      {
+        if(rank_in_world == ranks_in_world_remote[i])
+        {
+          ownership = priority? 1 : 0;
+          break;
+        }  
+      }
+    }
+    
+#ifdef _showinfo
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, priority = %d, ownership = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, priority, ownership);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+#endif
+
+
+    // step 4 : extract local_comm and create intercomm
+
+    bool is_involved = is_local_leader || (!is_local_leader && ep_rank_loc == 0 && rank_in_world != local_leader_rank_in_world);
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, is_involved = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, is_involved);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+    if(is_involved)
+    {
+      ::MPI_Group local_group;
+      ::MPI_Group extracted_group;
+      ::MPI_Comm *extracted_comm = new ::MPI_Comm;
+
+
+      ::MPI_Comm_group(to_mpi_comm(local_comm->mpi_comm), &local_group);
+
+      int *ownership_list = new int[mpi_size];
+      int *mpi_rank_list = new int[mpi_size];
+
+      ::MPI_Allgather(&ownership, 1, to_mpi_type(MPI_INT), ownership_list, 1, to_mpi_type(MPI_INT), to_mpi_comm(local_comm->mpi_comm));
+      ::MPI_Allgather(&mpi_rank,  1, to_mpi_type(MPI_INT), mpi_rank_list,  1, to_mpi_type(MPI_INT), to_mpi_comm(local_comm->mpi_comm));
+
+      
+      int n=0;
+      for(int i=0; i<mpi_size; i++)
+      { 
+        n+=ownership_list[i];
+      }
+
+      int *new_mpi_rank_list = new int[n];
+      int j=0;
+      for(int i=0; i<mpi_size; i++)
+      { 
+        if(ownership_list[i] !=0)
+        {
+          new_mpi_rank_list[j++] = mpi_rank_list[i];
+        }
+      }
+
+
+      ::MPI_Group_incl(local_group, n, new_mpi_rank_list, &extracted_group);
+
+      ::MPI_Comm_create(to_mpi_comm(local_comm->mpi_comm), extracted_group, extracted_comm);
+
+      ::MPI_Comm *mpi_inter_comm = new ::MPI_Comm;
+
+      int local_leader_rank_in_extracted_comm;
+
+      if(is_local_leader)
+      {
+        ::MPI_Comm_rank(*extracted_comm, &local_leader_rank_in_extracted_comm);
+      }
+
+      ::MPI_Bcast(&local_leader_rank_in_extracted_comm, 1, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, to_mpi_comm(local_comm->mpi_comm));
+
+      int local_leader_rank_in_peer_mpi;
+      int remote_leader_rank_in_peer_mpi;
+
+      ::MPI_Comm_rank(to_mpi_comm(peer_comm->mpi_comm), &local_leader_rank_in_peer_mpi);
+
+      if(is_local_leader)
+      {
+        MPI_Request request[2];
+        MPI_Status status[2];
+        MPI_Isend(&local_leader_rank_in_peer_mpi,  1, MPI_INT, remote_leader, tag, peer_comm, &request[0]);
+        MPI_Irecv(&remote_leader_rank_in_peer_mpi, 1, MPI_INT, remote_leader, tag, peer_comm, &request[1]);
+        MPI_Waitall(2, request, status);
+      }
+
+      ::MPI_Bcast(&remote_leader_rank_in_peer_mpi, 1, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, to_mpi_comm(local_comm->mpi_comm));      
+
+
+      if(ownership)
+        ::MPI_Intercomm_create(*extracted_comm, local_leader_rank_in_extracted_comm, to_mpi_comm(peer_comm->mpi_comm), remote_leader_rank_in_peer_mpi, tag, mpi_inter_comm);
+
+
+      // step 5 :: determine new num_ep
+
+      int num_ep_count=0;
+
+      for(int i=0; i<ep_size; i++)
+      {
+        if(rank_in_world == ranks_in_world_local[i])
+          num_ep_count++;
+      }
+      
+      for(int i=0; i<remote_ep_size; i++)
+      {
+        if(rank_in_world == ranks_in_world_remote[i])
+          num_ep_count++;
+      }
+
+      //printf("peer_rank = %d, num_ep_count = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, num_ep_count);
+
+
+      // step 6 : create endpoints from extracted_comm 
+
+      if(ownership)
+      {
+        MPI_Comm *ep_comm;
+        MPI_Info info;
+        MPI_Comm_create_endpoints(extracted_comm, num_ep_count, info, ep_comm);
+
+        for(int i=0; i<num_ep_count; i++)
+        {
+          ep_comm[i]->is_intercomm = true;
+          ep_comm[i]->ep_comm_ptr->comm_label = ranks_in_world_local[local_leader];
+          ep_comm[i]->ep_comm_ptr->intercomm = new ep_lib::ep_intercomm;
+          ep_comm[i]->ep_comm_ptr->intercomm->mpi_inter_comm = mpi_inter_comm;
+        }
+
+        //delete ep_comm[0]->ep_rank_map;
+
+
+        #pragma omp critical (write_to_tag_list)      
+        intercomm_list.push_back(make_pair( make_pair(tag, min(local_leader_rank_in_world, remote_leader_rank_in_world)) , make_pair(ep_comm , make_pair(num_ep_count, 0))));
+        #pragma omp flush
+#ifdef _showinfo
+        for(int i=0; i<num_ep_count; i++)
+          printf("peer_rank = %d, ep_comm = %p, ep_comm[%d] -> new_ep_rank = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_comm, i, ep_comm[i]->ep_comm_ptr->size_rank_info[0].first);
+#endif
+      }
+
+
+      delete ownership_list;
+      delete mpi_rank_list;
+      delete new_mpi_rank_list;
+
+    }
+
+    int repeated=0;
+    for(int i=0; i<remote_ep_size; i++)
+    {
+      if(rank_in_world == ranks_in_world_remote[i])
+        repeated++;
+    }
+
+    int my_turn = ownership? ep_rank_loc : ep_rank_loc+repeated;
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, ep_rank_loc = %d, ownership = %d, repeated = %d, my_turn = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, ep_rank_loc, ownership, repeated, my_turn);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+
+    #pragma omp flush
+    #pragma omp critical (read_from_intercomm_list)
+    {
+      bool flag=true;
+      while(flag)
+      {
+        for(std::list<std::pair<std::pair<int, int> , std::pair<MPI_Comm * , std::pair<int, int> > > >::iterator iter = intercomm_list.begin(); iter!=intercomm_list.end(); iter++)
+        {
+          if(iter->first == make_pair(tag, min(local_leader_rank_in_world, remote_leader_rank_in_world)))
+          {
+            *newintercomm = iter->second.first[my_turn];
+            
+            iter->second.second.second++;
+            
+            if(iter->second.second.first == iter->second.second.second)
+              intercomm_list.erase(iter);
+
+            flag = false;
+            break; 
+          }
+        }
+      }
+    }
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, test_rank = %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, (*newintercomm)->ep_comm_ptr->size_rank_info[0].first);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+
+#endif
+
+
+    // step 7 : create intercomm_rank_map for local leaders
+    
+    int my_quadruple[4];
+
+    my_quadruple[0] = ep_rank;
+    my_quadruple[1] = (*newintercomm)->ep_comm_ptr->size_rank_info[1].first;
+    my_quadruple[2] = (*newintercomm)->ep_comm_ptr->size_rank_info[2].first;
+    my_quadruple[3] = (*newintercomm)->ep_comm_ptr->comm_label;
+
+
+#ifdef _showinfo
+
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+    printf("peer_rank = %d, my_quadruple = %d %d %d %d\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, my_quadruple[0], my_quadruple[1], my_quadruple[2], my_quadruple[3]);
+    MPI_Barrier(peer_comm);
+    MPI_Barrier(peer_comm);
+#endif
+
+    int *local_quadruple_list;
+    int *remote_quadruple_list;
+    if(is_involved)
+    {
+      local_quadruple_list = new int[4*ep_size];
+      remote_quadruple_list = new int[4*remote_ep_size];
+
+    }
+
+    MPI_Gather(my_quadruple, 4, MPI_INT, local_quadruple_list, 4, MPI_INT, local_leader, local_comm);
+
+
+    if(is_local_leader)
+    {
+      MPI_Request request[2];
+      MPI_Status status[2];
+      MPI_Isend(local_quadruple_list,  4*ep_size,        MPI_INT, remote_leader, tag, peer_comm, &request[0]);
+      MPI_Irecv(remote_quadruple_list, 4*remote_ep_size, MPI_INT, remote_leader, tag, peer_comm, &request[1]);
+      MPI_Waitall(2, request, status);
+    }
+
+    if(is_involved)
+    {
+      ::MPI_Bcast(remote_quadruple_list, 4*remote_ep_size, to_mpi_type(MPI_INT), local_comm->ep_rank_map->at(local_leader).second, to_mpi_comm(local_comm->mpi_comm));
+      (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new INTERCOMM_RANK_MAP;
+      for(int i=0; i<remote_ep_size; i++)
+      {
+        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->insert(std::pair<int, std::pair< int, std::pair<int, int> > >(remote_quadruple_list[4*i], remote_quadruple_list[4*i+1], remote_quadruple_list[4*i+2], remote_quadruple_list[4*i+3]));
+      }
+    }
+
+
+    // step 8 : associate intercomm_rank_map to endpoints
+
+    int *leader_rank_in_world_local_gathered = new int[(*newintercomm)->ep_comm_ptr->size_rank_info[1].second];
+
+    MPI_Allgather_local(&local_leader_rank_in_world, 1, MPI_INT, leader_rank_in_world_local_gathered, *newintercomm);
+
+
+    int new_rank_loc = (*newintercomm)->ep_comm_ptr->size_rank_info[1].first;
+    int *new_rank_loc_local_gathered = new int[(*newintercomm)->ep_comm_ptr->size_rank_info[1].second];
+
+    MPI_Allgather_local(&new_rank_loc, 1, MPI_INT, new_rank_loc_local_gathered, *newintercomm);
+
+    //printf("peer_rank = %d, leader_rank_in_world_local_gathered = %d %d %d %d, new_rank_loc_local_gathered = %d %d %d %d\n", 
+    //  peer_comm->ep_comm_ptr->size_rank_info[0].first, leader_rank_in_world_local_gathered[0], leader_rank_in_world_local_gathered[1], leader_rank_in_world_local_gathered[2], leader_rank_in_world_local_gathered[3],
+    //  new_rank_loc_local_gathered[0], new_rank_loc_local_gathered[1], new_rank_loc_local_gathered[2], new_rank_loc_local_gathered[3]);
+
+    if(is_involved)
+    {
+      if((*newintercomm)->ep_comm_ptr->size_rank_info[1].first == 0)
+      {
+        (*newintercomm)->ep_rank_map->clear();
+        delete (*newintercomm)->ep_rank_map;
+      }
+      (*newintercomm)->ep_rank_map = new EP_RANK_MAP[ep_size];
+      *((*newintercomm)->ep_rank_map) = *(local_comm->ep_rank_map);
+    }
+
+    MPI_Barrier_local(*newintercomm);
+
+    
+    if(!is_involved)
+    {
+      int target;
+      for(int i=0; i<(*newintercomm)->ep_comm_ptr->size_rank_info[1].second; i++)
+      {
+        if(local_leader_rank_in_world == leader_rank_in_world_local_gathered[i])
+        {
+          target = i;
+          break;
+        }
+      }
+      (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = (*newintercomm)->ep_comm_ptr->comm_list[target]->ep_comm_ptr->intercomm->intercomm_rank_map;
+      (*newintercomm)->ep_rank_map = (*newintercomm)->ep_comm_ptr->comm_list[target]->ep_rank_map;
+    }
+
+
+    //printf("peer_rank = %d, intercomm_rank_map add = %p\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map);
+    
+
+    if(peer_comm->ep_comm_ptr->size_rank_info[0].first == 5)
+    {
+      int receiver = rand()%10;
+      printf("receiver = %d, intercomm_local_rank = %d, intercomm_mpi_rank = %d, comm_label = %d\n", receiver, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).first, 
+        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.first, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.second);
+    }
+
+    if(peer_comm->ep_comm_ptr->size_rank_info[0].first == 9)
+    {
+      int receiver = rand()%6;
+      printf("receiver = %d, intercomm_local_rank = %d, intercomm_mpi_rank = %d, comm_label = %d\n", receiver, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).first, 
+        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.first, (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(receiver).second.second);
+    }
+
+    //printf("peer_rank = %d, rank_map add = %p\n", peer_comm->ep_comm_ptr->size_rank_info[0].first, (*newintercomm)->ep_rank_map);
+
+
+    if(peer_comm->ep_comm_ptr->size_rank_info[0].first == 5)
+    {
+      for(int i=0; i<ep_size; i++)
+      {
+        printf("rank_map->at(%d) = %d, %d\n", i, (*newintercomm)->ep_rank_map->at(i).first, (*newintercomm)->ep_rank_map->at(i).second);
+      }
+    }
+
+
+    // clean up
+
+    delete ranks_in_world_local;
+    delete ranks_in_world_remote;
+
+    if(is_involved)
+    {
+      delete local_quadruple_list;
+      delete remote_quadruple_list;
+    }
+
+    delete leader_rank_in_world_local_gathered;
+    delete new_rank_loc_local_gathered;
+
+
+  }
+
+  int MPI_Intercomm_create_kernel_bkp(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm)
+  {
+    int ep_rank, ep_rank_loc, mpi_rank;
+    int ep_size, num_ep, mpi_size;
+
+    ep_rank = local_comm->ep_comm_ptr->size_rank_info[0].first;
+    ep_rank_loc = local_comm->ep_comm_ptr->size_rank_info[1].first;
+    mpi_rank = local_comm->ep_comm_ptr->size_rank_info[2].first;
+    ep_size = local_comm->ep_comm_ptr->size_rank_info[0].second;
+    num_ep = local_comm->ep_comm_ptr->size_rank_info[1].second;
+    mpi_size = local_comm->ep_comm_ptr->size_rank_info[2].second;
+
 
     std::vector<int> rank_info[4];  //! 0->rank_in_world of local_comm,  1->rank_in_local_parent of local_comm
@@ -531 +1064 @@
 
 
-    (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
-    (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
+    //(*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map = new RANK_MAP;
+    //(*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->resize(remote_intercomm_size);
 
 
@@ -558 +1091 @@
       for(ii=0; ii<vec2map.size(); ii++)
       {
-        (*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(vec2map[ii].first) = make_pair(vec2map[ii].second.first, vec2map[ii].second.second);
+        //(*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->at(vec2map[ii].first) = make_pair(vec2map[ii].second.first, vec2map[ii].second.second);
       }
       
@@ -564 +1097 @@
     }
 
-    MPI_Bcast((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, new_bcast_root, *newintercomm);
+    //MPI_Bcast((*newintercomm)->ep_comm_ptr->intercomm->intercomm_rank_map->data(), 2*remote_intercomm_size, MPI_INT, new_bcast_root, *newintercomm);
 
     (*newintercomm)->ep_comm_ptr->intercomm->local_comm = (local_comm->ep_comm_ptr->comm_list[ep_rank_loc]);

XIOS/dev/branch_openmp/extern/ep_dev/ep_split.cpp

@@ -222 +222 @@
       }
     }
+
+#ifdef _showinfo
+    printf("ep_rank = %d, color = %d, key = %d, new_ep_rank = %d, new_ep_rank_loc = %d\n", ep_rank, color, key, new_ep_rank, new_ep_rank_loc);
+#endif
+
+    MPI_Barrier(comm);
+    MPI_Barrier(comm);
+    MPI_Barrier(comm);
     
 

XIOS/dev/branch_openmp/extern/ep_dev/ep_type.hpp

@@ -42 +42 @@
 
 
-//typedef std::pair< int, int > SIZE_RANK_INFO; // < rank, size>
-
-//typedef std::vector< std::pair<int, int> > RANK_MAP;  // at(ep_rank) = <ep_rank_local, mpi_rank>
-//typedef std::map<int, std::pair<int, int> > EP_RANK_MAP;  // key(ep_rank) = <ep_rank_local, mpi_rank>
-
-//typedef std::vector<std::pair< std::pair<int, int>, std::pair<int, int> > > INTERCOMM_RANK_MAP;
-
 
 namespace ep_lib
@@ -55 +48 @@
   class ep_communicator;
   class ep_intercomm;
-  class OMPbarrier;
-  typedef ep_communicator* EP_Comm;
-  //class MPI_Comm;
 
-/*
-  class MPI_Status
-  {
-    public:
-
-      void* ep_datatype;
-      int ep_src;
-      int ep_tag;
-      void* mpi_status;
-  };
-
-  class MPI_Message
-  {
-    public:
-
-      void* mpi_message;
-      int ep_src;
-      int ep_tag;
-      void* mpi_status;
-
-      MPI_Message() {}
-      MPI_Message(void* message);
-  };
-
-  typedef std::list<MPI_Message > Message_list;
-*/
-
-  class OMPbarrier
-  {
-    private:
-      int nbThreads;          //<The number of threads for this barrier
-      int currentNbThread;    //<The current number of threads waiting
-      bool sense;             //<Direct barrier feedback protection
-      omp_lock_t mutex;       //<To have an atomic int
-
-      OMPbarrier(OMPbarrier&){}
-      OMPbarrier& operator=(OMPbarrier&){return *this;}
-
-    public:
-      /** Constructor with the number of threads */
-      explicit OMPbarrier(const int inNbThreads)
-          : nbThreads(inNbThreads), currentNbThread(0), sense(false) {
-          omp_init_lock( &mutex );
-      }
-
-      /** Destructor, release the omp lock */
-      ~OMPbarrier(){
-          omp_destroy_lock( &mutex );
-      }
-
-      /** Perform a barrier */
-      void wait(){
-          const bool mySense = sense;
-          omp_set_lock( &mutex );
-          const int nbThreadsArrived = (++currentNbThread);
-          omp_unset_lock( &mutex );
-
-          if(nbThreadsArrived == nbThreads) {
-              currentNbThread = 0;
-              sense = !sense;
-              #pragma omp flush
-          }
-          else {
-              volatile const bool* const ptSense = &sense;
-              while( (*ptSense) == mySense){
-              }
-          }
-      }
-
-
-      /** Change the number of threads */
-      void setNbThreads(const int inNbThread){
-          omp_set_lock( &mutex );
-          nbThreads = inNbThread;
-          omp_unset_lock( &mutex );
-      }
-  };
-
-  /*
-  class ep_intercomm
-  {
-    public:
-
-    void *mpi_inter_comm;
-
-    RANK_MAP *intercomm_rank_map;
-    RANK_MAP *local_rank_map;
-    RANK_MAP *remote_rank_map;
-
-
-    SIZE_RANK_INFO size_rank_info[3];
-
-
-    MPI_Comm *local_comm;
-    int intercomm_tag;
-
-    ep_intercomm();
-    //~ep_intercomm(){delete mpi_inter_comm;}
-    bool operator == (ep_intercomm right);
-    bool operator != (ep_intercomm right);
-    
-  };
-
-  class ep_communicator
-  {
-    public:
-
-    SIZE_RANK_INFO size_rank_info[3]; // 0: ep_rank,     ep_size
-                                      // 1: ep_rank_loc, num_ep
-                                      // 2: mpi_rank,    mpi_size
-
-
-    MPI_Comm *comm_list;
-
-    Message_list *message_queue;
-
-
-    int comm_label;
-
-    ep_intercomm *intercomm;
-
-    ep_communicator();
-    bool operator == (ep_communicator right);
-    bool operator != (ep_communicator right);
-    
-  };
-
-
-  struct BUFFER
-  {
-    void * void_buffer[12];
-  };
-
-
-  class MPI_Comm
-  {
-    public:
-
-    bool is_ep;
-    bool is_intercomm;
-
-    BUFFER     *my_buffer;
-    OMPbarrier *ep_barrier;
-    RANK_MAP   *rank_map;
-    void* mpi_comm;
-    EP_Comm ep_comm_ptr;
-    MPI_Comm *mem_bridge;
-    void* mpi_bridge;
-
-    MPI_Comm();  
-    MPI_Comm(void* comm);
-    //~MPI_Comm(){delete mpi_comm;}
-
-
-    bool operator == (MPI_Comm right);
-    bool operator != (MPI_Comm right);
-    bool is_null();
-
-  };
-
-
-  class MPI_Info
-  {
-    public:
-
-      void* mpi_info;
-
-      MPI_Info();
-      MPI_Info(void* info);
-  };
-
-
-  class MPI_Request
-  {
-    public:
-
-      void* mpi_request;
-
-      int type; //! type of the non-blocking communication. 1: Isend; 2:Irecv; 3:Imrecv; 4:Issend
-      void* buf;
-
-      int ep_src;
-      int ep_tag;
-      void* ep_datatype;
-
-      MPI_Comm comm;    //! EP communicator related to the communication
-
-      MPI_Request() {}
-      MPI_Request(void* request);
-      bool operator == (MPI_Request right);
-
-  };
-
-*/  
 
   class MPI_Aint
@@ -264 +60 @@
     MPI_Aint(int aint);
     MPI_Aint operator=(int a);
-    //MPI_Aint(int a): mpi_aint(a) {}
   };
 
@@ -275 +70 @@
     MPI_Fint() {}
     MPI_Fint(void* fint);
-    //MPI_Fint(int f): mpi_fint(f) {}
                                  
   };
@@ -288 +82 @@
   static std::list<std::pair<std::pair<int, int>, MPI_Comm * > > tag_list;
 
+  static std::list< std::pair<std::pair<int, int> , std::pair<MPI_Comm * , std::pair<int, int> > > > intercomm_list;
+
   static std::map<std::pair<int, int>, MPI_Comm >  fc_comm_map;
 
-//  static std::map<std::pair<int, int>, MPI_Comm >  *fc_comm_map_ptr;
-//  #pragma omp threadprivate(fc_comm_map_ptr)
-//            //    <MPI_Fint,thread_num>   EP_Comm
 
 }

XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

@@ -680 +680 @@
 
 
-    MPI_Barrier(comm);
-    {
-    int rank, size;
-    MPI_Comm_rank(comm, &rank);
-    MPI_Comm_size(comm, &size);
-
-    //int tab_color[16]={2,2,2,3,0,1,1,3,2,1,3,0,0,2,0,0};     // OK
-    //int tab_key[16]={3,11,10,5,6,8,15,7,2,1,9,13,4,14,12,0}; // OK 
-    
-
-    int tab_color[16]={2,2,1,1,1,1,0,1,1,1,0,1,1,1,2,0};
-    //int tab_key[16]={3,11,12,4,6,15,10,5,2,8,14,7,13,9,1,0};
-    int tab_key[16]={3,11,7,7,6,15,10,5,2,8,14,7,13,9,1,0};
-    
-    int color = tab_color[rank];
-    int key = tab_key[rank];
-  
-    MPI_Comm split_comm;
-    MPI_Comm_split(comm, color, key, &split_comm);
-
-
-    
-    int split_rank;
-    MPI_Comm_rank(split_comm, &split_rank);
-    
-    printf("rank = %d, color = %d, key = %d, split_rank = %d, local_rank=%d\n", rank, color, key, split_rank, split_comm->ep_comm_ptr->size_rank_info[1].first);
-
-
-    MPI_Barrier(comm);
-    if(rank == 0) printf("\tMPI_Comm_split OK\n");
-    MPI_Barrier(comm);
-/*
-    MPI_Comm inter_comm;
-    //MPI_Intercomm_create(sub_comm, 0, comm, (color+1)%2, 99, &inter_comm);
-    MPI_Intercomm_create(sub_comm, 0, comm, remote_leader, 99, &inter_comm);
-
-    MPI_Barrier(comm);
-    if(rank == 0) printf("\tMPI_Intercomm_create OK\n");
-    MPI_Barrier(comm);
-
-    
-
-    int high=color;
-    MPI_Comm intra_comm;
-    MPI_Intercomm_merge(inter_comm, high, &intra_comm);
-    
-    int intra_rank, intra_size;
-    MPI_Comm_rank(intra_comm, &intra_rank);
-    MPI_Comm_size(intra_comm, &intra_size);
-
-    MPI_Barrier(comm);
-    if(rank == 0) printf("\tMPI_Intercomm_merge OK\n");
-    MPI_Barrier(comm);
-    
-    
-
-    //check_test_gatherv(comm);
-*/
-    MPI_Barrier(comm);
-    MPI_Comm_free(&split_comm);
-
-
-    //MPI_Barrier(comm);
-    //MPI_Comm_free(&inter_comm);
-
-
+    // TEST OF COMM_SPLIT
+    {
+
+      MPI_Barrier(comm);
+      int rank, size;
+      MPI_Comm_rank(comm, &rank);
+      MPI_Comm_size(comm, &size);
+    
+      int color = rand()%3;
+      int key = rand()%5;
+      
+      int color2 = rand()%3;
+      int key2 = rand()%5;
+  
+      MPI_Comm split_comm;
+      MPI_Comm_split(comm, color, key, &split_comm);
+
+      
+      MPI_Comm split_comm2;
+      MPI_Comm_split(comm, color2, key2, &split_comm2);
+
+
+    
+      int split_rank, split_size;
+      MPI_Comm_rank(split_comm, &split_rank);
+      MPI_Comm_size(split_comm, &split_size);
+    
+#ifdef _Memory_check    
+      printf("rank = %d, color = %d, key = %d, split_rank = %d, local_rank=%d\n", rank, color, key, split_rank, split_comm->ep_comm_ptr->size_rank_info[1].first);
+#endif
+
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+      
+      if(rank == 0) printf("            \t tMPI_Comm_split \t OK\n");
+      
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+      
+      int bcast_buf_size=100;
+      
+      std::vector<int> bcast_buf(bcast_buf_size, 0);
+      
+      if(split_rank==0) bcast_buf.assign(bcast_buf_size, (color+1)*split_size);
+      
+      MPI_Bcast(bcast_buf.data(), bcast_buf_size, MPI_INT, 0, split_comm);
+      
+      int bcast_test = 0;
+      for(int i=0; i<bcast_buf_size; i++)
+      {
+        if(bcast_buf[i] != (color+1)*split_size)
+          bcast_test = 1;
+      }
+  
+      int bcast_result;
+  
+      MPI_Reduce(&bcast_test, &bcast_result, 1, MPI_INT, MPI_SUM, 0, comm);
+  
+      MPI_Barrier(split_comm);
+      MPI_Comm_free(&split_comm);
+      
+      MPI_Barrier(split_comm2);
+      MPI_Comm_free(&split_comm2);
+      
+      
+      if(bcast_result == 0 && rank == 0)  printf("            \t test MPI_Bcast for split comm\t OK\n");
+      if(bcast_result != 0 && rank == 0)  printf("            \t test MPI_Bcast for split comm\t FAILED %d\n", bcast_result);
+      
+    }
+    
+    MPI_Barrier(comm);
+    MPI_Barrier(comm);
+    MPI_Barrier(comm);
+    
+    // TESE OF INTERCOMM_CREATE
+    {
+      MPI_Barrier(comm);
+      
+      int rank, size;
+      MPI_Comm_rank(comm, &rank);
+      MPI_Comm_size(comm, &size);
+      
+      int tab_color[16] = {2, 2, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 1};
+      int tab_key[16]   = {2, 1, 4, 1, 0, 4, 3, 3, 4, 3, 4, 4, 4, 0, 0, 2};
+
+      int color = tab_color[rank];
+      int key = tab_key[rank];
+      
+  
+      MPI_Comm split_comm;
+      MPI_Comm_split(comm, color, key, &split_comm);
+
+    
+      int split_rank, split_size;
+      MPI_Comm_rank(split_comm, &split_rank);
+      MPI_Comm_size(split_comm, &split_size);
+
+
+      printf("rank = %d, color = %d, key = %d, split_rank = %d, local_rank=%d\n", rank, color, key, split_rank, split_comm->ep_comm_ptr->size_rank_info[1].first);
+
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+      
+      if(rank == 0) printf("            \t MPI_Comm_split \t OK\n");
+      
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+      
+      
+      
+      
+      
+      int local_leader = 0;
+      int remote_leader = color==2? 13: 4;
+
+      MPI_Comm peer_comm = comm;
+
+      MPI_Comm inter_comm;
+      MPI_Intercomm_create(split_comm, local_leader, peer_comm, remote_leader, 99, &inter_comm);
+      
+
+
+      MPI_Barrier(comm);
+      MPI_Barrier(comm);
+
+      MPI_Barrier(split_comm);
+      MPI_Comm_free(&split_comm);
+
+
+      /*
+      MPI_Barrier(comm);
+      if(rank == 0) printf("\tMPI_Intercomm_create OK\n");
+      MPI_Barrier(comm);
+
+    
+
+      int high=color;
+      MPI_Comm intra_comm;
+      MPI_Intercomm_merge(inter_comm, high, &intra_comm);
+    
+      int intra_rank, intra_size;
+      MPI_Comm_rank(intra_comm, &intra_rank);
+      MPI_Comm_size(intra_comm, &intra_size);
+
+      MPI_Barrier(comm);
+      if(rank == 0) printf("\tMPI_Intercomm_merge OK\n");
+      MPI_Barrier(comm);
+    
+    
+
+      //check_test_gatherv(comm);
+      
+      MPI_Barrier(comm);
+      MPI_Comm_free(&inter_comm);
+      */
+    }
+  
     MPI_Barrier(comm);
     MPI_Comm_free(&comm);
-  }
   }
 

XIOS/dev/branch_openmp/extern/ep_dev/makefile

@@ -1 +1 @@
 appname := EP_test
 
-CXX := mpiicc -D_intelmpi -D_noMemory_check -D_nocheck_sum -openmp
+CXX := mpiicc -D_intelmpi -D_noMemory_check -D_nocheck_sum -qopenmp
 CXXFLAGS := -std=c++11  -g
-#CXXFLAGS := -std=c++11  -g
 
 srcfiles := $(shell find . -name "*.cpp")
@@ -16 +15 @@
 depend: .depend
 
-.depend: $(srcfiles) 
+depend: $(srcfiles) 
         rm -f ./.depend
         $(CXX) $(CXXFLAGS) -MM $^>>./.depend;
@@ -26 +25 @@
         rm -f *~ .depend
 
-include .depend