Changeset 1506 for XIOS/dev/branch_openmp/extern/ep_dev/ep_split.cpp

Timestamp:

05/29/18 13:12:21 (6 years ago)

Author:

yushan

Message:

mpi_comm_split respect the norme of MPI: ranking according to the key argument. Key can be the same. In such case, the original ranking is used.

File:

• XIOS/dev/branch_openmp/extern/ep_dev/ep_split.cpp (modified) (7 diffs)

XIOS/dev/branch_openmp/extern/ep_dev/ep_split.cpp

@@ -131 +131 @@
     MPI_Allgather(&key, 1, MPI_INT, all_key.data(),1, MPI_INT, comm);
     MPI_Allgather_local(&key, 1, MPI_INT, all_key_loc.data(), comm);
+    
+    vector<int> all_ep_rank(ep_size);
+    vector<int> all_ep_rank_loc(num_ep);
+    
+    vector<int> colored_ep_rank[num_color];
+    vector<int> colored_ep_rank_loc[num_color];
+    
+    MPI_Allgather(&ep_rank, 1, MPI_INT, all_ep_rank.data(),1, MPI_INT, comm);
+    MPI_Allgather_local(&ep_rank, 1, MPI_INT, all_ep_rank_loc.data(), comm);
 
     for(int i=0; i<num_ep; i++)
@@ -139 +148 @@
         {
           colored_key_loc[j].push_back(all_key_loc[i]);
+          colored_ep_rank_loc[j].push_back(all_ep_rank_loc[i]);
         }
       }
@@ -150 +160 @@
         {
           colored_key[j].push_back(all_key[i]);
-        }
-      }
-    }
+          colored_ep_rank[j].push_back(all_ep_rank[i]);
+        }
+      }
+    }
+    
+    int my_offset=0;
+    for(int i=0; i<colored_key[number_of_color].size(); i++)
+    {
+      if(key == colored_key[number_of_color][i])
+      {
+        if(ep_rank != colored_ep_rank[number_of_color][i])
+        {
+          my_offset++;
+        }
+        else
+          break;
+      }
+    }
+    
+    int my_offset_loc=0;
+    for(int i=0; i<colored_key_loc[number_of_color].size(); i++)
+    {
+      if(key == colored_key_loc[number_of_color][i])
+      {
+        if(ep_rank != colored_ep_rank_loc[number_of_color][i])
+        {
+          my_offset_loc++;
+        }
+        else
+          break;
+      }
+    }
+    
+    
     
     for(int i=0; i<num_color; i++)
@@ -166 +207 @@
       if(key == colored_key[number_of_color][i])
       {
-        new_ep_rank = i;
+        new_ep_rank = i+my_offset;
         break;
       }
@@ -177 +218 @@
       if(key == colored_key_loc[number_of_color][i])
       {
-        new_ep_rank_loc = i;
+        new_ep_rank_loc = i+my_offset_loc;
         break;
       }
@@ -272 +313 @@
       }
     }
-    
-    /*for(int i=0; i<ep_size; i++)
+
+#ifdef _Memory_check   
+    for(int i=0; i<ep_size; i++)
     {
       MPI_Barrier(comm);
@@ -288 +330 @@
       MPI_Barrier(comm);
       MPI_Barrier(comm);
-    }*/   
+    }   
+#endif
     
     return 0;