Changeset 1506 for XIOS/dev/branch_openmp

Timestamp:

05/29/18 13:12:21 (6 years ago)

Author:

yushan

Message:

mpi_comm_split respect the norme of MPI: ranking according to the key argument. Key can be the same. In such case, the original ranking is used.

Location:

XIOS/dev/branch_openmp/extern/ep_dev

Files:

• ep_split.cpp (modified) (7 diffs)
• main.cpp (modified) (2 diffs)

XIOS/dev/branch_openmp/extern/ep_dev/ep_split.cpp

@@ -131 +131 @@
     MPI_Allgather(&key, 1, MPI_INT, all_key.data(),1, MPI_INT, comm);
     MPI_Allgather_local(&key, 1, MPI_INT, all_key_loc.data(), comm);
+    
+    vector<int> all_ep_rank(ep_size);
+    vector<int> all_ep_rank_loc(num_ep);
+    
+    vector<int> colored_ep_rank[num_color];
+    vector<int> colored_ep_rank_loc[num_color];
+    
+    MPI_Allgather(&ep_rank, 1, MPI_INT, all_ep_rank.data(),1, MPI_INT, comm);
+    MPI_Allgather_local(&ep_rank, 1, MPI_INT, all_ep_rank_loc.data(), comm);
 
     for(int i=0; i<num_ep; i++)
@@ -139 +148 @@
         {
           colored_key_loc[j].push_back(all_key_loc[i]);
+          colored_ep_rank_loc[j].push_back(all_ep_rank_loc[i]);
         }
       }
@@ -150 +160 @@
         {
           colored_key[j].push_back(all_key[i]);
-        }
-      }
-    }
+          colored_ep_rank[j].push_back(all_ep_rank[i]);
+        }
+      }
+    }
+    
+    int my_offset=0;
+    for(int i=0; i<colored_key[number_of_color].size(); i++)
+    {
+      if(key == colored_key[number_of_color][i])
+      {
+        if(ep_rank != colored_ep_rank[number_of_color][i])
+        {
+          my_offset++;
+        }
+        else
+          break;
+      }
+    }
+    
+    int my_offset_loc=0;
+    for(int i=0; i<colored_key_loc[number_of_color].size(); i++)
+    {
+      if(key == colored_key_loc[number_of_color][i])
+      {
+        if(ep_rank != colored_ep_rank_loc[number_of_color][i])
+        {
+          my_offset_loc++;
+        }
+        else
+          break;
+      }
+    }
+    
+    
     
     for(int i=0; i<num_color; i++)
@@ -166 +207 @@
       if(key == colored_key[number_of_color][i])
       {
-        new_ep_rank = i;
+        new_ep_rank = i+my_offset;
         break;
       }
@@ -177 +218 @@
       if(key == colored_key_loc[number_of_color][i])
       {
-        new_ep_rank_loc = i;
+        new_ep_rank_loc = i+my_offset_loc;
         break;
       }
@@ -272 +313 @@
       }
     }
-    
-    /*for(int i=0; i<ep_size; i++)
+
+#ifdef _Memory_check   
+    for(int i=0; i<ep_size; i++)
     {
       MPI_Barrier(comm);
@@ -288 +330 @@
       MPI_Barrier(comm);
       MPI_Barrier(comm);
-    }*/   
+    }   
+#endif
     
     return 0;

XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

@@ -686 +686 @@
     MPI_Comm_size(comm, &size);
 
-    //int color = rank%2;
-    int tab_color[16]={2,2,2,3,0,1,1,3,2,1,3,0,0,2,0,0};
-    int tab_key[16]={3,11,10,5,6,8,15,7,2,1,9,13,4,14,12,0};
+    //int tab_color[16]={2,2,2,3,0,1,1,3,2,1,3,0,0,2,0,0};     // OK
+    //int tab_key[16]={3,11,10,5,6,8,15,7,2,1,9,13,4,14,12,0}; // OK 
+    
+
+    int tab_color[16]={2,2,1,1,1,1,0,1,1,1,0,1,1,1,2,0};
+    //int tab_key[16]={3,11,12,4,6,15,10,5,2,8,14,7,13,9,1,0};
+    int tab_key[16]={3,11,7,7,6,15,10,5,2,8,14,7,13,9,1,0};
+    
     int color = tab_color[rank];
     int key = tab_key[rank];
@@ -700 +705 @@
     MPI_Comm_rank(split_comm, &split_rank);
     
-    printf("rank = %d, color = %d, key = %d, split_rank = %d\n", rank, color, key, split_rank);
+    printf("rank = %d, color = %d, key = %d, split_rank = %d, local_rank=%d\n", rank, color, key, split_rank, split_comm->ep_comm_ptr->size_rank_info[1].first);