Ignore:
Timestamp:
06/01/18 19:36:12 (4 years ago)
Author:
yushan
Message:

optimisation partially finished. To do : special case for intercomm_create and intercomm_merge

File:
1 edited

Legend:

Unmodified
Added
Removed
  • XIOS/dev/branch_openmp/extern/ep_dev/main.cpp

    r1517 r1518  
    826826      MPI_Barrier(comm); 
    827827 
    828       if(color==2 && split_rank==1) 
     828      if(color==2 && split_rank==0) 
    829829      { 
    830830        double sendbuf[9]={1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9}; 
    831831        MPI_Request send_request; 
    832832        MPI_Status send_status; 
    833         MPI_Isend(sendbuf, 9, MPI_DOUBLE, 5, 10, inter_comm, &send_request); 
     833        MPI_Isend(sendbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &send_request); 
    834834        MPI_Wait(&send_request, &send_status); 
    835835      } 
    836836 
    837       if(color==1 && split_rank==5) 
     837      if(color==1 && split_rank==0) 
    838838      { 
    839839        double recvbuf[9]; 
    840840        MPI_Request recv_request; 
    841841        MPI_Status recv_status; 
    842         MPI_Irecv(recvbuf, 9, MPI_DOUBLE, 1, 10, inter_comm, &recv_request); 
     842        MPI_Irecv(recvbuf, 9, MPI_DOUBLE, 0, 10, inter_comm, &recv_request); 
    843843        MPI_Wait(&recv_request, &recv_status); 
    844844        for(int i=0; i<9; i++) 
     
    867867      MPI_Comm_dup(inter_comm, &inter_comm_dup); 
    868868 
     869      int inter_comm_dup_size; 
     870      int inter_comm_dup_remote_size; 
     871      MPI_Comm_size(inter_comm_dup, &inter_comm_dup_size); 
     872 
     873      MPI_Comm_remote_size(inter_comm_dup, &inter_comm_dup_remote_size); 
     874 
     875      bool high = inter_comm_dup_size>inter_comm_dup_remote_size; 
     876 
     877 
     878      //printf("rank = %d, high = %d, inter_comm_dup_size = %d, inter_comm_dup_remote_size = %d\n", rank, high, inter_comm_dup_size, inter_comm_dup_remote_size); 
     879 
     880      MPI_Comm inter_comm_dup_merged; 
     881 
     882      MPI_Intercomm_merge(inter_comm_dup, high, &inter_comm_dup_merged); 
     883       
     884      int inter_comm_dup_merged_rank; 
     885      MPI_Comm_rank(inter_comm_dup_merged, &inter_comm_dup_merged_rank); 
     886 
     887      int inter_comm_dup_merged_size; 
     888      MPI_Comm_size(inter_comm_dup_merged, &inter_comm_dup_merged_size); 
     889 
     890      printf("rank = %d, inter_comm_dup_merged_rank = %d, inter_comm_dup_merged_size = %d\n", rank, inter_comm_dup_merged_rank, inter_comm_dup_merged_size); 
     891 
     892      // TEST OF GATHER 
     893      { 
     894        int gather_root = 99; 
     895   
     896        if(inter_comm_dup_merged_rank == 0)  
     897        { 
     898          gather_root = rand() % inter_comm_dup_merged_size; 
     899        } 
     900   
     901        MPI_Bcast(&gather_root, 1, MPI_INT, 0, inter_comm_dup_merged); 
     902 
     903        double sendbuf[2]; 
     904        sendbuf[0] = inter_comm_dup_merged_rank * 1.0; 
     905        sendbuf[1] = inter_comm_dup_merged_size * (-1.0); 
     906 
     907        std::vector<double>recvbuf(2*inter_comm_dup_merged_size, 0); 
     908 
     909        MPI_Gather(sendbuf, 2, MPI_DOUBLE, recvbuf.data(), 2, MPI_DOUBLE, gather_root, inter_comm_dup_merged); 
     910 
     911        bool gather_result = true; 
     912 
     913        if(inter_comm_dup_merged_rank == gather_root) 
     914        { 
     915          for(int i=0; i<inter_comm_dup_merged_size; i++) 
     916          { 
     917            if(abs(recvbuf[2*i] - i) > 1.e-10 || abs(recvbuf[2*i+1] + inter_comm_dup_merged_size) > 1.e-10) 
     918            { 
     919              gather_result = false; 
     920              break; 
     921            }   
     922          } 
     923 
     924          if(gather_result) printf("root = %d : \t test MPI_Gather for merged comm\t OK \n", gather_root); 
     925          else              printf("root = %d : \t test MPI_Gather for merged comm\t FAILED\n", gather_root); 
     926        } 
     927      } 
     928 
     929      // TEST OF ALLREDUCE 
     930      { 
     931  
     932        int sendbuf[2]; 
     933        sendbuf[0] = inter_comm_dup_merged_rank; 
     934        sendbuf[1] = -inter_comm_dup_merged_size; 
     935 
     936        std::vector<int>recvbuf(2, 0); 
     937 
     938        MPI_Op op = MPI_MIN; 
     939 
     940        MPI_Allreduce(sendbuf, recvbuf.data(), 2, MPI_INT, op, inter_comm_dup_merged); 
     941 
     942 
     943        int allreduce_test = 1; 
     944 
     945       
     946        if((op == MPI_SUM && (abs(recvbuf[0]-(inter_comm_dup_merged_size-1)*inter_comm_dup_merged_size/2) > 1.e-10 || abs(recvbuf[1] + inter_comm_dup_merged_size * inter_comm_dup_merged_size) > 1.e-10) ) || 
     947           (op == MPI_MAX && (abs(recvbuf[0]-(inter_comm_dup_merged_size-1)) > 1.e-10 || abs(recvbuf[1] + inter_comm_dup_merged_size) > 1.e-10) )               || 
     948           (op == MPI_MIN && (abs(recvbuf[0]) > 1.e-10 || abs(recvbuf[1] + inter_comm_dup_merged_size) > 1.e-10) ) ) 
     949        { 
     950          allreduce_test = 0; printf("%d %d\n", recvbuf[0], recvbuf[1]); 
     951        }   
     952       
     953 
     954        int allreduce_result; 
     955        MPI_Reduce(&allreduce_test, &allreduce_result, 1, MPI_INT, MPI_MIN, 0, inter_comm_dup_merged); 
     956 
     957        if(inter_comm_dup_merged_rank == 0 && allreduce_result)  printf("            \t test MPI_Allreduce for merged comm \t OK\n"); 
     958        if(inter_comm_dup_merged_rank == 0 && !allreduce_result) printf("            \t test MPI_Allreduce for merged comm \t FAILED\n"); 
     959     
     960      } 
     961 
     962      // TEST OF EXSCAN 
     963      { 
     964  
     965        std::vector<int>sendbuf(2, inter_comm_dup_merged_rank); 
     966        std::vector<int>recvbuf(2, -1); 
     967 
     968        MPI_Op op = MPI_SUM; 
     969            
     970 
     971        MPI_Exscan(sendbuf.data(), recvbuf.data(), 2, MPI_INT, op, inter_comm_dup_merged); 
     972 
     973        int exscan_test = 1; 
     974 
     975        if(inter_comm_dup_merged_rank >0) 
     976        if((op == MPI_SUM && (abs(recvbuf[0]-inter_comm_dup_merged_rank*(inter_comm_dup_merged_rank-1)/2) > 1.e-10 || abs(recvbuf[1]-inter_comm_dup_merged_rank*(inter_comm_dup_merged_rank-1)/2) > 1.e-10) ) || 
     977           (op == MPI_MIN && (abs(recvbuf[0] ) > 1.e-10 || abs(recvbuf[1]) > 1.e-10) )  ||  
     978           (op == MPI_MAX && (abs(recvbuf[0] - inter_comm_dup_merged_rank+1) > 1.e-10 || abs(recvbuf[1] - inter_comm_dup_merged_rank+1) > 1.e-10) ) ) 
     979        { 
     980          exscan_test = 0;  
     981        } 
     982 
     983        //printf(" ID=%d : sendbuf = (%d, %d), recvbuf = (%d, %d), exscan_test = %d \n", inter_comm_dup_merged_rank, sendbuf[0], sendbuf[1], recvbuf[0], recvbuf[1], exscan_test);  
     984         
     985        int exscan_result; 
     986        MPI_Reduce(&exscan_test, &exscan_result, 1, MPI_INT, MPI_MIN, 0, inter_comm_dup_merged); 
     987     
     988        if(inter_comm_dup_merged_rank == 0 && exscan_result)  printf("            \t test MPI_Exscan for merged comm \t OK\n"); 
     989        if(inter_comm_dup_merged_rank == 0 && !exscan_result) printf("            \t test MPI_Exscan for merged comm \t FAILED %d\n", exscan_result); 
     990      } 
     991 
     992      // TEST OF SCATTERV 
     993      { 
     994 
     995        int scatterv_root; 
     996   
     997        if(inter_comm_dup_merged_rank == 0) scatterv_root = rand() % inter_comm_dup_merged_size; 
     998   
     999        MPI_Bcast(&scatterv_root, 1, MPI_INT, 0, inter_comm_dup_merged); 
     1000  
     1001        std::vector<int>sendbuf(2*inter_comm_dup_merged_size, inter_comm_dup_merged_rank); 
     1002        std::vector<int>recvbuf(2, -1); 
     1003        std::vector<int>sendcounts(inter_comm_dup_merged_size, 2); 
     1004        std::vector<int>displs(inter_comm_dup_merged_size, 0); 
     1005 
     1006        for(int i=0; i<inter_comm_dup_merged_size; i++) displs[i] = 2*(inter_comm_dup_merged_size-1-i); 
     1007 
     1008        if(inter_comm_dup_merged_rank == scatterv_root)  
     1009        { 
     1010          for(int i=0; i<inter_comm_dup_merged_size; i++)  
     1011          { 
     1012            sendbuf[2*i] = i; 
     1013            sendbuf[2*i+1] = inter_comm_dup_merged_size; 
     1014          } 
     1015        } 
     1016 
     1017 
     1018        MPI_Scatterv(sendbuf.data(), sendcounts.data(), displs.data(), MPI_INT, recvbuf.data(), 2, MPI_INT, scatterv_root, inter_comm_dup_merged); 
     1019 
     1020        //printf("ID = %d : %d %d\n", inter_comm_dup_merged_rank, recvbuf[0], recvbuf[1]); 
     1021 
     1022        int scatterv_test = 1; 
     1023 
     1024      
     1025        if( abs(recvbuf[0]-(inter_comm_dup_merged_size-1-inter_comm_dup_merged_rank)) > 1.e-10 || abs(recvbuf[1]-inter_comm_dup_merged_size) > 1.e-10 ) 
     1026        { 
     1027          scatterv_test = 0; printf("%d %d  id = %d\n", recvbuf[0], recvbuf[1], inter_comm_dup_merged_rank); 
     1028        }   
     1029     
     1030 
     1031        int scatterv_result; 
     1032        MPI_Reduce(&scatterv_test, &scatterv_result, 1, MPI_INT, MPI_MIN, scatterv_root, inter_comm_dup_merged); 
     1033 
     1034        if(inter_comm_dup_merged_rank == scatterv_root && scatterv_result)  printf("root = %d : \t test MPI_Scatterv for merged comm \t OK\n", scatterv_root); 
     1035        if(inter_comm_dup_merged_rank == scatterv_root && !scatterv_result) printf("root = %d : \t test MPI_Scatterv for merged comm \t FAILED\n", scatterv_root); 
     1036      } 
     1037 
     1038 
     1039 
     1040 
    8691041      MPI_Barrier(comm); 
    8701042      MPI_Barrier(comm); 
     
    8721044 
    8731045      MPI_Comm_free(&inter_comm_dup); 
     1046 
     1047      MPI_Barrier(comm); 
     1048      MPI_Barrier(comm); 
     1049 
     1050 
     1051      MPI_Comm_free(&inter_comm_dup_merged); 
     1052 
    8741053 
    8751054 
Note: See TracChangeset for help on using the changeset viewer.