Changeset 151 for codes/icosagcm/trunk/src/transfert_mpi.f90
- Timestamp:
- 05/13/13 14:30:31 (11 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
codes/icosagcm/trunk/src/transfert_mpi.f90
r148 r151 1 1 MODULE transfert_mpi_mod 2 2 USE genmod 3 USE field_mod 3 4 4 5 TYPE array … … 13 14 REAL,POINTER :: buffer_r4(:,:,:) 14 15 END TYPE array 16 17 TYPE t_buffer 18 REAL,POINTER :: r2(:) 19 REAL,POINTER :: r3(:,:) 20 REAL,POINTER :: r4(:,:,:) 21 END TYPE t_buffer 15 22 16 23 TYPE t_request … … 41 48 TYPE(t_request),POINTER :: req_e0_vect(:) 42 49 50 TYPE t_message 51 TYPE(t_request), POINTER :: request(:) 52 INTEGER :: nreq 53 INTEGER, POINTER :: mpi_req(:) 54 INTEGER, POINTER :: status(:,:) 55 TYPE(t_buffer),POINTER :: buffers(:) 56 TYPE(t_field),POINTER :: field(:) 57 LOGICAL :: completed 58 LOGICAL :: pending 59 END TYPE t_message 43 60 44 61 CONTAINS 45 62 46 63 SUBROUTINE init_transfert 47 64 USE domain_mod … … 70 87 DO j=jj_begin,jj_end+1 71 88 CALL request_add_point(ind,ii_begin-1,j,req_i1) 72 ENDDO73 74 DO i=ii_begin,ii_end75 ! CALL request_add_point(ind,i,jj_begin,req_i1)76 ENDDO77 78 DO j=jj_begin,jj_end79 ! CALL request_add_point(ind,ii_end,j,req_i1)80 ENDDO81 82 DO i=ii_begin,ii_end83 ! CALL request_add_point(ind,i,jj_end,req_i1)84 ENDDO85 86 DO j=jj_begin,jj_end87 ! CALL request_add_point(ind,ii_begin,j,req_i1)88 89 ENDDO 89 90 … … 142 143 ENDDO 143 144 144 DO i=ii_begin+1,ii_end-1145 ! CALL request_add_point(ind,i,jj_begin,req_e1_scal,right)146 ! CALL request_add_point(ind,i,jj_end,req_e1_scal,right)147 ENDDO148 149 DO j=jj_begin+1,jj_end-1150 ! CALL request_add_point(ind,ii_begin,j,req_e1_scal,rup)151 ! CALL request_add_point(ind,ii_end,j,req_e1_scal,rup)152 ENDDO153 154 ! CALL request_add_point(ind,ii_begin+1,jj_begin,req_e1_scal,left)155 ! CALL request_add_point(ind,ii_begin,jj_begin+1,req_e1_scal,ldown)156 ! CALL request_add_point(ind,ii_begin+1,jj_end,req_e1_scal,left)157 ! CALL request_add_point(ind,ii_end,jj_begin+1,req_e1_scal,ldown)158 159 145 ENDDO 160 146 … … 211 197 ENDDO 212 198 213 DO i=ii_begin+1,ii_end-1214 ! CALL request_add_point(ind,i,jj_begin,req_e1_vect,right)215 ! CALL request_add_point(ind,i,jj_end,req_e1_vect,right)216 ENDDO217 218 DO j=jj_begin+1,jj_end-1219 ! CALL request_add_point(ind,ii_begin,j,req_e1_vect,rup)220 ! CALL request_add_point(ind,ii_end,j,req_e1_vect,rup)221 ENDDO222 223 ! CALL request_add_point(ind,ii_begin+1,jj_begin,req_e1_vect,left)224 ! CALL request_add_point(ind,ii_begin,jj_begin+1,req_e1_vect,ldown)225 ! CALL request_add_point(ind,ii_begin+1,jj_end,req_e1_vect,left)226 ! CALL request_add_point(ind,ii_end,jj_begin+1,req_e1_vect,ldown)227 199 228 200 ENDDO … … 322 294 target_j=>req%target_j 323 295 target_sign=>req%target_sign 324 ! req%max_size=req%max_size*2 296 325 297 ALLOCATE(req%src_domain(req%max_size*2)) 326 298 ALLOCATE(req%src_ind(req%max_size*2)) … … 610 582 611 583 584 SUBROUTINE init_message_seq(field, request, message) 585 USE field_mod 586 USE domain_mod 587 USE mpi_mod 588 USE mpipara 589 USE mpi_mod 590 IMPLICIT NONE 591 TYPE(t_field),POINTER :: field(:) 592 TYPE(t_request),POINTER :: request(:) 593 TYPE(t_message) :: message 594 595 !$OMP MASTER 596 message%request=>request 597 !$OMP END MASTER 598 !$OMP BARRIER 599 600 END SUBROUTINE init_message_seq 601 602 SUBROUTINE send_message_seq(field,message) 603 USE field_mod 604 USE domain_mod 605 USE mpi_mod 606 USE mpipara 607 USE omp_para 608 USE trace 609 IMPLICIT NONE 610 TYPE(t_field),POINTER :: field(:) 611 TYPE(t_message) :: message 612 613 CALL transfert_request_seq(field,message%request) 614 615 END SUBROUTINE send_message_seq 616 617 SUBROUTINE test_message_seq(message) 618 IMPLICIT NONE 619 TYPE(t_message) :: message 620 END SUBROUTINE test_message_seq 621 622 623 SUBROUTINE wait_message_seq(message) 624 IMPLICIT NONE 625 TYPE(t_message) :: message 626 627 END SUBROUTINE wait_message_seq 628 629 SUBROUTINE transfert_message_seq(field,message) 630 USE field_mod 631 USE domain_mod 632 USE mpi_mod 633 USE mpipara 634 USE omp_para 635 USE trace 636 IMPLICIT NONE 637 TYPE(t_field),POINTER :: field(:) 638 TYPE(t_message) :: message 639 640 CALL send_message_seq(field,message) 641 642 END SUBROUTINE transfert_message_seq 643 644 645 SUBROUTINE init_message_mpi(field,request, message) 646 USE field_mod 647 USE domain_mod 648 USE mpi_mod 649 USE mpipara 650 USE mpi_mod 651 IMPLICIT NONE 652 653 TYPE(t_field),POINTER :: field(:) 654 TYPE(t_request),POINTER :: request(:) 655 TYPE(t_message) :: message 656 657 TYPE(ARRAY),POINTER :: recv,send 658 TYPE(t_request),POINTER :: req 659 INTEGER :: irecv,isend 660 INTEGER :: ireq,nreq 661 INTEGER :: ind 662 INTEGER :: dim3,dim4 663 664 !$OMP MASTER 665 message%request=>request 666 nreq=sum(request(:)%nsend)+sum(request(:)%nrecv) 667 message%nreq=nreq 668 ALLOCATE(message%mpi_req(nreq)) 669 ALLOCATE(message%buffers(nreq)) 670 ALLOCATE(message%status(MPI_STATUS_SIZE,nreq)) 671 672 message%pending=.FALSE. 673 message%completed=.FALSE. 674 675 IF (field(1)%data_type==type_real) THEN 676 677 IF (field(1)%ndim==2) THEN 678 679 ireq=0 680 DO ind=1,ndomain 681 req=>request(ind) 682 683 DO isend=1,req%nsend 684 ireq=ireq+1 685 send=>req%send(isend) 686 CALL allocate_mpi_buffer(message%buffers(ireq)%r2,send%size) 687 ENDDO 688 689 DO irecv=1,req%nrecv 690 ireq=ireq+1 691 recv=>req%recv(irecv) 692 CALL allocate_mpi_buffer(message%buffers(ireq)%r2,recv%size) 693 ENDDO 694 695 ENDDO 696 697 698 ELSE IF (field(1)%ndim==3) THEN 699 700 ireq=0 701 DO ind=1,ndomain 702 dim3=size(field(ind)%rval3d,2) 703 req=>request(ind) 704 705 DO isend=1,req%nsend 706 ireq=ireq+1 707 send=>req%send(isend) 708 CALL allocate_mpi_buffer(message%buffers(ireq)%r3,send%size,dim3) 709 ENDDO 710 711 DO irecv=1,req%nrecv 712 ireq=ireq+1 713 recv=>req%recv(irecv) 714 CALL allocate_mpi_buffer(message%buffers(ireq)%r3,recv%size,dim3) 715 716 ENDDO 717 718 ENDDO 719 720 721 ELSE IF (field(1)%ndim==4) THEN 722 723 ireq=0 724 DO ind=1,ndomain 725 dim3=size(field(ind)%rval4d,2) 726 dim4=size(field(ind)%rval4d,3) 727 req=>request(ind) 728 729 DO isend=1,req%nsend 730 ireq=ireq+1 731 send=>req%send(isend) 732 CALL allocate_mpi_buffer(message%buffers(ireq)%r4,send%size,dim3,dim4) 733 ENDDO 734 735 DO irecv=1,req%nrecv 736 ireq=ireq+1 737 recv=>req%recv(irecv) 738 CALL allocate_mpi_buffer(message%buffers(ireq)%r4,recv%size,dim3,dim4) 739 ENDDO 740 741 ENDDO 742 743 ENDIF 744 ENDIF 745 !$OMP END MASTER 746 !$OMP BARRIER 747 END SUBROUTINE init_message_mpi 748 749 SUBROUTINE barrier 750 USE mpi_mod 751 USE mpipara 752 IMPLICIT NONE 753 754 CALL MPI_BARRIER(comm_icosa,ierr) 755 756 END SUBROUTINE barrier 757 758 SUBROUTINE transfert_message_mpi(field,message) 759 USE field_mod 760 IMPLICIT NONE 761 TYPE(t_field),POINTER :: field(:) 762 TYPE(t_message) :: message 763 764 CALL send_message_mpi(field,message) 765 CALL wait_message_mpi(message) 766 767 END SUBROUTINE transfert_message_mpi 768 769 770 SUBROUTINE send_message_mpi(field,message) 771 USE field_mod 772 USE domain_mod 773 USE mpi_mod 774 USE mpipara 775 USE omp_para 776 USE trace 777 IMPLICIT NONE 778 TYPE(t_field),POINTER :: field(:) 779 TYPE(t_message) :: message 780 REAL(rstd),POINTER :: rval2d(:) 781 REAL(rstd),POINTER :: rval3d(:,:) 782 REAL(rstd),POINTER :: rval4d(:,:,:) 783 REAL(rstd),POINTER :: buffer_r2(:) 784 REAL(rstd),POINTER :: buffer_r3(:,:) 785 REAL(rstd),POINTER :: buffer_r4(:,:,:) 786 INTEGER,POINTER :: value(:) 787 INTEGER,POINTER :: sgn(:) 788 TYPE(ARRAY),POINTER :: recv,send 789 TYPE(t_request),POINTER :: req 790 INTEGER, ALLOCATABLE :: mpi_req(:) 791 INTEGER, ALLOCATABLE :: status(:,:) 792 INTEGER :: irecv,isend 793 INTEGER :: ireq,nreq 794 INTEGER :: ind,n,l,m 795 INTEGER :: dim3,dim4 796 797 !$OMP BARRIER 798 799 CALL trace_start("transfert_mpi") 800 801 nreq=message%nreq 802 message%field=>field 803 804 !$OMP MASTER 805 message%completed=.FALSE. 806 message%pending=.TRUE. 807 !$OMP END MASTER 808 809 IF (field(1)%data_type==type_real) THEN 810 IF (field(1)%ndim==2) THEN 811 812 ireq=0 813 DO ind=1,ndomain 814 rval2d=>field(ind)%rval2d 815 816 req=>message%request(ind) 817 DO isend=1,req%nsend 818 ireq=ireq+1 819 send=>req%send(isend) 820 buffer_r2=>message%buffers(ireq)%r2 821 value=>send%value 822 823 CALL trace_in 824 825 !$OMP DO SCHEDULE(STATIC) 826 DO n=1,send%size 827 buffer_r2(n)=rval2d(value(n)) 828 ENDDO 829 830 CALL trace_out 831 832 !$OMP MASTER 833 CALL MPI_ISSEND(buffer_r2,send%size,MPI_REAL8,send%rank,ind,comm_icosa, message%mpi_req(ireq),ierr) 834 !$OMP END MASTER 835 ENDDO 836 837 DO irecv=1,req%nrecv 838 ireq=ireq+1 839 recv=>req%recv(irecv) 840 buffer_r2=>message%buffers(ireq)%r2 841 !$OMP MASTER 842 CALL MPI_IRECV(buffer_r2,recv%size,MPI_REAL8,recv%rank,recv%domain,comm_icosa, message%mpi_req(ireq),ierr) 843 !$OMP END MASTER 844 ENDDO 845 846 ENDDO 847 848 ELSE IF (field(1)%ndim==3) THEN 849 850 ireq=0 851 DO ind=1,ndomain 852 dim3=size(field(ind)%rval3d,2) 853 rval3d=>field(ind)%rval3d 854 req=>message%request(ind) 855 856 DO isend=1,req%nsend 857 ireq=ireq+1 858 send=>req%send(isend) 859 buffer_r3=>message%buffers(ireq)%r3 860 value=>send%value 861 862 CALL trace_in 863 864 !$OMP DO SCHEDULE(STATIC) 865 DO n=1,send%size 866 buffer_r3(n,:)=rval3d(value(n),:) 867 ENDDO 868 869 CALL trace_out 870 871 !$OMP MASTER 872 CALL MPI_ISSEND(buffer_r3,send%size*dim3,MPI_REAL8,send%rank,ind,comm_icosa, message%mpi_req(ireq),ierr) 873 !$OMP END MASTER 874 ENDDO 875 876 DO irecv=1,req%nrecv 877 ireq=ireq+1 878 recv=>req%recv(irecv) 879 buffer_r3=>message%buffers(ireq)%r3 880 !$OMP MASTER 881 CALL MPI_IRECV(buffer_r3,recv%size*dim3,MPI_REAL8,recv%rank,recv%domain,comm_icosa, message%mpi_req(ireq),ierr) 882 !$OMP END MASTER 883 884 ENDDO 885 886 ENDDO 887 888 ELSE IF (field(1)%ndim==4) THEN 889 890 ireq=0 891 DO ind=1,ndomain 892 dim3=size(field(ind)%rval4d,2) 893 dim4=size(field(ind)%rval4d,3) 894 rval4d=>field(ind)%rval4d 895 req=>message%request(ind) 896 897 DO isend=1,req%nsend 898 ireq=ireq+1 899 send=>req%send(isend) 900 buffer_r4=>message%buffers(ireq)%r4 901 value=>send%value 902 903 CALL trace_in 904 905 !$OMP DO SCHEDULE(STATIC) 906 DO n=1,send%size 907 buffer_r4(n,:,:)=rval4d(value(n),:,:) 908 ENDDO 909 910 CALL trace_out 911 912 !$OMP MASTER 913 CALL MPI_ISSEND(buffer_r4,send%size*dim3*dim4,MPI_REAL8,send%rank,ind,comm_icosa, message%mpi_req(ireq),ierr) 914 !$OMP END MASTER 915 ENDDO 916 917 DO irecv=1,req%nrecv 918 ireq=ireq+1 919 recv=>req%recv(irecv) 920 buffer_r4=>message%buffers(ireq)%r4 921 !$OMP MASTER 922 CALL MPI_IRECV(buffer_r4,recv%size*dim3*dim4,MPI_REAL8,recv%rank,recv%domain,comm_icosa, message%mpi_req(ireq),ierr) 923 !$OMP END MASTER 924 ENDDO 925 926 ENDDO 927 928 ENDIF 929 930 ENDIF 931 932 CALL trace_end("transfert_mpi") 933 !$OMP BARRIER 934 935 END SUBROUTINE send_message_mpi 936 937 SUBROUTINE test_message_mpi(message) 938 IMPLICIT NONE 939 TYPE(t_message) :: message 940 941 INTEGER :: ierr 942 !$OMP MASTER 943 IF (.NOT. message%pending) RETURN 944 !$OMP END MASTER 945 946 !$OMP MASTER 947 IF (.NOT. message%completed) CALL MPI_TESTALL(message%nreq,message%mpi_req,message%completed,message%status,ierr) 948 !$OMP END MASTER 949 END SUBROUTINE test_message_mpi 950 951 952 SUBROUTINE wait_message_mpi(message) 953 USE field_mod 954 USE domain_mod 955 USE mpi_mod 956 USE mpipara 957 USE omp_para 958 USE trace 959 IMPLICIT NONE 960 TYPE(t_message) :: message 961 962 TYPE(t_field),POINTER :: field(:) 963 REAL(rstd),POINTER :: rval2d(:) 964 REAL(rstd),POINTER :: rval3d(:,:) 965 REAL(rstd),POINTER :: rval4d(:,:,:) 966 REAL(rstd),POINTER :: buffer_r2(:) 967 REAL(rstd),POINTER :: buffer_r3(:,:) 968 REAL(rstd),POINTER :: buffer_r4(:,:,:) 969 INTEGER,POINTER :: value(:) 970 INTEGER,POINTER :: sgn(:) 971 TYPE(ARRAY),POINTER :: recv,send 972 TYPE(t_request),POINTER :: req 973 INTEGER, ALLOCATABLE :: mpi_req(:) 974 INTEGER, ALLOCATABLE :: status(:,:) 975 INTEGER :: irecv,isend 976 INTEGER :: ireq,nreq 977 INTEGER :: ind,n,l,m 978 INTEGER :: dim3,dim4 979 980 !$OMP BARRIER 981 982 CALL trace_start("transfert_mpi") 983 984 IF (.NOT. message%pending) RETURN 985 986 field=>message%field 987 nreq=message%nreq 988 989 IF (field(1)%data_type==type_real) THEN 990 IF (field(1)%ndim==2) THEN 991 992 !$OMP MASTER 993 IF (.NOT. message%completed) CALL MPI_WAITALL(nreq,message%mpi_req,message%status,ierr) 994 !$OMP END MASTER 995 !$OMP BARRIER 996 997 ireq=0 998 DO ind=1,ndomain 999 rval2d=>field(ind)%rval2d 1000 req=>message%request(ind) 1001 1002 DO isend=1,req%nsend 1003 ireq=ireq+1 1004 ENDDO 1005 1006 DO irecv=1,req%nrecv 1007 ireq=ireq+1 1008 recv=>req%recv(irecv) 1009 buffer_r2=>message%buffers(ireq)%r2 1010 value=>recv%value 1011 sgn=>recv%sign 1012 1013 CALL trace_in 1014 1015 !$OMP DO SCHEDULE(STATIC) 1016 DO n=1,recv%size 1017 rval2d(value(n))=buffer_r2(n)*sgn(n) 1018 ENDDO 1019 1020 CALL trace_out 1021 1022 ENDDO 1023 1024 ENDDO 1025 1026 1027 ELSE IF (field(1)%ndim==3) THEN 1028 1029 !$OMP MASTER 1030 IF (.NOT. message%completed) CALL MPI_WAITALL(nreq,message%mpi_req,message%status,ierr) 1031 !$OMP END MASTER 1032 !$OMP BARRIER 1033 1034 ireq=0 1035 DO ind=1,ndomain 1036 rval3d=>field(ind)%rval3d 1037 req=>message%request(ind) 1038 1039 DO isend=1,req%nsend 1040 ireq=ireq+1 1041 ENDDO 1042 1043 DO irecv=1,req%nrecv 1044 ireq=ireq+1 1045 recv=>req%recv(irecv) 1046 buffer_r3=>message%buffers(ireq)%r3 1047 value=>recv%value 1048 sgn=>recv%sign 1049 1050 CALL trace_in 1051 1052 !$OMP DO SCHEDULE(STATIC) 1053 DO n=1,recv%size 1054 rval3d(value(n),:)=buffer_r3(n,:)*sgn(n) 1055 ENDDO 1056 1057 CALL trace_out 1058 1059 ENDDO 1060 1061 ENDDO 1062 1063 ELSE IF (field(1)%ndim==4) THEN 1064 !$OMP MASTER 1065 IF (.NOT. message%completed) CALL MPI_WAITALL(nreq,message%mpi_req,message%status,ierr) 1066 !$OMP END MASTER 1067 !$OMP BARRIER 1068 1069 ireq=0 1070 DO ind=1,ndomain 1071 rval4d=>field(ind)%rval4d 1072 req=>message%request(ind) 1073 1074 DO isend=1,req%nsend 1075 ireq=ireq+1 1076 ENDDO 1077 1078 DO irecv=1,req%nrecv 1079 ireq=ireq+1 1080 recv=>req%recv(irecv) 1081 buffer_r4=>message%buffers(ireq)%r4 1082 value=>recv%value 1083 sgn=>recv%sign 1084 1085 CALL trace_in 1086 1087 !$OMP DO SCHEDULE(STATIC) 1088 DO n=1,recv%size 1089 rval4d(value(n),:,:)=buffer_r4(n,:,:)*sgn(n) 1090 ENDDO 1091 1092 CALL trace_out 1093 1094 ENDDO 1095 1096 ENDDO 1097 1098 ENDIF 1099 1100 ENDIF 1101 1102 !$OMP MASTER 1103 message%pending=.FALSE. 1104 !$OMP END MASTER 1105 1106 CALL trace_end("transfert_mpi") 1107 !$OMP BARRIER 1108 1109 END SUBROUTINE wait_message_mpi 1110 1111 612 1112 SUBROUTINE transfert_request_mpi(field,request) 613 1113 USE field_mod … … 637 1137 638 1138 CALL trace_start("transfert_mpi") 639 1139 640 1140 IF (field(1)%data_type==type_real) THEN 641 1141 IF (field(1)%ndim==2) THEN … … 675 1175 676 1176 CALL MPI_WAITALL(nreq,mpi_req,status,ierr) 677 ! DO ind=1,ndomain678 ! field(ind)%rval2d(:)=0.679 ! ENDDO680 1177 681 1178 DO ind=1,ndomain … … 739 1236 740 1237 CALL MPI_WAITALL(nreq,mpi_req,status,ierr) 741 ! DO ind=1,ndomain742 ! field(ind)%rval2d(:)=0.743 ! ENDDO744 1238 745 1239 DO ind=1,ndomain … … 803 1297 804 1298 CALL MPI_WAITALL(nreq,mpi_req,status,ierr) 805 ! DO ind=1,ndomain806 ! field(ind)%rval2d(:)=0.807 ! ENDDO808 1299 809 1300 DO ind=1,ndomain … … 837 1328 END SUBROUTINE transfert_request_mpi 838 1329 839 SUBROUTINE transfert_request (field,request)1330 SUBROUTINE transfert_request_seq(field,request) 840 1331 USE field_mod 841 1332 USE domain_mod … … 875 1366 ENDDO 876 1367 877 END SUBROUTINE transfert_request 1368 END SUBROUTINE transfert_request_seq 878 1369 879 1370 … … 949 1440 950 1441 END SUBROUTINE gather_field 951 1442 1443 1444 SUBROUTINE trace_in 1445 USE trace 1446 IMPLICIT NONE 1447 1448 CALL trace_start("transfert_buffer") 1449 END SUBROUTINE trace_in 1450 1451 SUBROUTINE trace_out 1452 USE trace 1453 IMPLICIT NONE 1454 1455 CALL trace_end("transfert_buffer") 1456 END SUBROUTINE trace_out 952 1457 953 1458 END MODULE transfert_mpi_mod
Note: See TracChangeset
for help on using the changeset viewer.