source: XIOS/dev/branch_openmp/Note/EP note.lyx @ 1620

Last change on this file since 1620 was 1548, checked in by yushan, 6 years ago

add documents for ep_lib

File size: 55.5 KB
Line 
1#LyX file created by tex2lyx 2.3
2\lyxformat 544
3\begin_document
4\begin_header
5\save_transient_properties true
6\origin /homel/ywang/Documents/MPI_Endpoints/Note/
7\textclass article
8\begin_preamble
9\usepackage{listings}
10\usepackage[usenames,dvipsnames,svgnames,table]{xcolor}
11
12
13% Title Page
14\title{Note for MPI Endpoints}
15\author{}
16
17
18
19\end_preamble
20\use_default_options false
21\maintain_unincluded_children false
22\language english
23\language_package none
24\inputencoding utf8
25\fontencoding default
26\font_roman "default" "default"
27\font_sans "default" "default"
28\font_typewriter "default" "default"
29\font_math "auto" "auto"
30\font_default_family default
31\use_non_tex_fonts false
32\font_sc false
33\font_osf false
34\font_sf_scale 100 100
35\font_tt_scale 100 100
36\use_microtype false
37\use_dash_ligatures true
38\graphics default
39\default_output_format default
40\output_sync 0
41\bibtex_command default
42\index_command default
43\paperfontsize 10
44\spacing single
45\use_hyperref false
46\papersize a4paper
47\use_geometry false
48\use_package amsmath 2
49\use_package amssymb 0
50\use_package cancel 0
51\use_package esint 1
52\use_package mathdots 0
53\use_package mathtools 0
54\use_package mhchem 0
55\use_package stackrel 0
56\use_package stmaryrd 0
57\use_package undertilde 0
58\cite_engine basic
59\cite_engine_type default
60\biblio_style plain
61\use_bibtopic false
62\use_indices false
63\paperorientation portrait
64\suppress_date false
65\justification true
66\use_refstyle 0
67\use_minted 0
68\index Index
69\shortcut idx
70\color #008000
71\end_index
72\secnumdepth 3
73\tocdepth 3
74\paragraph_separation indent
75\paragraph_indentation default
76\is_math_indent 0
77\math_numbering_side default
78\quotes_style english
79\dynamic_quotes 0
80\papercolumns 1
81\papersides 1
82\paperpagestyle default
83\tracking_changes false
84\output_changes false
85\html_math_output 0
86\html_css_as_file 0
87\html_be_strict false
88\end_header
89
90\begin_body
91
92\begin_layout Standard
93
94\begin_inset ERT
95status collapsed
96
97\begin_layout Plain Layout
98
99\backslash
100maketitle
101\end_layout
102
103\end_inset
104
105
106\end_layout
107
108\begin_layout Standard
109
110\begin_inset ERT
111status collapsed
112
113\begin_layout Plain Layout
114%
115\backslash
116begin{abstract}
117\end_layout
118
119\begin_layout Plain Layout
120
121\end_layout
122
123\end_inset
124
125
126\begin_inset ERT
127status collapsed
128
129\begin_layout Plain Layout
130%
131\backslash
132end{abstract}
133\end_layout
134
135\end_inset
136
137
138\end_layout
139
140\begin_layout Section
141Purpose
142\end_layout
143
144\begin_layout Standard
145Use threads as if they are MPI processes. Each thread will be assigned a rank and be associated with a endpoints communicator (EP_Comm). Convention: one OpenMP thread corresponds to one endpoint.
146\end_layout
147
148\begin_layout Section
149MPI Endpoints Semantics
150\end_layout
151
152\begin_layout Standard
153\align center
154
155\begin_inset Graphics
156        filename scheme.png
157        scale 40
158
159\end_inset
160
161 
162\end_layout
163
164\begin_layout Standard
165Endpoints are created from one MPI communicator and the number of available threads:
166\end_layout
167
168\begin_layout Verbatim
169int MPI_Comm_create_endpoints(MPI_Comm parent_comm, int num_ep,
170\end_layout
171
172\begin_layout Verbatim
173                              MPI_Info info, MPI_Comm out_comm_hdls[])
174\end_layout
175
176\begin_layout Verbatim
177 
178\end_layout
179
180\begin_layout Standard
181
182\begin_inset Quotes eld
183\end_inset
184
185In this collective call, a single output communicator is created, and an array of
186\begin_inset ERT
187status collapsed
188
189\begin_layout Plain Layout
190
191\backslash
192verb|num_ep|
193\end_layout
194
195\end_inset
196
197 handles to this new communicator are returned, where the
198\begin_inset Formula $i^{th}$
199\end_inset
200
201 handle corresponds to the
202\begin_inset Formula $i^{th}$
203\end_inset
204
205 rank requested by the caller of
206\begin_inset ERT
207status collapsed
208
209\begin_layout Plain Layout
210
211\backslash
212verb|MPI_Comm_create_endpoints|
213\end_layout
214
215\end_inset
216
217. Ranks in the output communicator are ordered sequentially and in the same order as the parent communicator. After it has been created, the output communicator behaves as a normal communicator, and MPI calls on each endpoint (i.e., communicator handle) behave as though they originated from a separate MPI process. In particular, collective calls must be made once per endpoint.
218\begin_inset Quotes erd
219\end_inset
220
221
222\begin_inset CommandInset citation
223LatexCommand cite
224after ""
225key "Dinan:2013"
226literal "true"
227
228\end_inset
229
230
231\end_layout
232
233\begin_layout Standard
234
235\begin_inset Quotes eld
236\end_inset
237
238Once created, endpoints behave as MPI processes. For example, all ranks in an endpoints communicator must participate in collective operations. A consequence of this semantic is that endpoints also have MPI process progress requirements; that operations on that endpoint are required to make progress only when an MPI operation (e.g.
239\begin_inset ERT
240status collapsed
241
242\begin_layout Plain Layout
243
244\backslash
245verb|MPI_Test|
246\end_layout
247
248\end_inset
249
250) is performed on that endpoint. This semantic enables an MPI implementation to logically separate endpoints, treat them independently within the progress engine, and eliminate synchronization in updating their state.
251\begin_inset Quotes erd
252\end_inset
253
254
255\begin_inset CommandInset citation
256LatexCommand cite
257after ""
258key "Sridharan:2014"
259literal "true"
260
261\end_inset
262
263
264\end_layout
265
266\begin_layout Section
267EP types
268\end_layout
269
270\begin_layout Subsection*
271MPI_Comm
272\end_layout
273
274\begin_layout Standard
275
276\begin_inset ERT
277status collapsed
278
279\begin_layout Plain Layout
280
281\backslash
282verb|MPI_Comm|
283\end_layout
284
285\end_inset
286
287 is composed by:
288\end_layout
289
290\begin_layout Itemize
291
292\begin_inset Argument item:1
293status open
294
295\begin_layout Plain Layout
296
297\begin_inset Formula $\bullet$
298\end_inset
299
300
301\end_layout
302
303\end_inset
304
305
306\begin_inset ERT
307status collapsed
308
309\begin_layout Plain Layout
310
311\backslash
312verb|bool is_ep|
313\end_layout
314
315\end_inset
316
317: true
318\begin_inset Formula $\implies$
319\end_inset
320
321 EP, false
322\begin_inset Formula $\implies$
323\end_inset
324
325 MPI classic;
326\end_layout
327
328\begin_layout Itemize
329
330\begin_inset Argument item:1
331status open
332
333\begin_layout Plain Layout
334
335\begin_inset Formula $\bullet$
336\end_inset
337
338
339\end_layout
340
341\end_inset
342
343
344\begin_inset ERT
345status collapsed
346
347\begin_layout Plain Layout
348
349\backslash
350verb|int mpi_comm|
351\end_layout
352
353\end_inset
354
355: handle to the parent MPI communicator;
356\end_layout
357
358\begin_layout Itemize
359
360\begin_inset Argument item:1
361status open
362
363\begin_layout Plain Layout
364
365\begin_inset Formula $\bullet$
366\end_inset
367
368
369\end_layout
370
371\end_inset
372
373
374\begin_inset ERT
375status collapsed
376
377\begin_layout Plain Layout
378
379\backslash
380verb|OMPbarrier *ep_barrier|
381\end_layout
382
383\end_inset
384
385: openMP barrier, used for in-process synchronization and is different from
386\begin_inset ERT
387status collapsed
388
389\begin_layout Plain Layout
390
391\backslash
392verb|omp barrier|
393\end_layout
394
395\end_inset
396
397;
398\end_layout
399
400\begin_layout Itemize
401
402\begin_inset Argument item:1
403status open
404
405\begin_layout Plain Layout
406
407\begin_inset Formula $\bullet$
408\end_inset
409
410
411\end_layout
412
413\end_inset
414
415
416\begin_inset ERT
417status collapsed
418
419\begin_layout Plain Layout
420
421\backslash
422verb|int[2] size_rank_info[3]|
423\end_layout
424
425\end_inset
426
427: topology information of the current endpoint:
428\end_layout
429
430\begin_deeper
431\begin_layout Itemize
432rank of parent MPI process;
433\end_layout
434
435\begin_layout Itemize
436size of parent MPI communicator;
437\end_layout
438
439\begin_layout Itemize
440rank of endpoint, returned by
441\begin_inset ERT
442status collapsed
443
444\begin_layout Plain Layout
445
446\backslash
447verb|MPI_Comm_rank|
448\end_layout
449
450\end_inset
451
452;
453\end_layout
454
455\begin_layout Itemize
456size of EP communicator, returned by
457\begin_inset ERT
458status collapsed
459
460\begin_layout Plain Layout
461
462\backslash
463verb|MPI_Comm_size|
464\end_layout
465
466\end_inset
467
468;
469\end_layout
470
471\begin_layout Itemize
472in-process rank of endpoint;
473\end_layout
474
475\begin_layout Itemize
476in-process size of EP communicator, also noted as the number of endpoints in one MPI process.
477\end_layout
478
479\end_deeper
480\begin_layout Itemize
481
482\begin_inset Argument item:1
483status open
484
485\begin_layout Plain Layout
486
487\begin_inset Formula $\bullet$
488\end_inset
489
490
491\end_layout
492
493\end_inset
494
495
496\begin_inset ERT
497status collapsed
498
499\begin_layout Plain Layout
500
501\backslash
502verb|MPI_Comm *comm_list|
503\end_layout
504
505\end_inset
506
507: pointer of the first endpoint communicator of one process;
508\end_layout
509
510\begin_layout Itemize
511
512\begin_inset Argument item:1
513status open
514
515\begin_layout Plain Layout
516
517\begin_inset Formula $\bullet$
518\end_inset
519
520
521\end_layout
522
523\end_inset
524
525
526\begin_inset ERT
527status collapsed
528
529\begin_layout Plain Layout
530
531\backslash
532verb|Message_list *message_queue|
533\end_layout
534
535\end_inset
536
537: location of in-coming messages for each endpoint;
538\end_layout
539
540\begin_layout Itemize
541
542\begin_inset Argument item:1
543status open
544
545\begin_layout Plain Layout
546
547\begin_inset Formula $\bullet$
548\end_inset
549
550
551\end_layout
552
553\end_inset
554
555
556\begin_inset ERT
557status collapsed
558
559\begin_layout Plain Layout
560
561\backslash
562verb|RANK_MAP *rank_map|
563\end_layout
564
565\end_inset
566
567: a map composed by an integer and a pair of integers. The integer key represents the rank of an endpoint. The mapped type (pair of integers) gives the in-process rank of the endpoint and the rank of its parent MPI process:
568\end_layout
569
570\begin_layout Verbatim
571rank_map->at(ep_rank)=(ep_rank_local, mpi_rank)
572\end_layout
573
574\begin_layout Verbatim
575 
576\end_layout
577
578\begin_layout Itemize
579
580\begin_inset Argument item:1
581status open
582
583\begin_layout Plain Layout
584
585\begin_inset Formula $\bullet$
586\end_inset
587
588
589\end_layout
590
591\end_inset
592
593
594\begin_inset ERT
595status collapsed
596
597\begin_layout Plain Layout
598
599\backslash
600verb|BUFFER *ep_buffer|
601\end_layout
602
603\end_inset
604
605: buffer (of type
606\begin_inset ERT
607status collapsed
608
609\begin_layout Plain Layout
610
611\backslash
612verb|int|
613\end_layout
614
615\end_inset
616
617,
618\begin_inset ERT
619status collapsed
620
621\begin_layout Plain Layout
622
623\backslash
624verb|float|
625\end_layout
626
627\end_inset
628
629,
630\begin_inset ERT
631status collapsed
632
633\begin_layout Plain Layout
634
635\backslash
636verb|double|
637\end_layout
638
639\end_inset
640
641,
642\begin_inset ERT
643status collapsed
644
645\begin_layout Plain Layout
646
647\backslash
648verb|char|
649\end_layout
650
651\end_inset
652
653,
654\begin_inset ERT
655status collapsed
656
657\begin_layout Plain Layout
658
659\backslash
660verb|long|
661\end_layout
662
663\end_inset
664
665, and
666\begin_inset ERT
667status collapsed
668
669\begin_layout Plain Layout
670
671\backslash
672verb|unsigned long|
673\end_layout
674
675\end_inset
676
677) used for in-process communication.
678\end_layout
679
680\begin_layout Subsection
681MPI_Request
682\end_layout
683
684\begin_layout Standard
685
686\begin_inset ERT
687status collapsed
688
689\begin_layout Plain Layout
690
691\backslash
692verb|MPI_Request|
693\end_layout
694
695\end_inset
696
697 is composed by:
698\end_layout
699
700\begin_layout Itemize
701
702\begin_inset Argument item:1
703status open
704
705\begin_layout Plain Layout
706
707\begin_inset Formula $\bullet$
708\end_inset
709
710
711\end_layout
712
713\end_inset
714
715
716\begin_inset ERT
717status collapsed
718
719\begin_layout Plain Layout
720
721\backslash
722verb|int mpi_request|
723\end_layout
724
725\end_inset
726
727: handle to the MPI request;
728\end_layout
729
730\begin_layout Itemize
731
732\begin_inset Argument item:1
733status open
734
735\begin_layout Plain Layout
736
737\begin_inset Formula $\bullet$
738\end_inset
739
740
741\end_layout
742
743\end_inset
744
745
746\begin_inset ERT
747status collapsed
748
749\begin_layout Plain Layout
750
751\backslash
752verb|int ep_datatype|
753\end_layout
754
755\end_inset
756
757: data type of the communication;
758\end_layout
759
760\begin_layout Itemize
761
762\begin_inset Argument item:1
763status open
764
765\begin_layout Plain Layout
766
767\begin_inset Formula $\bullet$
768\end_inset
769
770
771\end_layout
772
773\end_inset
774
775
776\begin_inset ERT
777status collapsed
778
779\begin_layout Plain Layout
780
781\backslash
782verb|MPI_Comm comm|
783\end_layout
784
785\end_inset
786
787: handle to the EP communicator;
788\end_layout
789
790\begin_layout Itemize
791
792\begin_inset Argument item:1
793status open
794
795\begin_layout Plain Layout
796
797\begin_inset Formula $\bullet$
798\end_inset
799
800
801\end_layout
802
803\end_inset
804
805
806\begin_inset ERT
807status collapsed
808
809\begin_layout Plain Layout
810
811\backslash
812verb|int ep_src|
813\end_layout
814
815\end_inset
816
817: rank of the source endpoint;
818\end_layout
819
820\begin_layout Itemize
821
822\begin_inset Argument item:1
823status open
824
825\begin_layout Plain Layout
826
827\begin_inset Formula $\bullet$
828\end_inset
829
830
831\end_layout
832
833\end_inset
834
835
836\begin_inset ERT
837status collapsed
838
839\begin_layout Plain Layout
840
841\backslash
842verb|int ep_tag|
843\end_layout
844
845\end_inset
846
847: tag of the communication.
848\end_layout
849
850\begin_layout Itemize
851
852\begin_inset Argument item:1
853status open
854
855\begin_layout Plain Layout
856
857\begin_inset Formula $\bullet$
858\end_inset
859
860
861\end_layout
862
863\end_inset
864
865
866\begin_inset ERT
867status collapsed
868
869\begin_layout Plain Layout
870
871\backslash
872verb|int type|
873\end_layout
874
875\end_inset
876
877: type of the communication:
878\end_layout
879
880\begin_deeper
881\begin_layout Itemize
8821
883\begin_inset Formula $\implies$
884\end_inset
885
886 non-blocking send;
887\end_layout
888
889\begin_layout Itemize
8902
891\begin_inset Formula $\implies$
892\end_inset
893
894 pending non-blocking receive;
895\end_layout
896
897\begin_layout Itemize
8983
899\begin_inset Formula $\implies$
900\end_inset
901
902 non-blocking matching receive.
903\end_layout
904
905\end_deeper
906\begin_layout Subsection
907MPI_Status
908\end_layout
909
910\begin_layout Standard
911
912\begin_inset ERT
913status collapsed
914
915\begin_layout Plain Layout
916
917\backslash
918verb|MPI_Status|
919\end_layout
920
921\end_inset
922
923 consists of:
924\end_layout
925
926\begin_layout Itemize
927
928\begin_inset Argument item:1
929status open
930
931\begin_layout Plain Layout
932
933\begin_inset Formula $\bullet$
934\end_inset
935
936
937\end_layout
938
939\end_inset
940
941
942\begin_inset ERT
943status collapsed
944
945\begin_layout Plain Layout
946
947\backslash
948verb|int mpi_status|
949\end_layout
950
951\end_inset
952
953: handle to the MPI status;
954\end_layout
955
956\begin_layout Itemize
957
958\begin_inset Argument item:1
959status open
960
961\begin_layout Plain Layout
962
963\begin_inset Formula $\bullet$
964\end_inset
965
966
967\end_layout
968
969\end_inset
970
971
972\begin_inset ERT
973status collapsed
974
975\begin_layout Plain Layout
976
977\backslash
978verb|int ep_datatype|
979\end_layout
980
981\end_inset
982
983: data type of the communication;
984\end_layout
985
986\begin_layout Itemize
987
988\begin_inset Argument item:1
989status open
990
991\begin_layout Plain Layout
992
993\begin_inset Formula $\bullet$
994\end_inset
995
996
997\end_layout
998
999\end_inset
1000
1001
1002\begin_inset ERT
1003status collapsed
1004
1005\begin_layout Plain Layout
1006
1007\backslash
1008verb|int ep_src|
1009\end_layout
1010
1011\end_inset
1012
1013: rank of the source endpoint;
1014\end_layout
1015
1016\begin_layout Itemize
1017
1018\begin_inset Argument item:1
1019status open
1020
1021\begin_layout Plain Layout
1022
1023\begin_inset Formula $\bullet$
1024\end_inset
1025
1026
1027\end_layout
1028
1029\end_inset
1030
1031
1032\begin_inset ERT
1033status collapsed
1034
1035\begin_layout Plain Layout
1036
1037\backslash
1038verb|int ep_tag|
1039\end_layout
1040
1041\end_inset
1042
1043: tag of the communication.
1044\end_layout
1045
1046\begin_layout Subsection
1047MPI_Message
1048\end_layout
1049
1050\begin_layout Standard
1051
1052\begin_inset ERT
1053status collapsed
1054
1055\begin_layout Plain Layout
1056
1057\backslash
1058verb|MPI_Message|
1059\end_layout
1060
1061\end_inset
1062
1063 includes:
1064\end_layout
1065
1066\begin_layout Itemize
1067
1068\begin_inset Argument item:1
1069status open
1070
1071\begin_layout Plain Layout
1072
1073\begin_inset Formula $\bullet$
1074\end_inset
1075
1076
1077\end_layout
1078
1079\end_inset
1080
1081
1082\begin_inset ERT
1083status collapsed
1084
1085\begin_layout Plain Layout
1086
1087\backslash
1088verb|int mpi_message|
1089\end_layout
1090
1091\end_inset
1092
1093: handle to the MPI message;
1094\end_layout
1095
1096\begin_layout Itemize
1097
1098\begin_inset Argument item:1
1099status open
1100
1101\begin_layout Plain Layout
1102
1103\begin_inset Formula $\bullet$
1104\end_inset
1105
1106
1107\end_layout
1108
1109\end_inset
1110
1111
1112\begin_inset ERT
1113status collapsed
1114
1115\begin_layout Plain Layout
1116
1117\backslash
1118verb|int ep_src|
1119\end_layout
1120
1121\end_inset
1122
1123: rank of the source endpoint;
1124\end_layout
1125
1126\begin_layout Itemize
1127
1128\begin_inset Argument item:1
1129status open
1130
1131\begin_layout Plain Layout
1132
1133\begin_inset Formula $\bullet$
1134\end_inset
1135
1136
1137\end_layout
1138
1139\end_inset
1140
1141
1142\begin_inset ERT
1143status collapsed
1144
1145\begin_layout Plain Layout
1146
1147\backslash
1148verb|int ep_tag|
1149\end_layout
1150
1151\end_inset
1152
1153: tag of the communication.
1154\end_layout
1155
1156\begin_layout Standard
1157Other types, such as
1158\begin_inset ERT
1159status collapsed
1160
1161\begin_layout Plain Layout
1162
1163\backslash
1164verb|MPI_Info|
1165\end_layout
1166
1167\end_inset
1168
1169,
1170\begin_inset ERT
1171status collapsed
1172
1173\begin_layout Plain Layout
1174
1175\backslash
1176verb|MPI_Aint|
1177\end_layout
1178
1179\end_inset
1180
1181, and
1182\begin_inset ERT
1183status collapsed
1184
1185\begin_layout Plain Layout
1186
1187\backslash
1188verb|MPI_Fint|
1189\end_layout
1190
1191\end_inset
1192
1193 are defined in the same way.
1194\end_layout
1195
1196\begin_layout Section
1197P2P communication
1198\end_layout
1199
1200\begin_layout Standard
1201All EP point-to-point communication use tag to distinguish the source and destination endpoint. To be able to add these extra information to tag, we require that the tag value is represented using 31 bits in the underlying MPI inmplemention.
1202\end_layout
1203
1204\begin_layout Standard
1205
1206\begin_inset Graphics
1207        filename tag.png
1208        scale 40
1209
1210\end_inset
1211
1212
1213\end_layout
1214
1215\begin_layout Standard
1216EP_tag is user defined. MPI_tag is internally computed and used inside MPI calls. Because of the extension of tag, wild-cards as
1217\begin_inset ERT
1218status collapsed
1219
1220\begin_layout Plain Layout
1221
1222\backslash
1223verb|MPI_ANY_SOURCE|
1224\end_layout
1225
1226\end_inset
1227
1228 and
1229\begin_inset ERT
1230status collapsed
1231
1232\begin_layout Plain Layout
1233
1234\backslash
1235verb|MPI_ANY_TAG|
1236\end_layout
1237
1238\end_inset
1239
1240 will not be usable directly. An extra step of tag analysis is needed which leads to the message dequeuing mechanism.
1241\end_layout
1242
1243\begin_layout Standard
1244\align center
1245
1246\begin_inset Graphics
1247        filename sendrecv.png
1248        scale 50
1249
1250\end_inset
1251
1252 
1253\end_layout
1254
1255\begin_layout Standard
1256In MPI environment, each MPI process has an incoming message queue. In EP case, messages for all threads inside one MPI process are stored in this MPI queue. With the MPI 3 standard, we use the
1257\begin_inset ERT
1258status collapsed
1259
1260\begin_layout Plain Layout
1261
1262\backslash
1263verb|MPI_Improbe|
1264\end_layout
1265
1266\end_inset
1267
1268 routine to inquire the message queue and relocate the incoming message in the local message queue for the corresponding thread/endpoint.
1269\end_layout
1270
1271\begin_layout Standard
1272
1273\begin_inset Graphics
1274        filename dequeue.png
1275        scale 30
1276
1277\end_inset
1278
1279
1280\end_layout
1281
1282\begin_layout Standard
1283
1284\begin_inset ERT
1285status collapsed
1286
1287\begin_layout Plain Layout
1288% Any EP calls will trigger the message dequeuing and the probing, (matched-)receiving operations are performed upon the local message queue.
1289\end_layout
1290
1291\begin_layout Plain Layout
1292
1293\end_layout
1294
1295\end_inset
1296
1297
1298\begin_inset ERT
1299status collapsed
1300
1301\begin_layout Plain Layout
1302%
1303\end_layout
1304
1305\begin_layout Plain Layout
1306
1307\end_layout
1308
1309\end_inset
1310
1311
1312\begin_inset ERT
1313status collapsed
1314
1315\begin_layout Plain Layout
1316% Example:
1317\backslash
1318verb|EP_Recv(src=2, tag=10, comm1)|:
1319\end_layout
1320
1321\begin_layout Plain Layout
1322
1323\end_layout
1324
1325\end_inset
1326
1327
1328\begin_inset ERT
1329status collapsed
1330
1331\begin_layout Plain Layout
1332%
1333\backslash
1334begin{itemize}
1335\end_layout
1336
1337\begin_layout Plain Layout
1338
1339\end_layout
1340
1341\end_inset
1342
1343
1344\begin_inset ERT
1345status collapsed
1346
1347\begin_layout Plain Layout
1348
1349\backslash
1350item[1.] Dequeue MPI message queue;
1351\end_layout
1352
1353\begin_layout Plain Layout
1354
1355\end_layout
1356
1357\end_inset
1358
1359
1360\begin_inset ERT
1361status collapsed
1362
1363\begin_layout Plain Layout
1364
1365\backslash
1366item[2.] call
1367\backslash
1368verb|EP_Improb(src=2, tag=10, comm1, message)|;
1369\end_layout
1370
1371\begin_layout Plain Layout
1372
1373\end_layout
1374
1375\end_inset
1376
1377
1378\begin_inset ERT
1379status collapsed
1380
1381\begin_layout Plain Layout
1382
1383\backslash
1384item[3.] if find corresponding triple (src, tag, comm1), call
1385\backslash
1386verb|EP_Mrecv(src=2, tag=10, comm1, message)|;
1387\end_layout
1388
1389\begin_layout Plain Layout
1390
1391\end_layout
1392
1393\end_inset
1394
1395
1396\begin_inset ERT
1397status collapsed
1398
1399\begin_layout Plain Layout
1400
1401\backslash
1402item[4.] else, repeat from step 2.
1403\end_layout
1404
1405\begin_layout Plain Layout
1406
1407\end_layout
1408
1409\end_inset
1410
1411
1412\begin_inset ERT
1413status collapsed
1414
1415\begin_layout Plain Layout
1416%
1417\backslash
1418end{itemize}
1419\end_layout
1420
1421\end_inset
1422
1423
1424\end_layout
1425
1426\begin_layout Paragraph
1427Messages are
1428\shape italic
1429non-overtaking
1430\shape default
1431
1432\end_layout
1433
1434\begin_layout Standard
1435Incoming messages' order is important! If one thread is receiving multiple messages from the same source with the same tag. The receive order should be the same order in which the messages are sent. That is to say, the n-th sent message should be the n-th received message.
1436\end_layout
1437
1438\begin_layout Paragraph
1439Progress
1440\end_layout
1441
1442\begin_layout Standard
1443
1444\begin_inset Quotes eld
1445\end_inset
1446
1447If a pair of matching send and receives have been initiated on two processes, then at least one of these two operations will complete, independently of other actions in the system: the send operation will complete, unless the receive is satisfied by another message, and completes; the receive operation will complete, unless the message sent is consumed by another matching receive that was posted at the same destination process.
1448\begin_inset Quotes erd
1449\end_inset
1450
1451 
1452\begin_inset CommandInset citation
1453LatexCommand cite
1454after ""
1455key "MPI"
1456literal "true"
1457
1458\end_inset
1459
1460
1461\end_layout
1462
1463\begin_layout Standard
1464When one
1465\begin_inset ERT
1466status collapsed
1467
1468\begin_layout Plain Layout
1469
1470\backslash
1471verb|EP_Irecv|
1472\end_layout
1473
1474\end_inset
1475
1476 is issued, we first dequeue the MPI incoming message queue and distribute all incoming messages to the local queues according to the destination identifier. Next, the nonblocking receive request is added at the end of the request pending list. Third, the pending list is checked and requests with matching source, tag, and communicator will be accomplished.
1477\end_layout
1478
1479\begin_layout Standard
1480Because of the importance of message order, some communication completion functions must be discussed here such as
1481\begin_inset ERT
1482status collapsed
1483
1484\begin_layout Plain Layout
1485
1486\backslash
1487verb|MPI_Test|
1488\end_layout
1489
1490\end_inset
1491
1492 and
1493\begin_inset ERT
1494status collapsed
1495
1496\begin_layout Plain Layout
1497
1498\backslash
1499verb|MPI_Wait|
1500\end_layout
1501
1502\end_inset
1503
1504.
1505\begin_inset Quotes eld
1506\end_inset
1507
1508The functions
1509\begin_inset ERT
1510status collapsed
1511
1512\begin_layout Plain Layout
1513
1514\backslash
1515verb|MPI_Wait|
1516\end_layout
1517
1518\end_inset
1519
1520 and
1521\begin_inset ERT
1522status collapsed
1523
1524\begin_layout Plain Layout
1525
1526\backslash
1527verb|MPI_Test|
1528\end_layout
1529
1530\end_inset
1531
1532 are used to complete a nonblocking communication. The completion of a send operation indicates that the sender is now free to update the locations in the send buffer (the send operation itself leaves the content of the send buffer unchanged). It does not indicate that the message has been received, rather, it may have been buffered by the communication subsystem. However, if a synchronous mode send was used, the completion of the send operation indicates that a matching receive was initiated, and that the message will eventually be received by this matching receive. The completion of a receive operation indicates that the receive buffer contains the received message, the receiver is now free to access it, and that the status object is set. It does not indicate that the matching send operation has completed (but indicates, of course, that the send was initiated).
1533\begin_inset Quotes erd
1534\end_inset
1535
1536 
1537\begin_inset CommandInset citation
1538LatexCommand cite
1539after ""
1540key "MPI"
1541literal "true"
1542
1543\end_inset
1544
1545
1546\end_layout
1547
1548\begin_layout Paragraph
1549Example 1
1550\end_layout
1551
1552\begin_layout Standard
1553
1554\begin_inset ERT
1555status collapsed
1556
1557\begin_layout Plain Layout
1558
1559\backslash
1560verb|MPI_Test(MPI_Request *request, int *flag, MPI_Status *status)|
1561\end_layout
1562
1563\end_inset
1564
1565 
1566\end_layout
1567
1568\begin_layout Itemize
1569
1570\begin_inset Argument item:1
1571status open
1572
1573\begin_layout Plain Layout
15741.
1575\end_layout
1576
1577\end_inset
1578
1579If
1580\begin_inset ERT
1581status collapsed
1582
1583\begin_layout Plain Layout
1584
1585\backslash
1586verb|request->type == 1|
1587\end_layout
1588
1589\end_inset
1590
1591, communication to be tested is indeed issued from a non-blocking send. The completion status is returned by:
1592\end_layout
1593
1594\begin_layout Verbatim
1595   MPI_Test(& request->mpi_request, flag, & status->mpi_status)
1596\end_layout
1597
1598\begin_layout Verbatim
1599 
1600\end_layout
1601
1602\begin_layout Itemize
1603
1604\begin_inset Argument item:1
1605status open
1606
1607\begin_layout Plain Layout
16082.
1609\end_layout
1610
1611\end_inset
1612
1613If
1614\begin_inset ERT
1615status collapsed
1616
1617\begin_layout Plain Layout
1618
1619\backslash
1620verb|request->type == 2|
1621\end_layout
1622
1623\end_inset
1624
1625, it means that a non-blocking receive is called but the corresponding message is not yet probed. The request is in the pending list thus not yet completed. All incoming message is once again probed and all pending requests are checked. If after the second check, the matching message is found, thus a
1626\begin_inset ERT
1627status collapsed
1628
1629\begin_layout Plain Layout
1630
1631\backslash
1632verb|MPI_Imrecv|
1633\end_layout
1634
1635\end_inset
1636
1637 is called and the type is set to 3. Otherwise, the type is still 2, then
1638\begin_inset ERT
1639status collapsed
1640
1641\begin_layout Plain Layout
1642
1643\backslash
1644verb|flag = false|
1645\end_layout
1646
1647\end_inset
1648
1649 is returned.
1650\end_layout
1651
1652\begin_layout Itemize
1653
1654\begin_inset Argument item:1
1655status open
1656
1657\begin_layout Plain Layout
16583.
1659\end_layout
1660
1661\end_inset
1662
1663If
1664\begin_inset ERT
1665status collapsed
1666
1667\begin_layout Plain Layout
1668
1669\backslash
1670verb|request->type == 3|
1671\end_layout
1672
1673\end_inset
1674
1675, this indcates that the request is issued from a non-blocking receive call and the matching message is probed thus the status of the communication lies in the status of the
1676\begin_inset ERT
1677status collapsed
1678
1679\begin_layout Plain Layout
1680
1681\backslash
1682verb|MPI_Imrecv|
1683\end_layout
1684
1685\end_inset
1686
1687 function. The completion result is returned by:
1688\end_layout
1689
1690\begin_layout Verbatim
1691   MPI_Test(& request->mpi_request, flag, & status->mpi_status)
1692\end_layout
1693
1694\begin_layout Verbatim
1695 
1696\end_layout
1697
1698\begin_layout Paragraph
1699Example 2
1700\end_layout
1701
1702\begin_layout Standard
1703
1704\begin_inset ERT
1705status collapsed
1706
1707\begin_layout Plain Layout
1708
1709\backslash
1710verb|MPI_Wait(MPI_Request *request, MPI_Status *status)|
1711\end_layout
1712
1713\end_inset
1714
1715 
1716\end_layout
1717
1718\begin_layout Itemize
1719
1720\begin_inset Argument item:1
1721status open
1722
1723\begin_layout Plain Layout
17241.
1725\end_layout
1726
1727\end_inset
1728
1729If
1730\begin_inset ERT
1731status collapsed
1732
1733\begin_layout Plain Layout
1734
1735\backslash
1736verb|request->type == 1|
1737\end_layout
1738
1739\end_inset
1740
1741, communication to be tested is indeed issued from a non-blocking send. Jump to step 4.
1742\end_layout
1743
1744\begin_layout Itemize
1745
1746\begin_inset Argument item:1
1747status open
1748
1749\begin_layout Plain Layout
17502.
1751\end_layout
1752
1753\end_inset
1754
1755If
1756\begin_inset ERT
1757status collapsed
1758
1759\begin_layout Plain Layout
1760
1761\backslash
1762verb|request->type == 2|
1763\end_layout
1764
1765\end_inset
1766
1767, it means that a non-blocking receive is called but the corresponding message is not yet probed. The request is in the pending list thus not yet completed. We repeat the incoming message probing and the pending request checking until the matching message is found, thus a
1768\begin_inset ERT
1769status collapsed
1770
1771\begin_layout Plain Layout
1772
1773\backslash
1774verb|MPI_Imrecv|
1775\end_layout
1776
1777\end_inset
1778
1779 is called and the type is set to 3. Jump to step 4.
1780\end_layout
1781
1782\begin_layout Itemize
1783
1784\begin_inset Argument item:1
1785status open
1786
1787\begin_layout Plain Layout
17883.
1789\end_layout
1790
1791\end_inset
1792
1793If
1794\begin_inset ERT
1795status collapsed
1796
1797\begin_layout Plain Layout
1798
1799\backslash
1800verb|request->type == 3|
1801\end_layout
1802
1803\end_inset
1804
1805, this indcates that the request is issued from a non-blocking receive call and the matching message is probed thus the status of the communication lies in the status of the
1806\begin_inset ERT
1807status collapsed
1808
1809\begin_layout Plain Layout
1810
1811\backslash
1812verb|MPI_Imrecv|
1813\end_layout
1814
1815\end_inset
1816
1817 function. Jump to step 4.
1818\end_layout
1819
1820\begin_layout Itemize
1821
1822\begin_inset Argument item:1
1823status open
1824
1825\begin_layout Plain Layout
18264.
1827\end_layout
1828
1829\end_inset
1830
1831We force the completion by calling:
1832\end_layout
1833
1834\begin_layout Verbatim
1835   MPI_Wat(& request->mpi_request, & status->mpi_status)
1836\end_layout
1837
1838\begin_layout Verbatim
1839 
1840\end_layout
1841
1842\begin_layout Section
1843Collective communication
1844\end_layout
1845
1846\begin_layout Standard
1847All MPI classic collective communications are performed as the following pattern:
1848\end_layout
1849
1850\begin_layout Itemize
1851
1852\begin_inset Argument item:1
1853status open
1854
1855\begin_layout Plain Layout
18561.
1857\end_layout
1858
1859\end_inset
1860
1861Intra-process communication using OpenMP.
1862\shape italic
1863e.g.
1864\shape default
1865 Collect data from slave threads to master thread.
1866\end_layout
1867
1868\begin_layout Itemize
1869
1870\begin_inset Argument item:1
1871status open
1872
1873\begin_layout Plain Layout
18742.
1875\end_layout
1876
1877\end_inset
1878
1879Inter-process communication using MPI collective calls on master threads.
1880\end_layout
1881
1882\begin_layout Itemize
1883
1884\begin_inset Argument item:1
1885status open
1886
1887\begin_layout Plain Layout
18883.
1889\end_layout
1890
1891\end_inset
1892
1893Intra-process communication using OpenMP.
1894\shape italic
1895e.g.
1896\shape default
1897 Distribute data from master thread to slave threads.
1898\end_layout
1899
1900\begin_layout Paragraph
1901Example 1
1902\end_layout
1903
1904\begin_layout Standard
1905
1906\begin_inset ERT
1907status collapsed
1908
1909\begin_layout Plain Layout
1910
1911\backslash
1912verb|EP_Bcast(buffer, count, datatype, root = 4, comm)|
1913\end_layout
1914
1915\end_inset
1916
1917 with
1918\begin_inset ERT
1919status collapsed
1920
1921\begin_layout Plain Layout
1922
1923\backslash
1924verb|comm|
1925\end_layout
1926
1927\end_inset
1928
1929 composed by 4 MPI processes and 3 threads per process:
1930\end_layout
1931
1932\begin_layout Standard
1933We can consider the communicator as
1934\begin_inset Formula $\{\underbrace{(0,1,2)}_\textrm{proc 0} \quad \underbrace{(3,\textcolor{red}{4},5)}_\textrm{proc
19351}\quad \underbrace{(6,7,8)}_\textrm{proc 2}\quad \underbrace{(9,10,11)}_\textrm{proc 3}\}$
1936\end_inset
1937
1938.
1939\end_layout
1940
1941\begin_layout Standard
1942This collective communication is performed by the following three steps:
1943\end_layout
1944
1945\begin_layout Itemize
1946
1947\begin_inset Argument item:1
1948status open
1949
1950\begin_layout Plain Layout
19511.
1952\end_layout
1953
1954\end_inset
1955
1956EP process with rank 4 send the buffer to EP process rank 3 which is a master thread.
1957\end_layout
1958
1959\begin_layout Itemize
1960
1961\begin_inset Argument item:1
1962status open
1963
1964\begin_layout Plain Layout
19652.
1966\end_layout
1967
1968\end_inset
1969
1970We call
1971\begin_inset ERT
1972status collapsed
1973
1974\begin_layout Plain Layout
1975
1976\backslash
1977verb|MPI_Bcast(buffer, count, datatype, mpi_root = 1, mpi_comm) |
1978\end_layout
1979
1980\end_inset
1981
1982.
1983\end_layout
1984
1985\begin_layout Itemize
1986
1987\begin_inset Argument item:1
1988status open
1989
1990\begin_layout Plain Layout
19913.
1992\end_layout
1993
1994\end_inset
1995
1996All master threads send the buffer to its slaves.
1997\end_layout
1998
1999\begin_layout Standard
2000\align center
2001
2002\begin_inset Graphics
2003        filename bcast.png
2004        scale 30
2005
2006\end_inset
2007
2008 
2009\end_layout
2010
2011\begin_layout Paragraph
2012Example 2
2013\end_layout
2014
2015\begin_layout Standard
2016
2017\begin_inset ERT
2018status collapsed
2019
2020\begin_layout Plain Layout
2021
2022\backslash
2023verb|EP_Allreduce(sendbuf, recvbuf, count, datatype, op, comm)|
2024\end_layout
2025
2026\end_inset
2027
2028 with
2029\begin_inset ERT
2030status collapsed
2031
2032\begin_layout Plain Layout
2033
2034\backslash
2035verb|comm|
2036\end_layout
2037
2038\end_inset
2039
2040 the same as in example 1.
2041\end_layout
2042
2043\begin_layout Standard
2044This collective communication is performed by the following three steps:
2045\end_layout
2046
2047\begin_layout Itemize
2048
2049\begin_inset Argument item:1
2050status open
2051
2052\begin_layout Plain Layout
20531.
2054\end_layout
2055
2056\end_inset
2057
2058We perform a intra-process
2059\begin_inset Quotes eld
2060\end_inset
2061
2062allreduce
2063\begin_inset Quotes erd
2064\end_inset
2065
2066 operation: master threads collect data from its slaves and perform the reduce operation.
2067\end_layout
2068
2069\begin_layout Itemize
2070
2071\begin_inset Argument item:1
2072status open
2073
2074\begin_layout Plain Layout
20752.
2076\end_layout
2077
2078\end_inset
2079
2080Master threads call the classic
2081\begin_inset ERT
2082status collapsed
2083
2084\begin_layout Plain Layout
2085
2086\backslash
2087verb|MPI_Allreduce|
2088\end_layout
2089
2090\end_inset
2091
2092 routine.
2093\end_layout
2094
2095\begin_layout Itemize
2096
2097\begin_inset Argument item:1
2098status open
2099
2100\begin_layout Plain Layout
21013.
2102\end_layout
2103
2104\end_inset
2105
2106All master threads send the updated reduced data to its slaves.
2107\end_layout
2108
2109\begin_layout Standard
2110\align center
2111
2112\begin_inset Graphics
2113        filename allreduce.png
2114        scale 30
2115
2116\end_inset
2117
2118 
2119\end_layout
2120
2121\begin_layout Standard
2122Other collective communications have the similar execution pattern.
2123\end_layout
2124
2125\begin_layout Section
2126Inter-communicator
2127\end_layout
2128
2129\begin_layout Standard
2130In XIOS, inter-communicator is an very important component. Thus, our EP library must support inter-communications.
2131\end_layout
2132
2133\begin_layout Subsection
2134The splitting of intra-communicator
2135\end_layout
2136
2137\begin_layout Standard
2138Before talking about the inter-communicator, we will start by splitting intra-communicator. The C prototype of the splitting routine is
2139\end_layout
2140
2141\begin_layout Verbatim
2142int  MPI_Comm_split(MPI_Comm comm, int color, int key,
2143\end_layout
2144
2145\begin_layout Verbatim
2146                                    MPI_Comm *newcomm)
2147\end_layout
2148
2149\begin_layout Verbatim
2150 
2151\end_layout
2152
2153\begin_layout Standard
2154
2155\begin_inset Quotes eld
2156\end_inset
2157
2158This function partitions the group associated with
2159\begin_inset ERT
2160status collapsed
2161
2162\begin_layout Plain Layout
2163
2164\backslash
2165verb|comm|
2166\end_layout
2167
2168\end_inset
2169
2170 into disjoint subgroups, one for each value of
2171\begin_inset ERT
2172status collapsed
2173
2174\begin_layout Plain Layout
2175
2176\backslash
2177verb|color|
2178\end_layout
2179
2180\end_inset
2181
2182. Each subgroup contains all processes of the same color. Within each subgroup, the processes are ranked in the order defined by the value of the argument
2183\begin_inset ERT
2184status collapsed
2185
2186\begin_layout Plain Layout
2187
2188\backslash
2189verb|key|
2190\end_layout
2191
2192\end_inset
2193
2194, with ties broken according to their rank in the old group. A new communicator is created for each subgroup and returned in
2195\begin_inset ERT
2196status collapsed
2197
2198\begin_layout Plain Layout
2199
2200\backslash
2201verb|newcomm|
2202\end_layout
2203
2204\end_inset
2205
2206. A process may supply the color value
2207\begin_inset ERT
2208status collapsed
2209
2210\begin_layout Plain Layout
2211
2212\backslash
2213verb|MPI_UNDEFINED|
2214\end_layout
2215
2216\end_inset
2217
2218, in which case
2219\begin_inset ERT
2220status collapsed
2221
2222\begin_layout Plain Layout
2223
2224\backslash
2225verb|newcomm|
2226\end_layout
2227
2228\end_inset
2229
2230 returns
2231\begin_inset ERT
2232status collapsed
2233
2234\begin_layout Plain Layout
2235
2236\backslash
2237verb|MPI_COMM_NULL|
2238\end_layout
2239
2240\end_inset
2241
2242. This is a collective call, but each process is permitted to provide different values for color and key.
2243\begin_inset Quotes erd
2244\end_inset
2245
2246
2247\begin_inset CommandInset citation
2248LatexCommand cite
2249after ""
2250key "MPI"
2251literal "true"
2252
2253\end_inset
2254
2255
2256\end_layout
2257
2258\begin_layout Standard
2259By definition of the routine, in the case of EP, each thread participating the split operation will have only one color (
2260\begin_inset ERT
2261status collapsed
2262
2263\begin_layout Plain Layout
2264
2265\backslash
2266verb|MPI_UNDEFINED|
2267\end_layout
2268
2269\end_inset
2270
2271 is also considered to be one color). However, in the process's point of view, it can have multiple colors as shown in the following figure.
2272\end_layout
2273
2274\begin_layout Standard
2275\align center
2276
2277\begin_inset Graphics
2278        filename split.png
2279        scale 40
2280
2281\end_inset
2282
2283 
2284\end_layout
2285
2286\begin_layout Standard
2287This figure shows the result of the EP communicator splitting. Here we used the EP rank as key to assign the new rank of the thread in the resulting split intra-communicator. If the key is anything else than the EP rank, we follow the convention that the key takes effect only inside a process. This means that the threads are at first ordered by the MPI process rank and then by the value of key.
2288\end_layout
2289
2290\begin_layout Standard
2291Due to the fact that one process can have multiple colors for its threads, the splitting operation is executed by the following steps:
2292\end_layout
2293
2294\begin_layout Itemize
2295
2296\begin_inset Argument item:1
2297status open
2298
2299\begin_layout Plain Layout
23001.
2301\end_layout
2302
2303\end_inset
2304
2305Master threads collect all colors from its slaves and communicate with each other to determine the total number of colors across the communicator.
2306\end_layout
2307
2308\begin_layout Itemize
2309
2310\begin_inset Argument item:1
2311status open
2312
2313\begin_layout Plain Layout
23142.
2315\end_layout
2316
2317\end_inset
2318
2319For each color, the master thread check all its slave threads to obtain the number of threads having the same color.
2320\end_layout
2321
2322\begin_layout Itemize
2323
2324\begin_inset Argument item:1
2325status open
2326
2327\begin_layout Plain Layout
23283.
2329\end_layout
2330
2331\end_inset
2332
2333If at least one of the slave threads holds the color, then the master thread takes this color. If not, the master thread takes color
2334\begin_inset ERT
2335status collapsed
2336
2337\begin_layout Plain Layout
2338
2339\backslash
2340verb|MPI_UNDEFINED|
2341\end_layout
2342
2343\end_inset
2344
2345. All master threads call classic communicator splitting routine with key
2346\begin_inset Formula $=$
2347\end_inset
2348
2349 MPI rank.
2350\end_layout
2351
2352\begin_layout Itemize
2353
2354\begin_inset Argument item:1
2355status open
2356
2357\begin_layout Plain Layout
23584.
2359\end_layout
2360
2361\end_inset
2362
2363For master threads holding a defined color, we execute the endpoint creation routine according to the number of slave threads holding the same color. The resulting EP communicators are then assigned to these slave threads.
2364\end_layout
2365
2366\begin_layout Standard
2367\align center
2368
2369\begin_inset Graphics
2370        filename split2.png
2371        scale 40
2372
2373\end_inset
2374
2375 
2376\end_layout
2377
2378\begin_layout Subsection
2379The creation of inter-communicator
2380\end_layout
2381
2382\begin_layout Standard
2383In XIOS, the inter-communicators are create by the routine
2384\begin_inset ERT
2385status collapsed
2386
2387\begin_layout Plain Layout
2388
2389\backslash
2390verb|MPI_Intercomm_create|
2391\end_layout
2392
2393\end_inset
2394
2395 which is used to bind two intra-communicators into an inter-communicator. The C prototype is
2396\end_layout
2397
2398\begin_layout Verbatim
2399int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader,
2400\end_layout
2401
2402\begin_layout Verbatim
2403                         MPI_Comm peer_comm, int remote_leader,
2404\end_layout
2405
2406\begin_layout Verbatim
2407                               int tag, MPI_Comm *newintercomm)
2408\end_layout
2409
2410\begin_layout Verbatim
2411 
2412\end_layout
2413
2414\begin_layout Standard
2415According to the MPI standard,
2416\begin_inset Quotes eld
2417\end_inset
2418
2419an inter-communication is a point-to-point communication between processes in different groups
2420\begin_inset Quotes erd
2421\end_inset
2422
2423.
2424\begin_inset Quotes eld
2425\end_inset
2426
2427All inter-communicator constructors are blocking except for
2428\begin_inset ERT
2429status collapsed
2430
2431\begin_layout Plain Layout
2432
2433\backslash
2434verb|MPI_COMM_IDUP|
2435\end_layout
2436
2437\end_inset
2438
2439 and require that the local and remote groups be disjoint.
2440\begin_inset Quotes erd
2441\end_inset
2442
2443
2444\end_layout
2445
2446\begin_layout Standard
2447As in EP the threads are considered as processes, the non-overlapping condition can be translated to
2448\begin_inset Quotes eld
2449\end_inset
2450
2451non-overlapping
2452\begin_inset Quotes erd
2453\end_inset
2454
2455 at the thread level which means that one thread can not belong to the local group and the remote group. However, the parent process of the thread can be overlapped. As the EP library is built upon an existing MPI implementation which follows the non-overlapping condition at the process level, we can have an issue in the case.
2456\end_layout
2457
2458\begin_layout Standard
2459Before digging into this issue, we shall at first look at the case where the non-overlapping condition is perfectly respected.
2460\end_layout
2461
2462\begin_layout Standard
2463\align center
2464
2465\begin_inset Graphics
2466        filename intercomm.png
2467        scale 30
2468
2469\end_inset
2470
2471 
2472\end_layout
2473
2474\begin_layout Standard
2475As shown in the figure, we have two intra-communicators A and B and they are totally disjoint both at the thread and process level. Each of the communicators has a local leader. We also assume that both leaders belong to a peer communicator and have rank 4 and 9 respectively.
2476\end_layout
2477
2478\begin_layout Standard
2479To create the inter-communicator, all threads from the left intra-comm call:
2480\end_layout
2481
2482\begin_layout Verbatim
2483MPI_Intercomm_create(commA, local_leader = 2, peer_comm,
2484\end_layout
2485
2486\begin_layout Verbatim
2487                     remote_leader = 9, tag, inter_comm)
2488\end_layout
2489
2490\begin_layout Standard
2491and for threads of the right intra-comm, they call:
2492\end_layout
2493
2494\begin_layout Verbatim
2495MPI_Intercomm_create(commB, local_leader = 3, peer_comm,
2496\end_layout
2497
2498\begin_layout Verbatim
2499                     remote_leader = 4, tag, inter_comm)
2500\end_layout
2501
2502\begin_layout Standard
2503To perform the inter-communicator creation, we follow the 3 steps:
2504\end_layout
2505
2506\begin_layout Itemize
2507
2508\begin_inset Argument item:1
2509status open
2510
2511\begin_layout Plain Layout
25121.
2513\end_layout
2514
2515\end_inset
2516
2517Determine the leaders and ranks at the process level;
2518\end_layout
2519
2520\begin_layout Itemize
2521
2522\begin_inset Argument item:1
2523status open
2524
2525\begin_layout Plain Layout
25262.
2527\end_layout
2528
2529\end_inset
2530
2531Call classic
2532\begin_inset ERT
2533status collapsed
2534
2535\begin_layout Plain Layout
2536
2537\backslash
2538verb|MPI_Intercomm_create|
2539\end_layout
2540
2541\end_inset
2542
2543;
2544\end_layout
2545
2546\begin_layout Itemize
2547
2548\begin_inset Argument item:1
2549status open
2550
2551\begin_layout Plain Layout
25523.
2553\end_layout
2554
2555\end_inset
2556
2557Create endpoints from process and assigned to threads.
2558\end_layout
2559
2560\begin_layout Standard
2561\align center
2562
2563\begin_inset Graphics
2564        filename intercomm_step.png
2565        scale 25
2566
2567\end_inset
2568
2569 
2570\end_layout
2571
2572\begin_layout Standard
2573If we have overlapped process in the creation of inter-communicator, we should add an
2574\shape italic
2575priority check
2576\shape default
2577 to assign the process to only one intra-communicator. Several possibilities:
2578\end_layout
2579
2580\begin_layout Itemize
2581
2582\begin_inset Argument item:1
2583status open
2584
2585\begin_layout Plain Layout
25861.
2587\end_layout
2588
2589\end_inset
2590
2591Process is shared and contains no local leader
2592\begin_inset Formula $\implies$
2593\end_inset
2594
2595 process belongs to group with higher rank in peer comm;
2596\end_layout
2597
2598\begin_layout Itemize
2599
2600\begin_inset Argument item:1
2601status open
2602
2603\begin_layout Plain Layout
26042.
2605\end_layout
2606
2607\end_inset
2608
2609Process is shared and contains one local leader
2610\begin_inset Formula $\implies$
2611\end_inset
2612
2613 process belongs to group with the leader;
2614\end_layout
2615
2616\begin_layout Itemize
2617
2618\begin_inset Argument item:1
2619status open
2620
2621\begin_layout Plain Layout
26223.
2623\end_layout
2624
2625\end_inset
2626
2627Process is shared and contains both local leaders : leader change is performed and the peer communicator is
2628\begin_inset ERT
2629status collapsed
2630
2631\begin_layout Plain Layout
2632
2633\backslash
2634verb|MPI_COMM_WORLD|
2635\end_layout
2636
2637\end_inset
2638
2639 and we note
2640\begin_inset Quotes eld
2641\end_inset
2642
2643group A
2644\begin_inset Quotes erd
2645\end_inset
2646
2647 the group with smaller peer rank and
2648\begin_inset Quotes eld
2649\end_inset
2650
2651group B
2652\begin_inset Quotes erd
2653\end_inset
2654
2655 the group with higher peer rank.
2656\end_layout
2657
2658\begin_deeper
2659\begin_layout Itemize
2660
2661\begin_inset Argument item:1
2662status open
2663
2664\begin_layout Plain Layout
26653a.
2666\end_layout
2667
2668\end_inset
2669
2670If group A has at least two processes, the leader of group A is changed to the master thread of the process with smallest rank except the overlapped process. The overlapped process belongs to group B.
2671\end_layout
2672
2673\begin_layout Itemize
2674
2675\begin_inset Argument item:1
2676status open
2677
2678\begin_layout Plain Layout
26793b.
2680\end_layout
2681
2682\end_inset
2683
2684If group A has only one processes, and group B has at least two processes, then the leader of group B is changed to the master thread of the process with smallest rank except the overlapped process. The overlapped process belongs to group A.
2685\end_layout
2686
2687\begin_layout Itemize
2688
2689\begin_inset Argument item:1
2690status open
2691
2692\begin_layout Plain Layout
26933c.
2694\end_layout
2695
2696\end_inset
2697
2698If both group A and group B have only one process, then an one-process intra-communicator is created though it will be considered (labeled) as an inter-communicator.
2699\end_layout
2700
2701\end_deeper
2702\begin_layout Standard
2703\align center
2704
2705\begin_inset Graphics
2706        filename intercomm2.png
2707        scale 25
2708
2709\end_inset
2710
2711 
2712\end_layout
2713
2714\begin_layout Subsection
2715The merge of inter-communicators
2716\end_layout
2717
2718\begin_layout Standard
2719
2720\begin_inset ERT
2721status collapsed
2722
2723\begin_layout Plain Layout
2724
2725\backslash
2726verb|MPI_Intercomm_Merge(MPI_Comm intercomm, int high, MPI_Comm *newintracomm)|
2727\end_layout
2728
2729\end_inset
2730
2731 creates an intra-communicator by merging the local and remote groups of an inter-communicator. All processes should provide the same
2732\begin_inset ERT
2733status collapsed
2734
2735\begin_layout Plain Layout
2736
2737\backslash
2738verb|high|
2739\end_layout
2740
2741\end_inset
2742
2743 value within each of the two groups. If processes in one group provided the value
2744\begin_inset ERT
2745status collapsed
2746
2747\begin_layout Plain Layout
2748
2749\backslash
2750verb|high=false|
2751\end_layout
2752
2753\end_inset
2754
2755 and processes in the other group provided the value
2756\begin_inset ERT
2757status collapsed
2758
2759\begin_layout Plain Layout
2760
2761\backslash
2762verb|high=true|
2763\end_layout
2764
2765\end_inset
2766
2767 then the union orders the “low” group before the “high” group. If all processes provided the same high argument then the order of the union is arbitrary. This call is blocking and collective within the union of the two groups.
2768\begin_inset CommandInset citation
2769LatexCommand cite
2770after ""
2771key "MPI"
2772literal "true"
2773
2774\end_inset
2775
2776
2777\end_layout
2778
2779\begin_layout Standard
2780This routine can be considered as the inverse of
2781\begin_inset ERT
2782status collapsed
2783
2784\begin_layout Plain Layout
2785
2786\backslash
2787verb|MPI_Intercomm_create|
2788\end_layout
2789
2790\end_inset
2791
2792. In the intercommunicator create function, all 5 cases are eventually transformed into the case where no MPI process is shared by two groups. It is from this case that the merge funtion takes place.
2793\end_layout
2794
2795\begin_layout Itemize
2796
2797\begin_inset Argument item:1
2798status open
2799
2800\begin_layout Plain Layout
28011.
2802\end_layout
2803
2804\end_inset
2805
2806The classic
2807\begin_inset ERT
2808status collapsed
2809
2810\begin_layout Plain Layout
2811
2812\backslash
2813verb|MPI_Intercomm_merge|
2814\end_layout
2815
2816\end_inset
2817
2818 is called and an MPI intracommunicator is created from the two disjoint groups and MPI processes are ordered by the high value of the local leader.
2819\end_layout
2820
2821\begin_layout Itemize
2822
2823\begin_inset Argument item:1
2824status open
2825
2826\begin_layout Plain Layout
28272.
2828\end_layout
2829
2830\end_inset
2831
2832Endpoints are created based on the MPI intracommunicator and the new EP ranks are orderd firstly according to the high value of each thread and then to the origianl EP ranks in the intercommunicators.
2833\end_layout
2834
2835\begin_layout Standard
2836\align center
2837
2838\begin_inset Graphics
2839        filename merge.png
2840        scale 25
2841
2842\end_inset
2843
2844 
2845\end_layout
2846
2847\begin_layout Section
2848P2P communication on inter-communicators
2849\end_layout
2850
2851\begin_layout Standard
2852In case of the intercommunicators, the
2853\begin_inset ERT
2854status collapsed
2855
2856\begin_layout Plain Layout
2857
2858\backslash
2859verb|MPI_Comm|
2860\end_layout
2861
2862\end_inset
2863
2864 class has 3 members to determine the topology along with the original
2865\begin_inset ERT
2866status collapsed
2867
2868\begin_layout Plain Layout
2869
2870\backslash
2871verb|rank_map|
2872\end_layout
2873
2874\end_inset
2875
2876:
2877\end_layout
2878
2879\begin_layout Itemize
2880
2881\begin_inset ERT
2882status collapsed
2883
2884\begin_layout Plain Layout
2885
2886\backslash
2887verb|RANK_MAP local_rank_map[size of commA]|
2888\end_layout
2889
2890\end_inset
2891
2892: composed of the EP rank in commA' or commB';
2893\end_layout
2894
2895\begin_layout Itemize
2896
2897\begin_inset ERT
2898status collapsed
2899
2900\begin_layout Plain Layout
2901
2902\backslash
2903verb|RANK_MAP remote_rank_map[size of commB]|
2904\end_layout
2905
2906\end_inset
2907
2908: =
2909\begin_inset ERT
2910status collapsed
2911
2912\begin_layout Plain Layout
2913
2914\backslash
2915verb|local_rank_map|
2916\end_layout
2917
2918\end_inset
2919
2920 of remote group;
2921\end_layout
2922
2923\begin_layout Itemize
2924
2925\begin_inset ERT
2926status collapsed
2927
2928\begin_layout Plain Layout
2929
2930\backslash
2931verb|RANK_MAP intercomm_rank_map[size of commB']|
2932\end_layout
2933
2934\end_inset
2935
2936: =
2937\begin_inset ERT
2938status collapsed
2939
2940\begin_layout Plain Layout
2941
2942\backslash
2943verb|rank_map|
2944\end_layout
2945
2946\end_inset
2947
2948 of remote group';
2949\end_layout
2950
2951\begin_layout Itemize
2952
2953\begin_inset ERT
2954status collapsed
2955
2956\begin_layout Plain Layout
2957
2958\backslash
2959verb|RANK_MAP rank_map|
2960\end_layout
2961
2962\end_inset
2963
2964: rank map of commA' or commB'.
2965\end_layout
2966
2967\begin_layout Standard
2968For example, in the following configuration:
2969\end_layout
2970
2971\begin_layout Standard
2972\align center
2973
2974\begin_inset Graphics
2975        filename ranks.png
2976        scale 30
2977
2978\end_inset
2979
2980 
2981\end_layout
2982
2983\begin_layout Standard
2984For all endpoints in commA,
2985\end_layout
2986
2987\begin_layout Verbatim
2988 local_rank_map={(rank in commA' or commB',
2989\end_layout
2990
2991\begin_layout Verbatim
2992                  rank of leader in MPI_Comm_world)}
2993\end_layout
2994
2995\begin_layout Verbatim
2996               ={(1,0), (0,1), (2,1), (4,1)}
2997\end_layout
2998
2999\begin_layout Verbatim
3000               
3001\end_layout
3002
3003\begin_layout Verbatim
3004 remote_rank_map={(remote endpoints' rank in commA' or commB',
3005\end_layout
3006
3007\begin_layout Verbatim
3008                   rank of remote leader in MPI_Comm_world)}
3009\end_layout
3010
3011\begin_layout Verbatim
3012                ={(0,0), (1,1), (3,1), (5,1)}
3013\end_layout
3014
3015\begin_layout Standard
3016For all endpoints in commA'
3017\end_layout
3018
3019\begin_layout Verbatim
3020 intercomm_rank_map={(remote endpoints local rank in commA' or commB',
3021\end_layout
3022
3023\begin_layout Verbatim
3024                      remote endpoints MPI rank in commA' or commB')}
3025\end_layout
3026
3027\begin_layout Verbatim
3028                   ={(0,0), (1,0)}
3029\end_layout
3030
3031\begin_layout Verbatim
3032 rank_map={(local rank in commA', mpi rank in commA')}
3033\end_layout
3034
3035\begin_layout Verbatim
3036         ={(0,0), (1,0), (0,1), (1,1), (0,2), (1,2)}
3037\end_layout
3038
3039\begin_layout Standard
3040For all endpoints in comm B,
3041\end_layout
3042
3043\begin_layout Verbatim
3044 local_rank_map={(rank in commA' or commB',
3045\end_layout
3046
3047\begin_layout Verbatim
3048                  rank of leader in MPI_Comm_world)}
3049\end_layout
3050
3051\begin_layout Verbatim
3052               ={(0,0), (1,1), (3,1), (5,1)}
3053\end_layout
3054
3055\begin_layout Verbatim
3056               
3057\end_layout
3058
3059\begin_layout Verbatim
3060 remote_rank_map={(remote endpoints' rank in commA' or commB',
3061\end_layout
3062
3063\begin_layout Verbatim
3064                   rank of remote leader in MPI_Comm_world)}
3065\end_layout
3066
3067\begin_layout Verbatim
3068                ={(1,0), (0,1), (2,1), (4,1)}
3069\end_layout
3070
3071\begin_layout Standard
3072For all endpoints in commB'
3073\end_layout
3074
3075\begin_layout Verbatim
3076 intercomm_rank_map={(remote endpoints local rank in commA' or commB',
3077\end_layout
3078
3079\begin_layout Verbatim
3080                      remote endpoints MPI rank in commA' or commB')}
3081\end_layout
3082
3083\begin_layout Verbatim
3084                   ={(0,0), (1,0), (0,1), (1,1), (0,2), (1,2)}
3085\end_layout
3086
3087\begin_layout Verbatim
3088 rank_map={(local rank in commB', mpi rank in commB')}
3089\end_layout
3090
3091\begin_layout Verbatim
3092         ={(0,0), (1,0)}
3093\end_layout
3094
3095\begin_layout Standard
3096When calling a p2p communication on an inter-communicator, we should:
3097\end_layout
3098
3099\begin_layout Itemize
3100
3101\begin_inset Argument item:1
3102status open
3103
3104\begin_layout Plain Layout
31051.
3106\end_layout
3107
3108\end_inset
3109
3110Determine if the source and the destination endpoints are in a same group by checking the
3111\begin_inset Quotes eld
3112\end_inset
3113
3114labels
3115\begin_inset Quotes erd
3116\end_inset
3117
3118.
3119\end_layout
3120
3121\begin_deeper
3122\begin_layout Itemize
3123
3124\begin_inset Argument item:1
3125status open
3126
3127\begin_layout Plain Layout
3128
3129\begin_inset Formula $\bullet$
3130\end_inset
3131
3132
3133\end_layout
3134
3135\end_inset
3136
3137
3138\begin_inset ERT
3139status collapsed
3140
3141\begin_layout Plain Layout
3142
3143\backslash
3144verb|src_label = local_rank_map->at(src).second|
3145\end_layout
3146
3147\end_inset
3148
3149 
3150\end_layout
3151
3152\begin_layout Itemize
3153
3154\begin_inset Argument item:1
3155status open
3156
3157\begin_layout Plain Layout
3158
3159\begin_inset Formula $\bullet$
3160\end_inset
3161
3162
3163\end_layout
3164
3165\end_inset
3166
3167
3168\begin_inset ERT
3169status collapsed
3170
3171\begin_layout Plain Layout
3172
3173\backslash
3174verb|dest_label = remote_rank_map->at(dest).second|
3175\end_layout
3176
3177\end_inset
3178
3179 
3180\end_layout
3181
3182\end_deeper
3183\begin_layout Itemize
3184
3185\begin_inset Argument item:1
3186status open
3187
3188\begin_layout Plain Layout
31892.
3190\end_layout
3191
3192\end_inset
3193
3194If
3195\begin_inset ERT
3196status collapsed
3197
3198\begin_layout Plain Layout
3199
3200\backslash
3201verb|src_label == dest_label|
3202\end_layout
3203
3204\end_inset
3205
3206, then the communication is in fact a intra-communication. The new source rank and destination rank, as well as the local ranks, are deduced by:
3207\end_layout
3208
3209\begin_layout Verbatim
3210      src_rank = local_rank_map->at(src).first
3211\end_layout
3212
3213\begin_layout Verbatim
3214      dest_rank = remote_rank_map->at(dest).first
3215\end_layout
3216
3217\begin_layout Verbatim
3218      src_rank_local = rank_map->at(src_rank).first
3219\end_layout
3220
3221\begin_layout Verbatim
3222      dest_rank_local = rank_map->at(dest_rank).first
3223\end_layout
3224
3225\begin_layout Verbatim
3226     
3227\end_layout
3228
3229\begin_layout Itemize
3230
3231\begin_inset Argument item:1
3232status open
3233
3234\begin_layout Plain Layout
32353.
3236\end_layout
3237
3238\end_inset
3239
3240If
3241\begin_inset ERT
3242status collapsed
3243
3244\begin_layout Plain Layout
3245
3246\backslash
3247verb|src_label != dest_label|
3248\end_layout
3249
3250\end_inset
3251
3252, then the inter-communication is required. The new ranks are obtained by:
3253\end_layout
3254
3255\begin_layout Verbatim
3256      src_rank = local_rank_map->at(src).first
3257\end_layout
3258
3259\begin_layout Verbatim
3260      dest_rank = remote_rank_map->at(dest).first
3261\end_layout
3262
3263\begin_layout Verbatim
3264      src_rank_local = intercomm_rank_map->at(src_rank).first
3265\end_layout
3266
3267\begin_layout Verbatim
3268      dest_rank_local = rank_map->at(dest_rank).first
3269\end_layout
3270
3271\begin_layout Verbatim
3272     
3273\end_layout
3274
3275\begin_layout Itemize
3276
3277\begin_inset Argument item:1
3278status open
3279
3280\begin_layout Plain Layout
32814.
3282\end_layout
3283
3284\end_inset
3285
3286Call MPI P2P function to start the communication.
3287\end_layout
3288
3289\begin_deeper
3290\begin_layout Itemize
3291
3292\begin_inset Argument item:1
3293status open
3294
3295\begin_layout Plain Layout
3296
3297\begin_inset Formula $\bullet$
3298\end_inset
3299
3300
3301\end_layout
3302
3303\end_inset
3304
3305If intra-communication,
3306\begin_inset ERT
3307status collapsed
3308
3309\begin_layout Plain Layout
3310
3311\backslash
3312verb|mpi_comm = commA'_mpi or commB'_mpi|
3313\end_layout
3314
3315\end_inset
3316
3317;
3318\end_layout
3319
3320\begin_layout Itemize
3321
3322\begin_inset Argument item:1
3323status open
3324
3325\begin_layout Plain Layout
3326
3327\begin_inset Formula $\bullet$
3328\end_inset
3329
3330
3331\end_layout
3332
3333\end_inset
3334
3335If inter-communication,
3336\begin_inset ERT
3337status collapsed
3338
3339\begin_layout Plain Layout
3340
3341\backslash
3342verb|mpi_comm = inter_comm_mpi|
3343\end_layout
3344
3345\end_inset
3346
3347.
3348\end_layout
3349
3350\end_deeper
3351\begin_layout Standard
3352\align center
3353
3354\begin_inset Graphics
3355        filename sendrecv2.png
3356        scale 30
3357
3358\end_inset
3359
3360 
3361\end_layout
3362
3363\begin_layout Section
3364One-sided communications
3365\end_layout
3366
3367\begin_layout Standard
3368The one-sided communication is a type of communcation which involves only one process to specify all communication parameters, both for the sending side and the receiving side
3369\begin_inset CommandInset citation
3370LatexCommand cite
3371after "Chapter~11"
3372key "MPI"
3373literal "true"
3374
3375\end_inset
3376
3377. To extend this type of communication in the context of endpoints, we encounter some limitations. In the current work, the one-sided communication can only be used in the client-server mode which means that RMA(remote memory access) can occur only between a server and a client.
3378\end_layout
3379
3380\begin_layout Standard
3381The construction of RMA windows is illustrated by the following figure:
3382\end_layout
3383
3384\begin_layout Standard
3385\align center
3386
3387\begin_inset Graphics
3388        filename RMA_schema.pdf
3389        scale 50
3390
3391\end_inset
3392
3393 
3394\end_layout
3395
3396\begin_layout Itemize
3397we determin the max number of threads N in the endpoint environment (N=3 in the example);
3398\end_layout
3399
3400\begin_layout Itemize
3401on the server side, N windows are declared and asociated with the same memory adress;
3402\end_layout
3403
3404\begin_layout Itemize
3405we start a loop : i = 0, ..., N-1
3406\end_layout
3407
3408\begin_deeper
3409\begin_layout Itemize
3410each endpoint with thread number i declares an RMA window;
3411\end_layout
3412
3413\begin_layout Itemize
3414the link between windows on the client side and the i-th window on the server side are created via
3415\begin_inset ERT
3416status collapsed
3417
3418\begin_layout Plain Layout
3419
3420\backslash
3421verb|MPI_Win_created|
3422\end_layout
3423
3424\end_inset
3425
3426;
3427\end_layout
3428
3429\begin_layout Itemize
3430if the number of threads on a certain process is less than N, then a
3431\begin_inset ERT
3432status collapsed
3433
3434\begin_layout Plain Layout
3435
3436\backslash
3437verb|NULL|
3438\end_layout
3439
3440\end_inset
3441
3442 pointer is used as memory adress.
3443\end_layout
3444
3445\end_deeper
3446\begin_layout Standard
3447With the RMA windows created, we can then perform some communications:
3448\begin_inset ERT
3449status collapsed
3450
3451\begin_layout Plain Layout
3452
3453\backslash
3454verb|MPI_Put|
3455\end_layout
3456
3457\end_inset
3458
3459,
3460\begin_inset ERT
3461status collapsed
3462
3463\begin_layout Plain Layout
3464
3465\backslash
3466verb|MPI_Get|
3467\end_layout
3468
3469\end_inset
3470
3471,
3472\begin_inset ERT
3473status collapsed
3474
3475\begin_layout Plain Layout
3476
3477\backslash
3478verb|MPI_Accumulate|
3479\end_layout
3480
3481\end_inset
3482
3483,
3484\begin_inset ERT
3485status collapsed
3486
3487\begin_layout Plain Layout
3488
3489\backslash
3490verb|MPI_Get_accumulate|
3491\end_layout
3492
3493\end_inset
3494
3495,
3496\begin_inset ERT
3497status collapsed
3498
3499\begin_layout Plain Layout
3500
3501\backslash
3502verb|MPI_Fetch_and_op|
3503\end_layout
3504
3505\end_inset
3506
3507,
3508\begin_inset ERT
3509status collapsed
3510
3511\begin_layout Plain Layout
3512
3513\backslash
3514verb|MPI_Compare_and_swap|
3515\end_layout
3516
3517\end_inset
3518
3519,
3520\shape italic
3521etc
3522\shape default
3523.
3524\end_layout
3525
3526\begin_layout Standard
3527The main idea of any of the mentioned communications is to identify the threads which are involved in the connection. For example, we want to perform a put operation from EP 2 to the server. We know that EP 2 is the thread 0 of process 1. Thus the 0-th window (win A) of the server side should be used. Once the sender and the receiver are identified, the
3528\begin_inset ERT
3529status collapsed
3530
3531\begin_layout Plain Layout
3532
3533\backslash
3534verb|MPI_Put|
3535\end_layout
3536
3537\end_inset
3538
3539 communication can be established.
3540\end_layout
3541
3542\begin_layout Standard
3543Other RMA functions, such as
3544\begin_inset ERT
3545status collapsed
3546
3547\begin_layout Plain Layout
3548
3549\backslash
3550verb|MPI_Win_allocate|
3551\end_layout
3552
3553\end_inset
3554
3555,
3556\begin_inset ERT
3557status collapsed
3558
3559\begin_layout Plain Layout
3560
3561\backslash
3562verb|MPI_win_Fence|
3563\end_layout
3564
3565\end_inset
3566
3567, and
3568\begin_inset ERT
3569status collapsed
3570
3571\begin_layout Plain Layout
3572
3573\backslash
3574verb|MPI_Win_free|
3575\end_layout
3576
3577\end_inset
3578
3579, remain nearly the same and we will skip the detail in this document.
3580\end_layout
3581
3582\begin_layout Standard
3583 
3584\begin_inset CommandInset bibtex
3585LatexCommand bibtex
3586bibfiles "reference"
3587options "plain"
3588
3589\end_inset
3590
3591
3592\end_layout
3593
3594\end_body
3595\end_document
Note: See TracBrowser for help on using the repository browser.