%!PS-Adobe-2.0 %%Creator: dvips 5.516 Copyright 1986, 1993 Radical Eye Software %%Title: mpi-report.dvi %%CreationDate: Tue Oct 19 01:18:18 1993 %%Pages: 196 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips mpi-report %DVIPSSource: TeX output 1993.10.19:0116 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /hsize 8.5 72 mul N /landplus90{false}def /@rigin{isls{[0 landplus90{1 -1}{-1 1} ifelse 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div hsize mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{ statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{/nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{/sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0]N df-tail}B /E{ pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get} B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]}if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{cc 1 add D }B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin /SDict 200 dict N SDict begin /@SpecialDefaults{/hs 612 N /vs 792 N /ho 0 N /vo 0 N /hsc 1 N /vsc 1 N /ang 0 N /CLIP 0 N /rwiSeen false N /rhiSeen false N /letter{}N /note{}N /a4{}N /legal{}N}B /@scaleunit 100 N /@hscale{@scaleunit div /hsc X}B /@vscale{@scaleunit div /vsc X}B /@hsize{/hs X /CLIP 1 N}B /@vsize{/vs X /CLIP 1 N}B /@clip{ /CLIP 2 N}B /@hoffset{/ho X}B /@voffset{/vo X}B /@angle{/ang X}B /@rwi{ 10 div /rwi X /rwiSeen true N}B /@rhi{10 div /rhi X /rhiSeen true N}B /@llx{/llx X}B /@lly{/lly X}B /@urx{/urx X}B /@ury{/ury X}B /magscale true def end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N /note{}N /legal{} N /od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto} }{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}N /txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N /cp {pop pop showpage pm restore}N end}if}if}N /normalscale{Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale}if 0 setgray} N /psfts{S 65781.76 div N}N /startTexFig{/psf$SavedState save N userdict maxlength dict begin /magscale false def normalscale currentpoint TR /psf$ury psfts /psf$urx psfts /psf$lly psfts /psf$llx psfts /psf$y psfts /psf$x psfts currentpoint /psf$cy X /psf$cx X /psf$sx psf$x psf$urx psf$llx sub div N /psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR /showpage{}N /erasepage{}N /copypage{}N /p 3 def @MacSetUp}N /doclip{ psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N /endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin /SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count /ocount X /dcount countdictstack N}N /@setspecial {CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR }{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if /showpage{}N /erasepage{}N /copypage{}N newpath }N /@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{ end}repeat grestore SpecialSave restore end}N /@defspecial{SDict begin} N /@fedspecial{end}B /li{lineto}B /rl{rlineto}B /rc{rcurveto}B /np{ /SaveX currentpoint /SaveY X N 1 setlinecap newpath}N /st{stroke SaveX SaveY moveto}N /fil{fill SaveX SaveY moveto}N /ellipse{/endangle X /startangle X /yrad X /xrad X /savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet TeXDict begin 40258437 52099154 1000 300 300 (/ogi/staff/otto/papers/Editor/MPI/draft6/mpi-report.dvi) @start /Fa 2 63 df<0000038000000F0000003C000000F0000003C000000F0000003C 000000F0000003C000000F0000003C000000F0000000F00000003C0000000F00000003C0 000000F00000003C0000000F00000003C0000000F00000003C0000000F00000003801918 7D9520>60 D 62 D E /Fb 3 113 df105 D109 D112 D E /Fc 2 82 df<03FFC00FFFF030E1F860C07CC0C03C81C01C01 C01801C01801C030018020018040038380033E000370000700000600000600000E00000C 00000C000018000016157F9318>80 D<003F0000FFC00307E00401E00800F01000F03000 70600070600070E00070E00060E00060F000C0F000807C31807FE3001F06000008000070 003FC0107FC0300FF86001FFC0003F0014187D931B>I E /Fd 3 82 df<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFF7FFE7FFE3FFC1FF80FF003C010 107E9115>15 D<003FFF0001FFFFC0071E0FE0081C03F0381C01F8703C00F8603C0078C0 3C0078003C00780038007000380070007800E0007800E0007801C0007001800070020000 F00C0000E0300000E7C00001EF000001E0000001C0000001C00000038000000380000007 80000007000000070000000E0000000C0000001D1E809B1D>80 D<0003FC00000FFF0000 303F8000400F80018007C0030007C0060003E00E0003E01C0001E01C0001E0380001E038 0001E0700001E0700001E0F00001C0F00001C0F00003C0F0000380F0000380F8000700F8 0006007C030E007F0C1C003FF818000FC020000000400000018000000E00001FF800203F F000E007FE01C000FFC180001FFE000003F8001B227C9C22>I E /Fe 1 1 df0 D E /Ff 1 106 df<040C00000000007058 98983030606464683006127E910B>105 D E /Fg 5 118 df<03F80FFC1C1C3808700060 00E000E000E000E00060007000380E1C1E0FFC03F00F107E8F14>99 D110 D<07C01FF03C78701C701CE00EE00E E00EE00EE00EE00E701C783C3C781FF007C00F107E8F14>I<030007000700070007007F FCFFFC07000700070007000700070007000700070E070E070E070C03FC00F00F157F9414 >116 DI E /Fh 11 90 df22 D<0038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 38003800380038003800380038FFF8FFF8FFF80D63808118>I<00001800003000006000 00E00000C0000180000380000700000600000E00000C00001C0000380000380000700000 700000E00000E00001E00001C00001C0000380000380000380000780000700000700000F 00000E00000E00001E00001E00001E00001C00001C00003C00003C00003C00003C000038 0000780000780000780000780000780000780000780000780000700000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0000070 00007800007800007800007800007800007800007800007800003800003C00003C00003C 00003C00001C00001C00001E00001E00001E00000E00000E00000F000007000007000007 800003800003800003800001C00001C00001E00000E00000E00000700000700000380000 3800001C00000C00000E000006000007000003800001800000C00000E000006000003000 0018157C768121>32 DI<00001C00003C00 00F80001E00003C0000780000F00000E00001E00003C00003C00003C0000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 F00000F00000F00001E00001E00003C0000380000700000E00001C0000780000E00000E0 00007800001C00000E000007000003800003C00001E00001E00000F00000F00000F00000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 7800007800003C00003C00003C00001E00000E00000F000007800003C00001E00000F800 003C00001C167C7B8121>40 D<0018007800F001E003C007800F001F001E003E003C007C 007C007800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F8 00F800F800F800F800F800F800F8000D25707E25>56 D58 D<007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C00F800F800F800F001F001E003E003C0078007000E001C 003800F000C000F00038001C000E000700078003C003E001E001F000F000F800F800F800 7C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C0E4D798025>60 D62 D80 D89 D E /Fi 7 116 df<60F0F070101020204040040A7D830A>59 D<003E000C000C000C000C0018001800180018073018F0307060706060C060C060C06080 C080C480C4C1C446C838700F177E9612>100 D<03000380030000000000000000000000 00001C002400460046008C000C0018001800180031003100320032001C0009177F960C> 105 D<00180038001000000000000000000000000001C002200430043008600060006000 6000C000C000C000C001800180018001806300E300C60078000D1D80960E>I<383C1E00 44C6630047028100460301008E0703000C0603000C0603000C060600180C0600180C0620 180C0C20180C0C4030180440301807801B0E7F8D1F>109 D<383C0044C6004702004602 008E06000C06000C06000C0C00180C00180C40181840181880300880300F00120E7F8D15 >I<07C00C201870187038001E000FC003E000606060E060C0C0C1803F000C0E7E8D10> 115 D E /Fj 6 94 df<07C018303018701C600C600CE00EE00EE00EE00EE00EE00EE00E E00EE00E600C600C701C30181C7007C00F157F9412>48 D<03000700FF00070007000700 070007000700070007000700070007000700070007000700070007007FF00C157E9412> I<0F8030E040708030C038E0384038003800700070006000C00180030006000C08080810 183FF07FF0FFF00D157E9412>I61 D91 D93 D E /Fk 3 55 df0 D<020002000200C218F2783AE00F800F80 3AE0F278C2180200020002000D0E7E8E12>3 D<0003000300060006000C000C00180018 003000300060006000C000C00180018003000300060006000C000C001800180030003000 60006000C0004000101E7B9600>54 D E /Fl 25 123 df<387C7E7E3E0E1E1C78F06007 0B798416>44 D<70F8F8F870000000000000000070F8F8F8700512789116>58 D<7F1FC0FFBFE07F1FC01C07001C07001C07001C07001C07001C07001C07001FFF001FFF 001FFF001C07001C07001C07001C07001C07001C07001C07001C07001C07007F1FC0FFBF E07F1FC013197F9816>72 D<1FFC003FFE007FFF00780F00F00780E00380E00380E00380 E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380F00780 F00780780F007FFF003FFE001FFC0011197E9816>79 D<07E3001FFF003FFF00781F00F0 0700E00700E00700E00000F000007800003F80001FF00007FC0000FE00000F0000070000 0380000380600380E00380E00700F80F00FFFE00FFFC00C7F00011197E9816>83 D<7FFFE0FFFFE0FFFFE0E0E0E0E0E0E0E0E0E0E0E0E000E00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00007FC000FFE 0007FC0013197F9816>I<1FE0003FF0007FF800783C00300E00000E00000E0003FE001F FE003E0E00700E00E00E00E00E00E00E00783E007FFFE03FE7E00F83E013127E9116>97 D<03F80FFC1FFE3C1E780C7000E000E000E000E000E000F000700778073E0E1FFC0FF803 F010127D9116>99 D<003F00007F00003F0000070000070000070000070003C7000FF700 1FFF003C1F00780F00700700E00700E00700E00700E00700E00700E00700700F00700F00 3C1F001FFFE00FE7F007C7E014197F9816>I<03E00FF81FFC3C1E780E7007E007FFFFFF FFFFFFE000E000700778073C0F1FFE0FFC03F010127D9116>I<001F00007F8000FF8001 E78001C30001C00001C0007FFF00FFFF00FFFF0001C00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C0003FFE007FFF003FFE0011197F9816>I< 7E0000FE00007E00000E00000E00000E00000E00000E3C000EFE000FFF000F87800F0380 0E03800E03800E03800E03800E03800E03800E03800E03800E03800E03807FC7F0FFE7F8 7FC7F01519809816>104 D<018003C003C0018000000000000000007FC07FC07FC001C0 01C001C001C001C001C001C001C001C001C001C001C07FFFFFFF7FFF101A7D9916>I<00 3000780078003000000000000000001FF81FF81FF8003800380038003800380038003800 38003800380038003800380038003800380038003800386070F0F0FFE07FC03F800D237E 9916>I108 DI<7E3C00FEFE007FFF000F87800F03800E03800E03800E03800E03800E03800E 03800E03800E03800E03800E03807FC7F0FFE7F87FC7F01512809116>I<03E0000FF800 1FFC003C1E00780F00700700E00380E00380E00380E00380E00380F00780700700780F00 3C1E001FFC000FF80003E00011127E9116>I<7E3E00FEFF007FFF800F83C00F00E00E00 E00E00700E00700E00700E00700E00700E00700E00E00F01E00F83C00FFF800EFF000E3C 000E00000E00000E00000E00000E00000E00007FC000FFE0007FC000141B809116>I114 D<0FEC3FFC7FFCF03CE01C E01C70007F801FF007F8003C600EE00EF00EF81EFFFCFFF8C7E00F127D9116>I<030000 0700000700000700000700007FFF00FFFF00FFFF00070000070000070000070000070000 07000007000007010007038007038007038007870003FE0001FC0000F80011177F9616> I<7E1F80FE3F807E1F800E03800E03800E03800E03800E03800E03800E03800E03800E03 800E03800E03800E0F800FFFF007FBF803E3F01512809116>I<7F1FC0FF9FE07F1FC01C 07000E07000E0E000E0E00070E00071C00071C00039C00039C0003980001B80001B80000 F00000F00000F00000E00000E00000E00001C00079C0007BC0007F80003F00003C000013 1B7F9116>121 D<3FFFC07FFFC07FFFC0700780700F00701E00003C0000780001F00003 E0000780000F00001E01C03C01C07801C0FFFFC0FFFFC0FFFFC012127F9116>I E /Fm 58 123 df<0001C001F1C003F1C007F0000700000E00000E00000E00000E00000E 00000E00000E0000FFF1C0FFF1C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0121E809D16>12 D<00E001C0038007000E000E001C001C003800380038007000700070007000E000E000E0 00E000E000E000E000E000E000E000E000E00070007000700070003800380038001C001C 000E000E000700038001C000E00B2A7E9E10>40 DI<018001C0018001806186F99F7DBE1FF807E007E01FF87DBEF99F618601800180 01C0018010127E9E15>I<00060000000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000FFFFFFE0FFFFFFE000 060000000600000006000000060000000600000006000000060000000600000006000000 0600000006000000060000000600001B1C7E9720>II< FFE0FFE00B0280890E>I<010007003F00FF00C700070007000700070007000700070007 000700070007000700070007000700070007000700070007000700FFF8FFF80D1C7C9B15 >49 D<07C01FF03878701C601EE00EC00F400F400700070007000F000E001E001C003C00 7800F001E001C0038007000E001C0038007000FFFFFFFF101C7E9B15>I<001C0000003E 0000003E0000002E0000006700000067000000E7800000C7800000C3800001C3C0000183 C0000181C0000381E0000381E0000700F0000700F0000600F0000E0078000FFFF8000FFF F8001C003C001C003C0018003C0038001E0038001E0070001F0070000F0070000F00E000 0780191D7F9C1C>65 DI<003FC000 FFF003C0F00780300F00001E00003C00003C0000780000780000780000F00000F00000F0 0000F00000F00000F00000F00000F00000F000007800007800007800003C00003C00001E 00000F000807801803C07800FFF0003F80151F7D9D1B>IIII<003F8001FFF003C0F80780380F00181E0000 3C00003C0000780000780000780000F00000F00000F00000F00000F00000F00000F007F8 F007F8F000387800387800387800383C00383C00381E00380F003807803803C0F801FFF0 003F80151F7D9D1C>III75 DIII<003F000001FFE00003FFF00007C0F8000F807C001E001E 003E001F003C000F00780007807800078078000780F00003C0F00003C0F00003C0F00003 C0F00003C0F00003C0F00003C0F00003C0F80007C078000780780007807C000F803C000F 003E001F001F003E000F807C0007C0F80003FFF00001FFE000003F00001A1F7E9D1F>I< FFFC00FFFF00F00F80F003C0F001E0F000F0F000F0F000F0F000F0F000F0F000F0F001E0 F003E0F00FC0FFFF80FFFE00F00000F00000F00000F00000F00000F00000F00000F00000 F00000F00000F00000F00000F00000141D7C9C1B>I82 D<03F8000FFE001C0F00380700700300600000E00000E00000E00000 E00000F000007800007F00003FE0001FFC0007FE0001FF00001F800007800003C00003C0 0001C00001C00001C00001C0C00180E00380F007007C0E001FFC0007F000121F7E9D17> IIIII<78000E007C001E00 3C003C001E0038000F0070000F00F0000781E00003C1C00001C3C00001E7800000F70000 007E0000003E0000003C0000003C0000007E00000077000000E7800001E3800003C1C000 0381E0000700F0000F00F8000E0078001C003C003C003E0078001F0070000F00F0000F80 191D7F9C1C>II91 D93 D<0FC03FF07FF87038401C001C001C00FC0FFC3FFC781C E01CE01CE01CF07C7FFC7FDC3F1C0E127E9114>97 DI<07E00FF81FFC3C1C70047000E000E000E000E000E0 00E000700070043C1C1FFC0FF807E00E127E9112>I<000E000E000E000E000E000E000E 000E000E000E000E0F8E1FEE3FFE7C3E700E700EE00EE00EE00EE00EE00EE00EF00E701E 7C3E3FFE1FEE0F8E0F1D7E9C15>I<07C01FE03FF078787018601CFFFCFFFCFFFCE000E0 00E000700070043C1C3FFC1FF807E00E127E9112>I<00FC01FC03FC07000E000E000E00 0E000E000E000E00FFE0FFE00E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E1D809C0D>I<03C3C00FFFC01FFFC01C3800381C00381C00381C00 381C00381C001C38001FF8001FF0003BC0003800003800001FFC001FFF003FFF80700780 E001C0E001C0E001C0F003C07C0F803FFF001FFE0007F800121B7F9115>III107 DIII<03F000 0FFC001FFE003C0F00780780700380E001C0E001C0E001C0E001C0E001C0F003C0700380 7807803C0F001FFE000FFC0003F00012127F9115>II114 D<1FC03FF07FF0F030E000E000F0007F003FC0 1FE000F0003800388038F078FFF07FE01FC00D127F9110>I<1C001C001C001C001C001C 00FFE0FFE01C001C001C001C001C001C001C001C001C001C001C001C001C201FF00FF007 C00C187F970F>IIII<7003807807003C 0E001C1C000E1C0007380003F00001E00001C00001E00003F0000738000E18000E1C001C 0E00380700700380F003C01212809113>II<7FFC7FFC7FFC007800F000E001E003C0038007000F001E001C003C007800FF FCFFFCFFFC0E127F9112>I E /Fn 3 104 df<3078F06005047C830D>46 D<01E007100C1018083810701070607F80E000E000E000E000E000E0086010602030C01F 000D127B9113>101 D<00F3018F030F06070E0E0C0E1C0E1C0E381C381C381C381C3838 30383038187818F00F700070007000E000E0C0C0E1C0C3007E00101A7D9113>103 D E /Fo 24 121 df<78FCFCFCFC7806067D850D>46 D<78FCFCFCFC7800000000000078 FCFCFCFC7806127D910D>58 D61 D<001FE02000FFF8E003F80FE007C003E00F8001E01F0000E03E0000E03E0000607E0000 607C000060FC000000FC000000FC000000FC000000FC000000FC000000FC000000FC0000 007C0000607E0000603E0000603E0000C01F0000C00F80018007C0030003F80E0000FFFC 00001FE0001B1C7D9B22>67 D73 D80 D<003FE00001F07C0003C01E000F800F801F0007 C01F0007C03E0003E07E0003F07C0001F07C0001F0FC0001F8FC0001F8FC0001F8FC0001 F8FC0001F8FC0001F8FC0001F8FC0001F87C0001F07C0001F07E0003F03E0003E03E0703 E01F08C7C00F906F8003D07E0001F87C00003FF8080000380800003C1800003FF800001F F800001FF000000FF0000007E0000003C01D247D9B24>I<0FF8001C1E003E0F803E0780 3E07C01C07C00007C0007FC007E7C01F07C03C07C07C07C0F807C0F807C0F807C0780BC0 3E13F80FE1F815127F9117>97 D<03FC000E0E001C1F003C1F00781F00780E00F80000F8 0000F80000F80000F80000F800007800007801803C01801C03000E0E0003F80011127E91 15>99 D<000FF0000FF00001F00001F00001F00001F00001F00001F00001F00001F00001 F001F9F00F07F01C03F03C01F07801F07801F0F801F0F801F0F801F0F801F0F801F0F801 F07801F07801F03C01F01C03F00F0FFE03F9FE171D7E9C1B>I<01FC000F07001C03803C 01C07801C07801E0F801E0F801E0FFFFE0F80000F80000F800007800007C00603C00601E 00C00F038001FC0013127F9116>I<03F8F00E0F381E0F381C07303C07803C07803C0780 3C07801C07001E0F000E0E001BF8001000001800001800001FFF001FFFC00FFFE01FFFF0 7801F8F00078F00078F000787000707800F01E03C007FF00151B7F9118>103 D<1E003F003F003F003F001E00000000000000000000000000FF00FF001F001F001F001F 001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B1E7F9D0E>105 D108 DII<01FC000F07801C01C03C01E07800F07800F0F800F8 F800F8F800F8F800F8F800F8F800F87800F07800F03C01E01E03C00F078001FC0015127F 9118>II114 D<1FD830786018E018E018F000FF807FE07FF01FF807FC007CC01CC01C E01CE018F830CFC00E127E9113>I<0300030003000300070007000F000F003FFCFFFC1F 001F001F001F001F001F001F001F001F001F0C1F0C1F0C1F0C0F08079803F00E1A7F9913 >III120 D E /Fp 89 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8F870000000 000070F8F8F870051C779B18>33 D<4010E038F078E038E038E038E038E038E038E038E0 38E038E03860300D0E7B9C18>I<030600078F00078F00078F00078F00078F00078F007F FFC0FFFFE0FFFFE07FFFC00F1E000F1E000F1E000F1E000F1E000F1E007FFFC0FFFFE0FF FFE07FFFC01E3C001E3C001E3C001E3C001E3C001E3C000C1800131C7E9B18>I<380300 7C07807C0780EE0F80EE0F00EE0F00EE1F00EE1E00EE1E00EE3E007C3C007C3C00387C00 00780000780000F80000F00001F00001E00001E00003E00003C00003C00007C000078380 0787C00F87C00F0EE00F0EE01F0EE01E0EE01E0EE03E0EE03C07C03C07C018038013247E 9F18>37 D<01C00007E0000FF0000E70001C38001C38001C38001C38001C73F01C73F01C E3F00FE3800FC7000F87000F07001F0E003F0E007B8E0073DC00E1DC00E0F800E0F800E0 7070E0787070FC707FFFE03FCFE00F03C0141C7F9B18>I<387C7C7E3E0E0E0E1C1C38F8 F0C0070E789B18>I<007000F001E003C007800F001E001C003800380070007000700070 00E000E000E000E000E000E000E000E0007000700070007000380038001C001E000F0007 8003C001F000F000700C24799F18>I<6000F00078003C001E000F000780038001C001C0 00E000E000E000E00070007000700070007000700070007000E000E000E000E001C001C0 038007800F001E003C007800F00060000C247C9F18>I<01C00001C00001C00001C000C1 C180F1C780F9CF807FFF001FFC0007F00007F0001FFC007FFF00F9CF80F1C780C1C18001 C00001C00001C00001C00011147D9718>I<00600000F00000F00000F00000F00000F000 00F00000F0007FFFC0FFFFE0FFFFE07FFFC000F00000F00000F00000F00000F00000F000 00F00000600013147E9718>I<1C3E7E7F3F1F070E1E7CF860080C788518>I<7FFF00FFFF 80FFFF807FFF0011047D8F18>I<3078FCFC78300606778518>I<00030000078000078000 0F80000F00001F00001E00001E00003E00003C00007C0000780000780000F80000F00001 F00001E00003E00003C00003C00007C0000780000F80000F00000F00001F00001E00003E 00003C00003C00007C0000780000F80000F00000F0000060000011247D9F18>I<01F000 07FC000FFE001F1F001C07003803807803C07001C07001C0E000E0E000E0E000E0E000E0 E000E0E000E0E000E0E000E0E000E0F001E07001C07001C07803C03803801C07001F1F00 0FFE0007FC0001F000131C7E9B18>I<01800380038007800F803F80FF80FB8043800380 0380038003800380038003800380038003800380038003800380038003807FFCFFFE7FFC 0F1C7B9B18>I<03F0000FFE003FFF007C0F807003C0E001C0F000E0F000E06000E00000 E00000E00001C00001C00003C0000780000F00001E00003C0000780000F00001E00007C0 000F80001E00E03C00E07FFFE0FFFFE07FFFE0131C7E9B18>I<07F8001FFE003FFF0078 07807803C07801C03001C00001C00003C0000380000F0003FF0003FE0003FF0000078000 03C00001C00000E00000E00000E0F000E0F000E0F001C0F003C07C07803FFF001FFE0003 F800131C7E9B18>I<001F00003F0000770000770000E70001E70001C700038700078700 0707000E07001E07003C0700380700780700F00700FFFFF8FFFFF8FFFFF8000700000700 000700000700000700000700007FF000FFF8007FF0151C7F9B18>I<1FFF803FFF803FFF 803800003800003800003800003800003800003800003800003BF8003FFE003FFF003C07 801803C00001C00000E00000E06000E0F000E0F000E0E001C07003C07C0F803FFF001FFC 0003F000131C7E9B18>I<007E0001FF0007FF800F83C01E03C01C03C038018038000070 0000700000E1F800E7FE00FFFF00FE0780F803C0F001C0F000E0E000E0F000E07000E070 00E07000E03801C03C03C01E07800FFF0007FE0001F800131C7E9B18>II<03F8000FFE001FFF003E0F803803807001C07001 C07001C07001C03803803C07801FFF0007FC000FFE001F1F003C07807001C0F001E0E000 E0E000E0E000E0E000E07001C07803C03E0F801FFF000FFE0003F800131C7E9B18>I<03 F0000FFC001FFE003C0F00780780700380E001C0E001C0E001C0E001E0E001E07001E078 03E03C0FE01FFFE00FFEE003F0E00000E00001C00001C00001C0300380780780780F0078 3E003FFC001FF00007C000131C7E9B18>I<3078FCFC783000000000000000003078FCFC 78300614779318>I<183C7E7E3C180000000000000000183C7E7E3E1E0E1C3C78F06007 1A789318>I<000300000780001F80003F00007E0001FC0003F00007E0001FC0003F0000 7E0000FC0000FC00007E00003F00001FC00007E00003F00001FC00007E00003F00001F80 00078000030011187D9918>I<7FFFC0FFFFE0FFFFE0FFFFE00000000000000000000000 00FFFFE0FFFFE0FFFFE07FFFC0130C7E9318>I<600000F00000FC00007E00003F00001F C00007E00003F00001FC00007E00003F00001F80001F80003F00007E0001FC0003F00007 E0001FC0003F00007E0000FC0000F0000060000011187D9918>I<00700000F80000F800 00D80000D80001DC0001DC0001DC00018C00038E00038E00038E00038E00030600070700 0707000707000707000FFF800FFF800FFF800E03800E03801C01C01C01C07F07F0FF8FF8 7F07F0151C7F9B18>65 DI<00F8E003FEE0 07FFE00F07E01E03E03C01E03800E07000E07000E0700000E00000E00000E00000E00000 E00000E00000E00000E000007000007000E07000E03800E03C00E01E01C00F07C007FF80 03FE0000F800131C7E9B18>I<7FF800FFFE007FFF001C0F801C03C01C03C01C01E01C00 E01C00E01C00F01C00701C00701C00701C00701C00701C00701C00701C00701C00F01C00 E01C00E01C01E01C01C01C03C01C0F807FFF00FFFE007FF800141C7F9B18>III<01F1 C003FDC00FFFC01F0FC01C03C03803C03801C07001C07001C0700000E00000E00000E000 00E00000E00000E00FF0E01FF0E00FF07001C07001C07003C03803C03803C01C07C01F0F C00FFFC003FDC001F1C0141C7E9B18>I<7F07F0FF8FF87F07F01C01C01C01C01C01C01C 01C01C01C01C01C01C01C01C01C01C01C01FFFC01FFFC01FFFC01C01C01C01C01C01C01C 01C01C01C01C01C01C01C01C01C01C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>I< 7FFF00FFFF807FFF0001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C0007FFF00FFFF807FFF00111C7D9B18>I<01FFC003FFC001FFC0000E00000E00000E 00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E 00000E00000E00000E00000E00F00E00F00E00F03C007FFC003FF0000FC000121C7D9B18 >I<7F07F0FF87F87F07F01C03C01C07801C07001C0E001C1E001C3C001C38001C70001C F0001DF0001DF0001FB8001FB8001F1C001E1C001C0E001C0E001C07001C07001C03801C 03801C01C07F03F0FF87F87F03F0151C7F9B18>I<7FE000FFE0007FE0000E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0000 0E00000E00000E00000E00700E00700E00700E00700E00707FFFF0FFFFF07FFFF0141C7F 9B18>II<7E07F0FF0FF87F07F01D81C01D 81C01D81C01DC1C01CC1C01CC1C01CE1C01CE1C01CE1C01C61C01C71C01C71C01C31C01C 39C01C39C01C39C01C19C01C19C01C1DC01C0DC01C0DC01C0DC07F07C0FF87C07F03C015 1C7F9B18>I<0FF8003FFE007FFF00780F00700700F00780E00380E00380E00380E00380 E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380 F00780700700780F007FFF003FFE000FF800111C7D9B18>II<0FF8003FFE007FFF00780F00700700F00780E00380E00380E00380E0 0380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E1E380E1 E380F0E78070F700787F007FFF003FFE000FFC00001C00001E00000E00000F0000070000 070011227D9B18>I<7FF800FFFE007FFF001C0F801C03801C03C01C01C01C01C01C01C0 1C03C01C03801C0F801FFF001FFE001FFE001C0F001C07001C03801C03801C03801C0380 1C03801C039C1C039C1C039C7F01F8FF81F87F00F0161C7F9B18>I<03F3801FFF803FFF 807C0F80700780E00380E00380E00380E000007000007800003F00001FF00007FE0000FF 00000F800003C00001C00000E00000E06000E0E000E0E001E0F001C0F80780FFFF80FFFE 00E7F800131C7E9B18>I<7FFFF8FFFFF8FFFFF8E07038E07038E07038E0703800700000 700000700000700000700000700000700000700000700000700000700000700000700000 700000700000700000700000700007FF0007FF0007FF00151C7F9B18>IIII<7F8FE07F9FE07F8FE00E07000F0700070E00078E00 039C0003DC0001F80001F80000F00000F00000700000F00000F80001F80001DC00039E00 038E00070F000707000E07800E03801E03C07F07F0FF8FF87F07F0151C7F9B18>II<3FFFE07FFFE07FFFE07001C07003C070078070 0700000F00001E00001C00003C0000780000700000F00001E00001C00003C00007800007 00000F00001E00E01C00E03C00E07800E07000E0FFFFE0FFFFE0FFFFE0131C7E9B18>I< FFF8FFF8FFF8E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000 E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000FFF8FFF8FFF8 0D24779F18>I<600000F00000F00000F800007800007C00003C00003C00003E00001E00 001F00000F00000F00000F800007800007C00003C00003C00003E00001E00001F00000F0 0000F800007800007800007C00003C00003E00001E00001E00001F00000F00000F800007 8000078000030011247D9F18>II<7FFF00FFFF80FFFF807FFF0011047D 7F18>95 D<061E3E387070E0E0E0F8FC7C7C38070E789E18>I<1FE0003FF8007FFC0078 1E00300E0000070000070000FF0007FF001FFF007F0700780700E00700E00700E00700F0 0F00781F003FFFF01FFBF007E1F014147D9318>I<7E0000FE00007E00000E00000E0000 0E00000E00000E00000E3E000EFF800FFFC00FC1E00F80E00F00700E00700E00380E0038 0E00380E00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF80063E00151C80 9B18>I<01FE0007FF001FFF803E0780380300700000700000E00000E00000E00000E000 00E00000E000007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<00 1F80003F80001F8000038000038000038000038000038003E3800FFB801FFF803C1F8038 0F80700780700380E00380E00380E00380E00380E00380E00380700780700780380F803C 1F801FFFF00FFBF803E3F0151C7E9B18>I<01F00007FC001FFE003E0F00380780700380 700380E001C0E001C0FFFFC0FFFFC0FFFFC0E000007000007001C03801C03E03C01FFF80 07FF0001FC0012147D9318>I<001F80007FC000FFE000E1E001C0C001C00001C00001C0 007FFFC0FFFFC0FFFFC001C00001C00001C00001C00001C00001C00001C00001C00001C0 0001C00001C00001C00001C00001C0007FFF007FFF007FFF00131C7F9B18>I<01E1F007 FFF80FFFF81E1E301C0E003807003807003807003807003807001C0E001E1E001FFC001F F80039E0003800001C00001FFE001FFFC03FFFE07801F0700070E00038E00038E00038E0 00387800F07E03F01FFFC00FFF8001FC00151F7F9318>I<7E0000FE00007E00000E0000 0E00000E00000E00000E00000E3E000EFF800FFFC00FC1C00F80E00F00E00E00E00E00E0 0E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFFE7FE7FC3FC 171C809B18>I<03800007C00007C00007C0000380000000000000000000000000007FC0 00FFC0007FC00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0 0001C00001C00001C00001C000FFFF00FFFF80FFFF00111D7C9C18>I<0038007C007C00 7C003800000000000000000FFC1FFC0FFC001C001C001C001C001C001C001C001C001C00 1C001C001C001C001C001C001C001C001C001C001C001C001C6038F078FFF07FE03F800E 277E9C18>II<7FE000FFE0007FE00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00000E0007FFFC0FFFFE07FFF C0131C7E9B18>I<7CE0E000FFFBF8007FFFF8001F1F1C001E1E1C001E1E1C001C1C1C00 1C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C00 1C1C1C007F1F1F00FFBFBF807F1F1F001914819318>I<7E3E00FEFF807FFFC00FC1C00F 80E00F00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E 00E07FC3FCFFE7FE7FC3FC1714809318>I<01F0000FFE001FFF003E0F803803807001C0 7001C0E000E0E000E0E000E0E000E0E000E0F001E07001C07803C03C07803E0F801FFF00 0FFE0001F00013147E9318>I<7E3E00FEFF807FFFC00FC1E00F80E00F00700E00700E00 380E00380E00380E00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF800E3E 000E00000E00000E00000E00000E00000E00000E00007FC000FFE0007FC000151E809318 >I<01E38007FB801FFF803E1F80380F80700780700780E00380E00380E00380E00380E0 0380E00380700780700780380F803C1F801FFF800FFB8003E38000038000038000038000 0380000380000380000380003FF8003FF8003FF8151E7E9318>I<7F87E0FF9FF07FBFF8 03F87803F03003E00003C00003C000038000038000038000038000038000038000038000 0380000380007FFE00FFFF007FFE0015147F9318>I<07F7003FFF007FFF00780F00E007 00E00700E007007C00007FE0001FFC0003FE00001F00600780E00380E00380F00380F80F 00FFFF00FFFC00E7F00011147D9318>I<0180000380000380000380000380007FFFC0FF FFC0FFFFC003800003800003800003800003800003800003800003800003800003804003 80E00380E00380E001C1C001FFC000FF80003E0013197F9818>I<7E07E0FE0FE07E07E0 0E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E0 0E01E00F03E007FFFC03FFFE01FCFC1714809318>I<7F8FF0FF8FF87F8FF01E03C00E03 800E03800E0380070700070700070700038E00038E00038E00038E0001DC0001DC0001DC 0000F80000F80000700015147F9318>II<7F8FF07F9FF07F8FF0070700078E00039E0001DC0001F800 00F80000700000F00000F80001DC00039E00038E000707000F07807F8FF0FF8FF87F8FF0 15147F9318>I<7F8FF0FF8FF87F8FF00E01C00E03800E03800703800707000707000387 00038600038E0001CE0001CE0000CC0000CC0000DC000078000078000078000070000070 0000700000F00000E00079E0007BC0007F80003F00001E0000151E7F9318>I<3FFFF07F FFF07FFFF07001E07003C0700780000F00001E00003C0000F80001F00003C0000780000F 00701E00703C0070780070FFFFF0FFFFF0FFFFF014147F9318>I<0007E0001FE0007FE0 00780000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000 01E0007FC000FF8000FF80007FC00001E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E000007800007FE0001FE00007E013247E9F18>I<60F0 F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0600424 769F18>I<7C0000FF0000FFC00003C00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000F000007FC0003FE0003FE0007FC000F00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00003C000FFC000FF 00007C000013247E9F18>I E /Fq 66 124 df<003F83F800FF8FF801FF9FF803E03E00 038038000780780007807800078078000780780007807800078078000780780007807800 FFFC7FC0FFFC7FC0FFFC7FC0078078000780780007807800078078000780780007807800 078078000780780007807800078078000780780007807800078078000780780007807800 078078000780780007807800078078001D2380A21C>11 D<000078007C7800FC7801FC78 03C000038000078000078000078000078000078000078000078000078000FFFC78FFFC78 FFFC78078078078078078078078078078078078078078078078078078078078078078078 078078078078078078078078078078078078078078078078152480A31A>I<0000C01800 0000C018000000C018000001803000000180300000018030000001803000000300600000 03006000000300600000030060000003006000000600C000000600C000000600C0000006 00C000000C018000FFFFFFFFC0FFFFFFFFC0001803000000180300000018030000001803 00000030060000003006000000300600000030060000FFFFFFFFC0FFFFFFFFC000600C00 0000C018000000C018000000C018000000C0180000018030000001803000000180300000 018030000003006000000300600000030060000003006000000600C000000600C0000006 00C00000222D7DA229>35 D<007000E001C003C0078007000F000E001E001C003C003C00 380078007800780078007000F000F000F000F000F000F000F000F000F000F000F000F000 F000F0007000780078007800780038003C003C001C001E000E000F000700078003C001C0 00E000700C327DA413>40 DI<787878781830306060E0050A7C830E>44 DII<000080000180000180000300 000300000300000600000600000600000C00000C00000C00001800001800001800003000 00300000300000600000600000600000C00000C00000C000018000018000018000018000 0300000300000300000600000600000600000C00000C00000C0000180000180000180000 300000300000300000600000600000600000C00000C00000C0000011317DA418>I<00C0 01C007C0FFC0FFC0FBC003C003C003C003C003C003C003C003C003C003C003C003C003C0 03C003C003C003C003C003C003C003C003C003C003C003C0FFFFFFFFFFFF10227CA118> 49 D<03F0000FFC001FFE003C1F003007807007C06003C0E003E0C001E04001E04001E0 0001E00001E00001E00003C00003C0000780000780000F00001E00003C0000780000F000 01E00001C0000380000700000E00001C0000380000700000FFFFE0FFFFE0FFFFE013227E A118>I<01F00007FC001FFF003E0F003807807003C02003C02003C00003C00003C00003 C0000780000780000F00001E0003FC0003F80003FE00000F000007800003C00003C00001 E00001E00001E00001E00001E08001E0C003C0E003C07007803C0F801FFF000FFC0003F0 0013237EA118>I<001F00001F00002F00002F00006F0000EF0000CF0001CF0001CF0003 8F00038F00078F00070F000F0F000E0F001E0F003C0F003C0F00780F00780F00F00F00FF FFF8FFFFF8FFFFF8000F00000F00000F00000F00000F00000F00000F00000F00000F0015 217FA018>I<3FFF803FFF803FFF803C00003C00003C00003C00003C00003C00003C0000 3C00003C00003CF8003FFE003FFF003F0F803E07803C03C03803C00001E00001E00001E0 0001E00001E00001E00001E04003C04003C0E003C07007807C1F003FFE000FFC0003F000 13227EA018>I<007E0001FF0003FF0007C1000F00001E00001E00003C00003C00007800 0078000078F800F1FE00F7FF00FF0F80FC0780F803C0F803C0F801C0F001E0F001E0F001 E0F001E0F001E07001E07001E07801E07803C03803C03C03801C07801F0F000FFE0007FC 0001F00013237EA118>II<01F00007FC000FFE001E0F003C07803C07807803C07803C07803C0 7803C07803C03803803C07801E0F000F1E0007FC0003F8000FFE001E0F003C07807803C0 7803C0F001E0F001E0F001E0F001E0F001E0F001E07803C07803C03C07803E0F801FFF00 07FC0001F00013237EA118>I<01F00007FC000FFE001E0F003C07003807807803807803 C0F003C0F001C0F001E0F001E0F001E0F001E0F001E0F001E07003E07803E07803E03C07 E03E1FE01FFDE00FF1E003E3C00003C00003C0000380000780000700000F00101E00183C 003FF8001FF00007C00013237EA118>I<001F0000001F0000003F8000003F8000003B80 00007BC0000073C0000071C00000F1E00000F1E00000E0E00001E0F00001E0F00001C0F0 0003C0780003C078000380780007803C0007803C0007003C000F001E000F001E000FFFFE 001FFFFF001FFFFF001C000F003C0007803C00078038000780780003C0780003C0700003 C0F00001E0F00001E0E00001E01B237EA220>65 DI<000FF000007FFC0000FFFF 0001F01F0003C00700078000000F0000001E0000003E0000003C0000003C000000780000 007800000078000000F0000000F0000000F0000000F0000000F0000000F0000000F00000 00F0000000F00000007800000078000000780000003C0000003C0000003E0000001E0000 000F0000000780008003C0038001F00F8000FFFF00007FFC00000FF00019257DA31F>I< FFFE0000FFFFC000FFFFE000F003F000F000F800F0007C00F0003E00F0001E00F0000F00 F0000F00F0000780F0000780F0000780F00003C0F00003C0F00003C0F00003C0F00003C0 F00003C0F00003C0F00003C0F00003C0F0000380F0000780F0000780F0000780F0000F00 F0001F00F0001E00F0003C00F000FC00F003F800FFFFE000FFFFC000FFFE00001A237BA2 23>III<000FF000003FFE0000FFFF8001F80F8003E00380078000000F 0000001E0000001E0000003C0000003C000000780000007800000078000000F0000000F0 000000F0000000F0000000F0000000F0000000F000FFC0F000FFC0F000FFC0780003C078 0003C0780003C03C0003C03C0003C01E0003C01E0003C00F0003C0078003C003E003C001 F807C000FFFFC0003FFF00000FF8001A257DA321>III76 DII<001FC000007FF00001FFFC0003F07E0007C01F000F800F801F0007C0 1E0003C03C0001E03C0001E0780000F0780000F0780000F070000070F0000078F0000078 F0000078F0000078F0000078F0000078F0000078F0000078F0000078780000F0780000F0 780000F07C0001F03C0001E03E0003E01E0003C01F0007C00F800F8007C01F0003F07E00 01FFFC00007FF000001FC0001D257DA324>II82 D<00FE0003FFC007FFE00F81E01E00603C00003C00007800007800007800007800007800 007C00003C00003F00001FC0000FFC0007FF0001FF80003FC00007E00001F00000F00000 F8000078000078000078000078000078000078C000F0E000F0F801E07E07C03FFF800FFF 0001FC0015257EA31B>IIIII<780003E07C0003C03E0007 C01E000F800F000F000F801F0007C03E0003C03C0003E07C0001F0780000F0F0000079F0 00007DE000003FC000001FC000001F8000000F0000001F8000001FC000003FC000007DE0 000079F00000F0F00001F0780001E03C0003C03E0007C01E0007800F000F000F801F0007 801E0003C03C0003E07C0001F0780000F0F80000F81D237FA220>I<07E01FF83FFC381E 201E000F000F000F000F00FF07FF1FFF3E0F780FF00FF00FF00FF00FF83F7FFF3FEF1F8F 10167E9517>97 DI<01FC0007FF000FFF801F03803C0180780000780000700000 F00000F00000F00000F00000F00000F000007800007800007800003C00401F03C00FFFC0 07FF8001FC0012167E9516>I<0003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C003E3C00FFBC01FFFC03F0FC03C07C07803C07803 C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07803C07803C03C07C03E0F C01FFFC00FFBC003E3C012237EA219>I<03F00007FC001FFE003E0F003C078078038078 0380F001C0FFFFC0FFFFC0FFFFC0F00000F00000F000007000007800007800003C00801F 07800FFF8007FF0001F80012167E9516>I<003F00FF01FF03C003800780078007800780 0780078007800780FFF8FFF8FFF807800780078007800780078007800780078007800780 07800780078007800780078007800780102380A20F>I<01F07807FFF80FFFF81F1F001E 0F003C07803C07803C07803C07803C07801E0F001F1F000FFE001FFC0019F00038000038 00003C00001FFE001FFFC01FFFE03FFFF07801F07800F8F00078F00078F00078F0007878 00F03E03E01FFFC00FFF8001FC0015217F9518>III<00F000F0 00F000F000000000000000000000000000000000000000F000F000F000F000F000F000F0 00F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F0 00F000F080F0E3E0FFE07FC01F000C2D83A20D>IIIII<01FC00 07FF000FFF801F07C03C01E07800F07800F0700070F00078F00078F00078F00078F00078 F000787800F07800F07C01F03E03E01F07C00FFF8007FF0001FC0015167F9518>II<03E3C007FBC00F FFC01F0FC03C07C07C03C07803C0F803C0F003C0F003C0F003C0F003C0F003C0F003C0F8 03C07803C07803C03C07C03F0FC01FFFC00FFBC003E3C00003C00003C00003C00003C000 03C00003C00003C00003C00003C00003C012207E9519>II<07F01FFC3FFE3C0E7806780078007C003F003FF01FF80FFC01FE001F000F000F00 0FC00FF81EFFFE3FFC0FF010167F9513>I<0F000F000F000F000F000F00FFF8FFF8FFF8 0F000F000F000F000F000F000F000F000F000F000F000F000F000F000F080F1C07FC07F8 03E00E1C7F9B12>IIII<7801F07C01E03E03C01E07C00F0780078F0007DE0003FC00 01FC0000F80000700000F80001FC0003DC00039E00078F000F07801E07801E03C03C01E0 7800F0F800F81516809516>II<7FFFC07FFFC07FFFC0000F80000F00001E00003E00007C0000780000 F00001F00003E00003C0000780000F80001F00001E00003C00007C0000FFFFC0FFFFC0FF FFC012167F9515>II E /Fr 68 125 df<007E1F0001C1B1800303E3C00703C3C00E03C1800E01C0000E01C0000E01C0 000E01C0000E01C0000E01C000FFFFFC000E01C0000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0007F87FC001A1D809C18>11 D<007E0001C1800301800703C00E03C0 0E01800E00000E00000E00000E00000E0000FFFFC00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C07F87F8 151D809C17>I<007FC001C1C00303C00703C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C0FFFFC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C00E01C00E01C00E01C00E01C00E01C07FCFF8151D809C17>I<003F07E00001C0 9C18000380F018000701F03C000E01E03C000E00E018000E00E000000E00E000000E00E0 00000E00E000000E00E00000FFFFFFFC000E00E01C000E00E01C000E00E01C000E00E01C 000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C00 0E00E01C000E00E01C000E00E01C000E00E01C000E00E01C007FC7FCFF80211D809C23> I<6060F0F0F8F86868080808080808101010102020404080800D0C7F9C15>34 D<004000800100020006000C000C0018001800300030007000600060006000E000E000E0 00E000E000E000E000E000E000E000E000E000600060006000700030003000180018000C 000C00060002000100008000400A2A7D9E10>40 D<800040002000100018000C000C0006 00060003000300038001800180018001C001C001C001C001C001C001C001C001C001C001 C001C0018001800180038003000300060006000C000C00180010002000400080000A2A7E 9E10>I<60F0F0701010101020204080040C7C830C>44 DI<60F0 F06004047C830C>I<00010003000600060006000C000C000C0018001800180030003000 300060006000C000C000C0018001800180030003000300060006000C000C000C00180018 001800300030003000600060006000C000C00010297E9E15>I<03C00C301818300C300C 700E60066006E007E007E007E007E007E007E007E007E007E007E007E007E00760066006 700E300C300C18180C3007E0101D7E9B15>I<030007003F00C700070007000700070007 00070007000700070007000700070007000700070007000700070007000700070007000F 80FFF80D1C7C9B15>I<07C01830201C400C400EF00FF80FF807F8077007000F000E000E 001C001C00380070006000C00180030006010C01180110023FFE7FFEFFFE101C7E9B15> I<07E01830201C201C781E780E781E381E001C001C00180030006007E00030001C001C00 0E000F000F700FF80FF80FF80FF00E401C201C183007E0101D7E9B15>I<60F0F0600000 000000000000000060F0F06004127C910C>58 D<60F0F0600000000000000000000060F0 F0701010101020204080041A7C910C>I<7FFFFFC0FFFFFFE00000000000000000000000 000000000000000000000000000000000000000000FFFFFFE07FFFFFC01B0C7E8F20>61 D<000600000006000000060000000F0000000F0000000F00000017800000178000001780 000023C0000023C0000023C0000041E0000041E0000041E0000080F0000080F0000180F8 000100780001FFF80003007C0002003C0002003C0006003E0004001E0004001E000C001F 001E001F00FF80FFF01C1D7F9C1F>65 DI< 001F808000E0618001801980070007800E0003801C0003801C0001803800018078000080 7800008070000080F0000000F0000000F0000000F0000000F0000000F0000000F0000000 F0000000700000807800008078000080380000801C0001001C0001000E00020007000400 0180080000E03000001FC000191E7E9C1E>IIII<001F8080 00E0618001801980070007800E0003801C0003801C000180380001807800008078000080 70000080F0000000F0000000F0000000F0000000F0000000F0000000F000FFF0F0000F80 700007807800078078000780380007801C0007801C0007800E00078007000B8001801180 00E06080001F80001C1E7E9C21>III75 DIII<003F 800000E0E0000380380007001C000E000E001C0007003C00078038000380780003C07800 03C0700001C0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0F000 01E0700001C0780003C0780003C0380003803C0007801C0007000E000E0007001C000380 380000E0E000003F80001B1E7E9C20>II< FFFF00000F01E0000F0078000F003C000F001C000F001E000F001E000F001E000F001E00 0F001C000F003C000F0078000F01E0000FFF00000F03C0000F00E0000F00F0000F007800 0F0078000F0078000F0078000F0078000F0078000F0078100F0078100F0038100F003C20 FFF01C20000007C01C1D7E9B1F>82 D<07E0801C1980300580700380600180E00180E000 80E00080E00080F00000F800007C00007FC0003FF8001FFE0007FF0000FF80000F800007 C00003C00001C08001C08001C08001C0C00180C00180E00300D00200CC0C0083F800121E 7E9C17>I<7FFFFFC0700F01C0600F00C0400F0040400F0040C00F0020800F0020800F00 20800F0020000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00 00000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00 00001F800003FFFC001B1C7F9B1E>II87 D89 D<08081010202040404040808080808080B0B0F8F8 787830300D0C7A9C15>92 D<1FC000307000783800781C00301C00001C00001C0001FC00 0F1C00381C00701C00601C00E01C40E01C40E01C40603C40304E801F870012127E9115> 97 DI<07E00C3018783078703060 00E000E000E000E000E000E00060007004300418080C3007C00E127E9112>I<003F0000 070000070000070000070000070000070000070000070000070000070003E7000C170018 0F00300700700700600700E00700E00700E00700E00700E00700E0070060070070070030 0700180F000C370007C7E0131D7E9C17>I<03E00C301818300C700E6006E006FFFEE000 E000E000E00060007002300218040C1803E00F127F9112>I<00F8018C071E061E0E0C0E 000E000E000E000E000E00FFE00E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E007FE00F1D809C0D>I<00038003C4C00C38C01C3880181800381C00 381C00381C00381C001818001C38000C300013C0001000003000001800001FF8001FFF00 1FFF803003806001C0C000C0C000C0C000C06001803003001C0E0007F800121C7F9215> II<18003C003C0018000000000000 000000000000000000FC001C001C001C001C001C001C001C001C001C001C001C001C001C 001C001C001C00FF80091D7F9C0C>I<00C001E001E000C0000000000000000000000000 00000FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E060E0F0C0F1C061803E000B25839C0D>IIIII<03F0000E1C00180600300300700380600180 E001C0E001C0E001C0E001C0E001C0E001C06001807003803003001806000E1C0003F000 12127F9115>II<03C1000C3300180B00300F0070070070 0700E00700E00700E00700E00700E00700E00700600700700700300F00180F000C370007 C700000700000700000700000700000700000700000700003FE0131A7E9116>II<1F9030704030C010C010E010F8007F803FE00FF000F880388018C018C018E010 D0608FC00D127F9110>I<04000400040004000C000C001C003C00FFE01C001C001C001C 001C001C001C001C001C001C101C101C101C101C100C100E2003C00C1A7F9910>IIII<7F8FF00F03800F030007020003840001C8 0001D80000F00000700000780000F800009C00010E00020E000607000403801E07C0FF0F F81512809116>II<7FFC70386038407040F040E041C003 C0038007000F040E041C043C0C380870087038FFF80E127F9112>I II E /Fs 51 124 df<0001000200040008001000200060 00C0018001800300070006000E000C001C0018003800380030007000700060006000E000 E000C000C000C000C000C000C000C000C000C000C000C000C000C0004000600060002000 100010000800102E79A113>40 D<00100000080000040000060000020000030000030000 030000010000018000018000018000018000018000018000018000038000038000038000 0300000300000300000700000700000600000600000E00000C00000C00001C0000180000 380000300000700000600000E00000C0000180000100000300000600000C000018000030 0000600000800000112E80A113>I<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I<70F8F8F0E005057B840E>I<000F800030E000E07001C0 700380300380380700380F00780F00780E00781E00781E00703C00F03C00F03C00F03C00 F07801E07801E07801E07801C07003C0F003C0F00380F00780F00700700700700E00701C 003038001870000FC000151F7C9D17>48 D<000200020006000E003C00DC031C001C0038 003800380038007000700070007000E000E000E000E001C001C001C001C0038003800380 03800780FFF80F1E7B9D17>I<001F000061800080E00100E00200700220700420700410 700820F00820F00820F00840E00881E00703C0000380000700000C000018000060000080 000300000400000800401000401000802001807E030047FF0041FE0080FC00807800141F 7C9D17>I<001F0000718000C0C00180C00380E00700E00F00E00F01E01E01E01E01E01E 01E01E01C01C03C01C03C01C03C01C07C01C0F800C0F8006378003C700000F00000E0000 0E00001C00601C00F03800F07000E0600080C0004380003E0000131F7B9D17>57 D<070F1F1F0E0000000000000000000070F8F8F0E008147B930E>I<0000020000000600 0000060000000E0000001E0000001E0000003F0000002F0000004F0000004F0000008F00 00010F0000010F0000020F0000020F0000040F00000C0F0000080F0000100F0000100F00 00200F80003FFF800040078000C007800080078001000780010007800200078002000780 060007801E000F80FF807FF81D207E9F22>65 D<01FFFFC0001E00F0001E0078001E0038 001E003C003C003C003C003C003C003C003C003C0078007800780078007800F0007801E0 00F0078000FFFE0000F00F8000F003C001E001C001E001E001E001E001E001E003C001E0 03C001E003C001E003C001C0078003C00780078007800F0007801E000F007800FFFFE000 1E1F7D9E20>I<0000FE0200078186001C004C0038003C0060003C00C0001C01C0001803 800018070000180F0000181E0000101E0000103C0000003C000000780000007800000078 00000078000000F0000000F0000000F0000000F0000000F0000080700000807000008070 0001003800010038000200180004000C001800060020000381C00000FE00001F217A9F21 >I<01FFFF80001E00E0001E0070001E0038001E001C003C001C003C000E003C000E003C 000E0078000E0078000E0078000E0078000E00F0001E00F0001E00F0001E00F0001E01E0 003C01E0003C01E0003C01E0007803C0007003C0007003C000E003C001C0078001C00780 038007800E0007801C000F007000FFFFC0001F1F7D9E22>I<01FFFFFE001E001C001E00 0C001E0004001E0004003C0004003C0004003C0004003C00040078080800780800007808 000078180000F0300000FFF00000F0300000F0300001E0200001E0200001E0200001E000 1003C0002003C0002003C0004003C00040078000800780018007800100078007000F001F 00FFFFFE001F1F7D9E1F>I<01FFFFFC001E0038001E0018001E0008001E0008003C0008 003C0008003C0008003C00080078001000780800007808000078080000F0100000F03000 00FFF00000F0300001E0200001E0200001E0200001E0200003C0000003C0000003C00000 03C00000078000000780000007800000078000000F800000FFF800001E1F7D9E1E>I<00 00FC040007030C001C00980030007800E0007801C000380380003003800030070000300E 0000301E0000201E0000203C0000003C00000078000000780000007800000078000000F0 000000F000FFF0F0000780F0000780F0000F0070000F0070000F0070000F0070001E0038 001E0018003E001C002E000E00CC000383040000FC00001E217A9F23>I<01FFF3FFE000 1F003E00001E003C00001E003C00001E003C00003C007800003C007800003C007800003C 007800007800F000007800F000007800F000007800F00000F001E00000FFFFE00000F001 E00000F001E00001E003C00001E003C00001E003C00001E003C00003C007800003C00780 0003C007800003C007800007800F000007800F000007800F000007800F00000F801F0000 FFF1FFE000231F7D9E22>I<01FFF0001F00001E00001E00001E00003C00003C00003C00 003C0000780000780000780000780000F00000F00000F00000F00001E00001E00001E000 01E00003C00003C00003C00003C0000780000780000780000780000F8000FFF800141F7D 9E12>I<01FE00007FC0001E0000FC00001E0000F8000017000178000017000178000027 0002F00000270004F00000270004F00000270008F00000470009E00000470011E0000047 0021E00000470021E00000870043C00000838043C00000838083C00000838083C0000103 810780000103820780000103820780000103840780000203840F00000203880F00000203 900F00000203900F00000401E01E00000401E01E00000401C01E00000C01801E00001C01 803E0000FF8103FFC0002A1F7D9E29>77 D<01FFFF80001E00E0001E0070001E0038001E 003C003C003C003C003C003C003C003C003C0078007800780078007800F0007800E000F0 03C000F00F0000FFFC0000F0000001E0000001E0000001E0000001E0000003C0000003C0 000003C0000003C00000078000000780000007800000078000000F800000FFF000001E1F 7D9E1F>80 D<0007E040001C18C0003005800060038000C0038001C00180018001000380 010003800100038001000380000003C0000003C0000003F8000001FF800001FFE000007F F000001FF0000001F8000000780000007800000038000000380020003800200038002000 300060007000600060006000E0007000C000E8038000C606000081F800001A217D9F1A> 83 D<0FFFFFF01E0780E0180780201007802020078020200F0020600F0020400F002040 0F0020801E0040001E0000001E0000001E0000003C0000003C0000003C0000003C000000 78000000780000007800000078000000F0000000F0000000F0000000F0000001E0000001 E0000001E0000001E0000003E00000FFFF00001C1F789E21>I<7FFC1FF807C003C00780 010007800100078001000F0002000F0002000F0002000F0002001E0004001E0004001E00 04001E0004003C0008003C0008003C0008003C0008007800100078001000780010007800 1000F0002000F0002000F0002000F0004000F00040007000800070010000300200003804 00000C18000007E000001D20779E22>II<00F1800389C00707800E03801C03803C038038070078070078070078 0700F00E00F00E00F00E00F00E20F01C40F01C40703C40705C40308C800F070013147C93 17>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E 380F380F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207B9F 15>I<007E0001C1000300800E07801E07801C07003C0200780000780000780000F00000 F00000F00000F00000F0000070010070020030040018380007C00011147C9315>I<0000 780003F80000700000700000700000700000E00000E00000E00000E00001C00001C000F1 C00389C00707800E03801C03803C0380380700780700780700780700F00E00F00E00F00E 00F00E20F01C40F01C40703C40705C40308C800F070015207C9F17>I<007C01C207010E 011C013C013802780C7BF07C00F000F000F000F0007000700170023804183807C010147C 9315>I<00007800019C00033C00033C000718000700000700000E00000E00000E00000E 00000E0001FFE0001C00001C00001C00001C000038000038000038000038000038000070 0000700000700000700000700000700000E00000E00000E00000E00000C00001C00001C0 000180003180007B0000F300006600003C00001629829F0E>I<003C6000E27001C1E003 80E00700E00F00E00E01C01E01C01E01C01E01C03C03803C03803C03803C03803C07003C 07001C0F001C17000C2E0003CE00000E00000E00001C00001C00301C00783800F0700060 E0003F8000141D7E9315>I<01E0000FE00001C00001C00001C00001C000038000038000 038000038000070000070000071E000763000E81800F01C00E01C00E01C01C03801C0380 1C03801C0380380700380700380700380E10700E20700C20701C20700C40E00CC0600700 14207D9F17>I<00C001E001E001C000000000000000000000000000000E003300230043 804300470087000E000E000E001C001C001C003840388030807080310033001C000B1F7C 9E0E>I<0001800003C00003C00003800000000000000000000000000000000000000000 00003C00004600008700008700010700010700020E00000E00000E00000E00001C00001C 00001C00001C0000380000380000380000380000700000700000700000700000E00000E0 0030E00079C000F180006300003C00001228829E0E>I<01E0000FE00001C00001C00001 C00001C0000380000380000380000380000700000700000703C00704200E08E00E11E00E 21E00E40C01C80001D00001E00001FC00038E00038700038700038384070708070708070 7080703100E03100601E0013207D9F15>I<03C01FC00380038003800380070007000700 07000E000E000E000E001C001C001C001C0038003800380038007000700070007100E200 E200E200E200640038000A207C9F0C>I<1C0F80F0002630C318004740640C004780680E 004700700E004700700E008E00E01C000E00E01C000E00E01C000E00E01C001C01C03800 1C01C038001C01C038001C01C0708038038071003803806100380380E100380380620070 07006600300300380021147C9325>I<1C0F802630C04740604780604700704700708E00 E00E00E00E00E00E00E01C01C01C01C01C01C01C03843803883803083807083803107003 303001C016147C931A>I<007C0001C3000301800E01C01E01C01C01E03C01E07801E078 01E07801E0F003C0F003C0F003C0F00780F00700700F00700E0030180018700007C00013 147C9317>I<01C1E002621804741C04781C04701E04701E08E01E00E01E00E01E00E01E 01C03C01C03C01C03C01C0380380780380700380E003C1C0072380071E00070000070000 0E00000E00000E00000E00001C00001C0000FFC000171D809317>I<00F0400388C00705 800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E00F01C 00F01C00703C00705C0030B8000F380000380000380000700000700000700000700000E0 0000E0000FFE00121D7C9315>I<1C1E002661004783804787804707804703008E00000E 00000E00000E00001C00001C00001C00001C000038000038000038000038000070000030 000011147C9313>I<00FC030206010C030C070C060C000F800FF007F803FC003E000E70 0EF00CF00CE008401020601F8010147D9313>I<018001C0038003800380038007000700 FFF007000E000E000E000E001C001C001C001C0038003800380038207040704070807080 31001E000C1C7C9B0F>I<0E00C03300E02301C04381C04301C04701C08703800E03800E 03800E03801C07001C07001C07001C07101C0E20180E20180E201C1E200C264007C38014 147C9318>I<0E03803307802307C04383C04301C04700C08700800E00800E00800E0080 1C01001C01001C01001C02001C02001C04001C04001C08000E300003C00012147C9315> I<0E00C1C03300E3C02301C3E04381C1E04301C0E04701C060870380400E0380400E0380 400E0380401C0700801C0700801C0700801C0701001C0701001C0602001C0F02000C0F04 000E13080003E1F0001B147C931E>I<0383800CC4401068E01071E02071E02070C040E0 0000E00000E00000E00001C00001C00001C00001C040638080F38080F38100E5810084C6 0078780013147D9315>I<0E00C03300E02301C04381C04301C04701C08703800E03800E 03800E03801C07001C07001C07001C07001C0E00180E00180E001C1E000C3C0007DC0000 1C00001C00003800F03800F07000E06000C0C0004380003E0000131D7C9316>I123 D E /Ft 51 121 df<00030004000800100020006000C00180010003 00060006000C000C001C0018001800380030003000700070006000600060006000E000E0 00E000E000E000E00060006000600060006000300030001000180008000C000400020001 00102E7CA112>40 D<008000C000600020003000180018000C000C000C000E0006000600 0600060006000600060006000600060006000E000E000E000C000C001C00180018003800 3000300060006000C000C001800100030006000C0018003000600080000F2E7FA112>I< 1C3E7E7E3A0202040408081020C0070E7D840D>44 DI<3078F8 787005057C840D>I<007E0001C3000301800701C00E00C00E00E01C00E01C00E03C01E0 3801E07801E07801E07801E07801E07801E0F003C0F003C0F003C0F003C0F003C0F00380 F00780E00780E00700E00700E00E00600E00701C003038003870000FC000131F7C9D17> 48 D<000C001C00FC0F380038003800380038003800700070007000700070007000E000 E000E000E000E000E001C001C001C001C001C001C0038003C0FFFE0F1E7C9D17>I<003F 8000C1E00100F00200780400780400780F007C0F807C0F807C0F00780600780000F80000 F00001E00001C0000380000700000E00001C0000380000600000C0000180000300200600 200800401000403FFFC07FFF80FFFF80161E7E9D17>I<007F000183C00201E00400F007 00F00F00F00F01F00F01F00001E00001E00003C0000380000700000E0000F800000E0000 07000007800007C00003C00007C03007C07807C0F807C0F807C0F00780800F00400E0020 1C0018780007E000141F7D9D17>I<0000600000600000E00001C00003C00005C0000DC0 0009C00011C000238000438000C380008380010380020380040700080700180700100700 200700400700FFFFF0000E00000E00000E00000E00000E00001C00001E0001FFE0141E7E 9D17>I<01803001FFE003FFC003FF0003FC000200000200000200000400000400000400 00047C000587000603800C01800801C00001C00001E00001E00001E00001E07003C0F803 C0F003C0E00380800780400700400E00201C0018700007C000141F7D9D17>I<000F8000 704000C0200180E00301E00701E00E00C01E00001C00003C000038000078F800790E007A 07007C0300F80380F80380F003C0F003C0F003C0F003C0F00780E00780E00780E00700E0 0F00600E00701C0030180018700007C000131F7C9D17>I<2000003FFFE07FFFC07FFF80 400100C00200800200800400000800001000002000004000004000008000018000030000 0300000700000600000E00000E00001E00001C00001C00003C00003C00003C0000780000 780000780000300000131F799D17>I<003F0000C1C00100600200600400300C00300C00 300C00300C00600E00600F80C00FC18007F60003FC0001FC0001FF00063F800C0F801007 C03003C06001C06000C0C000C0C000C0C000C0C00080C0010060030030040018180007E0 00141F7D9D17>I<007E0001C3000301800601C00E01C01C00C03C00E03C00E03C01E078 01E07801E07801E07801E07803E07803E03803C03807C01C0BC00C13C003E38000078000 0780000700000E00600E00F01C00F01800E0300080600041C0003F0000131F7C9D17>I< 07E01838201C401C701CF03CF03C603C0038007000E001C0018003000200060004000400 08000800080008000000000000000000000030007800F800780070000E20799F15>63 D<0000100000001800000038000000380000007800000078000000FC000001BC0000013C 0000033C0000023C0000063C0000043E0000081E0000081E0000101E0000101E0000201E 0000200F0000400F0000400F0000FFFF0000800F0001000F800100078002000780020007 8004000780040007800C0007C03E0007C0FF807FFC1E207E9F22>65 D<07FFFF00007C01C0003C01E0003C00F0007800F8007800F8007800F8007800F8007800 F8007800F000F001F000F001E000F003C000F00F8000FFFE0000F00F0001E007C001E003 C001E003E001E001E001E001E001E001E003C001E003C003E003C003E003C003C003C007 C003C00F8007800F0007803E00FFFFF0001D1F7E9E20>I<0001F808000E061800380138 007000F801E0007803C0007007800030078000300F0000301F0000301E0000303E000020 3C0000007C0000007C0000007C0000007C000000F8000000F8000000F8000000F8000000 F80000007800004078000080780000803C0000803C0001001C0002000E00020006000C00 0300100001C0E000003F00001D217B9F21>I<07FFFF00007C01E0003C00F0003C007800 78003C0078003C0078001E0078001E0078001E0078001F00F0001F00F0001F00F0001F00 F0001F00F0001F00F0001F01E0001E01E0003E01E0003E01E0003E01E0003C01E0007C03 C0007803C000F003C000F003C001E003C003C003C0078007800F0007803C00FFFFE00020 1F7E9E23>I<07FFFFF8007C0078003C0038003C00180078001800780008007800080078 0008007800080078080800F0100000F0100000F0100000F0300000FFF00000F0700001E0 200001E0200001E0200001E0200001E0000801E0001003C0001003C0001003C0002003C0 002003C0006003C000C0078001C0078007C0FFFFFF801D1F7E9E1F>I<07FFFFF8007C00 78003C0038003C001800780018007800080078000800780008007800080078000800F010 0000F0100000F0100000F0300000F0700000FFF00001E0600001E0200001E0200001E020 0001E0200001E0000003C0000003C0000003C0000003C0000003C0000003C00000078000 0007C00000FFFE00001D1F7E9E1E>I<0001FC04000F030C003C009C0070007C00E0003C 01C0003803800018078000180F0000181F0000181E0000183E0000103C0000007C000000 7C0000007C0000007C000000F8000000F8000000F8007FFCF80003E0780001E0780001E0 780003C0780003C03C0003C03C0003C01C0003C00E0007C007000B800380118001E06080 003F80001E217B9F24>I<07FFC7FFC0007C00F800003C007800003C007800007800F000 007800F000007800F000007800F000007800F000007800F00000F001E00000F001E00000 F001E00000F001E00000FFFFE00000F001E00001E003C00001E003C00001E003C00001E0 03C00001E003C00001E003C00003C007800003C007800003C007800003C007800003C007 800003C007800007800F000007C00F8000FFF8FFF800221F7E9E22>I<07FFE0007C0000 3C00003C0000780000780000780000780000780000780000F00000F00000F00000F00000 F00000F00001E00001E00001E00001E00001E00001E00003C00003C00003C00003C00003 C00003C00007800007C000FFFC00131F7F9E10>I<00FFFC0007C00003C00003C0000780 000780000780000780000780000780000F00000F00000F00000F00000F00000F00001E00 001E00001E00001E00001E00001E00003C00303C00783C00F83C00F83800F07800807000 40E00021C0001F000016207D9E17>I<07FFE07FE0007C001F00003C000C00003C001800 00780010000078004000007800800000780100000078020000007804000000F008000000 F010000000F060000000F0F0000000F1F0000000F278000001E478000001E878000001F0 3C000001E03C000001E01E000001E01E000003C00F000003C00F000003C00F000003C007 800003C007800003C003C000078003C00007C007E000FFFC3FFC00231F7E9E23>I<07FF F000007E0000003C0000003C000000780000007800000078000000780000007800000078 000000F0000000F0000000F0000000F0000000F0000000F0000001E0000001E0000001E0 000001E0000001E0008001E0010003C0010003C0010003C0030003C0020003C0060003C0 060007801E0007807C00FFFFFC00191F7E9E1C>I<07FC0000FFC0007C0000F800003C00 017800003C00017800004E0002F000004E0002F000004E0004F000004E0004F000004E00 08F000004E0008F00000870011E00000870011E00000870021E00000870021E000008700 41E00000838041E00001038083C00001038083C00001038103C00001038203C0000101C2 03C0000101C403C0000201C40780000201C80780000201C80780000201D00780000200F0 0780000600E00780000600E00F00000F00C00F8000FFE0C1FFF8002A1F7E9E2A>I<07FC 01FFC0003E003E00003E001800003E001800004F001000004F0010000047801000004780 10000043C010000043C010000083C020000081E020000081E020000080F020000080F020 000080782000010078400001007C400001003C400001003C400001001E400001001E4000 02000F800002000F800002000F800002000780000200078000060003800006000300000F 00010000FFE0010000221F7E9E22>I<0003F800001E0E000038070000E0038001C001C0 03C001E0078000E00F0000F00F0000F01E0000F01E0000F83E0000F83C0000F87C0000F8 7C0000F87C0000F87C0000F8F80001F0F80001F0F80001F0F80001F0F80003E0780003E0 780003C0780007C07C0007803C000F003C001E001E001C000E0038000700F00003C3C000 00FE00001D217B9F23>I<07FFFF00007C03C0003C01E0003C00F0007800F0007800F800 7800F8007800F8007800F8007800F000F001F000F001E000F003C000F0078000F00F0000 FFF80001E0000001E0000001E0000001E0000001E0000001E0000003C0000003C0000003 C0000003C0000003C0000003C000000780000007C00000FFFC00001D1F7E9E1F>I<0003 F800001E0E000038070000F0038001E001C003C001E0078001E00F0000F00F0000F01F00 00F01E0000F83E0000F83C0000F87C0000F87C0000F87C0000F87C0000F8F80001F0F800 01F0F80001F0F80001F0F80003E0780003E0780003C0780007C0781E07803C210F003C40 9E001E409C000E80F8000740F00003C1C04000FEC0400000E0400000E0800000E1800000 FF800000FF000000FF0000007E0000003C001D297B9F23>I<07FFFC00007C0700003C03 C0003C01E0007801E0007801F0007801F0007801F0007801F0007801E000F003E000F003 C000F0078000F00F0000F03C0000FFF00001E0300001E0380001E01C0001E01C0001E01C 0001E01E0003C03E0003C03E0003C03E0003C03E0003C03E0003C03E0207803E0407C01F 04FFFC0F18000003E01F207E9E21>I<003F040060CC01803C03801C03001C0700180600 080E00080E00080E00080E00000F00000F80000FE00007FE0003FF8001FFC0007FE00007 E00001E00000E00000F00000F04000E04000E04000E04000E06000C0600180E00380F803 00C60C0081F80016217D9F19>I<3FFFFFF03C0780F03007803060078030400F0010400F 0010C00F0010800F0010800F0010800F0010001E0000001E0000001E0000001E0000001E 0000001E0000003C0000003C0000003C0000003C0000003C0000003C0000007800000078 00000078000000780000007800000078000000F0000001F800007FFFE0001C1F7A9E21> IIII<03FFC0FFC0007F007E00003E003800001E003000001E002000000F004000000F0080 00000F81000000078200000007C600000003C400000003E800000001F000000001F00000 0000F000000000F800000000F8000000017C000000023C000000043C0000000C1E000000 081E000000101F000000200F000000400F800000C0078000008007C000010003C0000700 03E0001F8007E000FFE01FFE00221F7F9E22>II<03FFFFC003E007800380078007000F0006001E0004003C000C003C0008 0078000800F0000801F0000001E0000003C0000007800000078000000F0000001E000000 3C0000003C00000078000000F0010001E0010001E0020003C00200078002000F8006000F 0004001E000C003C001C007C003C007800F800FFFFF8001A1F7D9E1C>I<01FC07060E0F 1C0F380E78007000F000F000F000F000E000E000E000E000F0027004300818300FC01014 7C9314>99 D<0000700003F00000F00000700000700000E00000E00000E00000E00000E0 0000E00001C000F9C00305C00E03C01C03C03801C0780380700380F00380F00380F00380 F00380E00700E00700E00700E00700E00700700F00301E00186F000F8FE014207C9F19> I<00F800070E000E07001C0700380380780380700380F00380F00380FFFF80F00000E000 00E00000E00000E00000F001007002003004001C180007E00011147D9314>I<01C003E0 03E003C0018000000000000000000000000003801F800780038003800700070007000700 070007000E000E000E000E000E000E001C001E00FF800B1F7F9E0C>105 D<038F801F90E007A0E003C0600380600780E00700E00700E00700E00700E00700E00E01 C00E01C00E01C00E01C00E01C00E01C01C03801E03C0FFCFF815147E9319>110 D<00FC000387000E01801C00C03800E03800E07000F0F000F0F000F0F000F0F000F0E001 E0E001E0E001C0E003C0F00380700700380E001C1C0007E00014147D9317>I<00800100 01000100030007000F001E00FFF80E000E000E000E001C001C001C001C001C001C003800 38203820382038203840384018800F000D1C7C9B12>116 D<1C0380FC1F803C07801C03 801C0380380700380700380700380700380700380700700E00700E00700E00700E00701E 00701E00703C00305E001F9FC012147B9319>I<1FF0FF03C07801C06001C04000E08000 E180007300007600003C00003C00001C00002E00004E000087000107000203800603800C 01C03E03E0FF07FC18147F9318>120 D E /Fu 26 122 df<60F0F07010101020204080 040B7D830B>44 D<60F0F06004047D830B>46 D<078018603030303060186018E01CE01C E01CE01CE01CE01CE01CE01CE01CE01CE01CE01C6018601870383030186007800E187E97 13>48 D<03000700FF000700070007000700070007000700070007000700070007000700 0700070007000700070007000700FFF00C187D9713>I<0F8010E0207060787038203800 7800700070006000C00F8000E000700038003C003CE03CE03CC03C4038407030E00F800E 187E9713>51 D<00300030007000F000F001700370027004700C70087010703070207040 70C070FFFF00700070007000700070007007FF10187F9713>I<01E006100C1818383038 300070006000E000E7C0E860F030F018E018E01CE01CE01C601C601C7018301830301860 07C00E187E9713>54 D<078018603030201860186018601870103C303E600F8007C019F0 30F86038401CC00CC00CC00CC00C6008201018600FC00E187E9713>56 D<07801860303070306018E018E018E01CE01CE01C601C603C303C185C0F9C001C001800 18003870307060604021801F000E187E9713>I<1FFC00E000E000E000E000E000E000E0 00E000E000E000E000E000E000E000E000E000E000E000E040E0E0E0E0E041C061801E00 0E1A7D9914>74 D<007F000001C1C000070070000E0038001C001C003C001E0038000E00 78000F0070000700F0000780F0000780F0000780F0000780F0000780F0000780F0000780 F000078078000F0078000F0038000E003C001E001C001C000E0038000700700001C1C000 007F0000191A7E991E>79 D86 D<3F8070C070E020700070007007F01C7030707070 E070E071E071E0F171FB1E3C10107E8F13>97 DI<07F80C 1C381C30087000E000E000E000E000E000E0007000300438080C1807E00E107F8F11>I< 07C01C3030187018600CE00CFFFCE000E000E000E0006000300438080C1807E00E107F8F 11>101 D<01F0031807380E100E000E000E000E000E000E00FFC00E000E000E000E000E 000E000E000E000E000E000E000E000E000E007FE00D1A80990C>I<18003C003C001800 000000000000000000000000FC001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C00FF80091A80990A>105 D108 D110 D<07E01C38300C700E60 06E007E007E007E007E007E0076006700E381C1C3807E010107F8F13>I114 D<1F2060E04020C020C020F0007F003FC01FE000F080708030C030C020F0408F800C107F 8F0F>I<0400040004000C000C001C003C00FFC01C001C001C001C001C001C001C001C00 1C201C201C201C201C200E4003800B177F960F>II121 D E /Fv 63 123 df<00000F80003F8F80007F8F8000FF8F8001FF 8F8003E0000003C0000007C0000007C0000007C0000007C0000007C0000007C0000007C0 000007C0000007C00000FFFF8F80FFFF8F80FFFF8F8007C00F8007C00F8007C00F8007C0 0F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C0 0F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C0 0F8007C00F8007C00F80192B7FAA20>12 D<001E003C007800F001F001E003C007C00780 0F800F001F001F001E003E003E003E007C007C007C007C007C007800F800F800F800F800 F800F800F800F800F800F800F800F800F800F800F80078007C007C007C007C007C003E00 3E003E001E001F001F000F000F80078007C003C001E001F000F00078003C001E0F3D7CAC 17>40 DI44 DII<007F00 0001FFC00007FFF0000FFFF8000FC1F8001F007C003F007E003E003E003C001E007C001F 007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F 8078000F007C001F007C001F007C001F003E003E003E003E003F007E001F80FC000FC1F8 000FFFF80007FFF00001FFC000007F000019297EA71E>48 D<00180000380000F80007F8 00FFF800FFF800F8F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F8007FFFF07FFFF07FFF F014287BA71E>I<00FE0003FFC007FFE00FFFF01F03F83C00FC38007E78003E70003EF0 001FF0001F60001F20001F00001F00001F00001F00003E00003E00007C00007C0000F800 01F00001E00003C0000780000F00001E00003C0000780000F00001E00003C0000780000F 00001E00003C00007FFFFF7FFFFF7FFFFF7FFFFF18287EA71E>I<007F000001FFC00007 FFF0000FFFF8001FC1F8003E007C003C003E0078003E0038003E0010003E0000003E0000 003E0000003C0000007C000000FC000001F8000007F00000FFE00000FFC00000FFE00000 FFF0000001FC0000007C0000003E0000001F0000001F0000000F8000000F8000000F8000 000F8000000F8040000F8060001F00F0001F00F8003F007E007E003F81FC001FFFF8000F FFF00003FFE000007F000019297EA71E>I<0003F0000007F0000005F000000DF000000D F000001DF0000039F0000039F0000079F0000079F00000F1F00000F1F00001E1F00003E1 F00003E1F00007C1F00007C1F0000F81F0000F81F0001F01F0001F01F0003E01F0007C01 F0007C01F000F801F000FFFFFF80FFFFFF80FFFFFF80FFFFFF800001F0000001F0000001 F0000001F0000001F0000001F0000001F0000001F0000001F0000001F00019277EA61E> I<3FFFFC3FFFFC3FFFFC3FFFFC3E00003E00003E00003E00003E00003E00003E00003E00 003E00003E00003E3F003EFFC03FFFE03FFFF03FE1F83F807C3F003E3E003E00003E0000 1F00001F00001F00001F00001F00001F00001F20001F60003E70003EF8007C7C00FC3F03 F81FFFF00FFFE007FF8000FE0018287EA61E>I<000FF000003FFC0000FFFC0001FFFC00 03F80C0007E000000FC000000F8000001F0000001E0000003E0000003C0000007C000000 7C0000007C3FE000F8FFF000F9FFF800FBFFFC00FF807E00FF003E00FE003F00FC001F00 FC001F00FC000F80F8000F80F8000F80F8000F80F8000F8078000F807C000F807C000F80 7C000F003E001F003E001F001F003E001F807C000FC1FC0007FFF80003FFF00001FFC000 007F000019297EA71E>II<007F000001FF C00007FFF0000FFFF8001FC1FC003F007E003E003E007E003F007C001F007C001F007C00 1F007C001F007C001F003E003E003E003E001F007C000FC1F80007FFF00003FFE00003FF E0000FFFF8001FC1FC003F007E003E003E007C001F007C001F00F8000F80F8000F80F800 0F80F8000F80F8000F80F8000F807C001F007C001F007E003F003F007E001FC1FC000FFF F80007FFF00003FFE000007F000019297EA71E>I<007F000001FFC00003FFE0000FFFF0 000FC1F8001F007C003E007C007C003E007C001E007C001F00F8001F00F8001F00F8000F 00F8000F80F8000F80F8000F80F8000F80F8001F807C001F807C001F807E003F803E007F 803F00FF801FFFEF800FFFCF8007FF8F8003FE1F0000001F0000001F0000001E0000003E 0000003E0000007C0000007C000000F8001801F0001E07E0003FFFC0001FFF80000FFE00 0003F8000019297EA71E>I<03FC001FFF807FFFC0FFFFE0FC07F07001F04000F80000F8 0000F80000F80000F80001F80001F00007F0000FE0001FC0001F80003E00003E00007C00 00780000F80000F00000F00000F00000F00000F00001F00001F00001F000000000000000 00000000000000000000000000000001F00001F00001F00001F00001F000152A7DA91C> 63 D<0001F000000003F800000003F800000007FC00000007BC00000007BC0000000F3E 0000000F1E0000000F1E0000001F1F0000001E1F0000001E0F0000003E0F8000003C0F80 00003C078000007C07C000007807C00000F803E00000F803E00000F003E00001F001F000 01F001F00001E001F00003E000F80003E000F80003C000F80007FFFFFC0007FFFFFC000F FFFFFE000F80003E000F80003E001F00003F001F00001F001E00001F003E00000F803E00 000F803C00000F807C000007C07C000007C078000007C0F8000003E0F8000003E0232A7E A928>65 DI< 0001FF00000FFFE0003FFFF8007FFFF800FE01F801F8003003F0001007C000000F800000 1F8000001F0000003E0000003E0000007E0000007C0000007C0000007C000000F8000000 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 7C0000007C0000007C0000007E0000003E0000003E0000001F0000001F8000000F800000 07C0000003F0000401F8001C00FE00FC007FFFFC003FFFF8000FFFE00001FF001E2C7CAA 26>IIII<0001FF00000F FFE0003FFFFC007FFFFE00FF01FE01F8003E03F0000C07C000000FC000001F8000001F00 00003F0000003E0000007E0000007C0000007C0000007C000000F8000000F8000000F800 0000F8000000F8000000F8000000F8000000F8001FFEF8001FFEF8001FFE7C001FFE7C00 003E7C00003E7E00003E3E00003E3F00003E1F00003E1F80003E0FC0003E07C0003E03F0 003E01F8003E00FF00FE007FFFFE003FFFFC000FFFE00001FF001F2C7CAA28>III76 DII<0001FC0000000FFF8000003FFFE000007FFFF00001FE03FC0003F800FE0007E000 3F0007C0001F000F80000F801F000007C01F000007C03E000003E03E000003E07C000001 F07C000001F07C000001F078000000F0F8000000F8F8000000F8F8000000F8F8000000F8 F8000000F8F8000000F8F8000000F8F8000000F8F8000000F8F8000000F87C000001F07C 000001F07C000001F07E000003F03E000003E03F000007E01F000007C01F80000FC00FC0 001F8007E0003F0007F0007F0003F800FE0001FE03FC0000FFFFF800003FFFE000000FFF 80000001FC0000252C7DAA2C>II82 D<007FC00001FFF80007FFFE000FFFFF001FC07F003F000F007E0006007C0000 007C000000F8000000F8000000F8000000F8000000F8000000FC0000007E0000007F0000 003F8000001FF800000FFF800007FFE00003FFF80000FFFC00000FFE000000FF0000003F 0000001F8000000F8000000FC0000007C0000007C0000007C0000007C0000007C0000007 C000000F8060000F80F0001F00FC003F00FF80FE007FFFFC001FFFF80007FFE00000FF80 001A2C7DAA21>IIIII<01FE000FFF803FFFC03FFFE03C03 F03001F00001F80000F80000F80000F80000F80000F8007FF807FFF81FFFF83FE0F87F00 F8FC00F8F800F8F800F8F800F8FC01F87E07F87FFFF83FFFF81FFCF80FE0F8151B7E9A1D >97 DI<007FC001FFF007FF FC0FFFFC1FC07C1F00083E00007C00007C00007C0000F80000F80000F80000F80000F800 00F80000F800007C00007C00007E00003E00001F000C1FC07C0FFFFC07FFFC01FFF0007F 80161B7E9A1B>I<00003E00003E00003E00003E00003E00003E00003E00003E00003E00 003E00003E00003E00003E00003E00003E00FC3E03FF3E07FFFE0FFFFE1FC1FE3F007E3E 003E7C003E7C003EFC003EF8003EF8003EF8003EF8003EF8003EF8003EF8003EFC003E7C 003E7C003E3E007E3F00FE1FC1FE0FFFFE07FFBE03FF3E00FC3E172A7EA91F>I<007E00 03FF8007FFC00FFFE01F83F03F00F03E00787C00787C003878003CFFFFFCFFFFFCFFFFFC FFFFFCF80000F80000F800007800007C00007C00003E00003F000C1FC07C0FFFFC07FFFC 01FFF0007F80161B7E9A1B>I<001FC0007FC000FFC001FFC003F00003E00007C00007C0 0007C00007C00007C00007C00007C00007C00007C000FFFE00FFFE00FFFE0007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C000122A7FA912 >I<00F8078003FE7FC00FFFFFC01FFFFFC01F07C0003E03E0003E03E0007C01F0007C01 F0007C01F0007C01F0007C01F0007C01F0003E03E0003E03E0001F07C0001FFFC0003FFF 80003BFE000038F8000078000000780000003C0000003FFFC0003FFFF8001FFFFC001FFF FE003FFFFF007C007F00F8001F80F8000F80F8000F80F8000F80FC001F807E003F003F80 FE003FFFFE000FFFF80007FFF00000FF80001A287E9A1E>III<007C007C007C007C00 7C0000000000000000000000000000000000000000007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C007C40FCF1F8FFF8FFF07FE00F800E3683A910>III< F83F003F00F8FFC0FFC0FBFFE3FFE0FFFFF7FFF0FF83F783F0FE01FE01F8FC00FC00F8FC 00FC00F8FC00FC00F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800 F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800F8 00F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8251B7B9A30>II<007F000001FFC00007FFF0000FFFF8001FC1FC003F00 7E003E003E007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F80F800 0F80F8000F80F8000F807C001F007C001F007E003F003E003E003F007E001FC1FC000FFF F80007FFF00001FFC000007F0000191B7E9A1E>I I<00FC3E03FF3E07FFBE0FFFFE1FC1FE3F80FE3E007E7E003E7C003E7C003EF8003EF800 3EF8003EF8003EF8003EF8003EF8003EFC003E7C003E7E003E3E007E3F00FE1FC1FE1FFF FE0FFFBE03FF3E01F83E00003E00003E00003E00003E00003E00003E00003E00003E0000 3E00003E00003E00003E17277E9A1F>II<03FC001FFF803FFFC07FFFC07C07C0F80080F80000F80000F80000FC0000 7F80007FF8003FFE001FFF0007FF8000FFC0000FE00007E00003E00003E04003E0E007E0 FC0FC0FFFFC07FFF801FFE0003F800131B7E9A17>I<07C00007C00007C00007C00007C0 0007C00007C000FFFFC0FFFFC0FFFFC007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 4007E1C003FFE003FFE001FF8000FC0013227FA116>IIII<7C000FC03E001F803F001F001F803E000F807C0007C0FC0003E0F800 01F1F00001FBE00000FFC000007FC000003F8000001F0000001F0000003F8000007FC000 00FBC00000F3E00001F1F00003E0F80007C07C000F807C000F803E001F001F003E000F80 7E000FC0FC0007E01B1B809A1C>III E /Fw 19 121 df<00001E000000003E00000000FE00 000003FE0000003FFE0000FFFFFE0000FFFFFE0000FFFFFE0000FFCFFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00007FFFFFFFC07F FFFFFFC07FFFFFFFC07FFFFFFFC0223879B731>49 D<0007FE0000007FFFE00001FFFFF8 0003FFFFFE0007F01FFF000F8007FF801F0001FFC03E0000FFE07F8000FFF07FC0007FF0 FFE0007FF8FFF0003FF8FFF0003FFCFFF0003FFCFFF0003FFCFFF0003FFC7FE0003FFC3F C0003FFC1F80003FFC0000003FFC0000003FF80000007FF80000007FF00000007FF00000 00FFE0000000FFC0000001FF80000001FF00000003FE00000007FC00000007F80000000F F00000001FC00000003F800000007F00000000FC00000001F800000001F0003C0003E000 3C0007C0003C000F000078001E000078003C00007800780000F800F00000F801FFFFFFF8 03FFFFFFF007FFFFFFF00FFFFFFFF01FFFFFFFF03FFFFFFFF07FFFFFFFF0FFFFFFFFF0FF FFFFFFE0FFFFFFFFE0FFFFFFFFE026387BB731>I<0003FF8000001FFFF000007FFFFE00 00FE03FF0001F000FF8003C000FFC00780007FE00FF0007FF00FF8007FF01FFC007FF81F FE007FF81FFE007FF81FFE007FF81FFE007FF81FFE007FF80FFC007FF007F8007FF003F0 007FF0000000FFE0000000FFC0000001FF80000001FF00000003FE00000007FC0000001F F000000FFFC000000FFF8000000FFFF800000003FE00000000FF800000007FE00000003F F00000003FF80000003FFC0000001FFC0000001FFE0000001FFE0200001FFF1FC0001FFF 3FE0001FFF7FF0001FFF7FF0001FFFFFF8001FFFFFF8001FFFFFF8001FFEFFF8001FFEFF F0001FFE7FF0003FFC7FE0003FFC3FC0003FF81F80007FF01FE000FFE007FC03FFC003FF FFFF0001FFFFFE00003FFFF0000007FF800028397CB731>I<00000007C0000000000FC0 000000000FC0000000001FC0000000003FC0000000007FC000000000FFC000000000FFC0 00000001FFC000000003FFC000000007FFC00000000FFFC00000000FFFC00000001EFFC0 0000003CFFC00000007CFFC0000000F8FFC0000000F0FFC0000001E0FFC0000003C0FFC0 000007C0FFC000000F80FFC000000F00FFC000001E00FFC000003C00FFC000007C00FFC0 0000F800FFC00000F000FFC00001E000FFC00003C000FFC00007C000FFC0000F8000FFC0 000F0000FFC0001E0000FFC0003C0000FFC0007C0000FFC000F80000FFC000FFFFFFFFFF C0FFFFFFFFFFC0FFFFFFFFFFC0FFFFFFFFFFC0000001FFC000000001FFC000000001FFC0 00000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0 00000001FFC0000007FFFFFFC00007FFFFFFC00007FFFFFFC00007FFFFFFC02A377DB631 >I<04000000C00F800007C00FF8007FC00FFFFFFF800FFFFFFF000FFFFFFE000FFFFFFC 000FFFFFF8000FFFFFF0000FFFFFE0000FFFFF80000FFFFE00000FFFF800000F80000000 0F800000000F800000000F800000000F800000000F800000000F800000000F800000000F 81FF00000F8FFFE0000FBFFFF8000FFE03FE000FF000FF000FC000FF800F80007FC00F00 007FE00700007FF00000003FF00000003FF80000003FF80000003FF80000003FFC000000 3FFC0600003FFC1F80003FFC3FC0003FFC7FE0003FFCFFE0003FFCFFF0003FFCFFF0003F FCFFF0003FF8FFE0003FF8FFE0003FF87FC0007FF07F00007FF03C00007FE03E0000FFC0 1F0000FF800FC003FF0007F00FFE0003FFFFFC0001FFFFF000007FFFC000000FFC000026 397BB731>I<00000FF80000007FFF000003FFFF80000FFC07C0001FE001E0003FC001F0 007F0007F000FF000FF001FE001FF803FC003FF807FC003FF80FFC003FF80FF8003FF81F F8001FF01FF8000FE03FF80007C03FF00000003FF00000007FF00000007FF00000007FF0 0000007FF07FF000FFF0FFFE00FFF1F7FF00FFF3807F80FFF6003FE0FFFE001FF0FFFC00 1FF0FFFC000FF8FFF8000FFCFFF8000FFCFFF8000FFEFFF8000FFEFFF0000FFEFFF0000F FFFFF0000FFFFFF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF 3FF0000FFF3FF0000FFE3FF0000FFE1FF0000FFE1FF8000FFC0FF8000FFC0FF8001FF807 FC001FF003FC001FF001FE003FE000FF80FFC0007FFFFF00001FFFFE000007FFF8000000 FFC00028397CB731>I<1E00000000001F00000000001FF0000000001FFFFFFFFFC01FFF FFFFFFC01FFFFFFFFFC03FFFFFFFFFC03FFFFFFFFF803FFFFFFFFF003FFFFFFFFE003FFF FFFFFC003FFFFFFFF8003FFFFFFFF0007FFFFFFFF0007C000007E0007C00000FC0007800 001F80007800001F00007800003E0000F000007E0000F00000FC0000F00001F800000000 03F00000000003E00000000007E0000000000FC0000000000F80000000001F8000000000 3F80000000003F00000000007F00000000007F0000000000FF0000000000FE0000000001 FE0000000001FE0000000003FE0000000003FE0000000003FE0000000007FC0000000007 FC0000000007FC000000000FFC000000000FFC000000000FFC000000000FFC000000000F FC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001F FC000000001FFC000000001FFC000000001FFC000000000FF80000000007F00000000003 E00000002A3B7BB931>I<0001FF8000000FFFF800003FFFFE00007F00FF0000F8003F80 01F0000FC003E0000FE007C00007F007C00007F00FC00003F80FC00003F81FC00003F81F C00003F81FE00003F81FF00003F81FF80003F81FFC0007F01FFF0007F01FFFC007E00FFF F00FE00FFFFC1FC007FFFE3F8007FFFFFF0003FFFFFC0001FFFFF80000FFFFFE00007FFF FF00003FFFFF80003FFFFFE000FFFFFFF001FDFFFFF007F07FFFF80FE01FFFFC1FC007FF FC3F8001FFFE3F8000FFFE7F00003FFF7F00000FFFFE000003FFFE000001FFFE000000FF FE000000FFFE0000007FFE0000007FFE0000007EFE0000007E7F0000007E7F000000FC3F 800000FC3FC00001F81FE00003F00FF00007E007FE007FC003FFFFFF8000FFFFFE00003F FFF8000003FF800028397CB731>I<0001FF8000001FFFF000007FFFFC0000FF81FE0001 FE007F0007FC003F800FF8003FC00FF8001FE01FF8001FF03FF0001FF03FF0000FF87FF0 000FF87FF0000FFCFFF0000FFCFFF0000FFCFFF0000FFEFFF0000FFEFFF0000FFEFFF000 0FFEFFF0000FFEFFF0000FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0001FFF7FF0001F FF3FF0001FFF3FF0001FFF1FF0003FFF0FF8003FFF07F8007FFF03FC006FFF01FE01CFFF 00FFEF8FFF007FFF0FFF000FFE0FFE0000000FFE0000000FFE0000000FFE0000000FFC00 00000FFC03E0001FFC07F0001FF80FF8001FF81FFC001FF01FFC003FF01FFC003FE01FFC 003FC01FF8007FC00FF000FF800FE001FF0007C003FE0007F01FFC0003FFFFF00001FFFF C000007FFF0000000FF8000028397CB731>I<00000001F80000000000000001F8000000 0000000003FC0000000000000003FC0000000000000007FE0000000000000007FE000000 0000000007FE000000000000000FFF000000000000000FFF000000000000001FFF800000 000000001FFF800000000000001FFF800000000000003FFFC00000000000003FFFC00000 000000007FFFE00000000000007DFFE00000000000007DFFE0000000000000FDFFF00000 00000000F8FFF0000000000001F8FFF8000000000001F07FF8000000000001F07FF80000 00000003F07FFC000000000003E03FFC000000000007E03FFE000000000007C01FFE0000 0000000FC01FFF00000000000F801FFF00000000000F800FFF00000000001F800FFF8000 0000001F0007FF80000000003F0007FFC0000000003E0003FFC0000000003E0003FFC000 0000007E0003FFE0000000007C0001FFE000000000FC0001FFF000000000F80000FFF000 000000F80000FFF000000001FFFFFFFFF800000001FFFFFFFFF800000003FFFFFFFFFC00 000003FFFFFFFFFC00000003E000003FFC00000007E000003FFE00000007C000001FFE00 00000FC000001FFF0000000F8000000FFF0000001F8000000FFF8000001F0000000FFF80 00001F00000007FF8000003F00000007FFC000003E00000003FFC000007E00000003FFE0 0000FF00000001FFE000FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0FFFFFC0001FFFFFF F0FFFFFC0001FFFFFFF0443B7DBA4B>65 D<0000001FFF000030000001FFFFE000F00000 0FFFFFFC01F000007FFFFFFE03F00001FFFE007F87F00003FFE0000FCFF0000FFF000003 FFF0001FFC000001FFF0003FF80000007FF0007FF00000003FF000FFC00000003FF001FF C00000001FF003FF800000000FF007FF000000000FF00FFF0000000007F00FFE00000000 07F01FFE0000000003F01FFE0000000003F03FFC0000000003F03FFC0000000001F03FFC 0000000001F07FFC0000000001F07FF80000000001F07FF80000000000007FF800000000 0000FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF8 000000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF800000000 0000FFF8000000000000FFF80000000000007FF80000000000007FF80000000000007FF8 0000000000007FFC0000000000F03FFC0000000000F03FFC0000000000F03FFC00000000 00F01FFE0000000000F01FFE0000000001E00FFE0000000001E00FFF0000000001E007FF 0000000003C003FF8000000003C001FFC0000000078000FFE00000000F00007FF0000000 1F00003FF80000003E00001FFC0000007C00000FFF000001F8000003FFE00007F0000001 FFFE003FC00000007FFFFFFF000000000FFFFFFC0000000001FFFFF000000000001FFF00 00003C3D7BBB47>67 D<001FFF00000001FFFFF0000003FFFFFC000007F007FE00000FF8 01FF00001FFC00FF80001FFC007FC0001FFC007FE0001FFC003FE0000FF8003FF0000FF8 003FF00007F0003FF00001C0003FF0000000003FF0000000003FF0000000003FF0000000 FFFFF000000FFFFFF000007FF83FF00001FF803FF00007FE003FF0000FF8003FF0001FF0 003FF0003FE0003FF0007FE0003FF0007FE0003FF000FFC0003FF000FFC0003FF000FFC0 003FF000FFC0003FF000FFC0007FF0007FE0007FF0007FE000DFF0003FF0039FF8001FFC 0F0FFFF007FFFE0FFFF001FFFC07FFF0003FE000FFF02C267DA530>97 D<0001FFC000000FFFF800003FFFFE0000FF80FF0001FE003F8007FC001FC00FF8000FE0 0FF8000FF01FF00007F03FF00007F83FF00007F87FE00007F87FE00003FC7FE00003FC7F E00003FCFFE00003FCFFFFFFFFFCFFFFFFFFFCFFFFFFFFFCFFE0000000FFE0000000FFE0 000000FFE00000007FE00000007FE00000007FE00000003FE00000003FF000003C1FF000 003C1FF000003C0FF800007807FC0000F803FE0001F001FF0007E000FFC03FC0003FFFFF 000007FFFC000000FFE00026267DA52D>101 D<00FF00000000FFFF00000000FFFF0000 0000FFFF00000000FFFF0000000007FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF007FC00003FF01FFF80003FF07FFFC0003FF0F03FE0003FF1C01 FF0003FF3001FF8003FF6000FF8003FFE000FFC003FFC000FFC003FF8000FFC003FF8000 FFC003FF8000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3F FFFFFFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFF303C7CBB37>104 D<00FE007FC000FFFE01FFF800FFFE07FFFC00FFFE0F03FE00FFFE1C01FF0007FE3001FF 8003FE6000FF8003FEE000FFC003FEC000FFC003FF8000FFC003FF8000FFC003FF8000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFFFFFFFC3FFF FFFFFFFC3FFFFFFFFFFC3FFFFF30267CA537>110 D<00FF01FF8000FFFF0FFFF000FFFF 3FFFFC00FFFFFE03FF00FFFFF000FF8003FFC0007FC003FF80003FE003FF00003FF003FF 00001FF803FF00001FFC03FF00000FFC03FF00000FFE03FF00000FFE03FF000007FE03FF 000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF 000007FF03FF000007FF03FF000007FF03FF000007FE03FF000007FE03FF00000FFE03FF 00000FFC03FF00000FFC03FF00001FF803FF00001FF803FF00003FF003FF80003FE003FF C0007FC003FFF001FF8003FFFC07FF0003FF3FFFFC0003FF0FFFF00003FF01FF000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 00000000FFFFFC000000FFFFFC000000FFFFFC000000FFFFFC00000030377DA537>112 D<00FE03F000FFFE0FFE00FFFE1FFF00FFFE3C3F80FFFE707FC007FE60FFE003FEE0FFE0 03FEC0FFE003FFC0FFE003FF807FC003FF807FC003FF803F8003FF800E0003FF00000003 FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF 00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00 000003FF00000003FF00000003FF00000003FF00000003FF000000FFFFFE0000FFFFFE00 00FFFFFE0000FFFFFE000023267DA529>114 D<00078000000780000007800000078000 00078000000F8000000F8000000F8000000F8000001F8000001F8000003F8000003F8000 007F800000FF800001FF800007FF80001FFFFFF0FFFFFFF0FFFFFFF0FFFFFFF001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C00FF8078 00FFC078007FC070003FE0E0001FFFC00007FF800001FF001E377EB626>116 D120 D E /Fx 79 123 df<007F07F001FF1FF003FF3FF007807800070070000F00F0000F00F0000F00F0000F00 F0000F00F0000F00F0000F00F000FFF8FF80FFF8FF80FFF8FF800F00F0000F00F0000F00 F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00 F0000F00F0000F00F0000F00F0000F00F0000F00F0001C20809F1B>11 D<0000F000F8F001F8F003F8F00780000700000F00000F00000F00000F00000F00000F00 000F0000FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00 F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F0142180A018>I<00 F8F001F8F003F8F00780F00700F00F00F00F00F00F00F00F00F00F00F00F00F00F00F0FF F8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F 00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F01420809F18>I<783C783C783C 783C180C3018301860306030E0700E0A7F9F17>34 D<007000E001C00380078007000E00 1E001E003C003C003C0078007800780078007000F000F000F000F000F000F000F000F000 F000F000F000F000700078007800780078003C003C003C001E001E000E00070007800380 01C000E000700C2E7EA112>40 DI<018001C001800180C183E187F99F7DBE1FF807E007E01FF87DBEF99FE187C1 830180018001C0018010147DA117>I<0006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 00060000FFFFFFF0FFFFFFF0000600000006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 1C207D9A23>I<787878781830306060E0050A7D830D>II< F0F0F0F004047C830D>I<000100030003000600060006000C000C000C00180018001800 300030003000600060006000C000C000C00180018001800300030003000600060006000C 000C000C00180018001800300030003000600060006000C000C000C000102D7DA117>I< 03F00007F8001FFE001E1E003C0F00380700780780780780700380F003C0F003C0F003C0 F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0780780 7807807807803807003C0F001E1E001FFE000FFC0003F00012207E9E17>I<00C001C00F C0FFC0FFC0F3C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C0FFFEFFFEFFFE0F1F7C9E17>I<07F0000FFC001FFE 00383F00700F00600780E00780E003C04003C04003C00003C00003C00003C00007800007 80000F00000E00001C00003C0000780000E00001C0000380000700000E00001C00003800 00700000FFFFC0FFFFC0FFFFC0121F7E9E17>I<03F0000FFC001FFE003C1F00780F0030 0780200780000780000780000780000F00000F00003E0003FC0003F80003FC00001E0000 0F000007800003800003C00003C00003C00003C08003C0C003C0C00780700F807C1F003F FE000FFC0003F00012207E9E17>I<003E00003E00005E00005E0000DE0001DE00019E00 039E00039E00079E00071E000F1E000E1E001E1E003C1E003C1E00781E00781E00F01E00 FFFFF0FFFFF0FFFFF0001E00001E00001E00001E00001E00001E00001E00001E00141E7F 9D17>I<7FFF007FFF007FFF0078000078000078000078000078000078000078000079F0 007FFC007FFE007F1F007C07007C07807807800003C00003C00003C00003C00003C00003 C00003C0400780600780F00F007C1E003FFC001FF80007E000121F7E9D17>I<007C0001 FE0003FE000782000F00001E00003C00003C000078000078000078F800F3FC00F7FE00FE 1F00FC0F80F80780F80780F003C0F003C0F003C0F003C0F003C07003C07803C07803C078 07803807803C0F001E1E000FFE0007F80001F00012207E9E17>II<03F0000FFC001FFE003E1F003C0F007807807807 807807807807807807807807803C0F001E1E000FFC0007F8000FFC001F3E003C0F007807 80780780F003C0F003C0F003C0F003C0F003C0F003C07807807C0F803E1F001FFE000FFC 0003F00012207E9E17>I<03F00007F8000FFC001E1E003C0F00780700780780F00780F0 0380F003C0F003C0F003C0F003C0F003C0F003C07807C07807C07C0FC03E1FC01FFBC00F F3C007C780000780000780000700000F00001E00201E00307C007FF8003FF0000FC00012 207E9E17>II61 D<001F0000001F0000003F8000003B8000003B8000 007BC0000073C0000071C00000F1E00000E1E00000E0E00001E0F00001E0F00001C0F000 03C0780003C078000380780007803C0007803C0007003C000FFFFE000FFFFE000FFFFE00 1E000F001E000F003C000F803C0007803C000780780007C0780003C0780003C0F00003E0 1B207F9F1E>65 DI<001FC000FFF801FFFC03E03C07800C0F00001E00003E00003C00007C0000780000 780000780000F00000F00000F00000F00000F00000F00000F00000F00000780000780000 7800007C00003C00003E00001E00000F000207800E03E03E01FFFC00FFF0001FC017227D A01D>IIII<001FE000FFF801FFFE03E0 3E07800E0F00001E00003E00003C00007C0000780000780000780000F00000F00000F000 00F00000F00000F00000F003FEF003FE7803FE78001E78001E7C001E3C001E3E001E1E00 1E0F001E07801E03E03E01FFFE00FFF8001FC017227DA01E>III<003C003C003C003C003C003C 003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C 003C003C003C003CC078E078FFF07FE00FC00E217E9F15>IIII I<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C000F 00780007807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003 C0F00003C0F00003C0F00003C0F80007C07800078078000780780007803C000F003C000F 001E001E001F003E000F807C0007E1F80003FFF00000FFC000003F00001A227DA021>I< FFF800FFFF00FFFF80F00FC0F003E0F001E0F000F0F000F0F000F0F000F0F000F0F000F0 F000F0F001E0F003E0F00FC0FFFF80FFFF00FFF800F00000F00000F00000F00000F00000 F00000F00000F00000F00000F00000F00000F00000F0000014207B9F1D>I<003F000000 FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C000F007800078078 00078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F0 0003C0F00003C0F00003C07800078078000780781E07803C0F0F003C0F8F001E079E001F 03FE000F83FC0007E1F80003FFF00000FFF800003F780000007C0000003E0000001E0000 001F0000000F801A277DA021>II<01FC0007FF800FFFC01F03C03C00C03C0000780000780000780000 7800007800007C00003C00003F00001FE0000FFC0007FE0001FF00003F800007C00003C0 0003E00001E00001E00001E00001E00001E00001C0C003C0F007C0FC0F807FFF001FFE00 03F80013227EA019>I IIII<780007807C000F003E001F001E001E000F003C000F807C 000780780003C0F00003E1F00001F1E00000F3C000007FC000007F8000003F0000001F00 00001E0000003F0000007F8000007FC00000F3C00001F1E00001E0F00003C0F80007C078 0007803C000F003E001F001E001E000F003C000F807C0007C0780003C0F00003E01B207F 9F1E>IIII<381C3018301860306030C060F078F078F078F0780E0A799F17>II<07E03FF87FFC701E401F000F000F000F003F07FF1FFF7E0F F80FF00FF00FF00FF83F7FFF3FEF1F8F10147E9316>97 DI<03F00FFC1FFE3E0E3C0278007800F000F0 00F000F000F000F000780078003C013E0F1FFF0FFE03F010147E9314>I<000780000780 00078000078000078000078000078000078000078000078000078000078007C7800FF780 1FFF803E1F807C0780780780F80780F00780F00780F00780F00780F00780F00780F00780 780780780F803E1F801FFF800FF78007C78011207E9F17>I<03F0000FFC001FFE003E1F 003C0700780700700380FFFF80FFFF80FFFF80F00000F00000F000007000007800003C01 003E07001FFF0007FE0001F80011147F9314>I<007E01FE03FE078007000F000F000F00 0F000F000F000F00FFF0FFF0FFF00F000F000F000F000F000F000F000F000F000F000F00 0F000F000F000F000F000F000F20809F0E>I<03E0F00FFFF01FFFF03E3E003C1E00780F 00780F00780F00780F00780F003C1E003E3E001FFC003FF80033E0003000003800003FFE 003FFF801FFFC03FFFE07803F0F000F0F000F0F000F0F801F07E07E03FFFC00FFF0003FC 00141E7F9317>III<01E001E001E001E00000000000000000000000000000000001E0 01E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E0 01E001E001E001E001E001E0C3C0FFC0FF803F000B29839F0C>IIIII<01F80007FE001FFF803F0FC03C03C07801E07801E0 F000F0F000F0F000F0F000F0F000F0F000F07801E07801E03C03C03F0FC01FFF8007FE00 01F80014147F9317>II<03C7800F F7801FFF803E1F807C0F80780780780780F00780F00780F00780F00780F00780F00780F8 07807807807C0F803E1F801FFF800FF78007C78000078000078000078000078000078000 0780000780000780000780111D7E9317>II<07F01FFC3FFC78 0C7800780078007C003FC01FF00FF803F8007C003C003CC03CF07CFFF87FF00FC00E147F 9311>I<1E001E001E001E001E001E00FFF0FFF0FFF01E001E001E001E001E001E001E00 1E001E001E001E001E001E001E201FF00FF007C00C1A7F9910>IIII<7801E07C03C03E07801E0F000F0F00079E0003FC0003F800 01F80000F00001F00001F80003FC00079E000F0F000E0F001E07803C03C07801E0F801F0 1414809315>II<7FFF7FFF7FFF00 3E003C007800F800F001E003E007C007800F001F001E003C007C00FFFFFFFFFFFF10147F 9314>I E /Fy 33 123 df<007E01C007000E001C003C003800780078007FF0F000F000 F000700070007000300018000C1807E00F147E9312>15 D<70F8F8F87005057C840D>58 D<70F8FCFC74040404080810102040060E7C840D>I<000001C00000078000001E000000 78000001E00000078000000E00000038000000F0000003C000000F0000003C000000F000 0000F00000003C0000000F00000003C0000000F0000000380000000E0000000780000001 E0000000780000001E0000000780000001C01A1A7C9723>I<0001000300030006000600 06000C000C000C00180018001800300030003000600060006000C000C000C00180018001 800300030003000600060006000C000C000C001800180018003000300030006000600060 00C000C000C000102D7DA117>II<000002000000060000000E0000000E0000001E 0000001F0000002F0000002F0000004F0000008F0000008F0000010F0000010F0000020F 0000040F0000040F0000080F8000080780001007800020078000200780007FFF80004007 8000800780018007800100078002000780020007C0040003C00C0003C01E0007C0FF807F FC1E207E9F22>65 D<0FFFFFFC1E03C0381803C0181003C0082003C00820078008600780 084007800840078008800F0010000F0000000F0000000F0000001E0000001E0000001E00 00001E0000003C0000003C0000003C0000003C0000007800000078000000780000007800 0000F0000000F0000000F0000000F0000001F000007FFFC0001E1F7F9E1B>84 D<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E10F01C20F01C20703C20705C40308C400F078014147E9318>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E380F38 0F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207E9F14>I< 007C01C207010E0F1E0F1C0E3C04780078007800F000F000F000F000F000700170023004 18380FC010147E9314>I<0000780003F80000700000700000700000700000E00000E000 00E00000E00001C00001C000F1C00389C00707800E03801C03803C038038070078070078 0700780700F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F078015 207E9F18>I<007C01C207010E011C013C013802780C7BF07C00F000F000F000F0007000 700170023004183807C010147E9315>I<00007C0000CE00019E00039E00030C00070000 0700000700000700000E00000E00000E0000FFF0000E00000E00001C00001C00001C0000 1C00001C0000380000380000380000380000380000700000700000700000700000700000 E00000E00000E00000E00000C00001C000318000798000F300006200003C000017297E9F 16>I<001E3000713800E0F001C0700380700780700700E00F00E00F00E00F00E01E01C0 1E01C01E01C01E01C01E03801E03800E07800E0B8006170001E700000700000700000E00 000E00300E00781C00F038006070003FC000151D809316>I<01E0000FE00001C00001C0 0001C00001C000038000038000038000038000070000070000071F000761800E80C00F00 C00E00E00E00E01C01C01C01C01C01C01C01C0380380380380380380380704700708700E 08700E10700610E006206003C016207E9F1A>I<00E001E001E000C00000000000000000 0000000000000E00130023804380438043808700070007000E000E001C001C001C203840 38403840388019000E000B1F7E9E10>I<0000C00001E00001E00001C000000000000000 0000000000000000000000000000001E0000630000438000838001038001038002070000 0700000700000700000E00000E00000E00000E00001C00001C00001C00001C0000380000 380000380000380000700000700030700078E000F1C0006380003E00001328819E13>I< 01E0000FE00001C00001C00001C00001C000038000038000038000038000070000070000 0701E00706100E08700E10F00E20F00E40601C80001D00001E00001FC000387000383800 383800381C20703840703840703840701880E01880600F0014207E9F18>I<03C01FC003 8003800380038007000700070007000E000E000E000E001C001C001C001C003800380038 0038007000700070007100E200E200E200E200640038000A207E9F0E>I<1E07C07C0023 1861860023A032030043C034030043803803804380380380870070070007007007000700 70070007007007000E00E00E000E00E00E000E00E00E000E00E01C101C01C01C201C01C0 38201C01C038401C01C0184038038018801801800F0024147E9328>I<1E07802318C023 A06043C0704380704380708700E00700E00700E00700E00E01C00E01C00E01C00E03821C 03841C07041C07081C03083803101801E017147E931B>I<007C0001C3000301800E01C0 1E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780F00700700F00 700E0030180018700007C00013147E9316>I<03C1E004621804741C08781C08701E0870 1E10E01E00E01E00E01E00E01E01C03C01C03C01C03C01C0380380780380700380E003C1 C0072380071E000700000700000E00000E00000E00000E00001C00001C0000FFC000171D 819317>I<1E1E0023210023C38043C7804387804383008700000700000700000700000E 00000E00000E00000E00001C00001C00001C00001C000038000018000011147E9315> 114 D<007C018203010603060706060E00078007F803FC01FE001F00077007F006F006E0 04400820301FC010147E9315>I<00C000E001C001C001C001C003800380FFF803800700 0700070007000E000E000E000E001C001C001C001C10382038203820384018800F000D1C 7F9B10>I<0F00601180702180E021C0E041C0E04380E08381C00701C00701C00701C00E 03800E03800E03800E03840E07080C07080C07080E0F1006131003E1E016147E931A>I< 0F01801183C02183E021C1E041C0E04380608380400700400700400700400E00800E0080 0E00800E01000E01000C02000E04000E040006180001E00013147E9316>I<0F00606011 8070F02180E0F821C0E07841C0E0384380E0188381C0100701C0100701C0100701C0100E 0380200E0380200E0380200E0380400E0380400E0380800E078080060781000709860001 F078001D147E9321>I<03C1C00C62201034701038F02038F02038604070000070000070 0000700000E00000E00000E00000E02061C040F1C040F1C080E2C080446300383C001414 7E931A>I<0F00601180702180E021C0E041C0E04380E08381C00701C00701C00701C00E 03800E03800E03800E03800E07000C07000C07000E0F00061E0003EE00000E00000E0000 1C0078180078380070700060600021C0001F0000141D7E9316>I<01E02003F04007F8C0 0C1F8008010000020000040000080000100000600000C000010000020000040080080100 1003003F060061FC0040F80080700013147E9315>I E /Fz 54 122 df<000FF07F00007FF9FF8000F83FC7C001E07F8FC003E07F0FC007C07F0FC007C03F07 8007C01F000007C01F000007C01F000007C01F000007C01F0000FFFFFFF800FFFFFFF800 07C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007 C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C0 1F000007C01F00003FF8FFF0003FF8FFF0002220809F1F>11 D<000FE000007FF80000F8 1C0001E07C0003E07C0007C07C0007C07C0007C0380007C0000007C0000007C0000007C1 FE00FFFFFE00FFFFFE0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C0 3E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C0 3E003FF9FFC03FF9FFC01A20809F1D>I<000FF03FFC00007FFDFFFC0000F83FE0FC0001 E03F81FC0003E07F81FC0007C07F00FC0007C03F00FC0007C01F007C0007C01F007C0007 C01F007C0007C01F007C0007C01F007C00FFFFFFFFFC00FFFFFFFFFC0007C01F007C0007 C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007 C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007 C01F007C0007C01F007C0007C01F007C003FF8FFE3FF803FF8FFE3FF802920809F2C>15 D<387CFEFFFF7F3B030306060E0C18702008107C860F>44 DI<01FC0007FF001F07C01E03C03E03E07C01F07C01F07C01F0FC01F8FC01F8FC01 F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F87C01F07C01 F07C01F03E03E01E03C01F8FC007FF0001FC00151D7E9C1A>48 D<00E00001E0000FE000 FFE000F3E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E000 03E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E000 FFFF80FFFF80111D7C9C1A>I<07F0001FFE00383F007C1F80FE0FC0FE0FC0FE0FE0FE07 E07C07E03807E0000FE0000FC0000FC0001F80001F00003E0000780000F00000E00001C0 000380600700600E00601C00E01FFFC03FFFC07FFFC0FFFFC0FFFFC0131D7D9C1A>I<01 FC0007FF000E0F801E0FC03F07E03F07E03F07E03F07E01E0FC0000FC0000F80001F0001 FC0001FC00000F800007C00003E00003F00003F83803F87C03F8FE03F8FE03F8FE03F0FC 03F07807E03C0FC01FFF8003FC00151D7E9C1A>I<0001C00003C00007C00007C0000FC0 001FC0003BC00073C00063C000C3C00183C00383C00703C00E03C00C03C01803C03803C0 7003C0E003C0FFFFFEFFFFFE0007C00007C00007C00007C00007C00007C000FFFE00FFFE 171D7F9C1A>I<3803803FFF803FFF003FFE003FFC003FF0003F80003000003000003000 0030000033F80037FE003C1F00380F801007C00007C00007E00007E07807E0FC07E0FC07 E0FC07E0FC07C0780FC0600F80381F001FFC0007F000131D7D9C1A>I<003F0001FFC007 E0E00F81E01F03F01E03F03E03F07C03F07C01E07C0000FC1000FCFF00FDFFC0FD03E0FE 01F0FE01F0FC01F8FC01F8FC01F8FC01F87C01F87C01F87C01F83C01F03E01F01E03E00F 07C007FF8001FE00151D7E9C1A>I<6000007FFFF87FFFF87FFFF07FFFE07FFFC0E00180 C00300C00300C00600000C0000180000380000380000780000700000F00000F00001F000 01F00001F00001F00003F00003F00003F00003F00003F00003F00001E00000C000151E7D 9D1A>I<01FC0007FF000F07801E03C01C01E03C01E03C01E03E01E03F01E03FC3C01FE3 801FFF000FFE0007FF8007FFC01FFFE03C3FF0780FF07803F8F001F8F000F8F00078F000 78F000707800707C00E03E03C00FFF8003FC00151D7E9C1A>I<01FC000FFF001F07803E 03C07C03E07C01E0FC01F0FC01F0FC01F0FC01F8FC01F8FC01F8FC01F87C03F87C03F83E 05F81FFDF807F9F80041F80001F03C01F07E01F07E03E07E03E07E07C03C0780381F001F FC0007F000151D7E9C1A>I<387CFEFEFE7C38000000000000387CFEFEFE7C3807147C93 0F>I<07F8001FFE00381F80780F80FC0FC0FC0FC0FC0FC0780FC0301F80001F00003E00 007C0000700000E00000E00000C00000C00000C00000C00000C00000C000000000000000 00000000000001C00003E00007F00007F00007F00003E00001C00012207D9F19>63 D<0000E000000000E000000001F000000001F000000001F000000003F800000003F80000 0006FC00000006FC0000000EFE0000000C7E0000000C7E000000183F000000183F000000 303F800000301F800000701FC00000600FC00000600FC00000C007E00000FFFFE00001FF FFF000018003F000018003F000030001F800030001F800060001FC00060000FC000E0000 FE00FFE00FFFE0FFE00FFFE0231F7E9E28>65 DI<0007FC02003FFF0E00FE03DE03F000FE07E0003E0FC0001E1F80001E3F 00000E3F00000E7F0000067E0000067E000006FE000000FE000000FE000000FE000000FE 000000FE000000FE0000007E0000007E0000067F0000063F0000063F00000C1F80000C0F C0001807E0003803F0007000FE01C0003FFF800007FC001F1F7D9E26>I69 DI<0007FC0200003FFF0E0000FE03DE0003F000FE0007E0003E000FC0001E001F 80001E003F00000E003F00000E007F000006007E000006007E00000600FE00000000FE00 000000FE00000000FE00000000FE00000000FE003FFFE0FE003FFFE07E00007E007E0000 7E007F00007E003F00007E003F00007E001F80007E000FC0007E0007E0007E0003F000FE 0000FE01FE00003FFF8E000007FC0600231F7D9E29>I73 D76 DI80 D<03FC080FFF381E03F83800F8700078700038F00038F00018F000 18F80000FC00007FC0007FFE003FFF801FFFE00FFFF007FFF000FFF80007F80000FC0000 7C00003CC0003CC0003CC0003CE00038E00078F80070FE01E0E7FFC081FF00161F7D9E1D >83 D<7FFFFFFC7FFFFFFC7C07E07C7007E01C6007E00C6007E00CE007E00EC007E006C0 07E006C007E006C007E0060007E0000007E0000007E0000007E0000007E0000007E00000 07E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E00000 07E0000007E00003FFFFC003FFFFC01F1E7E9D24>I<07FC001FFF003F0F803F07C03F03 E03F03E00C03E00003E0007FE007FBE01F03E03C03E07C03E0F803E0F803E0F803E0FC05 E07E0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007FF801F0FC03E0FC03E0FC07C0FC07C0300FC00 00FC0000FC0000FC0000FC0000FC00007C00007E00003E00603F00C01F81C007FF0001FC 0013147E9317>I<0007F80007F80000F80000F80000F80000F80000F80000F80000F800 00F80000F80000F801F8F80FFEF81F83F83E01F87E00F87C00F87C00F8FC00F8FC00F8FC 00F8FC00F8FC00F8FC00F87C00F87C00F87E00F83E01F81F07F80FFEFF03F8FF18207E9F 1D>I<01FE0007FF800F83C01E01E03E00F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000 FC0000FC00007C00007C00003E00181E00180F807007FFE000FF8015147F9318>I<001F 8000FFC001F3E003E7E003C7E007C7E007C3C007C00007C00007C00007C00007C000FFFC 00FFFC0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C0003FFC003FFC0013207F9F10>I<01FC3C07FFFE0F 079E1E03DE3E03E03E03E03E03E03E03E03E03E01E03C00F07800FFF0009FC0018000018 00001C00001FFF800FFFF007FFF81FFFFC3C007C70003EF0001EF0001EF0001E78003C78 003C3F01F80FFFE001FF00171E7F931A>II<1C003E007F007F007F003E001C00000000000000000000 000000FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F 001F00FFE0FFE00B217EA00E>I<0038007C00FE00FE00FE007C00380000000000000000 0000000001FE01FE003E003E003E003E003E003E003E003E003E003E003E003E003E003E 003E003E003E003E003E003E303E783EFC3CFC7C78783FF01FC00F2A83A010>IIIII<01FF0007FFC01F83F03E00F83E00F8 7C007C7C007CFC007EFC007EFC007EFC007EFC007EFC007E7C007C7C007C3E00F83E00F8 1F83F007FFC001FF0017147F931A>II<01F81807FE381F87783F01F83E01F87E00F87C00F8FC00F8FC00F8FC00F8FC00F8FC 00F8FC00F87C00F87E00F87E00F83F01F81F87F80FFEF803F8F80000F80000F80000F800 00F80000F80000F80000F80007FF0007FF181D7E931C>II<0FE63FFE701E600EE006E006F800FFC07F F83FFC1FFE03FE001FC007C007E007F006F81EFFFCC7F010147E9315>I<018001800180 03800380038007800F803F80FFFCFFFC0F800F800F800F800F800F800F800F800F800F80 0F860F860F860F860F8607CC03F801F00F1D7F9C14>IIIIII E /FA 37 122 df<00000007FF800000000001FFFFF0000000000FFFFFFC000000003FFFFF FE00000000FFFC00FF00000001FFC0003F80000007FF00007FC000000FFE0001FFC00000 1FFC0001FFE000001FF80003FFE000003FF00003FFE000003FF00003FFE000007FE00003 FFE000007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE00001FFC00000 7FE00000FF8000007FE000003E0000007FE00000000000007FE00000000000007FE00000 000000007FE00000000000007FE00000000000007FE00000000000007FE0003FFFF000FF FFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFF FFF000007FF00000FFF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF0003FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03F FFFFE03FFFFFE03B487EC742>12 D<07C0001FF0003FFC007FFC007FFE00FFFE00FFFF00 FFFF00FFFF80FFFF80FFFF807FFF807FFF803FFF801FFF8007E780000780000780000F00 000F00000F00000F00001E00001E00003C00003C0000780000F80000F00001F00003E000 07C0000F80001F00001E00000C00001124788F21>44 D<0000000003E000000000000000 000007F000000000000000000007F00000000000000000000FF80000000000000000000F F80000000000000000000FF80000000000000000001FFC0000000000000000001FFC0000 000000000000003FFE0000000000000000003FFE0000000000000000007FFF0000000000 000000007FFF0000000000000000007FFF000000000000000000FFFF8000000000000000 00FFFF800000000000000001FFFFC00000000000000001FFFFC00000000000000001FFFF C00000000000000003FFFFE00000000000000003EFFFE00000000000000007EFFFF00000 000000000007CFFFF00000000000000007C7FFF0000000000000000FC7FFF80000000000 00000F83FFF8000000000000001F83FFFC000000000000001F03FFFC000000000000001F 01FFFC000000000000003F01FFFE000000000000003E00FFFE000000000000007E00FFFF 000000000000007C007FFF00000000000000FC007FFF80000000000000F8007FFF800000 00000000F8003FFF80000000000001F8003FFFC0000000000001F0001FFFC00000000000 03F0001FFFE0000000000003E0001FFFE0000000000003E0000FFFE0000000000007E000 0FFFF0000000000007C00007FFF000000000000FC00007FFF800000000000F800007FFF8 00000000000F800003FFF800000000001F800003FFFC00000000001F000001FFFC000000 00003FFFFFFFFFFE00000000003FFFFFFFFFFE00000000007FFFFFFFFFFF00000000007F FFFFFFFFFF00000000007FFFFFFFFFFF0000000000FC0000007FFF8000000000F8000000 3FFF8000000001F80000003FFFC000000001F00000003FFFC000000001F00000001FFFC0 00000003F00000001FFFE000000003E00000000FFFE000000007E00000000FFFF0000000 07C00000000FFFF000000007C000000007FFF00000000FC000000007FFF80000000F8000 000003FFF80000001F8000000003FFFC0000001F0000000003FFFC0000007FC000000001 FFFC0000FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF 80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF8051487CC75A>65 DI<00 0000003FFE00000E0000000FFFFFC0001E0000007FFFFFF8003E000003FFFFFFFE00FE00 000FFFFFFFFF81FE00003FFFF800FFC3FE0000FFFF80000FF7FE0001FFFC000003FFFE00 07FFF0000001FFFE000FFFC00000007FFE001FFF800000003FFE003FFF000000001FFE00 7FFE000000000FFE00FFFC0000000007FE01FFF80000000007FE03FFF00000000003FE03 FFF00000000001FE07FFE00000000001FE07FFE00000000000FE0FFFC00000000000FE0F FFC000000000007E1FFFC000000000007E1FFF8000000000007E3FFF8000000000007E3F FF8000000000003E3FFF8000000000003E7FFF8000000000003E7FFF0000000000003E7F FF000000000000007FFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF000000000000007FFF000000000000007F FF000000000000007FFF000000000000007FFF8000000000003E3FFF8000000000003E3F FF8000000000003E3FFF8000000000003E1FFF8000000000003E1FFFC000000000003E0F FFC000000000007C0FFFC000000000007C07FFE000000000007C07FFE00000000000F803 FFF00000000000F803FFF00000000001F801FFF80000000001F000FFFC0000000003E000 7FFE0000000007E0003FFF000000000FC0001FFF800000001F80000FFFC00000003F0000 07FFF0000000FE000001FFFC000001FC000000FFFF80000FF80000003FFFF8007FF00000 000FFFFFFFFFC000000003FFFFFFFF00000000007FFFFFFC00000000000FFFFFE0000000 0000003FFE000000474979C756>I69 DI<000000003FFE 00000E000000000FFFFFC0001E000000007FFFFFF8003E00000003FFFFFFFE00FE000000 0FFFFFFFFF81FE0000003FFFF800FFC3FE000000FFFF80000FF7FE000001FFFC000003FF FE000007FFF0000001FFFE00000FFFC00000007FFE00001FFF800000003FFE00003FFF00 0000001FFE00007FFE000000000FFE0000FFFC0000000007FE0001FFF80000000007FE00 03FFF00000000003FE0003FFF00000000001FE0007FFE00000000001FE0007FFE0000000 0000FE000FFFC00000000000FE000FFFC000000000007E001FFFC000000000007E001FFF 8000000000007E003FFF8000000000007E003FFF8000000000003E003FFF800000000000 3E007FFF8000000000003E007FFF0000000000003E007FFF00000000000000007FFF0000 000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000 FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF00000000 00000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF 0000000000000000FFFF0000000000000000FFFF0000007FFFFFFFFE7FFF0000007FFFFF FFFE7FFF0000007FFFFFFFFE7FFF8000007FFFFFFFFE7FFF8000007FFFFFFFFE3FFF8000 000000FFFE003FFF8000000000FFFE003FFF8000000000FFFE001FFF8000000000FFFE00 1FFFC000000000FFFE000FFFC000000000FFFE000FFFC000000000FFFE0007FFE0000000 00FFFE0007FFE000000000FFFE0003FFF000000000FFFE0003FFF800000000FFFE0001FF F800000000FFFE0000FFFC00000000FFFE00007FFE00000000FFFE00003FFF00000000FF FE00001FFF80000000FFFE00000FFFE0000001FFFE000007FFF0000003FFFE000001FFFC 000007FFFE000000FFFF80001FFFFE0000003FFFFC00FFC7FE0000000FFFFFFFFF83FE00 000003FFFFFFFE00FE000000007FFFFFF8003E000000000FFFFFE0000E00000000003FFE 000000004F4979C75D>I73 D76 DI80 D<00007FF00007000007FFFF000F00001FFFFFC01F00007FFFFFF03F0000 FFFFFFFC7F0003FFC00FFEFF0007FE0000FFFF0007FC00003FFF000FF000001FFF001FF0 000007FF001FE0000003FF003FE0000001FF003FC0000001FF007FC0000000FF007FC000 00007F007FC00000007F00FFC00000003F00FFC00000003F00FFC00000003F00FFE00000 003F00FFE00000001F00FFF00000001F00FFF80000001F00FFFC0000001F00FFFF000000 00007FFFC0000000007FFFFE000000007FFFFFE00000003FFFFFFF0000003FFFFFFFF000 001FFFFFFFFC00001FFFFFFFFF00000FFFFFFFFFC00007FFFFFFFFE00007FFFFFFFFF000 03FFFFFFFFF80000FFFFFFFFFC00007FFFFFFFFE00003FFFFFFFFE00000FFFFFFFFF0000 01FFFFFFFF0000003FFFFFFF80000001FFFFFF800000000FFFFFC000000000FFFFC00000 00001FFFC0000000000FFFE00000000007FFE00000000003FFE07800000001FFE0F80000 0000FFE0F800000000FFE0F8000000007FE0F8000000007FE0F8000000007FE0FC000000 007FE0FC000000007FC0FC000000007FC0FE000000007FC0FE000000007F80FF00000000 FF80FF80000000FF80FFC0000000FF00FFE0000001FE00FFF8000003FE00FFFE000007FC 00FFFF80000FF800FFFFFC007FF000FE3FFFFFFFE000FC0FFFFFFF8000F803FFFFFF0000 F0007FFFF80000E00003FFC00000334979C742>83 D<3FFFFFFFFFFFFFFFFF003FFFFFFF FFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF00 3FFF0003FFF8003FFF007FF80003FFF80007FF807FE00003FFF80001FF807FC00003FFF8 0000FF807F800003FFF800007F807F000003FFF800003F807F000003FFF800003F807E00 0003FFF800001F807E000003FFF800001F807E000003FFF800000F807C000003FFF80000 0F807C000003FFF800000F807C000003FFF800000F807C000003FFF800000F80FC000003 FFF800000FC0F8000003FFF8000007C0F8000003FFF8000007C0F8000003FFF8000007C0 F8000003FFF8000007C0F8000003FFF8000007C000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF80000000000000003FFF8000000000000 0003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003 FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800000000 00000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF80000000000000003FFF8000000000000 0003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003 FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800000000 00000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF8000000000003FFFFFFFFFFF800000003 FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF8 00004A467CC553>I<0007FFFC000000007FFFFFC0000001FFFFFFF8000003FFFFFFFE00 0007FE001FFF000007FF0003FFC0000FFF8001FFE0000FFF8000FFF0000FFF80007FF000 0FFF80007FF8000FFF80007FF80007FF00003FFC0007FF00003FFC0003FE00003FFC0000 F800003FFC00000000003FFC00000000003FFC00000000003FFC00000000003FFC000000 07FFFFFC000000FFFFFFFC000007FFFFFFFC00003FFFE03FFC0000FFFE003FFC0003FFF0 003FFC0007FFC0003FFC000FFF00003FFC001FFE00003FFC003FFC00003FFC007FF80000 3FFC007FF800003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000003F FC00FFF000003FFC00FFF000007FFC007FF80000FFFC007FF80001EFFC003FFC0003EFFC 003FFF0007CFFF000FFFC03F8FFFF807FFFFFF07FFFC01FFFFFC03FFFC007FFFF001FFFC 0003FF80007FF8362E7DAD3A>97 D<007FC00000000000FFFFC00000000000FFFFC00000 000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC00FFC00000001FFC07FFFC0000001FFC3FFFFF0000001FFCFFFFF FC000001FFDFF00FFF000001FFFF8003FF800001FFFE0001FFC00001FFF800007FE00001 FFF000007FF00001FFE000003FF80001FFE000001FFC0001FFE000001FFC0001FFE00000 1FFE0001FFE000001FFE0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001 FFE000000FFF0001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE00000 0FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001 FFE000000FFF8001FFE000000FFF8001FFE000000FFF0001FFE000000FFF0001FFE00000 0FFF0001FFE000001FFE0001FFE000001FFE0001FFE000001FFC0001FFE000001FFC0001 FFE000003FF80001FFF000003FF80001FFF800007FF00001FFFC0000FFE00001FFFE0001 FFC00001FFBF0007FF800001FF1FE01FFE000001FE0FFFFFFC000001FC03FFFFF0000001 F800FFFF80000001F0001FF800000039487CC742>I<00001FFFC0000000FFFFF8000007 FFFFFE00001FFFFFFF80007FFC00FFC000FFE001FFC001FFC003FFE003FF8003FFE007FF 0003FFE00FFE0003FFE00FFE0003FFE01FFC0001FFC01FFC0001FFC03FFC0000FF803FFC 00003E007FF8000000007FF8000000007FF800000000FFF800000000FFF800000000FFF8 00000000FFF800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF8 00000000FFF8000000007FF8000000007FF8000000007FFC000000003FFC000000003FFC 000000001FFC000000F81FFE000000F80FFE000000F80FFF000001F007FF800003F003FF C00007E001FFE0000FC000FFF0001F80007FFE00FF00001FFFFFFE000007FFFFF8000000 FFFFE00000001FFE00002D2E7CAD35>I<00000000007FC00000000000FFFFC000000000 00FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC00000000FFC01FFC0000000FFFF81FFC0000007FFFFE1FFC000001FFF FFF9FFC000007FFC03FFFFC00000FFF0007FFFC00001FFC0001FFFC00003FF80000FFFC0 0007FF000007FFC0000FFE000003FFC0000FFE000003FFC0001FFC000003FFC0001FFC00 0003FFC0003FFC000003FFC0003FFC000003FFC0007FF8000003FFC0007FF8000003FFC0 007FF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF800 0003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC0 00FFF8000003FFC000FFF8000003FFC0007FF8000003FFC0007FF8000003FFC0007FF800 0003FFC0003FF8000003FFC0003FFC000003FFC0003FFC000003FFC0001FFC000003FFC0 001FFC000003FFC0000FFE000007FFC00007FF00000FFFC00003FF00001FFFC00001FFC0 003FFFC00000FFE000FFFFE000007FF807FBFFFF80001FFFFFF3FFFF800007FFFFC3FFFF 800001FFFF03FFFF8000001FF803FFFF8039487CC742>I<00001FFE00000001FFFFE000 0007FFFFF800001FFFFFFE00007FFC07FF0000FFE001FF8001FFC0007FC003FF80003FE0 07FF00003FF00FFE00001FF01FFE00000FF81FFC00000FF83FFC00000FFC3FFC000007FC 7FFC000007FC7FF8000007FC7FF8000007FE7FF8000007FEFFF8000007FEFFF8000007FE FFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFCFFF800000000FFF800000000 FFF800000000FFF8000000007FF8000000007FF8000000007FFC000000003FFC00000000 3FFC000000003FFC0000001C1FFE0000003E0FFE0000003E07FF0000007E07FF000000FC 03FF800001F801FFC00003F0007FF0001FE0003FFE00FFC0001FFFFFFF800007FFFFFE00 0000FFFFF80000000FFF80002F2E7DAD36>I<000000FFC000000007FFF80000003FFFFC 000000FFFFFF000001FFC1FF000007FF03FF80000FFC03FF80000FF807FFC0001FF807FF C0003FF007FFC0003FF007FFC0003FE003FF80007FE003FF80007FE001FF00007FE000FE 00007FE0003800007FE0000000007FE0000000007FE0000000007FE0000000007FE00000 00007FE0000000007FE0000000007FE0000000007FE0000000007FE0000000FFFFFFFE00 00FFFFFFFE0000FFFFFFFE0000FFFFFFFE0000FFFFFFFE0000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF00000003FFFFFF800003FFFFFF800 003FFFFFF800003FFFFFF800003FFFFFF800002A487DC724>I<00000000001F8000007F F000FFE00007FFFF03FFF0001FFFFFC7FFF0007FFFFFFFC7F800FFE03FFE0FF801FF800F FC0FF803FF0007FE0FF807FE0003FF07F007FE0003FF07F00FFC0001FF81C00FFC0001FF 80000FFC0001FF80001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0 001FFC0001FFC0001FFC0001FFC0000FFC0001FF80000FFC0001FF80000FFC0001FF8000 07FE0003FF000007FE0003FF000003FF0007FE000001FF800FFC000000FFE03FF8000001 FFFFFFF0000001DFFFFFC0000003C7FFFF00000003C07FF000000007C0000000000007C0 000000000007C0000000000007C0000000000007E0000000000007F0000000000007F800 0000000007FFFFFFF0000007FFFFFFFF000003FFFFFFFFE00003FFFFFFFFF80001FFFFFF FFFE0001FFFFFFFFFF0000FFFFFFFFFF80007FFFFFFFFF8003FFFFFFFFFFC00FFFFFFFFF FFC01FF800001FFFE03FE0000001FFE07FC00000007FF07FC00000003FF0FF800000001F F0FF800000001FF0FF800000001FF0FF800000001FF0FF800000001FF07FC00000003FE0 7FC00000003FE03FE00000007FC03FF0000000FFC01FFC000003FF800FFF00000FFF0003 FFF000FFFC0000FFFFFFFFF000003FFFFFFFC0000007FFFFFE000000003FFFC000003544 7DAE3B>I<007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC000000000 00FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 01FFC0000001FFC00FFFF8000001FFC03FFFFE000001FFC0FFFFFF000001FFC1FC07FF80 0001FFC3E003FFC00001FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC 0000FFE00001FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FFF0 0001FFF00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 00FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFF C07FFFFFE03B487CC742>I<00FC0001FE0003FF0007FF800FFFC01FFFE01FFFE01FFFE0 1FFFE01FFFE01FFFE00FFFC007FF8003FF0001FE0000FC00000000000000000000000000 000000000000000000000000000000000000000000007FC0FFFFC0FFFFC0FFFFC0FFFFC0 FFFFC003FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0 01FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0 01FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF18497CC820>I<007FC000000000FFFFC000000000 FFFFC000000000FFFFC000000000FFFFC000000000FFFFC00000000003FFC00000000001 FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FF C00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC0 0000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC000 00000001FFC00000000001FFC00000000001FFC00000000001FFC0007FFFF801FFC0007F FFF801FFC0007FFFF801FFC0007FFFF801FFC0007FFFF801FFC0000FFE0001FFC00007F0 0001FFC0000FE00001FFC0003FC00001FFC0007F800001FFC000FE000001FFC001FC0000 01FFC007F8000001FFC00FF0000001FFC01FC0000001FFC03F80000001FFC0FF00000001 FFC1FE00000001FFC3FF00000001FFCFFF80000001FFDFFFC0000001FFFFFFC0000001FF FFFFE0000001FFFFFFF0000001FFFCFFF8000001FFF87FFC000001FFE03FFC000001FFC0 1FFE000001FFC01FFF000001FFC00FFF800001FFC007FFC00001FFC003FFC00001FFC001 FFE00001FFC001FFF00001FFC000FFF80001FFC0007FFC0001FFC0003FFC0001FFC0001F FE0001FFC0000FFF0001FFC0000FFF8001FFC0000FFFC0FFFFFF807FFFFFFFFFFF807FFF FFFFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFF38487CC73F>107 D<007FC000FFFFC000FFFFC000FFFFC000FFFFC000FFFFC00003FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC000FFFFFF80FFFFFF80FFFFFF80FFFFFF80FFFFFF 8019487CC720>I<007FC001FFC00000FFE00000FFFFC00FFFF80007FFFC0000FFFFC03F FFFE001FFFFF0000FFFFC0FFFFFF007FFFFF8000FFFFC1FC07FF80FE03FFC000FFFFC3E0 03FFC1F001FFE00003FFC7C001FFC3E000FFE00001FFCF0001FFE78000FFF00001FFDE00 00FFEF00007FF00001FFDC0000FFEE00007FF00001FFFC0000FFFE00007FF80001FFF800 00FFFC00007FF80001FFF00000FFF800007FF80001FFF00000FFF800007FF80001FFF000 00FFF800007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800FFFFFFC0 7FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC0 7FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF05C2E7CAD63>I<007FC001FFC00000FF FFC00FFFF80000FFFFC03FFFFE0000FFFFC0FFFFFF0000FFFFC1FC07FF8000FFFFC3E003 FFC00003FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC0000FFE00001 FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FFF00001FFF00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000FFFFFFC07F FFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B 2E7CAD42>I<00000FFF0000000000FFFFF000000007FFFFFE0000001FFFFFFF8000003F FC03FFC00000FFE0007FF00001FF80001FF80003FF00000FFC0007FE000007FE000FFE00 0007FF000FFC000003FF001FFC000003FF803FFC000003FFC03FF8000001FFC03FF80000 01FFC07FF8000001FFE07FF8000001FFE07FF8000001FFE0FFF8000001FFF0FFF8000001 FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FF F0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF07FF8000001FFE07FF8000001FFE0 7FF8000001FFE07FF8000001FFE03FFC000003FFC03FFC000003FFC01FFC000003FF801F FE000007FF800FFE000007FF0007FF00000FFE0003FF80001FFC0001FFC0003FF80000FF E0007FF000007FFC03FFE000001FFFFFFF80000007FFFFFE00000000FFFFF0000000000F FF000000342E7DAD3B>I<007FC00FFC000000FFFFC07FFFC00000FFFFC3FFFFF00000FF FFCFFFFFFC0000FFFFDFF01FFF0000FFFFFF8007FF800003FFFE0001FFC00001FFF80000 FFE00001FFF00000FFF00001FFE000007FF80001FFE000003FFC0001FFE000003FFC0001 FFE000003FFE0001FFE000001FFE0001FFE000001FFF0001FFE000001FFF0001FFE00000 1FFF0001FFE000000FFF0001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001 FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE00000 0FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF0001FFE000001FFF0001 FFE000001FFF0001FFE000001FFE0001FFE000001FFE0001FFE000003FFC0001FFE00000 3FFC0001FFE000007FF80001FFF000007FF80001FFF80000FFF00001FFFC0001FFE00001 FFFE0003FFC00001FFFF0007FF800001FFFFE03FFE000001FFEFFFFFFC000001FFE3FFFF F0000001FFE0FFFF80000001FFE01FF800000001FFE0000000000001FFE0000000000001 FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE00000 00000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001 FFE0000000000001FFE0000000000001FFE0000000000001FFE00000000000FFFFFFC000 000000FFFFFFC000000000FFFFFFC000000000FFFFFFC000000000FFFFFFC00000000039 427CAD42>I<00FF803F8000FFFF80FFF000FFFF83FFFC00FFFF87FFFE00FFFF8FC3FF00 FFFF8F07FF0003FF9E0FFF8001FFBC0FFF8001FFB80FFF8001FFF80FFF8001FFF00FFF80 01FFF007FF0001FFF007FF0001FFE003FE0001FFE000F80001FFE000000001FFE0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000292E7CAD31> 114 D<000FFF00E0007FFFF3E001FFFFFFE007FFFFFFE00FF800FFE01FC0001FE03F8000 0FE03F000007E07F000003E07F000003E0FF000003E0FF000003E0FF800003E0FFC00000 00FFF0000000FFFE000000FFFFF800007FFFFFC0007FFFFFF0003FFFFFFC001FFFFFFF00 0FFFFFFF8007FFFFFFC003FFFFFFE000FFFFFFF0003FFFFFF00003FFFFF800001FFFF800 0000FFFC0000001FFC7800000FFCF8000007FCF8000003FCFC000003FCFC000003FCFE00 0003F8FE000003F8FF000003F8FF800007F0FFC0000FF0FFF0001FE0FFFC00FFC0FFFFFF FF80FC7FFFFE00F81FFFF800E003FF8000262E7CAD2F>I<0001F000000001F000000001 F000000001F000000001F000000001F000000003F000000003F000000003F000000007F0 00000007F000000007F00000000FF00000000FF00000001FF00000003FF00000003FF000 00007FF0000001FFF0000003FFF000000FFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC0 FFFFFFFFC000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000 FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FF F0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0 000000FFF0000000FFF0000000FFF001F000FFF001F000FFF001F000FFF001F000FFF001 F000FFF001F000FFF001F000FFF001F000FFF001F0007FF001E0007FF803E0003FF803E0 003FFC07C0001FFE0F80000FFFFF800007FFFE000001FFFC0000001FF00024427EC12E> I<007FE000003FF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0 007FFFF000FFFFE0007FFFF00003FFE00001FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00001FFF00001FFE00001FFF00001FFE00001FFF0 0001FFE00003FFF00000FFE00007FFF00000FFE0000F7FF000007FE0001F7FF000007FF0 003E7FF800003FFC00FC7FFFE0001FFFFFF87FFFE00007FFFFE07FFFE00001FFFF807FFF E000003FFE007FFFE03B2E7CAD42>III<7FFFFF801FFFFF007FFFFF801FFFFF00 7FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF00007FF80001FF0000007FFC00 01FC0000003FFE0003F80000001FFF0007F00000000FFF0007E000000007FF800FC00000 0003FFC01F8000000003FFE03F8000000001FFF07F0000000000FFF8FE00000000007FF9 FC00000000003FFFF800000000003FFFF000000000001FFFE000000000000FFFC0000000 000007FFC0000000000003FFC0000000000001FFE0000000000001FFF0000000000001FF F8000000000003FFFC000000000003FFFE000000000007FFFE00000000000FEFFF000000 00001FCFFF80000000003F87FFC0000000007F03FFE000000000FE01FFE000000001FC00 FFF000000001F8007FF800000003F0007FFC00000007F0003FFE0000000FE0001FFF0000 001FC0000FFF0000003F800007FF800000FF800007FFC000FFFFF8003FFFFFC0FFFFF800 3FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC03A2E7EAD3F>I<7F FFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000 FFFF8000FFF000000FE00000FFF800000FC00000FFF800000FC000007FFC00000F800000 7FFC00001F8000003FFC00001F0000003FFE00003F0000001FFE00003E0000001FFF0000 7E0000000FFF00007C0000000FFF8000FC00000007FF8000F800000007FFC001F8000000 03FFC001F000000003FFE003F000000003FFE003F000000001FFF003E000000001FFF007 E000000000FFF007C000000000FFF80FC0000000007FF80F80000000007FFC1F80000000 003FFC1F00000000003FFE3F00000000001FFE3E00000000001FFF7E00000000000FFF7C 00000000000FFFFC00000000000FFFFC000000000007FFF8000000000007FFF800000000 0003FFF0000000000003FFF0000000000001FFE0000000000001FFE0000000000000FFC0 000000000000FFC00000000000007F800000000000007F800000000000003F0000000000 00003F000000000000003F000000000000003E000000000000007E000000000000007C00 000000000000FC000000001F8000F8000000003FC001F8000000007FE001F000000000FF F003F000000000FFF003E000000000FFF007E000000000FFF00FC000000000FFF01F8000 000000FFF03F80000000007FE07F00000000007F43FE00000000003FFFF800000000001F FFF0000000000007FFC0000000000001FE00000000000039427EAD3F>I E /FB 10 58 df<1F00318060C04040C060C060C060C060C060C060C060C060404060C0 31801F000B107F8F0F>48 D<0C003C00CC000C000C000C000C000C000C000C000C000C00 0C000C000C00FF8009107E8F0F>I<1F00618040C08060C0600060006000C00180030006 000C00102020207FC0FFC00B107F8F0F>I<1F00218060C060C000C0008001800F000080 00400060C060C060804060801F000B107F8F0F>I<0300030007000F000B001300330023 004300C300FFE003000300030003001FE00B107F8F0F>I<20803F002C00200020002000 2F0030802040006000600060C06080C061801F000B107F8F0F>I<0780184030C060C060 00C000CF00F080E040C060C060C060406060C030801F000B107F8F0F>I<40007FE07FC0 8080808001000200040004000C0008000800180018001800180018000B117E900F>I<1F 00318060C060C060C071803F000F00338061C0C060C060C060404060801F000B107F8F0F >I<1F00318060C0C040C060C060C06040E021E01E600060004060C0608043003E000B10 7F8F0F>I E /FC 14 104 df0 D<70F8F8F87005057C8D0D>I<400004C0000C6000183000301800600C00C00601800303 0001860000CC0000780000300000300000780000CC000186000303000601800C00C01800 60300030600018C0000C40000416187A9623>I<01800180018001800180C183F18F399C 0FF003C003C00FF0399CF18FC1830180018001800180018010147D9417>I<0001FE0000 0007FF8000001E01E000007800780000E0001C000180000600030000030006000001800C 000000C00C000000C0180000006030000000303000000030300000003060000000186000 0000186000000018C00000000CC00000000CC00000000CC00000000CC00000000CC00000 000CC00000000CC00000000CC00000000C60000000186000000018600000001830000000 303000000030300000003018000000600C000000C00C000000C006000001800300000300 018000060000E0001C000078007800001E01E0000007FF80000001FE0000262B7DA02D> 13 D<03C00FF01C38300C60066006C003C003C003C003C003C00360066006300C1C380F F003C010127D9317>I<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFFFFFFFF7FFE 7FFE3FFC1FF80FF003C010127D9317>I<000000C0000003C000000F0000003C000000F0 000003C00000070000001C00000078000001E00000078000001E00000078000000E00000 00780000001E0000000780000001E0000000780000001C0000000700000003C0000000F0 0000003C0000000F00000003C0000000C000000000000000000000000000000000000000 0000000000000000007FFFFF80FFFFFFC01A247C9C23>20 DI<003FF800 FFF803C0000700000C0000180000300000300000600000600000C00000C00000C00000FF FFF8FFFFF8C00000C00000C000006000006000003000003000001800000C000007000003 C00000FFF8003FF8151C7C981E>50 D<00000C00000C0000180000180000300000300000 600000600000C00000C0000180000180000180000300000300000600000600000C00000C 0000180000180000300000300000600000600000C00000C0000180000180000300000300 000600000600000600000C00000C0000180000180000300000300000600000600000C000 00400000162C7AA000>54 D<400002C00006C00006C00006C00006C00006C00006C00006 C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006 C0000660000C60000C3000181C00700F01E003FF8000FE00171C7D9A1E>91 D<000F0038006000E001C001C001C001C001C001C001C001C001C001C001C001C001C001 C001C0038007001E00F8001E000700038001C001C001C001C001C001C001C001C001C001 C001C001C001C001C001C000E000600038000F102D7DA117>102 DI E /FD 1 98 df<001800001800001800003C00003C00004E00004E00004E00008700008700 0187800103800103800201C00201C003FFC00400E00400E00800700800701800703C0078 FE01FF18177F961C>97 D E /FE 91 128 df<001F83E000F06E3001C078780380F87803 00F03007007000070070000700700007007000070070000700700007007000FFFFFF8007 007000070070000700700007007000070070000700700007007000070070000700700007 00700007007000070070000700700007007000070070000700700007007000070070007F E3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E007000007000007 0000070000070000070000FFFFE00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E07FC3FE17 20809F19>I<003FE000E0E001C1E00381E00700E00700E00700E00700E00700E00700E0 0700E00700E0FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E0 0700E00700E00700E00700E00700E00700E00700E00700E00700E07FE7FE1720809F19> I<001F81F80000F04F040001C07C06000380F80F000300F00F000700F00F000700700000 07007000000700700000070070000007007000000700700000FFFFFFFF00070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007000700700700070070070007007007000700700700070070 0700070070070007007007007FE3FE3FF02420809F26>I<001F81FF0000F06F070001C0 7C0F000380F80F000300F007000700700700070070070007007007000700700700070070 070007007007000700700700FFFFFFFF0007007007000700700700070070070007007007 000700700700070070070007007007000700700700070070070007007007000700700700 07007007000700700700070070070007007007000700700700070070070007007007007F E3FE3FF02420809F26>I22 D<7038F87CFC7EFC7E743A0402040204 020804080410081008201040200F0E7E9F17>34 D<000300C0000300C0000300C0000300 C00006018000060180000601800006018000060180000C0300000C0300000C0300000C03 00000C0300001806007FFFFFFCFFFFFFFE00300C0000300C0000300C0000300C00006018 00006018000060180000601800FFFFFFFE7FFFFFFC00C030000180600001806000018060 0001806000018060000300C0000300C0000300C0000300C0000300C00006018000060180 00060180001F297D9F26>I<007800000084000001840000030200000702000007020000 0702000007020000070400000704000007080000070800000310000003A00FFC03C003E0 038001C001C0008001C0010003E0010004E0020008F00200187004003078080070380800 701C1000F01E1000F00E2000F0074000F003C0087003C0087801C010380670301C183860 07E00F801E227EA023>38 D<70F8FCFC74040404080810102040060E7C9F0D>I<002000 4000800100020006000C000C00180018003000300030007000600060006000E000E000E0 00E000E000E000E000E000E000E000E000E0006000600060007000300030003000180018 000C000C000600020001000080004000200B2E7DA112>I<800040002000100008000C00 060006000300030001800180018001C000C000C000C000E000E000E000E000E000E000E0 00E000E000E000E000E000C000C000C001C001800180018003000300060006000C000800 10002000400080000B2E7DA112>I<01800180018001800180C183F18F399C0FF003C003 C00FF0399CF18FC1830180018001800180018010147DA117>I<00060000000600000006 000000060000000600000006000000060000000600000006000000060000000600000006 0000000600000006000000060000FFFFFFF0FFFFFFF00006000000060000000600000006 000000060000000600000006000000060000000600000006000000060000000600000006 000000060000000600001C207D9A23>I<70F8FCFC74040404080810102040060E7C840D> II<70F8F8F87005057C840D>I<00010003000300060006000600 0C000C000C00180018001800300030003000600060006000C000C000C001800180018003 00030003000600060006000C000C000C00180018001800300030003000600060006000C0 00C000C000102D7DA117>I<03F0000E1C001C0E00180600380700700380700380700380 700380F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0 F003C0F003C07003807003807003807807803807001806001C0E000E1C0003F000121F7E 9D17>I<018003800F80F380038003800380038003800380038003800380038003800380 03800380038003800380038003800380038003800380038007C0FFFE0F1E7C9D17>I<03 F0000C1C00100E00200700400780800780F007C0F803C0F803C0F803C02007C00007C000 0780000780000F00000E00001C0000380000700000600000C0000180000300000600400C 00401800401000803FFF807FFF80FFFF80121E7E9D17>I<03F0000C1C00100E00200F00 780F80780780780780380F80000F80000F00000F00000E00001C0000380003F000003C00 000E00000F000007800007800007C02007C0F807C0F807C0F807C0F00780400780400F00 200E001C3C0003F000121F7E9D17>I<000600000600000E00000E00001E00002E00002E 00004E00008E00008E00010E00020E00020E00040E00080E00080E00100E00200E00200E 00400E00C00E00FFFFF0000E00000E00000E00000E00000E00000E00000E0000FFE0141E 7F9D17>I<1803001FFE001FFC001FF8001FE00010000010000010000010000010000010 000011F000161C00180E001007001007800003800003800003C00003C00003C07003C0F0 03C0F003C0E00380400380400700200600100E000C380003E000121F7E9D17>I<007C00 0182000701000E03800C07801C0780380300380000780000700000700000F1F000F21C00 F40600F80700F80380F80380F003C0F003C0F003C0F003C0F003C07003C07003C0700380 3803803807001807000C0E00061C0001F000121F7E9D17>I<4000007FFFC07FFF807FFF 8040010080020080020080040000080000080000100000200000200000400000400000C0 0000C00001C0000180000380000380000380000380000780000780000780000780000780 00078000078000030000121F7D9D17>I<03F0000C0C0010060030030020018060018060 01806001807001807803003E03003F06001FC8000FF00003F80007FC000C7E00103F0030 0F806003804001C0C001C0C000C0C000C0C000C0C000806001802001001002000C0C0003 F000121F7E9D17>I<03F0000E18001C0C00380600380700700700700380F00380F00380 F003C0F003C0F003C0F003C0F003C07007C07007C03807C0180BC00E13C003E3C0000380 000380000380000700300700780600780E00700C002018001070000FC000121F7E9D17> I<70F8F8F8700000000000000000000070F8F8F87005147C930D>I<70F8F8F870000000 0000000000000070F0F8F878080808101010202040051D7C930D>I<7FFFFFE0FFFFFFF0 0000000000000000000000000000000000000000000000000000000000000000FFFFFFF0 7FFFFFE01C0C7D9023>61 D<0FC0307040384038E03CF03CF03C603C0038007000E000C0 01800180010003000200020002000200020002000000000000000000000007000F800F80 0F8007000E207D9F15>63 D<000100000003800000038000000380000007C0000007C000 0007C0000009E0000009E0000009E0000010F0000010F0000010F0000020780000207800 0020780000403C0000403C0000403C0000801E0000801E0000FFFE0001000F0001000F00 01000F00020007800200078002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F 22>65 DI<000FC040007030C001 C009C0038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078 000040F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8 000000780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003 80020001C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E007 0001E00F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8 000000F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C 0001E03C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000 781820000FE0001E217D9F24>I II<0FFF C0007C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C 00F83C00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007 001E0007803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001 F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003 E03C0003C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80 001C217D9F23>II<001F800000 F0F00001C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C 0003E0780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F8 0001F0F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E2047000F 204F0007A03E0001E0380000F0F010001FB01000003010000038300000387000003FF000 001FE000001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E000 80E00080E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007 800003C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C 0081F80012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F003080 0F0010800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F000000 0F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F000000 0F0000000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7F F83FF80FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C100000 3C2000003E4000001E4000000F8000000F8000000780000003C0000007E0000005E00000 09F0000018F8000010780000207C0000603C0000401E0000801F0001800F000100078002 0007C0070003C01F8007E0FFE01FFE1F1F7F9E22>II<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80 800F00001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C004 0780040F80040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8161F7D9E1C> II<080410082010201040204020804080408040B8 5CFC7EFC7E7C3E381C0F0E7B9F17>II<08102020 4040808080B8FCFC7C38060E7D9F0D>96 D<1FE000303000781800781C00300E00000E00 000E00000E0000FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E10781E10 3867200F83C014147E9317>I<0E0000FE00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00700E00700E00 780E00780E00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC300083E 0015207F9F19>I<03F80E0C1C1E381E380C70007000F000F000F000F000F000F0007000 7000380138011C020E0C03F010147E9314>I<000380003F800003800003800003800003 8000038000038000038000038000038000038003E380061B801C07803803803803807003 80700380F00380F00380F00380F00380F00380F003807003807003803803803807801C07 800E1B8003E3F815207E9F19>I<03F0000E1C001C0E00380700380700700700700380F0 0380F00380FFFF80F00000F00000F000007000007000003800801800800C010007060001 F80011147F9314>I<007C00C6018F038F07060700070007000700070007000700FFF007 00070007000700070007000700070007000700070007000700070007000700070007007F F01020809F0E>I<0000E003E3300E3C301C1C30380E00780F00780F00780F00780F0078 0F00380E001C1C001E380033E0002000002000003000003000003FFE001FFF800FFFC030 01E0600070C00030C00030C00030C000306000603000C01C038003FC00141F7F9417>I< 0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0000 0E3E000E43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16207F9F19>I<1C001E003E 001E001C000000000000000000000000000E007E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E00FFC00A1F809E0C>I<00E001F001F001F0 00E0000000000000000000000000007007F000F000700070007000700070007000700070 00700070007000700070007000700070007000700070007000706070F060F0C061803F00 0C28829E0E>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E0FF00E03C00E03000E02000E04000E08000E10000E30000E70000EF8 000F38000E1C000E1E000E0E000E07000E07800E03800E03C00E03E0FFCFF815207F9F18 >I<0E00FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B20809F0C> I<0E1F01F000FE618618000E81C81C000F00F00E000F00F00E000E00E00E000E00E00E00 0E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E 00E00E000E00E00E000E00E00E000E00E00E000E00E00E00FFE7FE7FE023147F9326>I< 0E3E00FE43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16147F9319>I<01F800070E 001C03803801C03801C07000E07000E0F000F0F000F0F000F0F000F0F000F0F000F07000 E07000E03801C03801C01C0380070E0001F80014147F9317>I<0E3E00FEC3800F01C00F 00E00E00E00E00F00E00700E00780E00780E00780E00780E00780E00780E00700E00F00E 00E00F01E00F01C00EC3000E3E000E00000E00000E00000E00000E00000E00000E00000E 0000FFE000151D7F9319>I<03E0800619801C05803C0780380380780380700380F00380 F00380F00380F00380F00380F003807003807803803803803807801C0B800E138003E380 000380000380000380000380000380000380000380000380003FF8151D7E9318>I<0E78 FE8C0F1E0F1E0F0C0E000E000E000E000E000E000E000E000E000E000E000E000E000E00 FFE00F147F9312>I<1F9030704030C010C010C010E00078007F803FE00FF00070803880 188018C018C018E030D0608F800D147E9312>I<020002000200060006000E000E003E00 FFF80E000E000E000E000E000E000E000E000E000E000E000E080E080E080E080E080610 031001E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E03C00603C0030DC001F1FC16 147F9319>I II<7FC3FC0F01E00701C007018003810001C20000E4 0000EC00007800003800003C00007C00004E000087000107000303800201C00601E01E01 E0FF07FE1714809318>II<3FFF38 0E200E201C40384078407000E001E001C00380078007010E011E011C0338027006700EFF FE10147F9314>III<30307878F87C 787830300E057C9E17>127 D E /FF 46 122 df<70F8FCFC7404040404080810102040 060F7C840E>44 DI<70F8F8F87005057C840E>I<01F000071C00 0C06001803003803803803807001C07001C07001C07001C0F001E0F001E0F001E0F001E0 F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E07001C07001C0 7001C07803C03803803803801C07000C0600071C0001F00013227EA018>48 D<008003800F80F380038003800380038003800380038003800380038003800380038003 80038003800380038003800380038003800380038003800380038007C0FFFE0F217CA018 >I<03F8000C1E001007002007804007C07807C07803C07807C03807C000078000078000 0700000F00000E0000380003F000001C00000F000007800007800003C00003C00003E020 03E07003E0F803E0F803E0F003C04003C0400780200780100F000C1C0003F00013227EA0 18>51 D<000200000600000E00000E00001E00001E00002E00004E00004E00008E00008E 00010E00020E00020E00040E00040E00080E00100E00100E00200E00200E00400E00800E 00FFFFF8000E00000E00000E00000E00000E00000E00000E00001F0001FFF015217FA018 >I<1000801E07001FFF001FFE001FF80013E00010000010000010000010000010000010 000010F800130E001407001803801003800001C00001C00001E00001E00001E00001E070 01E0F001E0F001E0E001C08001C04003C04003802007001006000C1C0003F00013227EA0 18>I<007E0001C1000300800601C00E03C01C03C0180180380000380000780000700000 700000F0F800F30C00F40600F40300F80380F801C0F001C0F001E0F001E0F001E0F001E0 F001E07001E07001E07001E03801C03801C01803801C03000C0600070C0001F00013227E A018>I<01F800060E000803001001802001802000C06000C06000C06000C07000C07801 803E01003F02001FC4000FF80003F80003FC00067F00083F80100F803007C06001C06000 E0C000E0C00060C00060C00060C000606000406000C03000801803000E0E0003F0001322 7EA018>56 D<01F000060C000C0600180700380380700380700380F001C0F001C0F001C0 F001E0F001E0F001E0F001E0F001E07001E07003E03803E01805E00C05E00619E003E1E0 0001C00001C00001C0000380000380300300780700780600700C002018001030000FC000 13227EA018>I<0001800000018000000180000003C0000003C0000003C0000005E00000 05E000000DF0000008F0000008F0000010F800001078000010780000203C0000203C0000 203C0000401E0000401E0000401E0000800F0000800F0000FFFF00010007800100078003 0007C0020003C0020003C0040003E0040001E0040001E00C0000F00C0000F03E0001F8FF 800FFF20237EA225>65 D<0007E0100038183000E0063001C00170038000F0070000F00E 0000701E0000701C0000303C0000303C0000307C0000107800001078000010F8000000F8 000000F8000000F8000000F8000000F8000000F8000000F800000078000000780000107C 0000103C0000103C0000101C0000201E0000200E000040070000400380008001C0010000 E0020000381C000007E0001C247DA223>67 D69 DI73 D77 DI<000FE00000783C0000E00E0003C0078007 8003C00F0001E00E0000E01E0000F03C0000783C0000787C00007C7C00007C7800003C78 00003CF800003EF800003EF800003EF800003EF800003EF800003EF800003EF800003EF8 00003E7800003C7C00007C7C00007C3C0000783E0000F81E0000F00F0001E00F0001E007 8003C003C0078000E00E0000783C00000FE0001F247DA226>II82 D<03F0200C0C601802603001E07000E0600060E00060E00060E00020E00020E00020F000 00F000007800007F00003FF0001FFE000FFF0003FF80003FC00007E00001E00000F00000 F0000070800070800070800070800070C00060C00060E000C0F000C0C80180C6070081FC 0014247DA21B>I<7FFFFFF87807807860078018400780084007800840078008C007800C 800780048007800480078004800780040007800000078000000780000007800000078000 000780000007800000078000000780000007800000078000000780000007800000078000 00078000000780000007800000078000000780000007800000078000000FC00003FFFF00 1E227EA123>I<0FE0001838003C0C003C0E0018070000070000070000070000FF0007C7 001E07003C0700780700700700F00708F00708F00708F00F087817083C23900FC1E01515 7E9418>97 D<0E0000FE00001E00000E00000E00000E00000E00000E00000E00000E0000 0E00000E00000E00000E00000E1F000E61C00E80600F00300E00380E003C0E001C0E001E 0E001E0E001E0E001E0E001E0E001E0E001E0E001C0E003C0E00380F00700C80600C41C0 083F0017237FA21B>I<01FE000703000C07801C0780380300780000700000F00000F000 00F00000F00000F00000F00000F000007000007800403800401C00800C010007060001F8 0012157E9416>I<0000E0000FE00001E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E001F8E00704E00C02E01C01E03800E07800E07000E0F0 00E0F000E0F000E0F000E0F000E0F000E0F000E07000E07800E03800E01801E00C02E007 0CF001F0FE17237EA21B>I<01FC000707000C03801C01C03801C07801E07000E0F000E0 FFFFE0F00000F00000F00000F00000F000007000007800203800201C00400E0080070300 00FC0013157F9416>I<003C00C6018F038F030F07000700070007000700070007000700 0700FFF80700070007000700070007000700070007000700070007000700070007000700 0700070007807FF8102380A20F>I<00007001F198071E180E0E181C07001C07003C0780 3C07803C07803C07801C07001C07000E0E000F1C0019F000100000100000180000180000 1FFE000FFFC00FFFE03800F0600030400018C00018C00018C000186000306000303800E0 0E038003FE0015217F9518>I<0E0000FE00001E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E1F800E60C00E80E00F00700F00700E00 700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00 700E00700E0070FFE7FF18237FA21B>I<1C001E003E001E001C00000000000000000000 000000000000000E00FE001E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E00FFC00A227FA10E>I<01C003E003E003E001C0000000000000 0000000000000000000001E00FE001E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E060E0F0C0F18061803E00 0B2C82A10F>I<0E0000FE00001E00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E03FC0E01F00E01C00E01800E02000E04000E08000E10 000E38000EF8000F1C000E1E000E0E000E07000E07800E03C00E01C00E01E00E00F00E00 F8FFE3FE17237FA21A>I<0E00FE001E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E00FFE00B237FA20E>I<0E1FC07F00FE60E183801E807201C00F003C00E0 0F003C00E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E 003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E00 3800E00E003800E0FFE3FF8FFE27157F942A>I<0E1F80FE60C01E80E00F00700F00700E 00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E 00700E00700E0070FFE7FF18157F941B>I<01FC000707000C01801800C03800E0700070 700070F00078F00078F00078F00078F00078F00078F000787000707800F03800E01C01C0 0E038007070001FC0015157F9418>I<0E1F00FE61C00E80600F00700E00380E003C0E00 1C0E001E0E001E0E001E0E001E0E001E0E001E0E001E0E003C0E003C0E00380F00700E80 E00E41C00E3F000E00000E00000E00000E00000E00000E00000E00000E00000E0000FFE0 00171F7F941B>I<0E3CFE461E8F0F0F0F060F000E000E000E000E000E000E000E000E00 0E000E000E000E000E000F00FFF010157F9413>114 D<0F8830786018C018C008C008E0 08F0007F803FE00FF001F8003C801C800C800CC00CC008E018D0308FC00E157E9413>I< 02000200020002000600060006000E001E003E00FFF80E000E000E000E000E000E000E00 0E000E000E000E000E040E040E040E040E040E040708030801F00E1F7F9E13>I<0E0070 FE07F01E00F00E00700E00700E00700E00700E00700E00700E00700E00700E00700E0070 0E00700E00700E00700E00F00E00F006017003827800FC7F18157F941B>III121 D E /FG 20 118 df45 D68 D73 D77 D80 D<007F802001FFE02007C078600F001C601E00 06E03C0003E0380001E0780000E0700000E070000060F0000060F0000060F0000020F000 0020F0000020F8000020F80000007C0000007E0000003F0000003FC000001FF800000FFF 800007FFF80003FFFC0000FFFF00000FFF800000FFC000001FE0000007E0000003F00000 01F0000000F0000000F8000000F88000007880000078800000788000007880000078C000 0078C0000070E00000F0E00000E0F00000E0F80001C0EC000380C7000700C1F01E00807F FC00800FF0001D337CB125>83 D<00FE00000303C0000C00E00010007000100038003C00 3C003E001C003E001E003E001E0008001E0000001E0000001E0000001E00000FFE0000FC 1E0003E01E000F801E001F001E003E001E003C001E007C001E00F8001E04F8001E04F800 1E04F8003E04F8003E0478003E047C005E043E008F080F0307F003FC03E01E1F7D9E21> 97 D<003F8000E0600380180700040F00041E001E1C003E3C003E7C003E7C0008780000 F80000F80000F80000F80000F80000F80000F80000F80000F800007800007C00007C0000 3C00011E00011E00020F000207000403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E0000001E0000001E0000001E0000001E0000001 E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001 E0000001E0001F81E000F061E001C019E0078005E00F0003E00E0003E01E0001E03C0001 E03C0001E07C0001E0780001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001 E0F80001E0F80001E0F80001E0780001E0780001E03C0001E03C0001E01C0001E01E0003 E00E0005E0070009E0038011F000E061FF003F81FF20327DB125>I<003F800000E0E000 0380380007003C000E001E001E001E001C000F003C000F007C000F0078000F8078000780 F8000780F8000780FFFFFF80F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000003C0000003C0000801E0000800E0001000F0002000780020001C00C00 00F03000001FC000191F7E9E1D>I<0007E0001C1000383800707C00E07C01E07C01C038 03C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C000 FFFFC0FFFFC003C00003C00003C00003C00003C00003C00003C00003C00003C00003C000 03C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C000 03C00003C00003C00003C00007E0007FFF007FFF0016327FB114>I<000000F0007F0308 01C1C41C0380E81C070070080F0078001E003C001E003C003E003E003E003E003E003E00 3E003E003E003E003E003E001E003C001E003C000F007800070070000780E00009C1C000 087F000018000000180000001800000018000000180000001C0000000E0000000FFFF800 07FFFF0003FFFF800E000FC0180001E0300000F070000070E0000038E0000038E0000038 E0000038E00000387000007070000070380000E01C0001C00700070001C01C00003FE000 1E2F7E9F21>I<07000F801F801F800F8007000000000000000000000000000000000000 00000000000780FF80FF800F800780078007800780078007800780078007800780078007 800780078007800780078007800780078007800780078007800FC0FFF8FFF80D307EAF12 >105 D<0780FE001FC000FF83078060F000FF8C03C18078000F9001E2003C0007A001E4 003C0007A000F4001E0007C000F8001E0007C000F8001E00078000F0001E00078000F000 1E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E 00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00 078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E0007 8000F0001E00078000F0001E00078000F0001E000FC001F8003F00FFFC1FFF83FFF0FFFC 1FFF83FFF0341F7E9E38>109 D<0780FE0000FF83078000FF8C03C0000F9001E00007A0 01E00007A000F00007C000F00007C000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0 00078000F000078000F000078000F000078000F000078000F000078000F000078000F000 078000F000078000F0000FC001F800FFFC1FFF80FFFC1FFF80211F7E9E25>I<001FC000 00F0780001C01C00070007000F0007801E0003C01C0001C03C0001E03C0001E0780000F0 780000F0780000F0F80000F8F80000F8F80000F8F80000F8F80000F8F80000F8F80000F8 F80000F8780000F07C0001F03C0001E03C0001E01E0003C01E0003C00F00078007800F00 01C01C0000F07800001FC0001D1F7E9E21>I<0783E0FF8C18FF907C0F907C07A07C07C0 3807C00007C00007C0000780000780000780000780000780000780000780000780000780 000780000780000780000780000780000780000780000780000780000780000FC000FFFE 00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030E00010E000 10E00010F00010F800007E00003FF0001FFF000FFFC003FFE0003FF00001F80000F88000 3C80003C80001CC0001CC0001CE0001CE00018F00038F00030CC0060C301C080FE00161F 7E9E1A>I<00400000400000400000400000400000C00000C00000C00001C00001C00003 C00007C0000FC0001FFFE0FFFFE003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C01003C01003C01003 C01003C01003C01003C01003C01001C02001E02000E0400078C0001F00142C7FAB19>I< 078000F000FF801FF000FF801FF0000F8001F000078000F000078000F000078000F00007 8000F000078000F000078000F000078000F000078000F000078000F000078000F0000780 00F000078000F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078001F000078001F000078001F000038002F00003C004F00001C008F8 00007030FF80001FC0FF80211F7E9E25>I E /FH 5 85 df<00000000C00000000000E0 0000000001E00000000003E00000000003E00000000007E00000000007E0000000000FE0 000000000FE0000000001FE0000000001FE00000000037E00000000067E00000000067E0 00000000C7E000000000C7F00000000183F00000000183F00000000303F00000000703F0 0000000603F00000000C03F00000000C03F00000001803F00000001803F00000003003F0 0000003003F00000006003F0000000C003F0000000C003F00000018003F00000018003F8 000003FFFFF8000003FFFFF80000060001F800000E0001F800000C0001F80000180001F8 0000180001F80000300001F80000300001F80000600001F80000E00001F80000C00001F8 0001C00001F80001C00001F80007C00001FC001FC00003FC00FFF8007FFFE0FFF8007FFF E02B327BB135>65 D<000FFFFFFE0000000FFFFFFF800000007F000FE00000007E0003F0 0000007E0000F80000007E0000FC0000007E00007C000000FC00003E000000FC00003E00 0000FC00003F000000FC00001F000001F800001F000001F800001F800001F800001F8000 01F800001F800003F000001F800003F000001F800003F000001F800003F000001F800007 E000003F800007E000003F800007E000003F800007E000003F80000FC000003F00000FC0 00007F00000FC000007F00000FC000007F00001F8000007E00001F800000FE00001F8000 00FE00001F800000FC00003F000001FC00003F000001F800003F000001F800003F000003 F000007E000003E000007E000007E000007E00000FC000007E00000F800000FC00001F80 0000FC00003F000000FC00007E000000FC0000FC000001F80001F0000001F80003E00000 01F8000FC0000003F8007F000000FFFFFFFC000000FFFFFFE000000031317BB036>68 D<000FFFFFFFFC000FFFFFFFFC00007F0001FC00007E00007C00007E00003C00007E0000 3C00007E0000180000FC0000180000FC0000180000FC0000180000FC0000180001F80000 180001F80000180001F80000180001F80000180003F00080100003F00180000003F00180 000003F00180000007E00300000007E00300000007E00700000007E01F0000000FFFFE00 00000FFFFE0000000FC01E0000000FC00E0000001F800C0000001F800C0000001F800C00 00001F800C0000003F00180000003F00080000003F00000000003F00000000007E000000 00007E00000000007E00000000007E0000000000FC0000000000FC0000000000FC000000 0000FC0000000001F80000000001F80000000001F80000000003F800000000FFFFF00000 00FFFFF00000002E317BB02F>70 D<000FFFFFF000000FFFFFFE0000007F003F8000007E 000FC000007E0007E000007E0003F000007E0001F80000FC0001F80000FC0001F80000FC 0001F80000FC0001F80001F80003F80001F80003F80001F80003F80001F80003F00003F0 0007F00003F00007E00003F0000FC00003F0000FC00007E0001F000007E0007E000007E0 00FC000007E007F000000FFFFFC000000FFFFF0000000FC00F8000000FC003C000001F80 03E000001F8001F000001F8001F000001F8001F800003F0001F800003F0001F800003F00 01F800003F0001F800007E0003F800007E0003F800007E0003F000007E0003F00000FC00 07F00000FC0007F00000FC0007F00800FC0007F00C01F80007F01801F80007F01801F800 03F03003F80003F030FFFFE001F0E0FFFFE000FFC0000000003F002E327BB034>82 D<07FFFFFFFFF00FFFFFFFFFF00FC00FE003F01E000FC000F01C000FC000E018000FC000 E038000FC0006030001F8000E030001F8000E060001F8000C060001F8000C060003F0000 C0C0003F0000C0C0003F0000C0C0003F0000C080007E00008000007E00000000007E0000 0000007E0000000000FC0000000000FC0000000000FC0000000000FC0000000001F80000 000001F80000000001F80000000001F80000000003F00000000003F00000000003F00000 000003F00000000007E00000000007E00000000007E00000000007E0000000000FC00000 00000FC0000000000FC0000000000FC0000000001F80000000001F80000000001F800000 00001F80000000003F00000000003F00000000003F0000000000FF00000000FFFFFF0000 00FFFFFF0000002C3173B033>84 D E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 795 908 a FH(D)26 b(R)g(A)f(F)h(T)225 999 y FG(Do)r(cumen)n(t) 20 b(for)i(a)f(Standard)g(Message-P)n(assing)f(In)n(terface)621 1194 y FF(Message)c(P)o(assing)h(In)o(terface)e(F)l(orum)793 1320 y(Octob)q(er)h(19,)h(1993)77 1378 y(This)g(w)o(ork)f(w)o(as)h (supp)q(orted)g(in)f(part)h(b)o(y)e(ARP)l(A)h(and)h(NSF)e(under)h(gran) o(t)h(ASC-9310330,)i(the)192 1436 y(National)d(Science)f(F)l(oundation) i(Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q(erativ)o(e) 76 1494 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o(y)e(the)h (Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)654 1552 y(through)f(Esprit)f(pro)s(ject)g(P6643.)p eop %%Page: 1 2 1 1 bop 166 45 a FE(The)14 b(Message)e(P)o(assing)h(In)o(terface)h(F)l (orum)f(\(MPIF\),)f(with)i(participation)g(from)f(o)o(v)o(er)f(40)h (organi-)75 102 y(zations,)h(has)f(b)q(een)i(meeting)f(since)g(Jan)o (uary)g(1993)e(to)h(discuss)h(and)g(de\014ne)h(a)e(set)g(of)g(library)i (in)o(terface)75 158 y(standards)h(for)h(message)f(passing.)25 b(MPIF)16 b(is)i(not)e(sanctioned)i(or)e(supp)q(orted)h(b)o(y)g(an)o(y) g(o\016cial)g(stan-)75 214 y(dards)e(organization.)166 271 y(This)22 b(is)f(a)g(draft)f(of)h(what)g(will)h(b)q(ecome)g(the)f (Final)h(Rep)q(ort,)h(V)l(ersion)f(1.0,)f(of)f(the)i(Message)75 327 y(P)o(assing)c(In)o(terface)h(F)l(orum.)29 b(This)19 b(do)q(cumen)o(t)g(con)o(tains)g(all)g(the)g(tec)o(hnical)h(features)e (prop)q(osed)h(for)75 384 y(the)c(in)o(terface.)20 b(This)c(cop)o(y)f (of)g(the)g(draft)g(w)o(as)f(pro)q(cessed)i(b)o(y)f(L)1174 378 y FD(a)1195 384 y FE(T)1220 398 y(E)1246 384 y(X)g(on)g(Octob)q(er) h(19,)e(1993.)166 440 y(MPIF)e(in)o(vites)h(commen)o(ts)e(on)h(the)g (tec)o(hnical)i(con)o(ten)o(t)d(of)h(MPI,)f(as)h(w)o(ell)h(as)e(on)h (the)g(editorial)i(pre-)75 497 y(sen)o(tation)h(in)h(the)f(do)q(cumen)o (t.)20 b(Commen)o(ts)14 b(receiv)o(ed)j(b)q(efore)e(Jan)o(uary)g(15,)f (1994)g(will)j(b)q(e)f(considered)75 553 y(in)g(pro)q(ducing)h(the)e (\014nal)h(draft)e(of)h(V)l(ersion)h(1.0)e(of)h(the)g(Message)g(P)o (assing)g(In)o(terface)g(Sp)q(eci\014cation.)166 610 y(The)i(goal)f(of)g(the)g(Message)g(P)o(assing)g(In)o(terface,)g (simply)i(stated,)e(is)g(to)g(dev)o(elop)h(a)g(widely)g(used)75 666 y(standard)g(for)f(writing)i(message-passing)f(programs.)24 b(As)17 b(suc)o(h)h(the)f(in)o(terface)g(should)h(establishing)75 723 y(a)d(practical,)h(p)q(ortable,)f(e\016cien)o(t,)g(and)h (\015exible)h(standard)e(for)f(message)h(passing.)179 1978 y(c)166 1979 y FC(\015)p FE(1993)g(Univ)o(ersit)o(y)h(of)f(T)l (ennessee,)h(Kno)o(xville,)h(T)l(ennessee.)22 b(P)o(ermission)16 b(to)f(cop)o(y)g(without)h(fee)75 2035 y(all)e(or)f(part)f(of)h(this)h (material)f(is)h(gran)o(ted,)e(pro)o(vided)i(the)g(Univ)o(ersit)o(y)f (of)g(T)l(ennessee)h(cop)o(yrigh)o(t)f(notice)75 2092 y(and)i(the)h(title)g(of)f(this)g(do)q(cumen)o(t)h(app)q(ear,)f(and)g (notice)h(is)g(giv)o(en)g(that)e(cop)o(ying)i(is)g(b)o(y)f(p)q (ermission)h(of)75 2148 y(the)f(Univ)o(ersit)o(y)h(of)f(T)l(ennessee.) -32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 3 2 2 bop 75 377 a FA(Con)m(ten)m(ts)75 645 y Fz(Ac)o(kno)o(wledgmen)o (ts)1362 b(2)75 747 y(1)42 b(In)o(tro)q(duction)19 b(to)f(MPI)1230 b(3)143 804 y FE(1.1)46 b(Ov)o(erview)16 b(and)f(Goals)38 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(3)143 860 y(1.2)46 b(Who)15 b(Should)h(Use)f(This)h(Standard?)32 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(5)143 917 y(1.3)46 b(What)14 b(Platforms)h(Are)g(T)l(argets)f(F)l(or)h(Implemen)o(tation?) 38 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(5)143 973 y(1.4)46 b(What)14 b(Is)i(Included)h(In)f(The)g (Standard?)42 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(5)143 1030 y(1.5)46 b(What)14 b(Is)i(Not)e(Included)k(In)e(The)f(Standard?)23 b Fy(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)91 b FE(6)143 1086 y(1.6)46 b(Organization)16 b(of)e(this)i(Do)q(cumen)o(t)34 b Fy(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(6)75 1188 y Fz(2)42 b(MPI)17 b(T)l(erms)f(and)i(Con)o(v)o(en)o (tions)1042 b(8)143 1245 y FE(2.1)46 b(Pro)q(cedure)16 b(Sp)q(eci\014cation)44 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)91 b FE(8)143 1302 y(2.2)46 b(Seman)o(tic)15 b(T)l(erms)g Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 91 b FE(8)143 1358 y(2.3)46 b(Data)14 b(T)o(yp)q(es)30 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)91 b FE(9)248 1415 y(2.3.1)50 b(Opaque)16 b(ob)s(jects)21 b Fy(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(9)248 1471 y(2.3.2)50 b(Arra)o(y)14 b(of)h(handles)35 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(10)248 1528 y(2.3.3)50 b(State)14 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(10)248 1584 y(2.3.4)50 b(Named)15 b(constan)o(ts)23 b Fy(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(10)248 1641 y(2.3.5)50 b(Choice)20 b Fy(:)i(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(10)143 1697 y(2.4)46 b(Language)15 b(Binding)45 b Fy(:)22 b(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(11)248 1754 y(2.4.1)50 b(Data)14 b(T)o(yp)q(es)i(and)f(Naming)g(Con)o(v)o(en)o (tions)39 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FE(11)248 1810 y(2.4.2)50 b(F)l(ortran)14 b(77)h(Binding)i(Issues)30 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(11)248 1867 y(2.4.3)50 b(C)15 b(Binding)i(Issues)31 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(12)143 1923 y(2.5)46 b(Pro)q(cesses)40 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b FE(12)143 1980 y(2.6)46 b(Error)14 b(Handling)32 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FE(13)143 2036 y(2.7)46 b(Implemen)o(tation)16 b(issues)36 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(14)248 2093 y(2.7.1)50 b(Indep)q(endence)19 b(of)14 b(basic)i(run)o(time)g(routines)44 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(14)248 2149 y(2.7.2)50 b(In)o(teraction)16 b(with)f(signals)h(in)g(POSIX)42 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)69 b FE(15)75 2251 y Fz(3)42 b(P)o(oin)o(t)17 b(to)h(P)o(oin)o(t)g(Comm)o(unication)975 b(16)143 2308 y FE(3.1)46 b(In)o(tro)q(duction)15 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(16)143 2364 y(3.2)46 b(Basic)16 b(send)f(op)q(eration)27 b Fy(:)c(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(16)248 2421 y(3.2.1)50 b(Message)15 b(data)26 b Fy(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b FE(17)248 2477 y(3.2.2)50 b(Message)15 b(en)o(v)o(elop)q(e)j Fy(:)k(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(18)143 2534 y(3.3)46 b(Basic)16 b(receiv)o(e)g(op)q(eration)h Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(19)248 2591 y(3.3.1)50 b(Return)16 b(status)j Fy(:)k(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(20)143 2647 y(3.4)46 b(Seman)o(tics)15 b(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)40 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b FE(21)143 2704 y(3.5)46 b(Data)14 b(T)o(yp)q(e)h(Matc)o(hing)22 b Fy(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)69 b FE(23)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 4 3 3 bop 143 45 a FE(3.6)46 b(Data)14 b(con)o(v)o(ersion)g Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FE(25)143 102 y(3.7)46 b(Comm)o(unication)15 b(Mo)q(des)i Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(26)143 159 y(3.8)46 b(Non)o(blo)q(c)o(king)16 b(comm)o(unication)j Fy(:)j(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(29)248 216 y(3.8.1)50 b(Comm)o(unication)16 b(Ob)s(jects)23 b Fy(:)g(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(29)248 272 y(3.8.2)50 b(Comm)o(unication)16 b(initiation)32 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FE(29)248 329 y(3.8.3)50 b(Comm)o(unication)16 b(Completion)i Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(32)248 386 y(3.8.4)50 b(Seman)o(tics)16 b(of)f(Non)o(blo)q(c)o(king)h(Comm)o(unications)36 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(33)248 443 y(3.8.5)50 b(Multiple)17 b(Completions)41 b Fy(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(35)143 500 y(3.9)46 b(Prob)q(e)15 b(and)g(Cancel)g Fy(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(38)143 556 y(3.10)23 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(ob)s(jects)i Fy(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(40)143 613 y(3.11)23 b(Send-receiv)o(e)d Fy(:)i(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(43)143 670 y(3.12)23 b(Null)16 b(pro)q(cesses)i Fy(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(46)143 727 y(3.13)23 b(Deriv)o(ed)15 b(datat)o(yp)q(es)42 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(46)248 784 y(3.13.1)27 b(Datat)o(yp)q(e)14 b(constructors)27 b Fy(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FE(47)248 840 y(3.13.2)27 b(Address)16 b(and)f(exten)o(t)g (functions)32 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(53)248 897 y(3.13.3)27 b(Lo)o(w)o(er-b)q(ound)16 b(and)f(upp)q(er-b)q(ound)j (mark)o(ers)33 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FE(55)248 954 y(3.13.4)27 b(Commit)15 b(and)g(free)28 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(55)248 1011 y(3.13.5)27 b(Use)16 b(of)e(general)i(datat)o(yp)q(es) f(in)h(comm)o(unication)36 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)69 b FE(57)248 1068 y(3.13.6)27 b(Examples)33 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(59)248 1124 y(3.13.7)27 b(Correct)15 b(use)g(of)g(addresses)26 b Fy(:)d(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(67)143 1181 y(3.14)23 b(Univ)o(ersal)16 b(comm)o(unication)g(functions)24 b Fy(:)f(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b FE(68)248 1238 y(3.14.1)27 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(ob)s(jects)e Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FE(69)248 1295 y(3.14.2)27 b(Non)o(blo)q(c)o(king) 17 b(comm)o(unication)e(initiation)31 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(69)248 1352 y(3.14.3)27 b(Comm)o(unication)16 b(completion)31 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(69)248 1408 y(3.14.4)27 b(Blo)q(c)o(king)17 b(comm)o(unication)g Fy(:)23 b(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b FE(70)248 1465 y(3.14.5)27 b(Prob)q(e)16 b(and)f(cancel)23 b Fy(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(70)248 1522 y(3.14.6)27 b(Return)16 b(status)j Fy(:)k(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(70)248 1579 y(3.14.7)27 b(send-receiv)o(e)17 b(and)e(exc)o(hange)22 b Fy(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(70)248 1636 y(3.14.8)27 b(Deriv)o(ed)16 b(datat)o(yp)q(es)37 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FE(70)75 1738 y Fz(4)42 b(Collectiv)o(e)19 b(Comm)o(unication)1077 b(71)143 1795 y FE(4.1)46 b(In)o(tro)q(duction) 15 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b FE(71)143 1852 y(4.2)46 b(Comm)o(unication)15 b(F)l(unctions)24 b Fy(:)f(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(72)143 1908 y(4.3)46 b(Barrier)15 b(sync)o(hronization)44 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(72)143 1965 y(4.4)46 b(Data)14 b(mo)o(v)o(e)g(functions)29 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(72)248 2022 y(4.4.1)50 b(Broadcast)24 b Fy(:)e(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(74)248 2079 y(4.4.2)50 b(Gather)13 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(74)248 2136 y(4.4.3)50 b(Examples)16 b(of)f(Usage)f(of)h Fx(MPI)p 921 2136 14 2 v 16 w(GA)l(THER)p FE(,)h Fx(MPI)p 1223 2136 V 16 w(GA)l(THERV)24 b Fy(:)e(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(76)248 2192 y(4.4.4)50 b(Scatter)c Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(83)248 2249 y(4.4.5)50 b(Examples)16 b(of)f(Usage)f(of)h Fx(MPI)p 921 2249 V 16 w(SCA)l(TTER)p FE(,)h Fx(MPI)p 1246 2249 V 16 w(SCA)l(TTERV)d Fy(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(85)248 2306 y(4.4.6)50 b(Gather-to-all)36 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b FE(88)248 2363 y(4.4.7)50 b(All-to-All)18 b(Scatter/Gather)32 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(90)143 2420 y(4.5)46 b(Global)16 b(Compute)e(Op)q(erations)20 b Fy(:)i(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(92)248 2476 y(4.5.1)50 b(Reduce)45 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(92)248 2533 y(4.5.2)50 b(User-Reduce)c Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b FE(95)248 2590 y(4.5.3)50 b(Reduce-Scatter)30 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(99)248 2647 y(4.5.4)50 b(Scan)25 b Fy(:)d(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(100)143 2704 y(4.6)g(Correctness)32 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(102)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 5 4 4 bop 248 45 a FE(4.6.1)50 b(Sync)o(hronization)17 b(Side-E\013ects)12 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(102)248 102 y(4.6.2)k(Multiple)17 b(Calls)f(to)f(Collectiv)o(e)31 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(103)75 205 y Fz(5)c(Groups,)17 b(Con)o(texts,)f(Comm)o(unicators,)g(and)i(Cac)o(heing)519 b(105)143 262 y FE(5.1)46 b(In)o(tro)q(duction)15 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b FE(105)248 319 y(5.1.1)k(Wh)o(y)15 b(w)o(e)g(need)h(libraries)30 b Fy(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(105)248 376 y(5.1.2)k(Wh)o(y)15 b(existing)h(systems)f(do)g(not)g(supp)q(ort)g (libraries)f Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(105)248 433 y(5.1.3)k(What)15 b(features)g(are)f(needed)j(to)d (supp)q(ort)i(libraries)23 b Fy(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)46 b FE(106)248 490 y(5.1.4)k(What)15 b(these)g(features)g (are)g(called)i(in)f(MPI)27 b Fy(:)c(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(106)143 547 y(5.2)g(Basic)16 b(Concepts)30 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FE(107)248 604 y(5.2.1)k(Groups)43 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b FE(107)248 661 y(5.2.2)k(Con)o(texts)c Fy(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(107)248 718 y(5.2.3)k(Comm)o(unicators)17 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)46 b FE(108)248 775 y(5.2.4)k(Prede\014ned)17 b(Comm)o(unicators)38 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(108)143 832 y(5.3)g(Group)15 b(Managemen)o(t)41 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(108)248 890 y(5.3.1)k(Group)15 b(Accessors)34 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(108)248 947 y(5.3.2)k(Group)15 b(Constructors)37 b Fy(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(109)248 1004 y(5.3.3)k(Group)15 b(Destructors)27 b Fy(:)c(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FE(113)143 1061 y(5.4)g(Comm)o(unicator)14 b(Managemen)o(t)i Fy(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(113)248 1118 y(5.4.1)k(Comm)o(unicator)15 b(Accessors)43 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(113)248 1175 y(5.4.2)k(Comm)o (unicator)15 b(Constructors)10 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(114)248 1232 y(5.4.3)k(Comm)o(unicator)15 b(Destructors)36 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(116)143 1289 y(5.5)g(In)o(ter-Comm)o (unication)14 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FE(116)248 1346 y(5.5.1)k(Comm)o(unicator)15 b(Accessors)43 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(118)248 1403 y(5.5.2)k(In)o(tercomm)o(unicator)15 b(Constructors)f(and)h (Destructors)j Fy(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(118)248 1460 y(5.5.3)k(Name)15 b(Service)25 b Fy(:)e(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(121)143 1517 y(5.6)g(Cac)o(heing)12 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(121)248 1574 y(5.6.1)k(F)l(unctionalit)o(y)32 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(122)143 1631 y(5.7)g(F)l (ormalizing)16 b(the)f(Lo)q(osely)h(Sync)o(hronous)g(Mo)q(del)g (\(Usage,)e(Safet)o(y\))j Fy(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(126)248 1688 y(5.7.1)k(Basic)16 b(Statemen)o(ts)22 b Fy(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(126)248 1745 y(5.7.2)k(Mo)q(dels)16 b(of)f(Execution)29 b Fy(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(126)143 1802 y(5.8)g(Motiv)m(ating)15 b(Examples)h Fy(:)23 b(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(128)248 1859 y(5.8.1)k(Curren)o (t)15 b(Practice)g(#1)27 b Fy(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(128)248 1916 y(5.8.2)k(Curren)o(t)15 b(Practice)g(#2)27 b Fy(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(129)248 1973 y(5.8.3)k(\(Appro)o(ximate\))15 b(Curren)o(t)f(Practice)i(#3)35 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b FE(129)248 2030 y(5.8.4)k(Example)16 b(#4)44 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(130)248 2087 y(5.8.5)k(Library)16 b(Example)g(#1)22 b Fy(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(131)248 2144 y(5.8.6)k(Library)16 b(Example)g(#2)22 b Fy(:)g(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FE(132)248 2201 y(5.8.7)k(In)o(ter-Comm)o (unication)16 b(Examples)g Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(134)75 2304 y Fz(6)c(Pro)q(cess)17 b(T)l(op)q(ologies)1224 b(140)143 2361 y FE(6.1)46 b(In)o(tro)q(duction)15 b Fy(:)22 b(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(140)143 2418 y(6.2)g(Virtual)16 b(T)l(op)q(ologies)37 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(140)143 2475 y(6.3)g(Em)o(b)q(edding)16 b(in)g(MPI)22 b Fy(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(141)143 2532 y(6.4)g(Ov)o(erview)16 b(of)e(the)i(prop)q(osed)f (MPI)g(functions)20 b Fy(:)j(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(141)143 2590 y(6.5)g(T)l(op)q(ology)15 b(Constructors)20 b Fy(:)i(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(142)248 2647 y(6.5.1)k(T)l(op)q (ology)15 b(inquiry)i(functions)42 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(145)248 2704 y(6.5.2)k(Cartesian)15 b(shift)h(co)q(ordinates)43 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(147)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 1 6 1 5 bop 248 45 a FE(6.5.3)50 b(P)o(artitioning)16 b(of)e(Cartesian)h (structures)45 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b FE(148)248 102 y(6.5.4)k(Lo)o(w{lev)o(el)16 b(top)q(ology)f(functions)37 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(149)143 158 y(6.6)g(A)15 b(simple)i(example)f Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(150)75 260 y Fz(7)c(MPI)17 b(En)o(vironmen)o(tal)g(Managemen)o(t) 883 b(152)143 316 y FE(7.1)46 b(Implemen)o(tation)16 b(information)22 b Fy(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(152)248 373 y(7.1.1)k(En)o(vironmen)o(tal)16 b(Inquiries)28 b Fy(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(152)248 429 y(7.1.2)k(Bu\013ering)42 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b FE(153)143 486 y(7.2)g(Error)14 b(handling)41 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FE(157)143 542 y(7.3)g(Startup)40 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FE(159)75 644 y Fz(8)c(Pro\014ling)18 b(In)o(terface)1238 b(161)143 701 y FE(8.1)46 b(Requiremen)o(ts)28 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FE(161)143 757 y(8.2)g(Discussion)21 b Fy(:)i(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(161)143 813 y(8.3)g(Logic)16 b(of)e(the)i(design)29 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(162)143 870 y(8.4)g(Examples)36 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(162)248 926 y(8.4.1)k(Pro\014ler)16 b(implemen)o(tation)35 b Fy(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(162)248 983 y(8.4.2)k(MPI)15 b(library)h(implemen)o(tation)h Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b FE(163)143 1039 y(8.5)g(Multiple)17 b(lev)o(els)f(of)f(in)o(terception)25 b Fy(:)d(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FE(164)75 1141 y Fz(9)c(Initial)20 b(Implemen)o(tation)e (Subset)967 b(165)143 1198 y FE(9.1)46 b(In)o(tro)q(duction)15 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FE(165)143 1254 y(9.2)g(Criteria)15 b(and)h(Rationale)28 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(165)143 1311 y(9.3)g(Subset)15 b(F)l(unctionalit)o(y)31 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(166)248 1367 y(9.3.1)k(P)o(oin)o(t)15 b(to)g(P)o(oin)o(t)f(F)l (unctionalit)o(y)i Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(166)248 1423 y(9.3.2)k(Collectiv)o(e)17 b(Comm)o(unication)e(F)l(unctionalit)o (y)24 b Fy(:)e(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)46 b FE(168)248 1480 y(9.3.3)k(Groups,)15 b(Con)o(texts,)e(and)j (Comm)o(unicators)e(F)l(unctionalit)o(y)j Fy(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)46 b FE(168)248 1536 y(9.3.4)k(T)l(op)q(ology)15 b(F)l(unctionalit)o(y)46 b Fy(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(169)248 1593 y(9.3.5)k(Language)15 b(Binding)41 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(170)248 1649 y(9.3.6)k(MPI)15 b(En)o(vironmen)o(tal)h(Managemen)o(t)f Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FE(170)248 1706 y(9.3.7)k(Pro\014ling)16 b(F)l(unctionalit)o(y)24 b Fy(:)e(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(170)143 1762 y(9.4)g(Subset)15 b(T)l(esting)42 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b FE(171)75 1864 y Fz(Bibliograph)o(y)1430 b(172)75 1966 y(A)28 b(Language)19 b(Binding)1242 b(174)143 2022 y FE(A.1)35 b(In)o(tro)q(duction)15 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(174)143 2079 y(A.2)35 b(De\014ned)16 b(Constan)o(ts)e(for)g(C)h(and)h(F)l (ortran)21 b Fy(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(174)143 2135 y(A.3)35 b(C)15 b(bindings)i(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o (unication)21 b Fy(:)i(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FE(176)143 2192 y(A.4)35 b(C)15 b(Bindings)i(for)d (Collectiv)o(e)j(Comm)o(unication)34 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(179)143 2248 y(A.5)35 b(C)15 b(Bindings)i(for)d(Groups,)h(Con)o (texts,)f(and)h(Comm)o(unicators)26 b Fy(:)c(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FE(180)143 2305 y(A.6)35 b(C)15 b(Bindings)i(for)d(Pro)q(cess)h(T)l(op)q(ologies)43 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(181)143 2361 y(A.7)35 b(C)15 b(bindings)i(for)d(En)o(vironmen)o(tal)i(Inquiry)42 b Fy(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b FE(181)143 2418 y(A.8)35 b(F)l(ortran)14 b(Bindings)j(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o(unication)39 b Fy(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(182)143 2474 y(A.9)35 b(F)l(ortran)14 b(Bindings)j(for)d (Collectiv)o(e)j(Comm)o(unication)23 b Fy(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(185)143 2531 y(A.10)12 b(F)l(ortran)i(Bindings)j(for)d(Groups,)h(Con)o(texts,)f(and) h(Comm)o(unicators)g Fy(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(186)143 2587 y(A.11)12 b(F)l(ortran)i(Bindings)j(for)d(Pro)q(cess) h(T)l(op)q(ologies)32 b Fy(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(188)143 2643 y(A.12)12 b(F)l(ortran)i(Bindings)j(for)d(En)o(vironmen)o(tal)i (Inquiry)24 b Fy(:)e(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FE(188)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 7 2 6 bop 75 75 a FA(Ac)m(kno)m(wledgmen)m(ts)166 282 y FE(The)19 b(tec)o(hnical)i(dev)o(elopmen)o(t)f(w)o(as)e(carried)i(out)f (b)o(y)g(subgroups,)h(whose)f(w)o(ork)g(w)o(as)f(review)o(ed)75 338 y(b)o(y)d(the)h(full)h(committee.)k(During)15 b(the)h(p)q(erio)q(d) h(of)e(dev)o(elopmen)o(t)h(of)f(the)h(Message)f(P)o(assing)g(In)o (terface)75 395 y(\(MPI\),)f(man)o(y)h(p)q(eople)i(serv)o(ed)e(in)h(p)q (ositions)g(of)f(resp)q(onsibilit)o(y:)143 483 y FC(\017)23 b FE(Jac)o(k)15 b(Dongarra,)e(Da)o(vid)i(W)l(alk)o(er,)g(Con)o(v)o (eners)g(and)g(Meeting)h(Chairs)143 572 y FC(\017)23 b FE(Ewing)15 b(Lusk,)h(Bob)f(Knigh)o(ten,)h(Min)o(utes)143 661 y FC(\017)23 b FE(Marc)14 b(Snir,)i(William)h(Gropp,)d(Ewing)i (Lusk,)f(P)o(oin)o(t)g(to)g(P)o(oin)o(t)f(Comm)o(unications)143 749 y FC(\017)23 b FE(Al)16 b(Geist,)e(Marc)h(Snir,)h(Stev)o(e)f(Otto,) f(Collectiv)o(e)j(Comm)o(unications)143 838 y FC(\017)23 b FE(Rolf)15 b(Hemp)q(el,)i(Pro)q(cess)e(T)l(op)q(ologies)143 927 y FC(\017)23 b FE(Ewing)15 b(Lusk,)h(Language)f(Binding)143 1015 y FC(\017)23 b FE(William)17 b(Gropp,)d(En)o(vironmen)o(tal)i (Managemen)o(t)e(and)h(Inquiry)143 1104 y FC(\017)23 b FE(James)15 b(Co)o(wnie,)g(Pro\014ling)143 1193 y FC(\017)23 b FE(T)l(on)o(y)17 b(Skjellum,)i(Lyndon)f(Clark)o(e,)g(Marc)e(Snir,)j (Ric)o(hard)f(Little\014eld,)j(Marc)16 b(Sears,)i(Groups,)189 1249 y(Con)o(texts,)13 b(and)j(Comm)o(unicators)143 1338 y FC(\017)23 b FE(Stev)o(en)15 b(Huss-Lederman,)h(Initial)h(Implemen)o (tation)f(Subset)143 1427 y FC(\017)23 b FE(Stev)o(e)15 b(Otto,)f(Editor)166 1515 y(The)22 b(follo)o(wing)h(list)f(includes)i (some)e(of)f(the)h(activ)o(e)g(participan)o(ts)h(in)f(the)g(MPI)g(pro)q (cess)g(not)75 1572 y(men)o(tioned)16 b(ab)q(o)o(v)o(e:)100 1705 y(Ed)f(Anderson)134 b(Jo)q(e)16 b(Baron)171 b(Eric)15 b(Barszcz)162 b(Scott)15 b(Berryman)100 1762 y(Rob)g(Bjornson)119 b(Anne)16 b(Elster)138 b(Jim)16 b(F)l(eeney)185 b(Vince)17 b(F)l(ernando)100 1818 y(Sam)e(Fineb)q(erg)118 b(Jon)16 b(Flo)o(w)o(er)154 b(Daniel)16 b(F)l(ry)o(e)179 b(Ian)16 b(Glendinning)100 1875 y(Adam)f(Green)o(b)q(erg)53 b(Rob)q(ert)16 b(Harrison)50 b(Leslie)17 b(Hart)189 b(T)l(om)15 b(Haupt)100 1931 y(Don)g(Heller)179 b(T)l(om)15 b(Henderson)62 b(Alex)16 b(Ho)246 b(C.T.)14 b(Ho)o(w)o(ard)g(Ho)100 1988 y(John)i(Kap)q(enga)106 b(Bob)16 b(Leary)166 b(Arth)o(ur)14 b(Maccab)q(e)84 b(P)o(eter)15 b(Madams)100 2044 y(Alan)h(Main)o(w)o(aring)49 b(Oliv)o(er)17 b(McBry)o(an)55 b(Phil)16 b(McKinley)123 b(Charles)16 b(Mosher)100 2100 y(Dan)f(Nessett)149 b(P)o(eter)15 b(P)o(ac)o(heco)94 b(Ho)o(w)o(ard)14 b(P)o(almer)100 b(P)o(aul)15 b(Pierce)100 2157 y(Sanja)o(y)g(Rank)m(a)120 b(P)o(eter)15 b(Rigsb)q(ee)102 b(Arc)o(h)15 b(Robison)139 b(Eric)o(h)16 b(Sc)o(hikuta)100 2213 y(Am)o(buj)f(Singh)133 b(Alan)16 b(Sussman)93 b(Rob)q(ert)15 b(T)l(omlinson)51 b(Rob)q(ert)16 b(G.)e(V)l(oigt)100 2270 y(Dennis)i(W)l(eeks)117 b(Stephen)17 b(Wheat)67 b(Stev)o(e)15 b(Zenith)166 2412 y(Oak)21 b(Ridge)h(National)g(Lab)q (oratory)e(made)h(the)g(draft)f(a)o(v)m(ailable)j(b)o(y)e(anon)o(ymous) g(FTP)f(mail)75 2469 y(serv)o(ers)15 b(and)g(w)o(as)g(instrumen)o(tal)g (in)h(distributing)h(the)e(do)q(cumen)o(t.)166 2525 y(MPI)22 b(op)q(erated)h(on)f(a)g(v)o(ery)g(tigh)o(t)g(budget)h(\(in)g(realit)o (y)l(,)h(it)f(had)f(no)g(budget)h(when)g(the)f(\014rst)75 2582 y(meeting)14 b(w)o(as)e(announced\).)20 b(D)o(ARP)l(A)13 b(and)g(NSF)g(ha)o(v)o(e)g(supp)q(orted)g(researc)o(h)g(at)f(v)m (arious)i(institutions)75 2638 y(that)i(ha)o(v)o(e)g(made)h(a)f(con)o (tribution)i(to)o(w)o(ards)d(tra)o(v)o(el)h(for)g(the)h(U.S.)f (academics.)25 b(Supp)q(ort)17 b(for)f(sev)o(eral)75 2695 y(Europ)q(ean)g(participan)o(ts)f(w)o(as)g(pro)o(vided)h(b)o(y)f (ESPRIT.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 8 3 7 bop 75 358 a Fw(Chapter)34 b(1)75 568 y FA(In)m(tro)s(duction)41 b(to)g(MPI)75 810 y Fv(1.1)59 b(Overview)19 b(and)g(Goals)75 915 y FE(Message)11 b(passing)i(is)f(a)g(paradigm)f(used)i(widely)g(on) f(certain)h(classes)f(of)f(parallel)j(mac)o(hines;)f(esp)q(ecially)75 972 y(those)j(with)i(distributed)g(memory)l(.)24 b(Although)17 b(there)g(are)f(man)o(y)h(v)m(ariations,)g(the)g(basic)g(concept)g(of) 75 1028 y(pro)q(cesses)12 b(comm)o(unicating)h(through)e(messages)g(is) i(w)o(ell)f(understo)q(o)q(d.)20 b(Ov)o(er)12 b(the)f(last)h(ten)g(y)o (ears,)f(sub-)75 1084 y(stan)o(tial)16 b(progress)g(has)g(b)q(een)h (made)f(in)h(casting)g(signi\014can)o(t)g(applications)h(in)f(this)f (paradigm.)23 b(Eac)o(h)75 1141 y(v)o(endor)c(has)h(implemen)o(ted)h (their)f(o)o(wn)f(v)m(arian)o(t.)33 b(More)19 b(recen)o(tly)l(,)j(sev)o (eral)d(systems)g(ha)o(v)o(e)h(demon-)75 1197 y(strated)12 b(that)h(a)g(message)f(passing)i(system)f(can)g(b)q(e)h(e\016cien)o (tly)h(and)e(b)q(e)h(p)q(ortably)f(implemen)o(ted.)21 b(It)13 b(is)75 1254 y(th)o(us)h(an)f(appropriate)h(time)g(to)f(try)g (to)g(de\014ne)i(b)q(oth)f(the)g(syn)o(tax)f(and)h(seman)o(tics)g(of)f (a)g(core)h(of)f(library)75 1310 y(routines)h(that)e(will)j(b)q(e)f (useful)h(to)e(a)g(wide)h(range)f(of)g(users)g(and)h(e\016cien)o(tly)g (implemen)o(table)i(on)d(a)g(wide)75 1367 y(range)i(of)g(computers.)166 1425 y(In)21 b(designing)i(MPI)e(w)o(e)f(ha)o(v)o(e)h(sough)o(t)f(to)g (mak)o(e)g(use)h(of)g(the)g(most)f(attractiv)o(e)f(features)i(of)f(a)75 1481 y(n)o(um)o(b)q(er)12 b(of)g(existing)h(message)e(passing)h (systems,)g(rather)f(than)h(selecting)h(one)f(of)g(them)g(and)g (adopting)75 1538 y(it)f(as)f(the)h(standard.)18 b(Th)o(us,)11 b(MPI)g(has)f(b)q(een)i(strongly)f(in\015uenced)i(b)o(y)d(w)o(ork)g(at) g(the)h(IBM)g(T.)f(J.)h(W)l(atson)75 1594 y(Researc)o(h)18 b(Cen)o(ter)g([1)o(,)g(2)o(],)g(In)o(tel's)g(NX/2)g([18)o(],)f(Express) h([17)o(],)g(nCUBE's)g(V)l(ertex)g([15)o(],)f(P4)h([5)o(],)g(and)75 1651 y(P)l(ARMA)o(CS)d([4,)f(6].)19 b(Other)c(imp)q(ortan)o(t)g(con)o (tributions)h(ha)o(v)o(e)e(come)h(from)f(Zip)q(co)q(de)j([19)o(,)d (20],)g(Chimp)75 1707 y([11)o(,)h(12)o(],)g(PVM)g([3)o(,)g(9)o(],)g (and)g(PICL)h([14)o(].)166 1765 y(The)d(MPI)g(standardization)g (e\013ort)f(in)o(v)o(olv)o(ed)i(ab)q(out)f(60)f(p)q(eople)i(from)f(40)f (organizations)h(mainly)75 1822 y(from)k(the)g(United)i(States)e(and)h (Europ)q(e.)27 b(Most)16 b(of)h(the)h(ma)s(jor)e(v)o(endors)h(of)g (concurren)o(t)h(computers)75 1878 y(w)o(ere)10 b(in)o(v)o(olv)o(ed)i (in)f(MPI,)g(along)f(with)i(researc)o(hers)e(from)g(univ)o(ersities,)j (go)o(v)o(ernmen)o(t)d(lab)q(oratories,)h(and)75 1935 y(industry)l(.)21 b(The)14 b(standardization)h(pro)q(cess)f(b)q(egan)h (with)f(the)h(W)l(orkshop)f(on)g(Standards)g(for)g(Message)75 1991 y(P)o(assing)j(in)i(a)e(Distributed)h(Memory)f(En)o(vironmen)o(t,) h(sp)q(onsored)g(b)o(y)f(the)h(Cen)o(ter)f(for)g(Researc)o(h)h(on)75 2048 y(P)o(arallel)11 b(Computing,)g(held)h(April)f(29-30,)f(1992,)g (in)h(Williamsburg,)h(Virginia)g([21)o(].)18 b(A)o(t)9 b(this)i(w)o(orkshop)75 2104 y(the)17 b(basic)h(features)f(essen)o (tial)h(to)f(a)g(standard)g(message)g(passing)g(in)o(terface)h(w)o(ere) f(discussed,)i(and)e(a)75 2161 y(w)o(orking)e(group)g(established)i(to) d(con)o(tin)o(ue)i(the)f(standardization)h(pro)q(cess.)166 2219 y(A)e(preliminary)i(draft)e(prop)q(osal,)g(kno)o(wn)g(as)g(MPI1,)g (w)o(as)f(put)h(forw)o(ard)f(b)o(y)h(Dongarra,)f(Hemp)q(el,)75 2275 y(Hey)l(,)i(and)g(W)l(alk)o(er)f(in)i(No)o(v)o(em)o(b)q(er)e (1992,)f(and)i(a)f(revised)i(v)o(ersion)f(w)o(as)f(completed)h(in)h(F)l (ebruary)e(1993)75 2332 y([10)o(].)20 b(MPI1)15 b(em)o(b)q(o)q(dies)i (the)f(main)g(features)f(that)g(w)o(ere)g(iden)o(ti\014ed)i(at)e(the)h (Williamsburg)h(w)o(orkshop)75 2388 y(as)g(b)q(eing)h(necessary)g(in)g (a)f(message)f(passing)i(standard.)26 b(This)17 b(prop)q(osal)h(w)o(as) e(in)o(tended)j(to)e(initiate)75 2445 y(discussion)i(of)e (standardization)h(issues)h(within)f(the)g(distributed)h(memory)e (concurren)o(t)g(computing)75 2501 y(comm)o(unit)o(y)l(,)12 b(and)f(has)h(serv)o(ed)f(as)g(a)g(basis)h(for)e(the)i(subsequen)o(t)g (MPI)f(standardization)h(pro)q(cess.)18 b(Since)75 2558 y(MPI1)g(w)o(as)f(primarily)i(in)o(tended)h(to)d(promote)h(discussion)i (and)e(\\get)f(the)h(ball)i(rolling,")f(it)g(fo)q(cuses)75 2614 y(mainly)g(on)f(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o(unications.)29 b(MPI1)17 b(do)q(es)i(not)e(include)k(an)o(y)c(collectiv)o(e)j(comm)o (u-)p 75 2661 720 2 v 127 2688 a FB(1)144 2704 y Fu(V)m(ersion)14 b(of)f(Octob)q(er)g(8,)g(1993)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 9 4 8 bop 75 -100 a FE(4)903 b Ft(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION)17 b(TO)e(MPI)75 45 y FE(nication)k(routines.)29 b(MPI1)18 b(brough)o(t)f(to)h(the)g(forefron)o(t)f(a)h(n)o(um)o(b)q(er)g(of)g (imp)q(ortan)o(t)g(standardization)75 102 y(issues,)f(and)g(has)f(serv) o(ed)h(as)f(a)g(catalyst)g(for)g(subsequen)o(t)i(progress,)d(ho)o(w)o (ev)o(er,)h(its)h(ma)s(jor)e(de\014ciency)75 158 y(is)h(that)e(the)h (managemen)o(t)g(of)g(resources)g(is)h(not)e(thread-safe.)166 218 y(In)k(No)o(v)o(em)o(b)q(er)e(1992,)g(a)h(meeting)h(of)e(the)h(MPI) g(w)o(orking)g(group)g(w)o(as)f(held)i(in)g(Minneap)q(olis,)i(at)75 275 y(whic)o(h)15 b(it)g(w)o(as)e(decided)k(to)c(place)j(the)e (standardization)h(pro)q(cess)g(on)f(a)g(more)g(formal)g(fo)q(oting,)g (and)h(to)75 331 y(generally)j(adopt)f(the)h(pro)q(cedures)g(and)f (organization)h(of)f(the)g(High)h(P)o(erformance)f(F)l(ortran)f(forum.) 75 388 y(Sub)q(committees)h(w)o(ere)e(formed)g(for)g(the)h(ma)s(jor)e (comp)q(onen)o(t)i(areas)f(of)g(the)h(standard,)f(and)g(an)h(email)75 444 y(discussion)23 b(service)f(established)g(for)f(eac)o(h.)37 b(In)22 b(addition,)h(the)e(goal)g(of)g(pro)q(ducing)h(a)f(draft)f(MPI) 75 501 y(standard)f(b)o(y)h(the)f(F)l(all)h(of)g(1993)e(w)o(as)h(set.) 32 b(T)l(o)19 b(ac)o(hiev)o(e)i(this)f(goal)f(the)g(MPI)h(w)o(orking)f (group)g(met)75 557 y(ev)o(ery)d(6)f(w)o(eeks)g(for)g(t)o(w)o(o)g(da)o (ys)g(throughout)g(the)h(\014rst)f(9)g(mon)o(ths)g(of)h(1993,)e(and)i (presen)o(ted)g(the)g(draft)75 614 y(MPI)c(standard)g(at)f(the)h(Sup)q (ercomputing)i(93)e(conference)g(in)h(No)o(v)o(em)o(b)q(er)f(1993.)18 b(These)12 b(meetings)h(and)75 670 y(the)18 b(email)h(discussion)g (together)e(constituted)i(the)f(MPI)f(forum,)h(mem)o(b)q(ership)h(of)e (whic)o(h)i(has)f(b)q(een)75 726 y(op)q(en)d(to)e(all)i(mem)o(b)q(ers)g (of)f(the)g(high)h(p)q(erformance)f(computing)h(comm)o(unit)o(y)l(.)20 b(The)14 b(main)h(adv)m(an)o(tages)75 783 y(of)f(establishing)i(a)e (message)g(passing)h(standard)f(are)g(p)q(ortabilit)o(y)i(and)e (ease-of-use.)20 b(In)15 b(a)f(distributed)75 839 y(memory)e(comm)o (unication)i(en)o(vironmen)o(t)f(in)g(whic)o(h)h(the)f(higher)g(lev)o (el)h(routines)f(and/or)f(abstractions)75 896 y(are)21 b(build)j(up)q(on)f(lo)o(w)o(er)e(lev)o(el)i(message)f(passing)g (routines)g(the)g(b)q(ene\014ts)h(of)e(standardization)h(are)75 952 y(particularly)c(apparen)o(t.)24 b(F)l(urthermore,)16 b(the)g(de\014nition)j(of)d(a)g(message)g(passing)h(standard,)f(suc)o (h)h(as)75 1009 y(that)f(prop)q(osed)h(here,)g(pro)o(vides)h(v)o (endors)e(with)h(a)g(clearly)h(de\014ned)g(base)f(set)f(of)h(routines)g (that)f(they)75 1065 y(can)d(implemen)o(t)g(e\016cien)o(tly)l(,)i(or)d (in)h(some)f(cases)h(pro)o(vide)g(hardw)o(are)e(supp)q(ort)i(for,)f (thereb)o(y)h(enhancing)75 1122 y(scalabilit)o(y)l(.)166 1182 y(The)18 b(goal)g(of)g(the)g(Message)g(P)o(assing)g(In)o(terface)g (simply)i(stated)d(is)i(to)f(dev)o(elop)h(a)f(widely)h(used)75 1238 y(standard)e(for)f(writing)i(message-passing)f(programs.)24 b(As)17 b(suc)o(h)h(the)f(in)o(terface)g(should)h(establishing)75 1295 y(a)d(practical,)h(p)q(ortable,)f(e\016cien)o(t,)g(and)h (\015exible)h(standard)e(for)f(message)h(passing.)166 1355 y(A)g(complete)h(list)g(of)f(goals)g(follo)o(w.)143 1481 y FC(\017)23 b FE(Design)11 b(an)f(application)i(programming)e(in) o(terface)h(\(not)f(necessarily)i(for)d(compilers)j(or)e(a)g(system)189 1537 y(implemen)o(tation)16 b(library\).)143 1647 y FC(\017)23 b FE(Allo)o(w)15 b(e\016cien)o(t)h(comm)o(unication:)21 b(Av)o(oid)16 b(memory)e(to)h(memory)g(cop)o(ying)g(and)h(allo)o(w)f(o) o(v)o(erlap)189 1703 y(of)c(computation)h(and)g(comm)o(unication)h(and) f(o\017oad)f(to)g(comm)o(unication)i(co-pro)q(cessor,)f(where)189 1760 y(a)o(v)m(ailable.)143 1869 y FC(\017)23 b FE(Allo)o(w)15 b(for)g(implemen)o(tation)h(that)f(can)g(b)q(e)h(used)g(in)g(a)f (heterogeneous)g(en)o(vironmen)o(t.)143 1978 y FC(\017)23 b FE(Allo)o(w)15 b(con)o(v)o(enien)o(t)h(C)f(and)h(F)l(ortran)e(77)g (bindings)j(for)e(in)o(terface.)143 2088 y FC(\017)23 b FE(Assume)15 b(a)g(reliable)i(comm)o(unication)f(in)o(terface:)k (User)15 b(need)h(not)f(cop)q(e)h(with)f(comm)o(unication)189 2144 y(failures.)21 b(Suc)o(h)15 b(failures)i(are)d(dealt)i(b)o(y)f (the)h(underlying)h(comm)o(unication)f(subsystem.)143 2254 y FC(\017)23 b FE(De\014ne)f(an)g(in)o(terface)g(that)f(is)h(not)f (to)q(o)g(di\013eren)o(t)h(from)f(curren)o(t)h(practice,)h(suc)o(h)f (as)g(PVM,)189 2310 y(Express,)14 b(P4,)h(etc)g(and)h(pro)o(vide)f (extension)h(that)f(allo)o(w)g(greater)f(\015exibili)q(t)o(y)l(.)143 2420 y FC(\017)23 b FE(De\014ne)d(an)f(in)o(terface)h(that)e(can)i(b)q (e)g(implemen)o(ted)h(on)e(man)o(y)g(v)o(endor's)g(platforms,)g(with)h (no)189 2476 y(signi\014can)o(t)c(c)o(hanges)f(in)h(the)f(underlying)i (comm)o(unication)f(and)g(system)e(soft)o(w)o(are.)143 2585 y FC(\017)23 b FE(Seman)o(tics)15 b(of)g(the)g(in)o(terface)h (should)g(b)q(e)g(language)f(indep)q(enden)o(t.)143 2695 y FC(\017)23 b FE(The)15 b(in)o(terface)h(should)g(b)q(e)g(designed)g (to)f(allo)o(w)g(for)g(thread-safet)o(y)l(.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 10 5 9 bop 75 -100 a Ft(1.2.)34 b(WHO)15 b(SHOULD)h(USE)g(THIS)g(ST)l(AND) o(ARD?)824 b FE(5)75 45 y Fv(1.2)59 b(Who)20 b(Should)g(Use)g(This)f (Standa)n(rd?)75 153 y FE(This)d(standard)f(is)h(in)o(tended)h(for)e (use)g(b)o(y)h(all)g(those)f(who)g(w)o(an)o(t)g(to)f(write)i(p)q (ortable)g(message-passing)75 210 y(programs)9 b(in)j(F)l(ortran)d(77)i (and)f(C.)h(This)g(includes)i(individual)h(application)e(programmers,)e (dev)o(elop)q(ers)75 266 y(of)16 b(soft)o(w)o(are)e(designed)k(to)d (run)h(on)g(parallel)i(mac)o(hines,)f(and)f(creators)f(of)h(en)o (vironmen)o(ts,)g(and)g(to)q(ols.)75 322 y(In)h(order)f(to)f(b)q(e)i (attractiv)o(e)e(to)h(this)g(wide)i(audience,)f(the)g(standard)e(m)o (ust)h(pro)o(vide)h(a)e(simple,)j(easy-)75 379 y(to-use)f(in)o(terface) g(for)f(the)g(basic)i(user)f(while)h(not)e(seman)o(tically)i (precluding)h(the)e(high-p)q(erformance)75 435 y(message-passing)e(op)q (erations)h(a)o(v)m(ailable)h(on)e(adv)m(anced)h(mac)o(hines.)75 598 y Fv(1.3)59 b(What)20 b(Platfo)n(rms)g(Are)g(T)-5 b(a)n(rgets)19 b(F)n(o)n(r)i(Implementat)o(ion?)75 706 y FE(The)c(attractiv)o(eness)f(of)g(the)h(message-passing)g(paradigm)f (at)g(least)h(partially)h(stems)e(from)g(its)h(wide)75 762 y(p)q(ortabilit)o(y)l(.)k(Programs)13 b(expressed)j(this)f(w)o(a)o (y)f(ma)o(y)g(run)h(on)g(distributed-memory)h(m)o(ultipro)q(cessors,)75 819 y(net)o(w)o(orks)i(of)i(w)o(orkstations,)e(and)i(com)o(binations)g (of)g(all)g(of)f(these.)34 b(In)20 b(addition,)h(shared-memory)75 875 y(implemen)o(tations)g(are)e(p)q(ossible.)34 b(The)20 b(paradigm)g(will)h(not)e(b)q(e)h(made)f(obsolete)h(b)o(y)g(arc)o (hitectures)75 931 y(com)o(bining)15 b(the)e(shared-)h(and)g (distributed-memory)h(views,)f(or)f(b)o(y)g(increases)i(in)f(net)o(w)o (ork)f(sp)q(eeds.)20 b(It)75 988 y(th)o(us)d(should)i(b)q(e)f(b)q(oth)g (p)q(ossible)h(and)f(useful)g(to)f(implemen)o(t)i(this)f(standard)f(on) h(a)f(great)g(v)m(ariet)o(y)g(of)75 1044 y(mac)o(hines,)i(including)h (those)e(\\mac)o(hines")g(consisting)g(of)g(collections)h(of)e(other)h (mac)o(hines,)g(parallel)75 1101 y(or)d(not,)f(connected)i(b)o(y)f(a)g (comm)o(unication)h(net)o(w)o(ork.)166 1161 y(The)21 b(in)o(terface)f(is)h(suitable)h(for)e(use)h(b)o(y)f(fully)i(general)f (MIMD)f(programs,)g(as)g(w)o(ell)h(as)f(those)75 1217 y(written)d(in)g(the)g(more)f(restricted)h(st)o(yle)g(of)f(SPMD.)g (Although)h(no)f(explicit)j(supp)q(ort)e(for)f(threads)g(is)75 1273 y(pro)o(vided,)e(the)f(in)o(terface)h(has)f(b)q(een)h(designed)g (so)f(as)g(not)f(to)h(prejudice)i(their)e(use.)20 b(With)13 b(this)h(v)o(ersion)75 1330 y(of)h(MPI)g(no)g(supp)q(ort)g(is)h(pro)o (vided)g(for)f(dynamic)h(spa)o(wning)f(of)g(tasks.)75 1492 y Fv(1.4)59 b(What)20 b(Is)f(Included)g(In)g(The)g(Standa)n(rd?)75 1600 y FE(The)c(standard)g(includes:)143 1723 y FC(\017)23 b FE(P)o(oin)o(t-to-p)q(oin)o(t)15 b(comm)o(unication)143 1830 y FC(\017)23 b FE(Collectiv)o(e)16 b(op)q(erations)143 1938 y FC(\017)23 b FE(Pro)q(cess)15 b(groups)143 2045 y FC(\017)23 b FE(Comm)o(unication)15 b(con)o(texts)143 2152 y FC(\017)23 b FE(Pro)q(cess)15 b(top)q(ology)143 2259 y FC(\017)23 b FE(Bindings)17 b(for)d(F)l(ortran)g(77)h(and)g(C) 143 2366 y FC(\017)23 b FE(En)o(vironmen)o(tal)15 b(Managemen)o(t)f (and)i(inquiry)143 2473 y FC(\017)23 b FE(Pro\014ling)16 b(in)o(terface)143 2581 y FC(\017)23 b FE(Subset)15 b(sp)q (eci\014cation)166 2704 y(In)h(addition,)g(w)o(e)f(ha)o(v)o(e)f(pro)o (vided)i(a)f(mo)q(del)h(implemen)o(tation.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 11 6 10 bop 75 -100 a FE(6)903 b Ft(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION) 17 b(TO)e(MPI)75 45 y Fv(1.5)59 b(What)20 b(Is)f(Not)h(Included)e(In)i (The)f(Standa)n(rd?)75 150 y FE(The)c(standard)g(do)q(es)h(not)f(sp)q (ecify:)143 264 y FC(\017)23 b FE(Explicit)17 b(shared-memory)e(op)q (erations)143 365 y FC(\017)23 b FE(Op)q(erations)c(that)e(require)i (more)f(op)q(erating)h(system)f(supp)q(ort)g(than)g(is)h(curren)o(tly)g (standard;)189 421 y(for)14 b(example,)i(in)o(terrupt-driv)o(en)g (receiv)o(es,)g(remote)f(execution,)h(or)e(activ)o(e)i(messages)143 522 y FC(\017)23 b FE(Program)13 b(construction)j(to)q(ols)143 622 y FC(\017)23 b FE(Debugging)15 b(facilities)143 723 y FC(\017)23 b FE(Auxiliary)17 b(functions)e(suc)o(h)h(as)f(timers)143 823 y FC(\017)23 b FE(Explicit)17 b(supp)q(ort)e(for)g(threads)143 924 y FC(\017)23 b FE(No)15 b(supp)q(ort)g(for)f(task)h(managemen)o(t) 143 1024 y FC(\017)23 b FE(I/O)15 b(functions)166 1139 y(There)g(are)f(man)o(y)h(features)f(that)g(ha)o(v)o(e)g(b)q(een)i (considered)g(and)f(not)g(included)i(in)f(this)f(standard.)75 1196 y(This)j(happ)q(ened)h(for)e(a)g(n)o(um)o(b)q(er)h(of)f(reasons,)g (one)g(of)g(whic)o(h)i(is)f(the)f(time)h(constrain)o(t)f(that)g(w)o(as) f(self)75 1252 y(imp)q(osed)g(in)h(\014nishing)g(the)e(standard.)20 b(F)l(eatures)15 b(that)f(are)h(not)g(included)j(can)e(alw)o(a)o(ys)e (b)q(e)i(o\013ered)f(as)75 1308 y(extensions)f(b)o(y)g(sp)q(eci\014c)h (implemen)o(tations.)21 b(P)o(erhaps)13 b(future)h(v)o(ersion)g(of)f (MPI)g(will)j(address)d(some)g(of)75 1365 y(these)i(issues.)75 1518 y Fv(1.6)59 b(Organization)20 b(of)g(this)f(Do)r(cument)75 1622 y FE(The)f(follo)o(wing)h(is)f(a)g(list)g(of)g(the)g(c)o(hapters)f (in)i(this)f(do)q(cumen)o(t,)h(along)f(with)g(a)f(brief)i(description)g (of)75 1679 y(eac)o(h.)143 1793 y FC(\017)k FE(Chapter)c(2,)i Fx(MPI)e(T)l(erms)h(and)g(Conventions)p FE(,)j(explains)e(notational)f (terms)g(and)g(con)o(v)o(en)o(tions)189 1850 y(used)15 b(throughout)g(the)g(MPI)g(do)q(cumen)o(t.)143 1950 y FC(\017)23 b FE(Chapter)16 b(3,)g Fx(P)o(oint)h(to)f(P)o(oint)h (Communication)p FE(,)f(de\014nes)h(the)g(basic,)g(pairwise)g(comm)o (unication)189 2007 y(subset)h(of)f(MPI.)g Fs(send)g FE(and)h Fs(r)n(e)n(cv)f FE(are)h(found)g(here,)g(along)g(with)g(man)o (y)f(asso)q(ciated)h(functions)189 2063 y(designed)e(to)f(mak)o(e)f (basic)i(comm)o(unication)g(p)q(o)o(w)o(erful)g(and)f(e\016cien)o(t.) 143 2164 y FC(\017)23 b FE(Chapter)12 b(4,)h Fx(Collective)g (Communications)p FE(,)f(de\014nes)i(pro)q(cess-group)f(collectiv)o(e)h (comm)o(unication)189 2220 y(op)q(erations.)28 b(W)l(ell)19 b(kno)o(wn)f(examples)h(of)e(this)i(are)e(barrier)i(and)f(broadcast)f (o)o(v)o(er)g(a)h(group)f(of)189 2277 y(pro)q(cesses)e(\(not)g (necessarily)h(all)g(the)g(pro)q(cesses\).)143 2377 y FC(\017)23 b FE(Chapter)16 b(5,)g Fx(Groups,)h(Contexts,)i(and)e (Communicato)o(rs)p FE(,)e(sho)o(ws)h(ho)o(w)g(groups)g(of)g(pro)q (cesses)h(are)189 2434 y(formed)c(and)h(manipulated,)i(ho)o(w)d(unique) i(comm)o(unication)g(con)o(texts)e(are)h(obtained,)g(and)g(ho)o(w)189 2490 y(the)h(t)o(w)o(o)f(are)h(b)q(ound)h(together)e(in)o(to)h(a)g Fs(c)n(ommunic)n(ator)p FE(.)143 2591 y FC(\017)23 b FE(Chapter)18 b(6,)h Fx(Pro)q(cess)h(T)l(op)q(ologies)p FE(,)f(explains)h(a)f(set)f(of)h(utilit)o(y)h(functions)f(mean)o(t)f (to)h(assist)f(in)189 2647 y(the)13 b(mapping)i(of)e(pro)q(cess)g (groups)h(\(a)f(linearly)i(ordered)f(set\))f(to)g(ric)o(her)h(top)q (ological)g(structures)189 2704 y(suc)o(h)h(as)g(m)o(ulti-dimensional)j (grids.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 12 7 11 bop 75 -100 a Ft(1.6.)34 b(OR)o(GANIZA)l(TION)17 b(OF)e(THIS)h(DOCUMENT)810 b FE(7)143 45 y FC(\017)23 b FE(Chapter)10 b(7,)h Fx(MPI)g(Environmental)g(Management)p FE(,)g(explains)h(ho)o(w)f(the)g(programmer)e(can)i(manage)189 102 y(and)22 b(mak)o(e)f(inquiries)j(of)d(the)h(curren)o(t)f(MPI)h(en)o (vironmen)o(t.)40 b(These)22 b(functions)g(are)g(needed)189 158 y(for)g(the)h(writing)g(of)f(correct,)i(robust)e(programs,)i(and)f (are)f(esp)q(ecially)j(imp)q(ortan)o(t)e(for)f(the)189 214 y(construction)15 b(of)g(highly-p)q(ortable,)i(message-passing)e (programs.)143 308 y FC(\017)23 b FE(Chapter)c(8,)i Fx(Pro\014ling)f (Interface)p FE(,)h(explains)g(a)f(simple)i(name)e(shifting)h(con)o(v)o (en)o(tion)f(that)f(an)o(y)189 365 y(MPI)i(implemen)o(tation)i(m)o(ust) e(supp)q(ort.)38 b(One)22 b(motiv)m(ation)g(for)f(this)h(is)g(the)f (abilit)o(y)i(to)e(put)189 421 y(p)q(erformance)c(pro\014ling)i(calls)f (in)o(to)g(MPI)f(without)h(the)f(need)i(for)d(access)i(to)f(the)g(MPI)h (source)189 478 y(co)q(de.)31 b(The)19 b(name)g(shift)g(is)h(merely)f (an)g(in)o(terface,)h(it)f(sa)o(ys)f(nothing)i(ab)q(out)e(ho)o(w)h(the) g(actual)189 534 y(pro\014ling)h(should)g(b)q(e)f(done)h(and)f(in)h (fact,)f(the)g(name)g(shift)g(can)g(usefully)i(b)q(e)e(used)h(for)e (other)189 591 y(purp)q(oses.)143 684 y FC(\017)23 b FE(Chapter)10 b(9,)h Fx(Initial)f(Implementation)g(Subset)p FE(,)j(suggests)d(to)g(MPI)h(implemen)o(tors)g(a)f(\\core")g(subset)189 741 y(of)16 b(MPI)h(that)g(is)g(useful,)h(in)o(ternally)h(consisten)o (t,)e(and)g(should)h(app)q(ear)g(\014rst)e(in)i(the)f(ev)o(olution)189 797 y(of)d(an)h(MPI)g(implemen)o(tation.)21 b(The)15 b(subset)g(is)h(de\014ned)g(so)e(that)h(consisten)o(t)g(implemen)o (tations)189 854 y(can)d(app)q(ear)g(rapidly)l(,)i(and)e(p)q(ortable)h (parallel)h(programming)d(with)i(MPI)f(can)g(start)f(in)i(a)f(timely) 189 910 y(manner.)143 1004 y FC(\017)23 b FE(Annex)f(A,)i Fx(Language)d(Bindings)p FE(,)k(giv)o(es)d(sp)q(eci\014c)i(syn)o(tax)d (in)i(F)l(ortran)e(and)h(C,)f(for)g(all)i(MPI)189 1060 y(functions,)15 b(constan)o(ts,)f(and)h(t)o(yp)q(es.)143 1154 y FC(\017)23 b FE(The)g Fx(MPI)g(F)o(unction)i(Index)f FE(is)g(a)g(simple)h(index)f(sho)o(wing)g(the)g(lo)q(cation)g(of)f(the) h(language-)189 1211 y(indep)q(enden)o(t)17 b(de\014nition)g(of)e(eac)o (h)g(MPI)g(function.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 8 13 8 12 bop 75 357 a Fw(Chapter)34 b(2)75 565 y FA(MPI)40 b(T)-10 b(erms)39 b(and)g(Con)m(v)m(en)m(tions)75 788 y FE(This)11 b(c)o(hapter)g(explains)h(notational)e(terms)g(and)h(con)o (v)o(en)o(tions)g(used)g(throughout)f(the)g(MPI)h(do)q(cumen)o(t,)75 844 y(some)k(of)g(the)g(c)o(hoices)h(that)e(ha)o(v)o(e)h(b)q(een)i (made,)d(and)i(the)f(rationale)h(b)q(ehind)h(those)e(c)o(hoices.)75 989 y Fv(2.1)59 b(Pro)r(cedure)19 b(Sp)r(eci\014cation)75 1092 y FE(MPI)g(pro)q(cedures)h(are)f(sp)q(eci\014ed)j(using)e(a)f (language)g(indep)q(enden)o(t)j(notation.)31 b(The)20 b(argumen)o(ts)e(of)75 1148 y(pro)q(cedure)e(calls)g(are)f(mark)o(ed)g (as)g Fr(IN)p FE(,)g Fr(OUT)g FE(or)g Fr(INOUT)p FE(.)g(The)h(meanings) f(of)g(these)h(are:)143 1243 y FC(\017)23 b FE(the)15 b(call)h(uses)g(but)f(do)q(es)h(not)e(up)q(date)i(a)f(argumen)o(t)f (mark)o(ed)h Fr(IN)p FE(,)143 1339 y FC(\017)23 b FE(the)15 b(call)h(ma)o(y)f(up)q(date)h(a)e(argumen)o(t)h(mark)o(ed)g Fr(OUT)p FE(,)143 1434 y FC(\017)23 b FE(the)15 b(call)h(b)q(oth)g (uses)f(and)g(up)q(dates)h(a)f(argumen)o(t)f(mark)o(ed)h Fr(INOUT)p FE(.)166 1529 y(There)i(is)h(one)f(sp)q(ecial)h(case)f(|)h (if)f(an)g(argumen)o(t)f(is)h(a)g(handle)h(to)e(an)h(opaque)g(ob)s (ject,)g(and)g(the)75 1586 y(ob)s(ject)g(is)h(up)q(dated)g(b)o(y)f(the) h(pro)q(cedure)g(call,)h(then)e(the)h(argumen)o(t)f(is)g(mark)o(ed)g Fr(OUT)p FE(.)h(This)g(is)g(done)75 1642 y(ev)o(en)f(though)g(the)g (handle)h(itself)f(is)h(not)e(mo)q(di\014ed)i(|)f(w)o(e)g(use)g(the)g Fr(OUT)g FE(attribute)g(to)f(denote)h(that)75 1699 y(what)e(the)g (handle)h Fs(r)n(efer)n(enc)n(es)e FE(is)h(up)q(dated.)166 1755 y(The)h(de\014nition)j(of)c(MPI)i(tries)f(to)g(a)o(v)o(oid,)g(to)f (the)i(largest)f(p)q(ossible)i(exten)o(t,)e(the)g(use)h(of)e Fr(INOUT)75 1812 y FE(argumen)o(ts,)f(b)q(ecause)i(suc)o(h)g(use)f(is)h (error-prone,)e(esp)q(ecially)k(for)d(scalar)g(argumen)o(ts.)166 1869 y(A)d(common)f(o)q(ccurrence)i(for)f(MPI)f(functions)i(is)f(an)g (argumen)o(t)f(that)g(is)i(used)f(as)g Fr(IN)g FE(b)o(y)g(some)f(pro-) 75 1925 y(cesses)h(and)h Fr(OUT)f FE(b)o(y)g(other)g(pro)q(cesses.)19 b(Suc)o(h)13 b(argumen)o(ts)e(are,)h(syn)o(tactically)l(,)i(an)e Fr(INOUT)g FE(argumen)o(t,)75 1982 y(although,)j(seman)o(tically)l(,)h (they)f(are)g(not)g(used)h(in)g(one)f(call)i(b)q(oth)e(for)g(input)h (and)f(for)g(output.)166 2038 y(Another)h(frequen)o(t)g(situation)h (arises)g(when)f(an)g(argumen)o(t)g(v)m(alue)h(is)g(needed)h(only)f(b)o (y)f(a)g(subset)75 2095 y(of)f(the)g(pro)q(cesses.)20 b(When)c(an)f(argumen)o(t)f(is)i(not)f(signi\014can)o(t)h(at)e(a)h(pro) q(cess)h(then)f(an)g(arbitrary)g(v)m(alue)75 2151 y(can)g(b)q(e)h (passed)g(as)e(argumen)o(t.)75 2297 y Fv(2.2)59 b(Semantic)18 b(T)-5 b(erms)75 2399 y FE(When)12 b(discussing)i(p)q(oin)o(t-to-p)q (oin)o(t)f(and)f(collectiv)o(e)h(comm)o(unications)g(the)f(follo)o (wing)h(terms)e(are)h(used:)75 2507 y Fz(non)o(blo)q(c)o(king)24 b FE(if)e(the)h(pro)q(cedure)g(ma)o(y)f(return)g(b)q(efore)h(the)f (related)h(comm)o(unication)g(op)q(eration)189 2563 y(completes,)13 b(and)g(b)q(efore)f(the)h(user)f(is)h(allo)o(w)o(ed)g(to)f(re-use)h (resources)f(\(suc)o(h)h(as)f(bu\013ers\))g(sp)q(eci\014ed)189 2620 y(in)k(the)f(call;)p 75 2661 720 2 v 127 2688 a FB(1)144 2704 y Fu(V)m(ersion)f(as)f(of)g(Octob)q(er)g(10,)g(1993)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 9 14 9 13 bop 75 -100 a Ft(2.3.)34 b(D)o(A)l(T)l(A)15 b(TYPES)1375 b FE(9)75 45 y Fz(lo)q(cally-blo)r(c)o(king)26 b FE(if)18 b(return)g(from)f(the)h(pro)q(cedure)h(indicates)g(the)f(user)g(is)h (allo)o(w)o(ed)f(to)f(re-use)i(re-)189 102 y(sources)c(sp)q(eci\014ed)i (in)f(the)f(call;)75 211 y Fz(globally-blo)q(cking)25 b FE(if)c(return)f(from)g(the)g(pro)q(cedure)i(indicates)f(the)g(comm)o (unication)g(op)q(eration)189 267 y(has)15 b(completed.)166 393 y(When)i(discussing)i(the)e(b)q(eha)o(vior)h(of)e(an)o(y)h(MPI)g (function,)h(not)e(necessarily)j(a)d(comm)o(unication)75 450 y(function,)g(the)f(follo)o(wing)h(terms)f(are)f(used.)75 575 y Fz(lo)q(cal)25 b FE(An)16 b(op)q(eration)g(is)g Fz(lo)q(cal)i FE(if)e(its)g(completion)h(dep)q(ends)h(only)e(on)g(the)f (lo)q(cal)i(executing)g(pro)q(cess.)189 632 y(Suc)o(h)e(an)h(op)q (eration)f(do)q(es)h(not)e(require)i(comm)o(unication)g(with)g(another) f(user)g(pro)q(cess.)75 741 y Fz(non-lo)q(cal)26 b FE(An)c(op)q (eration)g(is)g Fz(non-lo)q(cal)j FE(if)d(its)g(completion)h(ma)o(y)e (dep)q(end)i(on)f(comm)o(unication)189 798 y(o)q(ccurring)16 b(with)f(another)g(user)g(pro)q(cess.)75 907 y Fz(collectiv)o(e)24 b FE(An)17 b(op)q(eration)g(is)g Fz(collectiv)o(e)i FE(if)e(all)g(pro)q (cesses)g(in)h(a)e(pro)q(cess)h(group)g(need)g(to)f(p)q(erform)189 964 y(the)f(op)q(eration.)75 1129 y Fv(2.3)59 b(Data)19 b(T)-5 b(yp)r(es)75 1240 y Fq(2.3.1)49 b(Opaque)16 b(objects)75 1333 y FE(MPI)23 b(pro)q(cedures)g(use)g Fs(op)n(aque)h(obje)n(cts)p FE(.)42 b(These)23 b(hide)h(the)f(in)o(ternal)h(represen)o(tation)e (used,)j(th)o(us)75 1390 y(allo)o(wing:)c(similar)16 b(calls)h(in)f(C)f(and)g(F)l(ortran;)f(the)h(a)o(v)o(oidance)h(of)f (con\015icts)h(with)f(the)h(t)o(yping)f(rules)h(in)75 1446 y(these)f(languages;)g(future)g(extension)h(of)f(functionalit)o(y) l(.)21 b(The)16 b(mec)o(hanism)f(for)g(opaque)g(ob)s(jects)f(used)75 1502 y(here)i(lo)q(osely)g(follo)o(ws)f(the)g(POSIX)i(F)l(ortran)d (binding)j(standard.)166 1563 y(An)g(opaque)g(ob)s(ject)f(can)g(b)q(e)i (allo)q(cated)f(on)g(user)g(storage)e(or)h(on)h(system)f(\(i.e.)24 b(MPI)17 b(managed\))75 1619 y(storage.)23 b(W)l(e)17 b(call)h(the)f(former)f(a)g Fs(user)i(obje)n(ct)e FE(and)h(the)g (latter)g(a)f Fs(system)h(obje)n(ct)p FE(.)24 b(User)17 b(ob)s(jects)f(are)75 1676 y(allo)q(cated)h(and)f(deallo)q(cated)h (explicitly)h(b)o(y)e(the)g(user)g(as)f(a)h(regular)f(v)m(ariable.)23 b(The)16 b(t)o(yp)q(e)g(de\014nitions)75 1732 y(for)g(suc)o(h)h(ob)s (jects)f(are)h(pro)o(vided)g(b)o(y)g(MPI.)f(The)h(ob)s(jects)f(can)h(b) q(e)g(accessed)g(and)g(mo)q(di\014ed)h(only)g(via)75 1789 y(MPI)d(functions)h(that)f(tak)o(e)f(suc)o(h)i(ob)s(jects)e(as)h (argumen)o(ts.)166 1849 y(System)d(ob)s(jects)g(are)g(accessed)i(only)f (via)f Fs(hand)r(les)p FE(.)19 b(In)13 b(C,)f(a)g(di\013eren)o(t)h(t)o (yp)q(e)g(of)f(handle)i(is)f(de\014ned)75 1905 y(b)o(y)i(MPI)g(for)f (eac)o(h)h(t)o(yp)q(e)g(of)g(opaque)g(system)f(ob)s(ject.)20 b(In)15 b(F)l(ortran,)f(all)i(handles)g(ha)o(v)o(e)e(t)o(yp)q(e)h Fp(INTEGER)p FE(.)75 1962 y(The)g(allo)q(cation)g(and)g(deallo)q (cation)h(of)e(suc)o(h)h(an)f(ob)s(ject,)g(and)g(an)o(y)h(access)f(or)g (up)q(date)h(to)f(it,)g(are)h(done)75 2018 y(only)h(via)f(MPI)g(calls)i (that)d(accept)i(as)e(argumen)o(ts)h(handles)h(to)f(suc)o(h)g(ob)s (jects.)166 2079 y(An)f(opaque)f(ob)s(ject,)g(and)h(the)g(handle)g(to)f (suc)o(h)h(an)g(ob)s(ject,)e(are)i(signi\014can)o(t)g(only)g(at)f(the)h (pro)q(cess)75 2135 y(where)h(the)h(ob)s(ject)e(w)o(as)h(created,)g (and)g(cannot)g(b)q(e)h(transferred)f(to)f(another)h(pro)q(cess.)166 2195 y(Opaque)h(system)e(ob)s(jects)h(are)g(allo)q(cated)h(and)f (deallo)q(cated)i(b)o(y)e(calls)h(that)f(are)f(sp)q(eci\014c)j(to)e (eac)o(h)75 2252 y(ob)s(ject)k(t)o(yp)q(e.)32 b(These)20 b(are)f(listed)i(in)f(the)f(sections)h(where)g(the)f(ob)s(jects)g(are)g (describ)q(ed.)35 b(A)19 b(call)i(to)75 2308 y(deallo)q(cate)15 b(an)f(opaque)h(system)e(ob)s(ject)h(do)q(es)g(not)g(necessarily)h (deallo)q(cate)h(the)e(ob)s(ject)f(immediately)l(,)75 2365 y(but)e(ma)o(y)f(only)h(mark)g(it)g(as)f(candidate)i(for)e(deallo) q(cation.)20 b(The)11 b(ob)s(ject)f(will)j(b)q(e)e(deallo)q(cated)h(as) f(as)f(so)q(on)75 2421 y(as)17 b(there)g(there)g(is)h(no)f(p)q(ending)i (op)q(eration)e(that)g(is)g(using)h(this)g(ob)s(ject;)f(the)g(handle)i (then)e(b)q(ecomes)75 2478 y(n)o(ull.)22 b(F)l(or)15 b(example,)h(if)f(a)g(comm)o(unication)i(ob)s(ject)d(is)i(freed)g (after)f(the)g(comm)o(unication)h(started)f(but)75 2534 y(b)q(efore)h(it)h(completed,)g(then)f(the)h(p)q(ending)h(op)q(eration) e(is)h(not)e(a\013ected;)h(one)h(can)f(use)g(the)h(handle)g(in)75 2591 y(calls)i(to)f Fx(MPI)p 323 2591 14 2 v 16 w(W)l(AIT)g FE(or)g Fx(MPI)p 611 2591 V 16 w(ST)l(A)l(TUS)p FE(.)h(In)g(fact,)g (one)f(m)o(ust)g(complete)h(the)g(comm)o(unication)g(with)75 2647 y(suc)o(h)12 b(a)f(call.)20 b(Ho)o(w)o(ev)o(er,)11 b(once)h(the)f(op)q(eration)h(completes,)h(it)f(is)g(erroneous)f(to)g (re-use)h(the)g(freed)g(handle)75 2704 y(for)j(a)f(new)i(comm)o (unication.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 10 15 10 14 bop 75 -100 a FE(10)692 b Ft(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)166 45 y Fo(Implemen)o(tati)o(on)g (note:)47 b Fr(Some)15 b(ob)r(jects,)j(suc)o(h)f(as)g(datat)o(yp)q(e)f (ob)r(jects)i(\(see)g(section)f(3.13\),)f(can)g(b)q(e)75 95 y(used)j(b)o(y)f(sev)o(eral,)h(distinct,)f(concurren)o(t)i(comm)o (unication)15 b(op)q(erations.)30 b(Th)o(us,)19 b(to)f(\014nd)g(out)g (whether)h(suc)o(h)75 145 y(ob)r(ject)c(can)f(b)q(e)g(freed,)h(one)f (needs)h(a)e(reference)k(coun)o(t)d(of)f(p)q(ending)h(op)q(erations.) 166 202 y(MPI)d(do)q(es)h(not)f(k)o(eep)h(coun)o(t)f(of)f(references)k (to)d(an)g(opaque)g(MPI)g(ob)r(ject)h(that)f(are)h(created)g(b)o(y)f (assigning)f(to)75 258 y(one)j(handle)f(the)h(v)n(alue)e(of)h(another,) h(or)f(are)h(lost)f(when)h(a)f(handle)g(v)n(ariable)f(is)i(deallo)q (cated)f(\()p Fn(e.g.)p Fr(,)g(at)g(pro)q(cedure)75 314 y(exit\).)24 b(In)16 b(the)g(\014rst)h(case,)g(a)e(call)g(to)h Fm(MPI)p 756 314 13 2 v 14 w(GROUP)p 912 314 V 13 w(FREE)f Fr(ma)o(y)f(lea)o(v)o(e)h(a)h(dangling)e(reference;)19 b(in)c(the)i(second)75 371 y(case)d(an)e(opaque)h(MPI)g(ob)r(ject)g(ma) o(y)e(b)q(e)j(left)e(with)g(no)h(handle)g(to)f(it,)g(and)h(no)f(p)q (ossibilit)o(y)g(of)g(deallo)q(cating)g(it.)17 b(It)75 427 y(is)d(the)h(user)h(resp)q(onsibilit)o(y)e(to)g(a)o(v)o(oid)f (these)j(situations.)j(MPI)c(k)o(eeps)g(trac)o(k)g(only)f(of)g(c)o (hanges)h(in)f(the)h(n)o(um)o(b)q(er)75 484 y(of)e(activ)o(e)h (references)j(to)d(the)g(MPI)g(ob)r(ject)h(that)f(are)g(a)g(result)g (of)g(explicit)f(MPI)h(calls.)166 623 y FE(MPI)19 b(ma)o(y)g(pro)o (vide)h(prede\014ned)h(opaque)f(ob)s(jects)f(and)g(prede\014ned,)j (static)d(handles)i(to)e(these)75 680 y(ob)s(jects.)g(Suc)o(h)d(ob)s (jects)f(ma)o(y)f(not)h(b)q(e)h(destro)o(y)o(ed.)75 804 y Fq(2.3.2)49 b(Arra)o(y)16 b(of)h(handles)75 890 y FE(An)f(MPI)g(call) g(ma)o(y)f(need)i(an)e(argumen)o(t)g(that)g(is)h(an)g Fs(arr)n(ay-of-hand)r(les)p FE(.)22 b(The)16 b(arra)o(y-of-handles)f (is)h(a)75 947 y(regular)f(arra)o(y)e(with)j(en)o(tries)f(that)f(are)g (handles)i(to)e(ob)s(jects)g(of)h(the)g(same)f(t)o(yp)q(e)h(in)h (consecutiv)o(e)f(lo)q(ca-)75 1003 y(tions)h(in)g(the)f(arra)o(y)l(.)20 b(Whenev)o(er)c(suc)o(h)g(an)f(arra)o(y)f(is)i(used,)g(an)f(additional) i Fx(len)f FE(argumen)o(t)f(is)h(required)75 1060 y(to)e(indicate)j (the)d(n)o(um)o(b)q(er)i(of)e(v)m(alid)i(en)o(tries)g(\(unless)f(this)h (n)o(um)o(b)q(er)f(can)g(b)q(e)g(deriv)o(ed)h(otherwise\).)k(The)75 1116 y(v)m(alid)e(en)o(tries)f(are)g(at)f(the)h(b)q(eggining)h(of)e (the)h(arra)o(y;)f Fx(len)h FE(indecates)h(ho)o(w)e(man)o(y)g(of)g (them)h(there)g(are,)75 1173 y(and)e(need)h(not)f(b)q(e)h(the)f(en)o (tire)h(size)g(of)f(the)g(arra)o(y)l(.)75 1296 y Fq(2.3.3)49 b(State)75 1383 y FE(MPI)15 b(pro)q(cedures)h(use)g(at)e(v)m(arious)i (places)g(argumen)o(ts)f(with)g Fs(state)g FE(t)o(yp)q(es.)20 b(The)c(v)m(alues)g(of)f(suc)o(h)h(data)75 1439 y(t)o(yp)q(e)i(are)h (all)g(iden)o(ti\014ed)h(b)o(y)f(names,)f(and)h(no)f(op)q(eration)h(is) g(de\014ned)h(on)e(them.)29 b(F)l(or)18 b(example,)i(the)75 1496 y Fx(MPI)p 160 1496 14 2 v 16 w(SET)p 259 1496 V 17 w(ERRORMODE)c FE(routine)f(has)f(a)g(state)g(t)o(yp)q(e)h(argumen)o (t)e(with)i(v)m(alues)h Fm(MPI)p 1572 1496 13 2 v 14 w(ERRORSF)m(A)m(T)m(AL)p FE(,)75 1552 y Fm(MPI)p 152 1552 V 14 w(ERRORSRETURN)p FE(,)e(etc.)75 1676 y Fq(2.3.4)49 b(Named)16 b(constants)75 1763 y FE(MPI)e(pro)q(cedures)g(sometimes)g (assign)f(a)h(sp)q(ecial)h(meaning)f(to)f(a)h(sp)q(ecial)h(v)m(alue)g (of)e(a)g(basic)i(t)o(yp)q(e)e(argu-)75 1819 y(men)o(t;)k(e.g.)24 b Fx(tag)17 b FE(is)g(an)g(in)o(teger)g(v)m(alued)i(argumen)o(t)d(of)g (p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)f(op)q(erations,)75 1876 y(with)f(a)g(sp)q(ecial)i(wild-card)f(v)m(alue,)g Fm(MPI)p 775 1876 V 15 w(ANY)p 875 1876 V 14 w(T)m(A)o(G)p FE(.)e(Suc)o(h)i(argumen)o(ts)e(will)i(ha)o(v)o(e)f(a)g(range)f(of)h (regular)75 1932 y(v)m(alues,)i(whic)o(h)g(is)f(a)f(prop)q(er)h (subrange)g(of)f(the)h(range)g(of)f(v)m(alues)i(of)e(the)h(corresp)q (onding)h(basic)f(t)o(yp)q(e;)75 1989 y(sp)q(ecial)f(v)m(alues)g(\(suc) o(h)f(as)f Fm(MPI)p 609 1989 V 14 w(ANY)p 708 1989 V 15 w(T)m(A)o(G)p FE(\))f(will)j(b)q(e)f(outside)g(the)g(regular)g (range.)k(The)c(range)f(of)g(regu-)75 2045 y(lar)f(v)m(alues)i(can)e(b) q(e)h(queried,)g(and)f(sometimes)g(set,)g(using)h(en)o(vironmen)o(tal)g (inquiry)h(or)d(en)o(vironmen)o(tal)75 2102 y(managemen)o(t)i (functions)i(\(Chapter)f(7\).)166 2241 y Fo(Implemen)o(tati)o(on)d (note:)35 b Fr(Implemen)o(tors)11 b(should)i(detect,)i(whenev)o(er)g(p) q(ossible,)e(illegal)e(uses)k(of)d(\\sp)q(e-)75 2298 y(cial)i(v)n(alues".)22 b(Th)o(us,)16 b(the)f(use)i(of)d(the)i Fm(MPI)p 780 2298 V 14 w(ANY)p 879 2298 V 15 w(T)m(A)o(G)e Fr(v)n(alue)h(to)g(tag)g(a)g(message)g(sen)o(t)h(should)f(b)q(e)h (\015agged)f(as)75 2354 y(an)f(error.)75 2561 y Fq(2.3.5)49 b(Choice)75 2647 y FE(MPI)11 b(functions)h(sometimes)f(use)h(argumen)o (ts)e(with)h(a)g Fs(choic)n(e)g FE(\(or)g(union\))g(data)g(t)o(yp)q(e.) 19 b(Distinct)11 b(calls)i(to)75 2704 y(the)f(same)g(routine)g(ma)o(y)g (pass)f(b)o(y)h(reference)h(actual)f(argumen)o(ts)f(of)h(di\013eren)o (t)g(t)o(yp)q(es.)19 b(The)12 b(mec)o(hanism)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 11 16 11 15 bop 75 -100 a Ft(2.4.)34 b(LANGUA)o(GE)15 b(BINDING)1165 b FE(11)75 45 y(for)15 b(pro)o(viding)h(suc)o(h)f(argumen)o(ts)g(will)h (di\013er)g(from)e(language)i(to)e(language.)75 208 y Fv(2.4)59 b(Language)19 b(Binding)75 316 y FE(This)k(section)g (de\014nes)h(the)e(rules)i(for)e(MPI)g(language)h(binding)h(in)g (general)f(and)f(for)g(F)l(ortran)g(77)75 373 y(and)17 b(ANSI)h(C)f(in)h(particular.)26 b(De\014ned)17 b(here)h(are)f(v)m (arious)g(ob)s(ject)g(represen)o(tations,)g(as)f(w)o(ell)i(as)f(the)75 429 y(naming)k(con)o(v)o(en)o(tions)f(used)g(for)g(expressing)h(this)f (standard.)35 b(The)20 b(actual)g(calling)i(sequences)f(are)75 486 y(de\014ned)c(elsewhere.)166 546 y(It)23 b(is)g(exp)q(ected)h(that) e(an)o(y)g(F)l(ortran)g(90)g(and)h(C++)g(implemen)o(tations)h(use)f (the)g(F)l(ortran)f(77)75 602 y(and)f(ANSI)h(C)e(bindings,)k(resp)q (ectiv)o(ely)l(.)39 b(Although)22 b(w)o(e)e(consider)i(it)f(premature)g (to)f(de\014ne)i(other)75 658 y(bindings)e(to)d(F)l(ortran)g(90)g(and)h (C++,)h(the)f(curren)o(t)f(bindings)j(are)e(designed)h(to)e(encourage,) i(rather)75 715 y(than)c(discourage,)g(exp)q(erimen)o(tation)i(with)e (b)q(etter)g(bindings)i(whic)o(h)f(migh)o(t)f(b)q(e)h(adopted)g(later.) 166 775 y(There)i(are)g(sev)o(eral)g(imp)q(ortan)o(t)f(language)h (binding)i(issues)f(not)e(addressed)i(b)o(y)e(this)i(standard.)75 831 y(It)f(is)g(b)q(ey)o(ond)h(the)f(scop)q(e)g(of)g(this)g(standard)g (to)f(discuss)i(the)f(in)o(terop)q(erabilit)o(y)i(of)d(message)h (passing)75 888 y(b)q(et)o(w)o(een)c(languages.)19 b(Although)c(it)e(w) o(ould)i(b)q(e)f(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)h(to)e(guaran) o(tee)g(that)f(messages)75 944 y(sen)o(t)20 b(b)q(et)o(w)o(een)h (languages)f(b)q(e)h(prop)q(erly)g(in)o(terpreted,)h(the)e(matc)o(hing) g(of)g(t)o(yp)q(es)g(across)g(languages)75 1001 y(prev)o(en)o(ts)15 b(the)h(inclusion)i(of)d(suc)o(h)h(a)f(guaran)o(tee)g(in)i(this)f (standard.)k(It)c(is)g(fully)h(exp)q(ected)g(that)d(man)o(y)75 1057 y(implemen)o(tations)i(will)g(ha)o(v)o(e)f(suc)o(h)g(features,)f (and)h(that)f(suc)o(h)h(features)g(are)f(a)h(sign)g(of)f(the)h(qualit)o (y)g(of)75 1114 y(the)g(implemen)o(tation.)75 1255 y Fq(2.4.1)49 b(Data)17 b(T)l(yp)q(es)f(and)h(Naming)g(Conventions)75 1348 y FE(Handles)e(are)f(represen)o(ted)h(as)e(in)o(tegers)i(in)g(F)l (ortran)e(and)h(as)g(p)q(oin)o(ters)g(to)g(structures)f(in)i(C.)f(A)g (sp)q(ecial)75 1404 y(v)m(alue)g Fm(MPI)p 267 1404 13 2 v 14 w(NULL)g FE(is)f(used)g(to)g(represen)o(t)f(n)o(ull)j(handles)e (in)h(F)l(ortran;)e(in)i(C,)e(the)h(constan)o(t)f Fm(NULL)h FE(is)g(used)75 1460 y(as)i(usual.)166 1520 y(T)o(yp)q(ed)h(comm)o (unication)g(routines)g(supp)q(ort)g Fz(c)o(haracter)p FE(,)g Fz(double)i(precision)p FE(,)e Fz(real)p FE(,)g(and)g Fz(in-)75 1577 y(teger)d FE(data)f(in)h(F)l(ortran.)18 b Fz(in)o(t)p FE(,)13 b Fz(double)p FE(,)h Fz(real)p FE(,)f Fz(short)p FE(,)g Fz(c)o(har)p FE(,)f(and)h Fz(long)h FE(are)e(supp)q(orted)h(in)h(C,)e(along)75 1633 y(with)k Fz(unsigned)j(in)o(t)p FE(,)d Fz(unsigned)i(long)p FE(,)f Fz(unsigned)h(short)p FE(,)e(and)g Fz(unsigned)i(c)o(har)p FE(.)k(In)16 b(addition,)75 1690 y(a)e(sp)q(ecial)j Fz(b)o(yte)d FE(t)o(yp)q(e)g(is)h(supp)q(orted)h(in)f(b)q(oth)g(languages.)k(This)d (t)o(yp)q(e)e(allo)o(ws)h(comm)o(unication)g(to)f(b)q(e)75 1746 y(p)q(erformed)f(with)g(no)f(t)o(yp)q(e)h(con)o(v)o(ersion)f(at)g (all.)20 b(In)13 b(C,)f(the)h(de\014nitions)h(of)f(named)f(constan)o (ts,)g(function)75 1803 y(protot)o(yp)q(es,)k(and)h(aggregate)f(t)o(yp) q(e)g(information)h(m)o(ust)g(b)q(e)g(supplied)j(in)d(an)g(include)i (\014le)f Fp(mpi.h)p FE(.)24 b(In)75 1859 y(F)l(ortran,)c(the)h (corresp)q(onding)h(include)g(\014le)g(for)e(named)h(constan)o(ts)e(\() p Fp(PARAMETER)p FE(s\))g(will)j(b)q(e)f(called)75 1916 y Fp(mpif.h)p FE(.)75 2057 y Fq(2.4.2)49 b(F)o(o)o(rtran)15 b(77)i(Binding)h(Issues)75 2150 y FE(All)13 b(MPI)e(names)h(ha)o(v)o(e) f(an)h Fp(MPI)p 618 2150 15 2 v 28 w FE(pre\014x,)g(and)g(all)g(c)o (haracters)f(are)g(capitals.)20 b(All)13 b(MPI)e(F)l(ortran)g(subrou-) 75 2206 y(tines)16 b(ha)o(v)o(e)g(a)f(return)h(co)q(de)g(in)h(the)f (last)f(argumen)o(t.)21 b(A)16 b(few)g(MPI)f(op)q(erations)h(are)g (functions,)g(whic)o(h)75 2262 y(do)g(not)g(ha)o(v)o(e)f(the)i(return)f (co)q(de)g(argumen)o(t.)22 b(The)16 b(return)g(co)q(de)h(v)m(alue)g (for)f(successful)h(completion)g(is)75 2319 y Fm(MPI)p 152 2319 13 2 v 14 w(SUCCESS)p FE(;)11 b(error)h(co)q(des)i(are)e (implemen)o(tation)j(dep)q(enden)o(t.)20 b(Unless)14 b(explicitly)i(stated,)c(the)h(MPI)75 2375 y(F77)i(binding)k(is)e (consisten)o(t)f(with)h(ANSI)g(standard)f(F)l(ortran)f(77.)23 b(There)16 b(are)g(sev)o(eral)h(p)q(oin)o(ts)g(where)75 2432 y(this)d(standard)f(div)o(erges)g(from)g(the)g(ANSI)h(F)l(ortran)e (77)h(standard.)19 b(These)14 b(exceptions)g(are)f(consisten)o(t)75 2488 y(with)j(common)e(practice)i(in)g(the)g(F)l(ortran)e(comm)o(unit)o (y)l(.)20 b(In)15 b(particular:)143 2596 y FC(\017)23 b FE(MPI)15 b(Iden)o(ti\014ers)h(are)f(limited)i(to)e(thirt)o(y)l(,)g (not)f(six,)i(signi\014can)o(t)g(c)o(haracters.)143 2704 y FC(\017)23 b FE(MPI)15 b(Iden)o(ti\014ers)h(ma)o(y)f(con)o(tain)g (underscores)h(after)e(the)i(\014rst)e(c)o(haracter.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 12 17 12 16 bop 75 -100 a FE(12)692 b Ft(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)290 81 y Fp(double)23 b(precision)f(a)290 138 y(integer)h(b)290 194 y(call)g (MPI_send\(a,...\))385 251 y(.)385 307 y(.)290 363 y(call)g (MPI_send\(b,...\))75 511 y FE(Figure)e(2.1:)29 b(This)21 b(is)g(an)f(example)h(of)f(calling)i(a)e(routine)h(with)g(mismatc)o (hed)f(formal)g(and)h(actual)75 568 y(argumen)o(ts.)143 706 y FC(\017)i FE(The)15 b(same)g(MPI)g(subroutine)h(ma)o(y)e(b)q(e)i (called)h(with)e(di\013eren)o(t)h(argumen)o(t)e(t)o(yp)q(es.)20 b(An)15 b(example)189 762 y(is)h(sho)o(wn)f(in)h(Figure)g(2.4.2.)j (This)d(violates)g(the)g(letter)f(of)g(the)h(F)l(ortran)e(standard,)h (but)g(suc)o(h)h(a)189 819 y(violation)g(is)f(common)g(practice.)20 b(The)15 b(alternativ)o(e)g(w)o(ould)h(b)q(e)f(to)g(ha)o(v)o(e)f(a)h (separate)f(v)o(ersion)h(of)189 875 y Fx(MPI)p 274 875 14 2 v 15 w(SEND)p FE(,)h(for)e(example,)i(for)e(eac)o(h)i(data)e(t)o (yp)q(e.)143 978 y FC(\017)23 b FE(Although)12 b(not)f(required,)i(it)f (is)g(strongly)f(suggested)h(that)f(named)h(MPI)f(constan)o(ts)g(b)q(e) h(pro)o(vided)189 1035 y(in)18 b(an)g(include)i(\014le,)g(called)f Fp(mpif.h)p FE(.)27 b(On)19 b(systems)e(that)g(do)h(not)f(supp)q(ort)h (include)j(\014les,)e(the)189 1091 y(implemen)o(tation)d(should)g(sp)q (edify)h(the)e(v)m(alues)i(of)d(named)i(constan)o(ts.)75 1271 y Fq(2.4.3)49 b(C)16 b(Binding)i(Issues)75 1361 y FE(W)l(e)23 b(use)g(the)f(ANSI)i(C)e(declaration)h(format.)41 b(All)24 b(MPI)f(names)f(ha)o(v)o(e)g(an)h Fp(MPI)p 1548 1361 15 2 v 39 w FE(pre\014x,)i(de\014ned)75 1418 y(constan)o(ts)15 b(are)h(in)h(all)g(capital)g(letters,)f(and)g(de\014ned)i(t)o(yp)q(es)e (and)g(functions)h(ha)o(v)o(e)f(one)g(capital)h(letter)75 1474 y(after)d(the)i(pre\014x.)166 1533 y(Almost)24 b(all)h(C)e (functions)i(return)f(an)g(error)f(co)q(de.)47 b(The)24 b(successful)h(return)f(co)q(de)h(will)g(b)q(e)75 1589 y Fp(MPI)p 150 1589 V 17 w(SUCCESS)p FE(,)16 b(but)i(failure)h(return)f (co)q(des)g(are)f(implemen)o(tation)i(dep)q(enden)o(t.)29 b(A)18 b(few)g(C)f(functions)75 1646 y(will)i(return)f(v)m(alues,)h(so) e(that)g(they)g(can)h(b)q(e)g(implemen)o(ted)i(as)d(macros.)26 b(Logical)18 b(\015ags)g(are)f(in)o(tegers)75 1702 y(with)i(v)m(alues)g (1)f(and)g(0)g(meaning)h(\\true")f(and)g(\\false")g(resp)q(ectiv)o(ely) l(.)31 b(Bu\013er)18 b(p)q(oin)o(ters)h(are)f(of)g(t)o(yp)q(e)75 1759 y Fp(void*)p FE(.)75 1915 y Fv(2.5)59 b(Pro)r(cesses)75 2021 y FE(An)22 b(MPI)h(program)e(is)h(executed)h(b)o(y)g(sev)o(eral)f (autonomous)f(pro)q(cesses)i(that)e(eac)o(h)i(execute)f(their)75 2078 y(o)o(wn)17 b(co)q(de,)h(in)h(an)e(MIMD)g(st)o(yle.)27 b(The)18 b(co)q(des)g(executed)h(b)o(y)e(eac)o(h)h(pro)q(cess)g(need)g (not)f(b)q(e)h(iden)o(tical.)75 2134 y(The)i(pro)q(cesses)g(comm)o (unicate)g(via)g(calls)g(to)f(MPI)h(comm)o(unication)g(primitiv)o(es.) 35 b(T)o(ypically)l(,)22 b(eac)o(h)75 2191 y(pro)q(cessor)17 b(executes)i(in)f(its)g(o)o(wn)f(address)g(space,)h(although)g (shared-memory)g(implemen)o(tations)g(of)75 2247 y(MPI)i(are)f(p)q (ossible.)35 b(This)20 b(do)q(cumen)o(t)g(sp)q(eci\014es)i(the)d(b)q (eha)o(vior)h(of)g(a)f(parallel)i(program)e(assuming)75 2304 y(that)c(only)g(MPI)h(calls)g(are)f(used)h(for)e(comm)o (unication.)21 b(The)16 b(in)o(teraction)f(of)g(an)g(MPI)g(program)g (with)75 2360 y(other)g(p)q(ossible)i(means)e(of)g(comm)o(unication)h (\(e.g.,)d(shared)j(memory\))e(is)i(not)e(sp)q(eci\014ed.)166 2419 y(MPI)d(do)q(es)g(not)g(sp)q(ecify)h(the)f(execution)h(mo)q(del)g (for)e(eac)o(h)h(pro)q(cess.)19 b(A)11 b(pro)q(cess)g(can)h(b)q(e)f (sequen)o(tial,)75 2475 y(or)f(can)h(b)q(e)h(m)o(ulti-threaded,)g(with) f(threads)g(p)q(ossibly)h(executing)g(concurren)o(tly)l(.)20 b(Care)10 b(has)g(b)q(een)i(tak)o(en)75 2532 y(to)j(mak)o(e)f(MPI)h (\\thread-safe",)f(b)o(y)i(a)o(v)o(oiding)f(the)g(use)h(of)f(implicit)i (global)f(states.)166 2591 y(MPI)i(do)q(es)g(not)f(sp)q(ecify)j(either) e(the)g(in)o(teraction)h(of)e(pro)q(cesses)h(with)h(signals,)g(in)f(a)g (UNIX)g(en-)75 2647 y(vironmen)o(t,)e(or)g(other)f(ev)o(en)o(ts)h(that) g(do)g(not)f(relate)i(to)e(MPI)h(comm)o(unication.)23 b(That)16 b(is,)g(signals)h(are)75 2704 y(not)c(signi\014can)o(t)h (from)e(the)h(view)h(p)q(oin)o(t)g(of)e(MPI,)h(and)g(implemen)o(tors)h (should)g(attempt)e(to)h(implemen)o(t)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 13 18 13 17 bop 75 -100 a Ft(2.6.)34 b(ERR)o(OR)17 b(HANDLING)1217 b FE(13)75 45 y(MPI)14 b(so)g(that)f(signals)i(are)f(transparen)o(t:)19 b(an)14 b(MPI)g(call)h(susp)q(ended)h(b)o(y)e(a)g(signal)h(should)g (resume)g(and)75 102 y(complete)h(after)e(the)i(signal)g(is)f(handled.) 166 162 y(MPI)j(do)q(es)g(not)f(pro)o(vide)h(mec)o(hanisms)g(to)f(sp)q (ecify)i(the)f(initial)i(allo)q(cation)f(of)e(pro)q(cesses)h(to)f(an)75 218 y(MPI)d(computation)g(and)g(their)g(binding)i(to)d(ph)o(ysical)j (pro)q(cessors.)j(It)14 b(is)g(exp)q(ected)h(that)f(v)o(endors)f(will) 75 275 y(pro)o(vide)20 b(mec)o(hanisms)h(to)e(do)g(so)h(either)g(at)f (load)h(time)g(or)g(at)f(run)h(time.)34 b(Suc)o(h)20 b(mec)o(hanisms)g(will)75 331 y(allo)o(w)c(the)f(sp)q(eci\014cation)j (of)d(the)h(initial)i(n)o(um)o(b)q(er)d(of)h(required)g(pro)q(cesses,)g (the)g(co)q(de)g(to)f(b)q(e)h(executed)75 387 y(b)o(y)k(eac)o(h)h (initial)i(pro)q(cess,)e(and)g(the)g(allo)q(cation)g(of)f(pro)q(cesses) h(to)f(pro)q(cessors.)36 b(Also,)21 b(the)g(curren)o(t)75 444 y(prop)q(osal)f(do)q(es)h(not)e(pro)o(vide)i(for)e(dynamic)i (creation)f(or)g(deletion)h(of)f(pro)q(cesses)g(during)h(program)75 500 y(execution)c(\(the)f(total)f(n)o(um)o(b)q(er)i(of)f(pro)q(cesses)g (is)h(\014xed\),)f(although)g(it)h(is)f(in)o(tended)i(to)d(b)q(e)i (consisten)o(t)75 557 y(with)h(suc)o(h)f(extensions.)27 b(Finally)l(,)19 b(the)e(curren)o(t)g(prop)q(osal)h(do)q(es)f(not)g(sp) q(ecify)h(a)f(naming)h(sc)o(heme)g(for)75 613 y(pro)q(cesses.)33 b(W)l(e)20 b(prop)q(ose)f(to)g(alw)o(a)o(ys)g(iden)o(tify)i(pro)q (cesses)f(according)g(to)f(their)h(relativ)o(e)g(rank)f(in)h(a)75 670 y(group,)15 b(so)f(that,)g(e\013ectiv)o(ely)l(,)i(pro)q(cesses)g (are)f(iden)o(ti\014ed)i(b)o(y)e(consecutiv)o(e)h(in)o(tegers.)75 834 y Fv(2.6)59 b(Erro)n(r)21 b(Handling)75 942 y FE(MPI)11 b(pro)o(vides)h(the)g(user)f(with)h(reliable)i(message)d(transmission.) 19 b(A)11 b(message)g(sen)o(t)g(is)h(alw)o(a)o(ys)f(receiv)o(ed)75 999 y(correctly)l(,)17 b(and)g(the)g(user)g(do)q(es)g(not)f(need)i(to)e (c)o(hec)o(k)h(for)f(transmission)h(errors,)f(time-outs,)h(or)f(other) 75 1055 y(error)11 b(conditions.)20 b(In)13 b(other)f(w)o(ords,)f(MPI)h (do)q(es)g(not)g(pro)o(vide)g(mec)o(hanisms)h(for)e(dealing)j(with)e (failures)75 1112 y(in)17 b(the)f(comm)o(unication)h(system.)k(If)c (the)f(MPI)g(implemen)o(tation)h(is)f(built)i(on)e(an)g(unreliable)i (under-)75 1168 y(lying)h(mec)o(hanism,)f(then)g(it)f(is)h(the)g(job)f (of)g(the)g(implemen)o(tor)i(of)e(the)g(MPI)h(subsystem)f(to)g (insulate)75 1225 y(the)c(user)h(from)e(this)i(unreliabilit)o(y)l(,)i (or)d(to)f(re\015ect)i(unreco)o(v)o(erable)f(errors)g(as)f(global)i (program)e(failures.)75 1281 y(Similarly)l(,)i(MPI)e(itself)h(pro)o (vides)f(no)g(mec)o(hanisms)h(for)e(handling)i(no)q(de)g(failures.)20 b(\(A)11 b(mec)o(hanism)i(that)75 1338 y(allo)o(ws)i(the)h(user)f(to)g (restrict)g(the)g(scop)q(e)h(of)e(an)i(unreco)o(v)o(erable)f(error)g (is)h(presen)o(ted)f(in)h(section)g(7.2.\))166 1398 y(Of)g(course,)h (MPI)f(programs)f(ma)o(y)h(still)i(b)q(e)f(erroneous.)23 b(A)16 b Fz(program)i(error)d FE(can)i(o)q(ccur)f(when)75 1454 y(an)g(MPI)g(call)g(is)h(called)g(with)f(an)g(incorrect)g(argumen) o(t)f(\(non-existing)i(destination)g(in)g(a)e(send)i(op)q(er-)75 1511 y(ation,)g(bu\013er)g(to)q(o)f(small)h(in)h(a)e(receiv)o(e)i(op)q (eration,)f(etc.\))25 b(This)17 b(t)o(yp)q(e)g(of)f(error)g(w)o(ould)h (o)q(ccur)h(in)f(an)o(y)75 1567 y(implemen)o(tation.)31 b(In)19 b(addition,)h(a)f Fz(resource)h(error)d FE(ma)o(y)h(o)q(ccur)h (when)g(a)f(program)f(exceeds)j(the)75 1624 y(amoun)o(t)d(of)f(a)o(v)m (ailable)j(system)e(resources)h(\(n)o(um)o(b)q(er)f(of)g(p)q(ending)i (messages,)e(system)g(bu\013ers,)g(etc.\).)75 1680 y(The)h(o)q (ccurrence)h(of)f(this)g(t)o(yp)q(e)g(of)g(error)f(dep)q(ends)j(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(in)h(the)75 1737 y(system)13 b(and)h(the)g(resource)f(allo)q(cation)i(mec)o(hanism) f(used;)h(this)f(ma)o(y)f(di\013er)h(from)f(system)g(to)g(system.)75 1793 y(The)i(recommended)g(implemen)o(tation)g(will)h(pro)o(vide)f (generous)g(limits)g(on)g(some)f(of)g(the)g(resources)g(so)75 1849 y(as)f(to)g(alleviate)h(the)g(p)q(ortabilit)o(y)g(problem)g(this)g (represen)o(ts.)19 b(A)14 b(program)e(is)i Fz(safe)f FE(if)h(it)f(is)h(not)f(sub)s(ject)75 1906 y(to)i(resource)g(errors,)f Fz(unsafe)p FE(,)h(otherwise.)166 1966 y(All)d(MPI)e(calls)i(return)f (an)f(error)g(v)m(alue)i(that)e(indicates)i(successful)g(completion)f (of)f(the)h(op)q(eration,)75 2022 y(or)j(the)g(error)f(condition)j (that)d(o)q(ccurred,)i(otherwise.)20 b(Ho)o(w)o(ev)o(er,)13 b(MPI)h(do)q(es)g(not)g(require)h(the)f(user)h(to)75 2079 y(c)o(hec)o(k)h(for)e(error)h(v)m(alues.)22 b(It)15 b(is)h(recommended)h(that)d(an)i(implemen)o(tation)g(pro)o(vide)g(an)g (en)o(vironmen)o(t)75 2135 y(for)f(MPI)h(execution)g(that)f(can)h(b)q (e)g(set)g(so)f(that)g(an)o(y)g(error)g(encoun)o(tered)h(in)h(an)e(MPI) h(call)g(will)i(cause)75 2192 y(a)d(program)g(to)g(fail)i(with)f(a)f (suitable)i(error)e(message)g(returned)h(to)f(the)h(user.)22 b(The)16 b(en)o(vironmen)o(t)g(for)75 2248 y(an)f(MPI)g(execution)h (can)f(also)f(b)q(e)i(set)f(so)f(that)g(\(non-fatal\))g(errors)h(do)f (not)h(cause)g(a)g(program)e(to)i(fail,)75 2305 y(and)j(the)f(user)g (can)h(pro)o(vide)g(his)g(or)f(her)g(o)o(wn)g(error)g(reco)o(v)o(ery)f (co)q(de.)27 b(A)18 b(mec)o(hanism)g(that)e(ac)o(hiev)o(es)75 2361 y(this)g(in)g(a)f(POSIX)h(en)o(vironmen)o(t)f(is)h(describ)q(ed)h (in)f(section)g(7.2.)166 2421 y(MPI)d(calls)h(ma)o(y)f(initiate)h(op)q (erations)g(that)e(con)o(tin)o(ue)i(async)o(hronously)g(after)e(the)i (call)g(returned.)75 2478 y(Th)o(us,)22 b(the)g(op)q(eration)f(ma)o(y)g (return)g(with)h(a)f(co)q(de)h(indicating)h(successful)g(completion,)g (y)o(et)e(later)75 2534 y(cause)15 b(an)g(error)g(exception)h(to)e(b)q (e)i(raised.)k(If)15 b(there)g(is)h(a)f(subsequen)o(t)g(call)h(that)f (relates)g(to)f(the)h(same)75 2591 y(op)q(eration)f(\(e.g.,)f(a)g(call) i(that)e(v)o(eri\014es)i(that)e(an)h(async)o(hronous)f(op)q(eration)h (has)g(completed\))h(then)f(the)75 2647 y(error)g(argumen)o(t)g(asso)q (ciated)h(with)g(this)g(call)g(will)i(b)q(e)e(used)g(to)f(indicate)i (the)f(nature)f(of)h(the)f(error.)19 b(In)75 2704 y(a)14 b(few)h(cases,)f(the)g(error)g(ma)o(y)g(o)q(ccur)h(after)f(all)h(calls) h(that)d(relate)i(to)f(the)h(op)q(eration)f(ha)o(v)o(e)g(completed,)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 14 19 14 18 bop 75 -100 a FE(14)692 b Ft(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y FE(so)d(that)g(no)g(error) g(v)m(alue)i(can)e(b)q(e)h(used)g(to)f(indicate)i(the)f(nature)f(of)g (the)h(error)e(\(e.g.,)h(an)g(error)g(in)h(a)f(send)75 102 y(with)j(the)g(ready)f(mo)q(de\).)20 b(Suc)o(h)15 b(an)g(error)f(m)o(ust)g(b)q(e)i(treated)e(as)g(fatal,)g(since)i (information)f(cannot)f(b)q(e)75 158 y(returned)i(for)e(the)h(user)h (to)e(reco)o(v)o(er)h(from)f(it.)166 214 y(The)h(b)q(eha)o(vior)h(of)f (erroneous)g(programs)f(is)i(not)f(de\014ned)i(b)o(y)e(this)h(do)q (cumen)o(t.)k(The)c(desired)g(b)q(e-)75 271 y(ha)o(vior)g(is)h(that)f (a)g(relev)m(an)o(t)h(error)f(co)q(de)h(b)q(e)g(returned,)g(if)g(no)g (acceptable)g(b)q(eha)o(vior)g(can)g(b)q(e)g(de\014ned;)75 327 y(ho)o(w)o(ev)o(er,)12 b(e\016ciency)i(considerations)f(ma)o(y)f (dictate)g(that)g(the)h(system)f(do)q(es)g(not)g(detect)h(certain)g(t)o (yp)q(es)75 384 y(of)k(errors.)26 b(Also,)18 b(implemen)o(tations)h(ma) o(y)d(go)h(b)q(ey)o(ond)h(this)g(do)q(cumen)o(t)g(in)g(supp)q(orting)g (in)h(a)e(mean-)75 440 y(ningful)g(manners)e(co)q(des)h(that)e(are)h (de\014ned)i(here)e(to)g(b)q(e)h(erroneous.)166 497 y(MPI,)10 b(lik)o(e)h(an)o(y)f(other)g(similar)h(message-passing)f(library)l(,)i (allo)o(ws)e(users)g(to)g(write)g(non-deterministic)75 553 y(programs,)16 b(where)h(race)f(conditions)i(exist)g(and)f (di\013eren)o(t)g(timing)g(of)f(comm)o(unication)i(ma)o(y)e(lead)i(to) 75 610 y(di\013eren)o(t)g(results.)26 b(There)18 b(is)g(no)f (requiremen)o(t)h(that)f(rep)q(eated)h(executions)g(of)f(the)g(same)g (suc)o(h)h(pro-)75 666 y(gram)e(will)i(yield)g(the)f(same)f(result,)h (ev)o(en)g(when)g(rep)q(eated)g(with)g(the)g(same)f(ph)o(ysical)i (con\014guration)75 723 y(|)d(neither)g(user)f(nor)g(MPI)g(can)g(b)q(e) h(exp)q(ected)g(to)f(ha)o(v)o(e)g(full)h(con)o(trol)f(of)g(the)g (execution)h(en)o(vironmen)o(t.)75 779 y(Ho)o(w)o(ev)o(er,)e(rep)q (eatabilit)o(y)j(is)f(a)f(desirable)i(qualit)o(y-of-implemen)o(tation)g (goal.)j(In)c(particular,)g(it)g(is)g(ex-)75 835 y(p)q(ected)i(that)f (MPI)h(implemen)o(tations)g(will)h(a)o(v)o(oid)f(in)o(tro)q(ducing)g (additional)h(non-determinism.)26 b(F)l(or)75 892 y(example,)19 b(while)g(MPI)f(can)g(tak)o(e)f(adv)m(an)o(tage)g(of)g(asso)q(ciativit) o(y)h(and)g(comm)o(utativit)o(y)f(in)i(reordering)75 948 y(the)f(execution)g(of)f(a)h(global)g(reduce)g(op)q(eration,)g(it)g (is)g(desired)h(\(but)e(not)g(required\))i(that)e(the)g(same)75 1005 y(op)q(erations)e(order)g(b)q(e)h(used)g(in)g(rep)q(eated)g (executions.)75 1145 y Fv(2.7)59 b(Implementation)17 b(issues)75 1247 y FE(There)c(are)f(a)g(n)o(um)o(b)q(er)h(of)f(areas)f (where)i(an)f(MPI)h(implemen)o(tation)g(ma)o(y)f(in)o(teract)g(with)h (the)g(op)q(erating)75 1303 y(en)o(vironmen)o(t)k(and)f(system.)23 b(While)18 b(MPI)e(do)q(es)h(not)f(mandate)g(that)f(an)o(y)h(services)h (\(suc)o(h)g(as)f(I/O)g(or)75 1360 y(signal)h(handling\))g(b)q(e)f(pro) o(vided,)g(it)g(do)q(es)g(strongly)g(suggest)f(the)h(b)q(eha)o(vior)g (to)f(b)q(e)h(pro)o(vided)h(if)f(those)75 1416 y(services)g(are)g(a)o (v)m(ailable.)23 b(This)16 b(is)g(an)f(imp)q(ortan)o(t)h(p)q(oin)o(t)g (in)g(ac)o(hieving)h(p)q(ortabilit)o(y)g(across)d(platforms)75 1472 y(that)g(pro)o(vide)i(the)g(same)e(set)h(of)g(services.)75 1591 y Fq(2.7.1)49 b(Indep)q(endence)15 b(of)i(basic)f(runtime)g (routines)75 1677 y FE(MPI)d(programs)e(require)j(that)e(library)i (routines)f(that)f(are)h(part)f(of)g(the)h(basic)h(language)f(en)o (vironmen)o(t)75 1734 y(\(suc)o(h)i(as)g Fp(date)f FE(and)i Fp(write)e FE(in)i(F)l(ortran)e(and)h Fp(printf)f FE(and)h Fp(malloc)g FE(in)h(ANSI)f(C\))g(and)g(are)g(executed)75 1790 y(after)g Fp(MPI)p 258 1790 15 2 v 17 w(INIT)g FE(and)i(b)q(efore) f Fp(MPI)p 685 1790 V 17 w(FINALIZE)f FE(op)q(erate)g(indep)q(enden)o (tly)k(and)e(that)e(their)h Fs(c)n(ompletion)75 1847 y FE(is)g(indep)q(enden)o(t)h(of)e(the)g(action)h(of)e(other)h(pro)q (cesses)h(in)g(an)f(MPI)g(program.)166 1903 y(Note)h(that)g(this)h(in)g (no)g(w)o(a)o(y)e(prev)o(en)o(ts)i(the)f(creation)h(of)f(library)i (routines)f(that)f(pro)o(vide)h(paral-)75 1959 y(lel)j(services)g (whose)f(op)q(eration)g(is)g(collectiv)o(e.)33 b(Ho)o(w)o(ev)o(er,)18 b(the)h(follo)o(wing)g(program)f(is)i(exp)q(ected)g(to)75 2016 y(complete)c(in)g(an)f(ANSI)h(C)f(en)o(vironmen)o(t)h(regardless)f (of)g(the)g(size)h(of)f Fp(MPI)p 1371 2016 V 17 w(COMM)p 1484 2016 V 16 w(WORLD)p FE(:)75 2095 y Fp(int)23 b(rank;)75 2151 y(MPI_Init\()g(argc,)g(argv)g(\);)75 2207 y(MPI_Comm_rank\()f (MPI_COMM_WORLD,)g(&rank)h(\);)75 2264 y(if)h(\(rank)f(==)g(0\))h (printf\()f("Starting)f(program\\n")h(\);)75 2320 y(MPI_Finalize\(\);) 75 2399 y FE(The)15 b(corresp)q(onding)i(F)l(ortran)d(77)g(program)g (is)i(also)f(exp)q(ected)i(to)d(complete.)166 2456 y(As)g(an)h(example) g(of)f(what)g(is)h Fs(not)j FE(required)e(is)f(an)o(y)f(particular)h (ordering)g(of)f(the)g(action)h(of)f(these)75 2512 y(routines)19 b(when)h(called)g(b)o(y)f(sev)o(eral)h(tasks.)30 b(F)l(or)18 b(example,)j(MPI)e(mak)o(es)f(neither)i(requiremen)o(ts)f(or)75 2568 y(recommendations)c(for)g(the)g(output)g(from)75 2647 y Fp(MPI_Comm_rank\()22 b(MPI_COMM_WORLD,)g(&rank)h(\);)75 2704 y(printf\()g("Output)g(from)g(task)g(rank)h(\045d\\n",)f(rank)g (\);)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 15 20 15 19 bop 75 -100 a Ft(2.7.)34 b(IMPLEMENT)l(A)l(TION)16 b(ISSUES)1045 b FE(15)166 45 y(In)18 b(addition,)h(calls)f(that)f(fail) h(b)q(ecause)g(of)f(resource)h(exhaustion)g(or)f(other)g(error)g(are)g (not)g(con-)75 102 y(sidered)h(a)f(violation)h(of)e(the)h(requiremen)o (ts)h(here)f(\(ho)o(w)o(ev)o(er,)f(they)h(are)g(required)h(to)e (complete,)i(just)75 158 y(not)d(to)f(complete)i(successfully\).)75 280 y Fq(2.7.2)49 b(Interaction)16 b(with)g(signals)i(in)f(POSIX)75 366 y FE(The)e(in)o(ten)o(t)g(of)f(MPI)h(is)g(to)f(b)q(e)h(thread)g (and)f(signal)i(safe.)j(This)d(has)e(a)g(n)o(um)o(b)q(er)h(of)g(subtle) g(e\013ects.)20 b(F)l(or)75 422 y(example,)g(on)e(Unix)h(systems,)g(a)f (catc)o(hable)h(signal)g(suc)o(h)g(as)f(SIGALRM)h(\(an)f(alarm)h (signal\))g(m)o(ust)75 478 y(not)f(cause)g(an)h(MPI)f(routine)g(to)g(b) q(eha)o(v)o(e)h(di\013eren)o(tly)g(than)f(it)g(w)o(ould)h(ha)o(v)o(e)f (in)h(the)f(absence)h(of)f(the)75 535 y(signal.)37 b(Of)21 b(course,)h(if)f(the)g(signal)h(handler)f(issues)h(MPI)f(calls)g(or)g (c)o(hanges)f(the)h(en)o(vironmen)o(t)g(in)75 591 y(whic)o(h)c(the)g (MPI)g(routine)g(is)g(op)q(erating)f(\(for)g(example,)h(consuming)h (all)f(a)o(v)m(ailable)h(memory)e(space\),)75 648 y(the)e(MPI)f (routine)h(should)h(b)q(eha)o(v)o(e)f(as)f(appropriate)h(for)f(that)g (situation)h(\(in)g(particular,)g(in)g(this)g(case,)75 704 y(the)h(b)q(eha)o(vior)h(should)g(b)q(e)g(the)f(same)g(as)g(for)g (a)g(m)o(ultithreaded)h(MPI)f(implemen)o(tation\).)166 761 y(A)20 b(second)g(e\013ect)g(is)g(that)g(a)f(signal)i(handler)g (that)e(p)q(erforms)h(MPI)g(calls)h(m)o(ust)e(not)h(in)o(terfere)75 817 y(with)c(the)h(op)q(eration)f(of)g(MPI.)f(F)l(or)h(example,)g(an)h (MPI)f(receiv)o(e)h(of)e(an)o(y)h(t)o(yp)q(e)g(that)g(o)q(ccurs)g (within)h(a)75 874 y(signal)c(handler)f(m)o(ust)f(not)h(cause)g (erroneous)f(b)q(eha)o(vior)i(b)o(y)e(the)h(MPI)g(implemen)o(tation.)20 b(Note)11 b(that)g(an)75 930 y(implemen)o(tation)16 b(is)g(p)q (ermitted)g(to)e(prohibit)i(the)f(use)h(of)e(MPI)h(calls)h(from)e (within)j(a)d(signal)i(handler,)75 987 y(and)f(is)h(not)f(required)h (to)f(detect)g(suc)o(h)h(use.)166 1043 y(It)11 b(is)g(highly)h (desirable)g(that)e(MPI)g(not)g(use)h Fp(SIGALRM)p FE(,)e Fp(SIGFPE)p FE(,)h(and)g Fp(SIGIO)p FE(.)g(An)h(implemen)o(tation)75 1099 y(is)j Fs(r)n(e)n(quir)n(e)n(d)k FE(to)c(clearly)h(do)q(cumen)o(t) f(all)h(of)f(the)g(signals)h(that)e(the)h(MPI)g(implemen)o(tation)h (uses;)f(a)g(go)q(o)q(d)75 1156 y(place)i(for)f(this)g(information)h (is)f(a)g(Unix)h(`)p Fp(man)p FE(')e(page)h(on)h Fp(MPI)p FE(.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 16 21 16 20 bop 75 380 a Fw(Chapter)34 b(3)75 610 y FA(P)m(oin)m(t)41 b(to)f(P)m(oin)m(t)h(Comm)m(unication)75 874 y Fv(3.1)59 b(Intro)r(duction)75 1021 y FE(Sending)13 b(and)f(receiving)h(of)e (messages)f(b)o(y)i(pro)q(cesses)g(is)g(the)f(basic)h(MPI)g(comm)o (unication)g(mec)o(hanism.)75 1077 y(All)20 b(other)f(comm)o(unication) h(functions)g(can)f(b)q(e)h(de\014ned)h(and)e(implemen)o(ted)i(on)e (top)g(of)g(this)g(basic)75 1134 y(comm)o(unication)e(la)o(y)o(er,)f (using)h(few)f(auxiliary)i(data)e(managemen)o(t)f(routines,)i(and)g (using)g(the)f(group)75 1190 y(and)f(comm)o(unicator)g(manaipulation)i (functions.)166 1270 y(The)i(basic)g(p)q(oin)o(t)g(to)f(p)q(oin)o(t)h (comm)o(unication)g(op)q(erations)g(are)f Fz(send)g FE(and)h Fz(receiv)o(e)p FE(.)29 b(A)18 b Fz(send)75 1326 y FE(op)q(eration)13 b(creates)f(and)h(sends)g(a)g(message.)18 b(The)13 b(op)q(eration)g(sp) q(eci\014es)i(a)d Fz(send)i(bu\013er)e FE(in)i(the)e(sender)75 1383 y(memory)f(from)g(whic)o(h)h(the)g(message)f(data)g(is)h(tak)o (en.)18 b(In)12 b(addition,)h(the)f(send)g(op)q(eration)g(asso)q (ciates)f(an)75 1439 y Fz(en)o(v)o(elop)q(e)16 b FE(with)g(the)g (message.)22 b(This)16 b(en)o(v)o(elop)q(e)h(sp)q(eci\014es)h(the)e (message)g(destination)h(and)f(con)o(tains)75 1496 y(distinguishing)h (information)d(that)f(can)i(b)q(e)g(used)f(b)o(y)g(the)g Fz(receiv)o(e)g FE(op)q(eration)g(to)g(select)h(a)e(particular)75 1552 y(message.)166 1632 y(A)22 b Fz(receiv)o(e)f FE(op)q(eration)h (consumes)g(a)f(message.)39 b(The)22 b(message)f(to)g(b)q(e)i(receiv)o (ed)f(is)h(selected)75 1688 y(according)13 b(to)f(the)g(v)m(alue)i(on)e (its)g(en)o(v)o(elop)q(e,)i(and)f(the)f(message)g(data)g(is)h(put)f(in) o(to)g(the)h Fz(receiv)o(e)g(bu\013er)p FE(.)166 1768 y(The)f(next)g(sections)h(describ)q(e)g(the)g(basic)f(\(blo)q(c)o (king\))h(send)g(and)f(receiv)o(e)h(op)q(erations.)19 b(W)l(e)12 b(discuss)75 1824 y(send,)21 b(receiv)o(e,)g(basic)f(comm)o (unication)g(seman)o(tics,)h(t)o(yp)q(e)e(matc)o(hing)h(requiremen)o (ts,)h(t)o(yp)q(e)e(con)o(v)o(er-)75 1881 y(sion)14 b(in)h (heterogeneous)f(en)o(vironmen)o(ts,)f(and)h(more)g(general)g(comm)o (unication)g(mo)q(des.)20 b(Non)o(blo)q(c)o(king)75 1937 y(comm)o(unication)c(is)h(addressed)f(next,)f(follo)o(w)o(ed)h(b)o(y)g (c)o(hannel-lik)o(e)i(constructs)e(and)f(send-receiv)o(e)j(op-)75 1994 y(erations.)i(W)l(e)15 b(then)g(consider)h(general)f(datat)o(yp)q (es)f(that)g(allo)o(w)h(one)g(to)g(transfer)f(heterogeneous)h(and)75 2050 y(noncon)o(tiguous)f(data,)f(and)h(conclude)h(with)f(a)g (description)h(of)e(an)h(implemen)o(tation)h(of)e(MPI)g(p)q(oin)o(t)i (to)75 2106 y(p)q(oin)o(t)h(comm)o(unication)g(using)g(a)e(small)i(n)o (um)o(b)q(er)g(of)f(primitiv)o(es.)75 2382 y Fv(3.2)59 b(Basic)19 b(send)h(op)r(eration)75 2529 y FE(The)15 b(syn)o(tax)g(of)g(the)g(simplest)h(send)g(op)q(eration)f(is)h(giv)o (en)g(b)q(elo)o(w.)p 75 2661 720 2 v 127 2688 a FB(1)144 2704 y Fu(V)m(ersion)e(of)f(Oct)f(14,)h(1993)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 17 22 17 21 bop 75 -100 a Ft(3.2.)34 b(BASIC)16 b(SEND)f(OPERA)l(TION)1075 b FE(17)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(SEND\(buf,)16 b(count,)g(datat)o(yp)q(e,)g(dest,)g(tag,)f(comm\))117 124 y Fr(IN)155 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 204 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)g(in)g(send)h(bu\013er)h (\(nonnegativ)o(e)e(in)o(te-)905 260 y(ger\))117 339 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)117 419 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o (teger\))117 498 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 577 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))75 704 y Fp(int)23 b(MPI)p 245 704 15 2 v 17 w(Send\(void*)g(buf,)g(int)g(count,)h(MPI)p 979 704 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 760 y(int)h(tag,)f(MPI)p 683 760 V 17 w(Comm)g(comm\))75 849 y(MPI)p 150 849 V 17 w(SEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)h(COMM,)f(IERROR\))170 905 y()g(BUF\(*\))170 962 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 1126 y Fq(3.2.1)49 b(Message)16 b(data)75 1215 y FE(The)e(send)g (bu\013er)g(sp)q(eci\014ed)i(b)o(y)e(the)f Fx(MPI)p 803 1215 14 2 v 16 w(SEND)h FE(op)q(eration)g(consists)g(of)f Fx(count)i FE(successiv)o(e)g(en)o(tries)f(of)75 1272 y(the)f(t)o(yp)q(e)h(indicated)h(b)o(y)e Fx(datat)o(yp)q(e)p FE(,)i(starting)e(with)h(the)f(en)o(try)g(at)g(address)h Fx(buf)p FE(.)20 b(Note)13 b(that)g(w)o(e)g(sp)q(ecify)75 1328 y(the)19 b(message)f(length)i(in)g(terms)e(of)g(n)o(um)o(b)q(er)h (of)g Fs(elements)p FE(,)f(not)g(n)o(um)o(b)q(er)i(of)e Fs(bytes)p FE(.)30 b(The)19 b(former)f(is)75 1385 y(mac)o(hine)e(indep) q(enden)o(t)i(and)d(closer)h(to)e(the)h(application)i(lev)o(el.)166 1443 y(The)g(data)f(part)g(of)g(the)g(message)h(consists)f(of)h(a)f (sequence)i(of)e Fx(count)i FE(v)m(alues,)f(eac)o(h)g(of)f(the)h(t)o (yp)q(e)75 1500 y(indicated)j(b)o(y)e Fx(datat)o(yp)q(e)p FE(.)31 b Fx(count)20 b FE(ma)o(y)d(b)q(e)i(zero,)g(in)g(whic)o(h)g (case)g(the)f(data)g(part)f(of)h(the)h(message)e(is)75 1556 y(empt)o(y)l(.)30 b(The)18 b(basic)h(datat)o(yp)q(es)f(that)g(can) h(b)q(e)g(sp)q(eci\014ed)h(for)e(message)g(data)g(v)m(alues)i(corresp)q (ond)f(to)75 1613 y(the)14 b(basic)h(datat)o(yp)q(es)f(of)f(the)i(host) e(language.)20 b(The)15 b(p)q(ossible)g(v)m(alues)h(of)e(this)g (parameter)g(for)f(F)l(ortran)75 1669 y(and)i(the)h(corresp)q(onding)g (F)l(ortran)e(t)o(yp)q(es)h(are)g(listed)h(b)q(elo)o(w)p 472 1844 1006 2 v 471 1900 2 57 v 497 1884 a(MPI)f(datat)o(yp)q(e)p 1046 1900 V 294 w(F)l(ortran)f(datat)o(yp)q(e)p 1477 1900 V 472 1902 1006 2 v 471 1959 2 57 v 497 1942 a Fx(MPI)p 582 1942 14 2 v 16 w(INTEGER)p 1046 1959 2 57 v 287 w Fp(INTEGER)p 1477 1959 V 471 2015 V 497 1998 a Fx(MPI)p 582 1998 14 2 v 16 w(REAL)p 1046 2015 2 57 v 365 w Fp(REAL)p 1477 2015 V 471 2072 V 497 2055 a Fx(MPI)p 582 2055 14 2 v 16 w(DOUBLE)p 777 2055 V 17 w(PRECISION)p 1046 2072 2 57 v 50 w Fp(DOUBLE)23 b(PRECISION)p 1477 2072 V 471 2128 V 497 2111 a Fx(MPI)p 582 2111 14 2 v 16 w(COMPLEX)p 1046 2128 2 57 v 263 w Fp(COMPLEX)p 1477 2128 V 471 2184 V 497 2167 a Fx(MPI)p 582 2167 14 2 v 16 w(DOUBLE)p 777 2167 V 17 w(COMPLEX)p 1046 2184 2 57 v 67 w Fp(DOUBLE)g(COMPLEX)p 1477 2184 V 471 2241 V 497 2224 a Fx(MPI)p 582 2224 14 2 v 16 w(LOGICAL)p 1046 2241 2 57 v 291 w Fp(LOGICAL)p 1477 2241 V 471 2297 V 497 2280 a Fx(MPI)p 582 2280 14 2 v 16 w(CHARA)o(CTER)p 1046 2297 2 57 v 211 w Fp(CHARACTER\(1\))p 1477 2297 V 471 2354 V 497 2337 a Fx(MPI)p 582 2337 14 2 v 16 w(BYTE)p 1046 2354 2 57 v 1477 2354 V 472 2355 1006 2 v 75 2549 a FE(The)13 b(p)q(ossible)h(v)m(alues)f(for)f(this)g (parameter)g(for)g(C)g(and)g(the)h(corresp)q(onding)g(C)f(t)o(yp)q(es)g (are)g(listed)i(b)q(elo)o(w.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 18 23 18 22 bop 75 -100 a FE(18)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)p 466 5 1019 2 v 465 62 2 57 v 491 45 a FE(MPI)f(datat)o(yp)q(e)p 1004 62 V 259 w(C)g(datat)o(yp)q(e)p 1483 62 V 466 63 1019 2 v 465 120 2 57 v 491 103 a Fx(MPI)p 576 103 14 2 v 16 w(CHAR)p 1004 120 2 57 v 321 w Fp(signed)23 b(char)p 1483 120 V 465 176 V 491 159 a Fx(MPI)p 576 159 14 2 v 16 w(SHORT)p 1004 176 2 57 v 291 w Fp(signed)g(short)p 1483 176 V 465 233 V 491 216 a Fx(MPI)p 576 216 14 2 v 16 w(INT)p 1004 233 2 57 v 365 w Fp(signed)g(int)p 1483 233 V 465 289 V 491 272 a Fx(MPI)p 576 272 14 2 v 16 w(LONG)p 1004 289 2 57 v 321 w Fp(signed)g(long)g(int)p 1483 289 V 465 346 V 491 329 a Fx(MPI)p 576 329 14 2 v 16 w(UNSIGNED)p 815 329 V 17 w(CHAR)p 1004 346 2 57 v 81 w Fp(unsigned)f(char)p 1483 346 V 465 402 V 491 385 a Fx(MPI)p 576 385 14 2 v 16 w(UNSIGNED)p 815 385 V 17 w(SHORT)p 1004 402 2 57 v 51 w Fp(unsigned)g(short)i(int)p 1483 402 V 465 459 V 491 442 a Fx(MPI)p 576 442 14 2 v 16 w(UNSIGNED)p 1004 459 2 57 v 218 w Fp(unsigned)e(int)p 1483 459 V 465 515 V 491 498 a Fx(MPI)p 576 498 14 2 v 16 w(UNSIGNED)p 815 498 V 17 w(LONG)p 1004 515 2 57 v 81 w Fp(unsigned)g(long)i(int)p 1483 515 V 465 571 V 491 555 a Fx(MPI)p 576 555 14 2 v 16 w(FLO)o(A)l(T)p 1004 571 2 57 v 301 w Fp(float)p 1483 571 V 465 628 V 491 611 a Fx(MPI)p 576 611 14 2 v 16 w(DOUBLE)p 1004 628 2 57 v 262 w Fp(double)p 1483 628 V 465 684 V 491 667 a Fx(MPI)p 576 667 14 2 v 16 w(LONG)p 712 667 V 16 w(DOUBLE)p 1004 684 2 57 v 126 w Fp(long)f(double)p 1483 684 V 465 741 V 491 724 a Fx(MPI)p 576 724 14 2 v 16 w(BYTE)p 1004 741 2 57 v 1483 741 V 466 742 1019 2 v 166 925 a FE(The)16 b(datat)o(yp)q(e)f Fx(MPI)p 536 925 14 2 v 15 w(BYTE)i FE(do)q(es)e(not)g(corresp)q(ond)h(to)f(a)g(F)l (ortran)g(or)g(C)g(datat)o(yp)q(e.)20 b(A)c(v)m(alue)g(of)75 981 y(t)o(yp)q(e)i Fx(MPI)p 265 981 V 15 w(BYTE)h FE(consists)f(of)f(8) g(binary)h(digits.)28 b(A)18 b(b)o(yte)f(is)h(unin)o(terpreted)h(and)f (is)g(di\013eren)o(t)g(from)75 1038 y(a)f(c)o(haracter.)25 b(Di\013eren)o(t)17 b(mac)o(hines)g(ma)o(y)g(ha)o(v)o(e)g(di\013eren)o (t)g(represen)o(tations)g(for)g(c)o(haracters,)f(or)h(ma)o(y)75 1094 y(use)g(more)g(than)g(one)g(b)o(yte)g(to)g(represen)o(t)g(c)o (haracters.)25 b(On)18 b(the)f(other)g(hand,)g(a)g(b)o(yte)g(has)g(the) g(same)75 1150 y(binary)f(v)m(alue)g(on)f(all)h(mac)o(hines.)166 1210 y(MPI)11 b(supp)q(orts)g(the)f(basic)i(datat)o(yp)q(es)e(of)g(F)l (ortran)g(77)g(and)h(ANSI)h(C.)e(Additional)j(MPI)e(datat)o(yp)q(es)75 1266 y(should)32 b(b)q(e)g(pro)o(vided)f(if)h(the)f(host)f(language)h (has)g(additional)i(data)d(t)o(yp)q(es:)51 b(F)l(or)30 b(example,)75 1323 y Fx(MPI)p 160 1323 V 16 w(LONG)p 296 1323 V 17 w(LONG)p 433 1323 V 17 w(INT)p FE(,)16 b(for)g(64)g(bit)h(in)o(tegers)g(in)g(C;)f Fx(MPI)p 1128 1323 V 16 w(REAL2)p FE(,)h Fx(MPI)p 1390 1323 V 15 w(REAL4)g FE(and)g Fx(MPI)p 1728 1323 V 16 w(REAL8)75 1379 y FE(for)d(2,4)g(or)g (8)g(b)o(yte)h(reals,)f(in)i(F)l(ortran;)d Fx(MPI)p 843 1379 V 16 w(INTEGER1)i(MPI)p 1168 1379 V 16 w(INTEGER2)g FE(and)g Fx(MPI)p 1581 1379 V 16 w(INTEGER4)g FE(for)75 1436 y(1,)g(2)f(or)h(4)g(b)o(yte)g(in)o(tegers)g(in)h(F)l(ortran;)e (etc.)75 1573 y Fq(3.2.2)49 b(Message)16 b(envelop)q(e)75 1665 y FE(In)g(addition)g(to)e(the)h(data)g(part,)f(messages)g(con)o (tain)h(information)h(that)e(can)h(b)q(e)h(used)f(to)g(distinguish)75 1721 y(messages)h(and)g(selectiv)o(ely)i(receiv)o(e)f(them.)23 b(This)17 b(information)f(is)h(con)o(tained)g(in)g(a)f(\014xed)h(n)o (um)o(b)q(er)f(of)75 1778 y(\014xed-format)f(\014elds,)h(whic)o(h)g(w)o (e)f(collectiv)o(ely)i(call)g(the)e Fz(message)i(en)o(v)o(elop)q(e)p FE(.)i(These)d(\014elds)g(are)75 1883 y Fz(source)75 1988 y(destination)75 2093 y(tag)75 2198 y(con)o(text)166 2303 y FE(The)c(message)g(source)g(is)h(implicitly)i(determined)f(b)o (y)e(the)g(iden)o(tit)o(y)h(of)f(the)g(message)g(sender.)20 b(The)75 2359 y(other)15 b(\014elds)h(are)f(sp)q(eci\014ed)i(b)o(y)f (parameters)e(in)i(the)f(send)h(op)q(eration.)166 2419 y(The)k(in)o(teger-v)m(alued)i(message)d(tag)g(is)i(sp)q(eci\014ed)h(b) o(y)d(the)h Fx(tag)g FE(parameter.)33 b(This)21 b(in)o(teger)f(can)75 2475 y(b)q(e)d(used)g(b)o(y)g(the)f(program)f(to)h(distinguish)j (di\013eren)o(t)d(t)o(yp)q(es)h(of)f(messages.)23 b(The)17 b(range)f(of)g(v)m(alid)i(tag)75 2531 y(v)m(alues)e(is)g(implemen)o (tation)g(dep)q(enden)o(t)g(and)f(can)g(b)q(e)h(queried)g(using)g(the)f Fx(T)l(agRange)g FE(en)o(vironmen)o(tal)75 2588 y(inquiry)i(function,)e (as)g(describ)q(ed)i(in)f(Chapter)f(7.)166 2647 y(The)20 b(con)o(text)f(of)g(the)g(message)h(sen)o(t)f(is)h(sp)q(eci\014ed)i(b)o (y)d(the)h Fx(comm)d FE(parameter.)33 b(The)19 b(message)75 2704 y(carries)c(the)h(con)o(text)e(asso)q(ciated)i(with)f(this)h(comm) o(unicator)f(\(see)g(Chapter)g(5\).)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 19 24 19 23 bop 75 -100 a Ft(3.3.)34 b(BASIC)16 b(RECEIVE)g(OPERA)l(TION)990 b FE(19)166 45 y(The)13 b(message)f(destination)h(is)h(sp)q(eci\014ed)g (b)o(y)f(the)f Fx(dest)j FE(parameter)c(as)i(a)f(rank)g(within)i(the)f (pro)q(cess)75 102 y(group)j(asso)q(ciated)h(with)g(that)f(same)h(comm) o(unicator)f(\(remote)g(pro)q(cess)h(group,)f(for)g(in)o(tercomm)o (uni-)75 158 y(cators\).)i(The)d(range)f(of)f(v)m(alid)j(v)m(alues)g (is)e Fp(0,)24 b(...)47 b(,)24 b(n-1)p FE(,)14 b(where)g Fp(n)g FE(is)h(the)f(n)o(um)o(b)q(er)h(of)f(pro)q(cesses)g(in)75 214 y(this)k(group.)25 b(Th)o(us,)17 b(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unications)g(do)f(not)g(use)g(absolute)h(addresses,)f(but)g (only)75 271 y(relativ)o(e)f(ranks)f(within)h(a)f(group.)k(This)d(pro)o (vides)g(imp)q(ortan)o(t)f(mo)q(dularit)o(y)l(.)166 330 y(The)j(message)f(en)o(v)o(elop)q(e)i(w)o(ould)f(normally)g(b)q(e)h (enco)q(ded)g(b)o(y)e(a)g(\014xed-length)j(message)d(header.)75 387 y(Ho)o(w)o(ev)o(er,)c(the)i(actual)f(mec)o(hanism)h(used)g(to)f (asso)q(ciate)g(an)g(en)o(v)o(elop)q(e)i(with)e(a)g(message)g(is)h (implemen-)75 443 y(tation)d(dep)q(enden)o(t:)20 b(some)12 b(of)g(the)g(information)g(\(e.g.,)g(source)g(or)g(destination\))g(ma)o (y)g(b)q(e)h(implicit,)i(and)75 499 y(need)f(not)g(b)q(e)g(explicitly)i (carried)e(b)o(y)g(messages;)f(pro)q(cesses)h(ma)o(y)f(b)q(e)h(iden)o (ti\014ed)i(b)o(y)d(relativ)o(e)h(ranks,)f(or)75 556 y(absolute)j(ids;)f(etc.)75 715 y Fv(3.3)59 b(Basic)19 b(receive)g(op)r(eration)75 822 y FE(The)c(syn)o(tax)g(of)g(the)g (simplest)h(receiv)o(e)g(op)q(eration)g(is)f(giv)o(en)h(b)q(elo)o(w.)75 928 y Fx(MPI)p 160 928 14 2 v 16 w(RECV)g(\(buf,)f(count,)h(datat)o(yp) q(e,)h(source,)e(tag,)g(comm,)e(status\))117 1008 y Fr(OUT)108 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g (\(c)o(hoice\))117 1089 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1169 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen)o(t)e(\(state\))117 1250 y(IN)155 b Fx(source)468 b Fr(rank)14 b(of)f(source)j(\(in)o (teger\))117 1331 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 1411 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 1492 y(OUT)108 b Fx(status)476 b Fr(status)15 b(ob)r(ject)75 1619 y Fp(int)23 b(MPI)p 245 1619 15 2 v 17 w(Recv\(void*)g(buf,)g(int)g (count,)h(MPI)p 979 1619 V 16 w(Datatype)f(datatype,)g(int)g(source,) 393 1676 y(int)h(tag,)f(MPI)p 683 1676 V 17 w(Comm)g(comm,)g(MPI)p 1034 1676 V 17 w(Status)g(*status\))75 1765 y(MPI)p 150 1765 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS,)g(IERROR\))170 1821 y()g(BUF\(*\))170 1878 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS\(MPI)p 1463 1878 V 16 w(STATUS)p 1623 1878 V 17 w(SIZE\),)170 1934 y(IERROR)166 2023 y FE(The)15 b(receiv)o(e)h (bu\013er)f(consists)h(of)e(the)h(storage)f(con)o(taining)i Fx(count)g FE(consecutiv)o(e)g(elemen)o(ts)g(of)f(the)75 2080 y(t)o(yp)q(e)21 b(sp)q(eci\014ed)h(b)o(y)f Fx(datat)o(yp)q(e)p FE(,)i(starting)d(at)g(address)g Fx(buf)p FE(.)37 b(The)21 b(length)g(of)f(the)h(receiv)o(ed)g(message)75 2136 y(m)o(ust)15 b(b)q(e)i(less)g(then)f(or)f(equal)i(to)e(the)h(length)h(of)f(the)g (receiv)o(e)g(bu\013er.)23 b(I.e.,)15 b(all)i(incoming)g(data)f(m)o (ust)75 2193 y(\014t,)23 b(without)e(truncation,)i(in)o(to)e(the)h (receiv)o(e)g(bu\013er.)39 b(The)22 b Fx(MPI)p 1262 2193 14 2 v 16 w(PROBE)g FE(function)h(describ)q(ed)g(in)75 2249 y(section)16 b(3.9)e(can)h(b)q(e)h(used)g(to)f(receiv)o(e)h (messages)e(of)h(unkno)o(wn)h(length.)166 2308 y(The)j(selection)h(of)f (a)f(message)h(b)o(y)f(a)h(receiv)o(e)h(op)q(eration)f(is)g(done)g (uniquely)i(according)f(to)e(the)75 2365 y(v)m(alue)i(of)f(the)g (message)g(en)o(v)o(elop)q(e.)32 b(The)19 b(receiv)o(e)h(op)q(eration)g (sp)q(eci\014es)h(an)d Fz(en)o(v)o(elop)q(e)k(pattern)p FE(;)f(a)75 2421 y(message)13 b(can)g(b)q(e)g(receiv)o(ed)h(b)o(y)f (that)g(receiv)o(e)h(op)q(eration)f(only)g(if)h(its)f(en)o(v)o(elop)q (e)h(matc)o(hes)f(that)f(pattern.)75 2478 y(A)19 b(pattern)g(sp)q (eci\014es)i(v)m(alues)g(for)e(the)g Fx(source)p FE(,)i Fx(tag)e FE(and)h Fx(context)g FE(\014elds)h(of)e(the)g(message)g(en)o (v)o(elop)q(e.)75 2534 y(The)d(receiv)o(er)h(ma)o(y)e(sp)q(ecify)j(a)e (wildcard)h Fm(MPI)p 889 2534 13 2 v 14 w(ANY)p 988 2534 V 15 w(SOURCE)d FE(v)m(alue)k(for)d Fx(source)p FE(,)h(and/or)g(a)g (wildcard)75 2591 y Fm(MPI)p 152 2591 V 14 w(ANY)p 251 2591 V 15 w(T)m(A)o(G)c FE(v)m(alue)j(for)d Fx(tag)p FE(,)i(indicating)h(that)d(an)o(y)h(source)h(and/or)e(tag)h(are)g (acceptable.)20 b(It)13 b(cannot)75 2647 y(sp)q(ecify)h(a)f(wildcard)h (v)m(alue)g(for)f Fx(context)p FE(.)20 b(Th)o(us,)13 b(a)g(message)f(can)i(b)q(e)f(receiv)o(ed)h(b)o(y)f(a)g(receiv)o(e)h (op)q(eration)75 2704 y(only)23 b(if)g(it)g(is)g(addressed)h(to)e(the)g (receiving)j(pro)q(cess,)f(has)f(a)f(matc)o(hing)h(con)o(text,)g(has)g (matc)o(hing)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 20 25 20 24 bop 75 -100 a FE(20)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(source)e(unless)g (source=)p Fm(MPI)p 581 45 13 2 v 15 w(ANY)p 681 45 V 15 w(SOURCE)d FE(in)k(the)e(pattern,)g(and)h(has)g(a)f(matc)o(hing)g (tag)g(unless)i(tag=)75 102 y Fm(MPI)p 152 102 V 14 w(ANY)p 251 102 V 15 w(T)m(A)o(G)f FE(in)i(the)f(pattern.)166 159 y(The)c(message)g(tag)f(is)i(sp)q(eci\014ed)h(b)o(y)e(the)g Fx(tag)g FE(parameter)f(of)h(the)g(receiv)o(e)h(op)q(eration.)19 b(The)11 b(message)75 215 y Fx(context)19 b FE(is)e(the)h(con)o(text)e (attac)o(hed)h(with)h(the)f(comm)o(unicator)g(sp)q(eci\014ed)i(b)o(y)e (the)h(parameter)e Fx(comm)p FE(.)75 272 y(The)h(message)g(source,)h (if)f(di\013eren)o(t)h(from)32 b Fm(MPI)p 929 272 V 14 w(ANY)p 1028 272 V 15 w(SOURCE)p FE(,)15 b(is)j(sp)q(eci\014ed)h(as)e (a)g(rank)f(within)j(the)75 328 y(pro)q(cess)g(group)f(asso)q(ciated)h (with)g(that)e(same)h(comm)o(unicator)h(\(remote)e(pro)q(cess)i(group,) g(for)f(in)o(ter-)75 385 y(comm)o(unicators\).)k(Th)o(us,)17 b(the)f(range)g(of)g(v)m(alid)i(v)m(alues)f(for)f(the)g Fx(source)h FE(parameter)f(is)h FC(f)23 b Fp(0,)h(...)47 b(,)75 441 y(n-1)23 b FC(g[f)h Fp(MPI)p 345 441 15 2 v 17 w(ANY)p 434 441 V 17 w(SOURCE)f FC(g)p FE(,)14 b(where)i Fp(n)f FE(is)g(the)h(n)o(um)o(b)q(er)f(of)g(pro)q(cesses)h(in)g(this)f (group.)166 498 y(Note)j(the)i(asymmetry)d(b)q(et)o(w)o(een)j(send)f (and)g(receiv)o(e)h(op)q(erations:)28 b(A)19 b(receiv)o(e)h(op)q (eration)f(ma)o(y)75 555 y(accept)13 b(messages)f(from)g(an)h (arbitrary)f(sender;)i(on)f(the)g(other)f(hand,)i(a)e(send)h(op)q (eration)g(m)o(ust)g(sp)q(ecify)75 611 y(a)i(unique)j(receiv)o(er.)k (This)16 b(matc)o(hes)f(a)h(\\push")f(comm)o(unication)i(mec)o(hanism,) f(where)g(data)f(transfer)75 668 y(is)i(e\013ected)h(b)o(y)f(the)g (sender)g(\(rather)f(than)h(a)g(\\pull")h(mec)o(hanism,)g(where)f(data) f(transfer)g(is)i(e\013ected)75 724 y(b)o(y)d(the)g(receiv)o(er\).)166 781 y(Source)h(=)f(destination)h(is)g(allo)o(w)o(ed:)k(a)15 b(pro)q(cess)g(can)h(send)g(a)e(message)h(to)g(itself.)75 908 y Fq(3.3.1)49 b(Return)16 b(status)75 995 y FE(The)c(source)h(or)e (the)i(tag)e(of)h(a)g(receiv)o(ed)h(message)f(ma)o(y)g(not)f(b)q(e)i (kno)o(wn)f(if)h(wildcard)g(v)m(alues)h(where)e(used)75 1051 y(in)i(the)f(receiv)o(e)h(op)q(eration.)19 b(Also,)13 b(the)g(actual)g(length)h(of)f(the)g(message)f(receiv)o(ed)i(ma)o(y)e (not)h(b)q(e)h(kno)o(wn.)75 1108 y(Th)o(us,)21 b(this)g(information)g (needs)g(to)f(b)q(e)h(returned)g(b)o(y)f(the)h(receiv)o(e)g(op)q (eration.)36 b(The)21 b(information)75 1164 y(is)e(returned)f(b)o(y)g (the)g Fx(status)i FE(parameter)e(of)f(the)h Fx(MPI)p 1030 1164 14 2 v 16 w(RECV)h FE(function.)29 b(This)19 b(is)g(a)e(parameter)h(of)f(a)75 1221 y(sp)q(ecial)d(MPI-de\014ned)g(t) o(yp)q(e.)19 b(Status)13 b(v)m(ariables)g(need)h(to)e(b)q(e)h (explicitly)i(allo)q(cated)f(b)o(y)e(the)h(user)g({)f(they)75 1277 y(are)j(not)g(system)f(ob)s(jects.)166 1334 y(The)g(status)f(v)m (ariable)i(can)f(b)q(e)g(\\deco)q(ded")h(to)e(retriev)o(e)h(the)g Fx(count)p FE(,)g Fx(source)h FE(and)f Fx(tag)f FE(\014elds,)i(using)75 1391 y(the)g(query)g(functions)h(listed)g(b)q(elo)o(w.)k(The)15 b(information)g(returned)h(b)o(y)f(these)g(query)g(functions)g(is)h (the)75 1447 y(information)h(last)g(stored)g(in)h(the)f Fx(status)i FE(v)m(ariable)f(b)o(y)f(a)g(receiv)o(e)h(function.)26 b(It)17 b(is)h(erroneous)e(to)h(call)75 1504 y(these)e(query)h (functions)g(if)f(the)h Fx(status)h FE(v)m(ariable)f(w)o(as)f(nev)o(er) g(set)g(b)o(y)g(a)g(receiv)o(e.)75 1608 y Fx(MPI)p 160 1608 V 16 w(GET)p 264 1608 V 17 w(SOURCE\(status,)j(source\))117 1686 y Fr(IN)155 b Fx(status)476 b Fr(return)15 b(status)g(of)e(receiv) o(e)i(op)q(eration)f(\(Status\))117 1763 y(OUT)108 b Fx(source)468 b Fr(source)15 b(rank)f(\(in)o(teger\))75 1888 y Fp(int)23 b(MPI)p 245 1888 15 2 v 17 w(Get)p 334 1888 V 17 w(source\(MPI)p 591 1888 V 16 w(Status)g(status,)g(int)g (*source\))75 1975 y(MPI)p 150 1975 V 17 w(GET)p 239 1975 V 17 w(SOURCE\(STATUS,)e(SOURCE,)i(IERROR\))170 2032 y(INTEGER)g(STATUS\(MPI)p 604 2032 V 16 w(STATUS)p 764 2032 V 17 w(SIZE\),)g(SOURCE,)f(IERROR)166 2119 y FE(Returns)13 b(the)g(rank)g(of)f(the)h(message)f(source)h(in)h(the)f (group)f(asso)q(ciated)h(with)g(the)g(comm)o(unicator)75 2176 y(that)h(w)o(as)h(used)h(to)e(receiv)o(e)i(\(remote)f(group,)f (for)h(in)o(tercomm)o(unicators\).)75 2280 y Fx(MPI)p 160 2280 14 2 v 16 w(GET)p 264 2280 V 17 w(T)l(A)o(G\(status,)i(tag\)) 117 2358 y Fr(IN)155 b Fx(status)476 b Fr(return)15 b(status)g(of)e (receiv)o(e)i(op)q(eration)f(\(Status\))117 2435 y(OUT)108 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))75 2560 y Fp(int)23 b(MPI)p 245 2560 15 2 v 17 w(Get)p 334 2560 V 17 w(tag\(MPI)p 519 2560 V 16 w(Status)g(status,)g(int)h(*tag\)) 75 2647 y(MPI)p 150 2647 V 17 w(GET)p 239 2647 V 17 w(TAG\(STATUS,)e (TAG,)h(IERROR\))170 2704 y(INTEGER)g(STATUS\(MPI)p 604 2704 V 16 w(STATUS)p 764 2704 V 17 w(SIZE\),)g(TAG,)g(IERROR)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 21 26 21 25 bop 75 -100 a Ft(3.4.)34 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)450 b FE(21)166 45 y(Returns)16 b(the)f(tag)f(of)h (receiv)o(ed)h(message.)75 152 y Fx(MPI)p 160 152 14 2 v 16 w(GET)p 264 152 V 17 w(COUNT\(status,)h(datat)o(yp)q(e,)f (count\))117 233 y Fr(IN)155 b Fx(status)476 b Fr(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 315 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen)o(t)117 397 y(OUT)108 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(elemen)o(ts)f(\(in)o(teger\))75 525 y Fp(int)23 b(MPI)p 245 525 15 2 v 17 w(Get)p 334 525 V 17 w(count\(MPI)p 567 525 V 16 w(Status)g(status,)g(MPI)p 1013 525 V 17 w(Datatype)f(datatype,)h(int)h(*count\))75 615 y(MPI)p 150 615 V 17 w(GET)p 239 615 V 17 w(COUNT\(STATUS,)e (DATATYPE,)g(COUNT,)h(IERROR\))170 671 y(INTEGER)g(STATUS\(MPI)p 604 671 V 16 w(STATUS)p 764 671 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 761 y FE(Returns)15 b(the)f(n)o(um)o(b)q(er)h(of)f(elemen)o (ts)h(receiv)o(ed.)21 b(\(Here,)14 b(again,)h(w)o(e)f(coun)o(t)g Fs(elements)p FE(,)f(not)h Fs(bytes)p FE(.\))75 818 y(The)19 b Fx(datat)o(yp)q(e)h FE(argumen)o(t)e(should)i(matc)o(h)e(the)g (argumen)o(t)g(pro)o(vided)i(b)o(y)e(the)h(receiv)o(e)g(call)h(that)e (set)75 874 y(the)d Fx(status)i FE(v)m(ariable.)166 934 y(Note)e(that)g(it)h(is)g(not)f(mandatory)g(to)g(query)g(the)h(return)g (status)e(after)h(a)g(receiv)o(e.)22 b(The)16 b(receiv)o(er)75 991 y(will)h(use)e(these)h(calls,)g(or)e(some)h(of)g(them,)g(only)h (when)f(the)h(information)f(they)g(return)g(is)h(needed.)166 1127 y Fo(Implemen)o(tati)o(on)c(note:)166 1187 y Fr(One)j(exp)q(ects)g (deco)q(de)h(functions)e(to)f(b)q(e)i(in-lined)e(in)g(man)o(y)f (implemen)o(tations.)166 1329 y FE(All)19 b(send)g(and)g(receiv)o(e)g (op)q(erations)f(use)h(the)f Fx(buf)p FE(,)h Fx(count)p FE(,)h Fx(datat)o(yp)q(e)p FE(,)g Fx(source)p FE(,)f Fx(dest)p FE(,)h Fx(tag)p FE(,)f Fx(comm)75 1386 y FE(and)d Fx(status)i FE(parameters)d(in)i(the)f(same)f(w)o(a)o(y)g(as)h(the)g (basic)g Fx(MPI)p 1204 1386 14 2 v 16 w(SEND)g FE(and)g Fx(MPI)p 1524 1386 V 16 w(RECV)h FE(op)q(erations)75 1442 y(describ)q(ed)g(in)f(this)g(section.)75 1605 y Fv(3.4)59 b(Semantics)18 b(of)i(p)r(oint-to-p)r(oint)e(communication)75 1713 y FE(A)g(v)m(alid)h(MPI)e(implemen)o(tation)i(guaran)o(tees)e (certain)h(general)g(prop)q(erties)g(of)f(p)q(oin)o(t-to-p)q(oin)o(t)h (com-)75 1770 y(m)o(unication,)e(whic)o(h)g(are)f(describ)q(ed)i(in)f (this)f(section.)75 1910 y Fx(Order)46 b FE(Messages)14 b(are)h Fs(non-overtaking)p FE(,)g(within)h(eac)o(h)f(con)o(text:)20 b(if)c(t)o(w)o(o)d(messages)i(are)g(successiv)o(ely)75 1966 y(sen)o(t)10 b(from)g(the)g(same)g(source,)h(to)e(the)i(same)f (destination,)h(with)g(the)f(same)g(con)o(text,)h(then)f(the)g (messages)75 2023 y(are)17 b(receiv)o(ed)i(in)f(the)f(order)g(they)h(w) o(ere)f(sen)o(t.)26 b(I.e.,)17 b(if)h(the)f(receiv)o(er)i(p)q(osts)e(t) o(w)o(o)f(successiv)o(e)i(receiv)o(es)75 2079 y(that)g(b)q(oth)g(matc)o (h)g(either)i(message,)e(then)h(the)f(\014rst)h(receiv)o(e)g(will)h (receiv)o(e)g(the)e(\014rst)g(message,)h(and)75 2136 y(the)e(second)h(receiv)o(e)h(will)g(receiv)o(e)f(the)f(second)h (message.)26 b(This)18 b(requiremen)o(t)g(facilitates)g(matc)o(hing)75 2192 y(of)e(sends)h(to)f(receiv)o(es.)25 b(It)16 b(guaran)o(tees)g (that)g(message-passing)g(co)q(de)h(is)g(deterministic,)i(if)e(pro)q (cesses)75 2248 y(are)e(single-threaded)i(and)e(wildcard)i Fm(MPI)p 817 2248 13 2 v 14 w(ANY)p 916 2248 V 15 w(SOURCE)c FE(is)j(not)f(used)h(in)g(receiv)o(es.)166 2308 y(If)h(a)g(pro)q(cess)h (has)f(a)g(single)h(thread)f(of)g(execution,)i(then)e(an)o(y)g(t)o(w)o (o)f(comm)o(unications)i(executed)75 2365 y(b)o(y)e(this)h(pro)q(cess)g (are)f(ordered.)23 b(On)17 b(the)f(other)g(hand,)h(if)g(the)f(pro)q (cess)h(is)g(m)o(ulti-threaded,)g(then)g(the)75 2421 y(seman)o(tics)e(of)f(thread)h(execution)h(ma)o(y)e(not)g(de\014ne)i(a) e(relativ)o(e)i(order)e(b)q(et)o(w)o(een)h(t)o(w)o(o)e(send)j(op)q (erations)75 2478 y(executed)g(b)o(y)f(t)o(w)o(o)f(distinct)j(threads:) i(the)d(op)q(erations)f(are)g(logically)i(concurren)o(t,)e(ev)o(en)h (though)f(one)75 2534 y(ph)o(ysically)22 b(precedes)e(the)g(other.)33 b(In)20 b(suc)o(h)g(a)f(case,)i(the)e(t)o(w)o(o)g(messages)g(sen)o(t)g (can)h(b)q(e)g(receiv)o(ed)h(in)75 2591 y(an)o(y)f(order.)37 b(Similarly)l(,)24 b(if)d(t)o(w)o(o)e(receiv)o(e)j(op)q(erations)f (that)f(are)g(logically)j(concurren)o(t)e(receiv)o(e)g(t)o(w)o(o)75 2647 y(successiv)o(ely)i(sen)o(t)d(messages,)h(then)g(the)g(t)o(w)o(o)f (messages)g(can)h(matc)o(h)f(the)h(t)o(w)o(o)e(receiv)o(es)j(in)f (either)75 2704 y(order.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 22 27 22 26 bop 75 -100 a FE(22)613 b Ft(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fx(Progress)45 b FE(If)13 b(a)g(pair)g(of)f(matc)o(hing)h(send)h(and)f(receiv)o(es)g (ha)o(v)o(e)g(b)q(een)h(initiated)g(on)f(t)o(w)o(o)e(pro)q(cesses,)j (then)75 102 y(at)k(least)h(one)h(of)e(these)h(t)o(w)o(o)f(op)q (erations)h(will)i(complete,)f(indep)q(enden)o(tly)i(of)c(other)h (actions)g(in)h(the)75 158 y(system:)e(the)13 b(send)h(op)q(eration)f (will)h(complete,)g(unless)g(the)f(receiv)o(e)h(is)f(satis\014ed)h(b)o (y)e(another)h(message,)75 214 y(and)h(completes;)g(the)g(receiv)o(e)h (op)q(eration)f(will)h(complete,)g(unless)g(the)f(message)f(sen)o(t)g (is)i(consumed)f(b)o(y)75 271 y(another)h(matc)o(hing)g(receiv)o(e)h (that)f(w)o(as)f(p)q(osted)h(at)g(the)g(same)g(destination)h(pro)q (cess.)75 391 y Fx(F)o(airness)46 b FE(MPI)15 b(mak)o(es)g(no)g(guaran) o(tee)g(of)g Fs(fairness)f FE(in)i(the)g(handling)h(of)e(comm)o (unication.)21 b(Supp)q(ose)75 447 y(that)c(a)f(send)i(w)o(as)f(p)q (osted.)26 b(Then)18 b(it)f(is)h(p)q(ossible)h(that)e(the)g (destination)h(pro)q(cess)g(rep)q(eatedly)g(p)q(osts)75 503 y(a)d(receiv)o(e)h(that)f(matc)o(hes)f(this)i(send,)g(y)o(et)e(the) i(message)f(is)g(nev)o(er)h(receiv)o(ed,)g(b)q(ecause)g(it)g(is)f(eac)o (h)h(time)75 560 y(o)o(v)o(ertak)o(en)e(b)o(y)i(another)f(message,)g (sen)o(t)h(from)f(another)g(source.)21 b(Similarly)l(,)d(supp)q(ose)f (that)e(a)g(receiv)o(e)75 616 y(w)o(as)f(p)q(osted)i(b)o(y)f(a)g(m)o (ulti-threaded)h(pro)q(cess.)k(Then)c(it)f(is)h(p)q(ossible)h(that)d (messages)h(that)f(matc)o(h)h(this)75 673 y(receiv)o(e)20 b(are)g(rep)q(eatedly)g(receiv)o(ed,)i(y)o(et)d(the)g(receiv)o(e)i(is)f (nev)o(er)f(satis\014ed,)i(b)q(ecause)f(it)g(is)g(o)o(v)o(ertak)o(en)75 729 y(b)o(y)15 b(other)h(receiv)o(es)g(p)q(osted)g(at)e(this)i(no)q(de) h(\(b)o(y)e(other)g(executing)h(threads\).)21 b(It)15 b(is)h(the)g(programmer's)75 786 y(resp)q(onsibilit)o(y)i(to)c(prev)o (en)o(t)h(starv)m(ation)g(in)h(suc)o(h)g(situations.)75 905 y Fx(Resource)e(limitations)45 b FE(The)13 b(curren)o(t)f(practice) h(for)f(man)o(y)g(commercial)i(message-passing)e(libraries)i(is)75 962 y(that)f(\(short\))f(messages)h(are)g(bu\013ered)g(b)o(y)h(the)f (system,)g(th)o(us)g(allo)o(wing)h(blo)q(c)o(king)h(send)f(op)q (erations)g(to)75 1018 y(complete)d(ahead)f(of)g(the)h(matc)o(hing)f (receiv)o(es.)19 b(It)11 b(is)f(exp)q(ected)i(that)e(man)o(y)f(MPI)i (implemen)o(tations)g(will)75 1075 y(follo)o(w)j(this)g(practice,)g (and)f(pro)o(vide)h(the)g(same)f(lev)o(el)i(of)e(bu\013ering)h(that)f (is)h(a)o(v)m(ailable)h(on)f(the)f(libraries)75 1131 y(they)21 b(replace.)37 b(Ho)o(w)o(ev)o(er,)20 b(message)g(bu\013ering) h(is)h(not)e(a)g(univ)o(ersal)i(practice.)36 b(Ev)o(en)21 b(on)f(systems)75 1188 y(where)e(bu\013ering)g(o)q(ccur,)g(the)f(amoun) o(t)g(of)g(bu\013er)h(space)f(a)o(v)m(ailable)j(and)d(the)h(w)o(a)o(y)e (it)i(is)g(allo)q(cated)g(is)75 1244 y(b)q(ound)e(to)f(b)q(e)h (implemen)o(tation)g(dep)q(enden)o(t.)166 1300 y(Therefore,)24 b(message)e(bu\013ering)h(is)g(not)f(mandated)g(b)o(y)g(MPI)h(and)f(is) h(seen)g(as)f(a)g(qualit)o(y)h(of)75 1357 y(implemen)o(tation)d(issue.) 32 b(A)19 b(v)m(alid)i(MPI)e(implemen)o(tation)h(of)f Fx(MPI)p 1285 1357 14 2 v 15 w(SEND)h FE(is)f(to)g(blo)q(c)o(k)g(the)g (sender)75 1413 y(un)o(til)k(a)f(matc)o(hing)h(receiv)o(e)g(has)f(b)q (een)h(initiated.)43 b(In)23 b(general,)h(the)e(programmer)f(can)h(mak) o(e)g(no)75 1470 y(assumptions)f(on)g(the)g(a)o(v)m(ailabilit)o(y)j(of) c(bu\013er)h(space,)i(and)e(ho)o(w)f(this)i(space)f(is)h(allo)q(cated.) 38 b(Th)o(us,)75 1526 y(p)q(ortable)14 b(\(safe\))e(MPI)h(co)q(de)h (should)g(w)o(ork)f(under)h(the)f(assumption)g(that)g(an)g(arbitrary)g (subset)g(of)g(the)75 1583 y(send)18 b(op)q(erations)f(are)g(going)g (to)g(return)g(b)q(efore)h(a)f(matc)o(hing)g(receiv)o(e)h(is)g(p)q (osted,)g(and)f(the)g(rest)g(will)75 1639 y(blo)q(c)o(k)f(un)o(til)g(a) f(matc)o(hing)g(receiv)o(e)i(is)e(p)q(osted.)166 1696 y(MPI)g(implemen)o(tation)i(will)h(pro)o(vide)e(information)g(on)f(the) h(amoun)o(t)f(of)g(a)o(v)m(ailable)i(bu\013er)f(space)75 1752 y(and)f(on)h(the)f(bu\013ering)h(p)q(olicy)g(via)g(the)f(en)o (vironmen)o(tal)h(enquiries)h(describ)q(ed)g(in)f(Chapter)f(7.)166 1809 y(Examples)h(\(in)o(v)o(olving)g(t)o(w)o(o)e(pro)q(cesses)h(with)h (ranks)f(0)f(and)i(1\))166 1865 y(The)f(follo)o(wing)h(program)e(is)i (safe,)f(and)g(should)h(alw)o(a)o(ys)f(succeed.)75 2013 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2069 y(IF)h(\(rank.EQ.0\))123 2126 y(THEN)170 2182 y(CALL)g (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\)) 170 2239 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(status,)g(ierr\))123 2295 y(ELSE)95 b(!)23 b(rank.EQ.1)170 2352 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(status,)g(ierr\))170 2408 y(CALL)h(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))75 2464 y(END)g(IF)166 2556 y FE(The)15 b(follo)o(wing)h(program)e(is)i (erroneous,)f(and)g(should)h(alw)o(a)o(ys)f(fail.)75 2704 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 23 28 23 27 bop 75 -100 a Ft(3.5.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) 1105 b FE(23)75 45 y Fp(IF)24 b(\(rank.EQ.0\))123 102 y(THEN)170 158 y(CALL)g(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h (tag,)f(comm,)g(status,)g(ierr\))170 214 y(CALL)h(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))123 271 y(ELSE)95 b(!)23 b(rank.EQ.1)170 327 y(CALL)h(MPI_RECV\(recvbuf,)d (count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(status,)g(ierr\))170 384 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))75 440 y(END)g(IF)166 538 y FE(The)15 b(receiv)o(e)g(op)q(eration)g(of)f(the)h(\014rst)f(pro)q(cess)h(m)o (ust)f(complete)i(b)q(efore)f(its)f(send,)h(and)g(can)g(com-)75 594 y(plete)j(only)g(if)g(the)f(matc)o(hing)g(send)h(of)f(the)g(second) h(pro)q(cessor)f(is)h(executed;)g(the)g(receiv)o(e)g(op)q(eration)75 651 y(of)c(the)h(second)g(pro)q(cess)g(m)o(ust)f(complete)h(b)q(efore)g (its)f(send)i(and)e(can)h(complete)g(only)g(if)g(the)g(matc)o(hing)75 707 y(send)h(of)f(the)g(\014rst)g(pro)q(cess)g(is)h(executed.)21 b(This)15 b(program)f(will)j(deadlo)q(c)o(k.)166 765 y(The)10 b(follo)o(wing)h(program)f(is)g(unsafe,)h(and)g(ma)o(y)e (succeed)j(or)e(fail,)h(dep)q(ending)i(on)d(implemen)o(tation.)75 933 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 990 y(IF)h(\(rank.EQ.0\))123 1046 y(THEN)170 1103 y(CALL)g (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\)) 170 1159 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(status,)g(ierr\))123 1216 y(ELSE)95 b(!)23 b(rank.EQ.1)170 1272 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))170 1329 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(0,)h(tag,)f(comm,)g(status,)g(ierr\))75 1385 y(END)g(IF)166 1553 y FE(The)11 b(message)g(sen)o(t)g(b)o(y)g(eac) o(h)g(pro)q(cess)g(has)g(to)f(b)q(e)i(copied)g(out)f(b)q(efore)g(the)g (send)h(op)q(eration)f(returns)75 1609 y(and)k(the)g(receiv)o(e)h(op)q (eration)f(starts.)k(F)l(or)14 b(the)h(program)f(to)g(complete,)h(it)h (is)f(necessary)g(that)g(at)f(least)75 1665 y(one)i(of)g(the)h(t)o(w)o (o)d(messages)i(sen)o(t)g(is)h(bu\013ered.)23 b(Th)o(us,)16 b(this)h(program)e(can)i(succeed)g(only)g(if)g(the)f(MPI)75 1722 y(implemen)o(tation)e(bu\013ers)f(messages,)g(and)g(the)g(comm)o (unication)h(system)f(has)g(su\016cien)o(t)h(bu\013er)f(space)75 1778 y(to)i(bu\013er)g Fx(count)h FE(w)o(ords)f(of)f(data.)75 1927 y Fv(3.5)59 b(Data)19 b(T)-5 b(yp)r(e)19 b(Matching)75 2031 y FE(One)d(can)f(think)h(of)f(message)g(transmission)g(as)g (consisting)h(of)f(three)g(phases:)131 2128 y(1.)22 b(Data)14 b(is)h(pulled)j(out)d(of)f(the)i(send)f(bu\013er)h(and)f(a)g(message)g (is)g(assem)o(bled)131 2226 y(2.)22 b(A)15 b(message)g(is)g (transferred)g(from)f(sender)i(to)f(receiv)o(er)131 2324 y(3.)22 b(Data)14 b(is)h(pulled)j(from)c(the)h(incoming)i(message)e (and)g(disassem)o(bled)i(in)o(to)e(the)g(receiv)o(e)h(bu\013er)166 2421 y(T)o(yp)q(e)k(matc)o(hing)f(has)h(to)e(b)q(e)j(observ)o(ed)e(at)g (eac)o(h)g(of)h(these)f(three)h(phases:)29 b(The)19 b(t)o(yp)q(e)h(of)f (eac)o(h)75 2478 y(v)m(ariable)i(in)f(the)g(sender)g(bu\013er)g(has)g (to)f(matc)o(h)g(the)g(t)o(yp)q(e)h(sp)q(eci\014ed)i(for)d(that)g(en)o (try)g(b)o(y)h(the)f(send)75 2534 y(op)q(eration;)d(the)f(t)o(yp)q(e)h (sp)q(eci\014ed)i(b)o(y)e(the)f(send)i(op)q(eration)e(has)h(to)f(matc)o (h)g(the)h(t)o(yp)q(e)f(sp)q(eci\014ed)j(b)o(y)e(the)75 2591 y(receiv)o(e)g(op)q(eration;)e(and)h(the)g(t)o(yp)q(e)f(of)h(eac)o (h)f(v)m(ariable)i(in)g(the)e(receiv)o(e)i(bu\013er)f(has)f(to)g(matc)o (h)g(the)h(t)o(yp)q(e)75 2647 y(sp)q(eci\014ed)f(for)e(that)g(en)o(try) g(b)o(y)g(the)g(receiv)o(e)h(op)q(eration.)20 b(A)12 b(program)f(that)h(fails)h(to)e(observ)o(e)h(these)h(three)75 2704 y(rules)j(is)g(erroneous.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 24 29 24 28 bop 75 -100 a FE(24)613 b Ft(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FE(T)l(o)f (de\014ne)i(t)o(yp)q(e)f(matc)o(hing)g(more)f(precisely)l(,)j(w)o(e)d (need)i(to)e(deal)h(with)g(t)o(w)o(o)f(issues:)22 b(matc)o(hing)15 b(of)75 102 y(t)o(yp)q(es)e(of)g(the)h(host)f(language)g(with)h(t)o(yp) q(es)g(sp)q(eci\014ed)h(in)f(comm)o(unication)h(op)q(erations;)e(and)h (matc)o(hing)75 158 y(of)h(t)o(yp)q(es)g(at)g(sender)g(and)h(receiv)o (er.)166 214 y(A)h(t)o(yp)q(e)f(sp)q(eci\014ed)j(for)d(an)h(en)o(try)f (b)o(y)h(a)f(send)h(op)q(eration)g(matc)o(hes)f(the)h(t)o(yp)q(e)g(sp)q (eci\014ed)i(for)d(that)75 271 y(en)o(try)11 b(b)o(y)h(a)f(receiv)o(e)h (op)q(eration)g(if)g(b)q(oth)f(op)q(erations)h(use)g(iden)o(tical)h (names:)36 b Fx(MPI)p 1500 271 14 2 v 16 w(INTEGER)12 b FE(matc)o(hes)75 327 y Fx(MPI)p 160 327 V 16 w(INTEGER)p FE(,)j Fx(MPI)p 475 327 V 16 w(REAL)h FE(matc)o(hes)30 b Fx(MPI)p 891 327 V 15 w(REAL)p FE(,)16 b(and)f(so)g(on.)166 384 y(The)k(t)o(yp)q(e)g(of)f(a)g(v)m(ariable)i(in)g(a)e(host)h (program)e(matc)o(hes)h(the)h(t)o(yp)q(e)g(sp)q(eci\014ed)i(in)e(the)g (comm)o(u-)75 440 y(nication)g(op)q(eration)g(if)f(the)h(datat)o(yp)q (e)f(name)g(used)h(b)o(y)f(that)f(op)q(eration)i(corresp)q(onds)f(to)g (the)g(basic)75 497 y(t)o(yp)q(e)i(of)g(the)g(host)g(program)f(v)m (ariable:)31 b(an)20 b(en)o(try)f(with)i(t)o(yp)q(e)f(name)40 b Fx(MPI)p 1449 497 V 16 w(INTEGER)21 b FE(matc)o(hes)e(a)75 553 y(F)l(ortran)c(v)m(ariable)j(of)f(t)o(yp)q(e)f Fp(INTEGER)p FE(,)g(an)g(en)o(try)g(with)h(t)o(yp)q(e)g(name)g Fx(MPI)p 1368 553 V 16 w(REAL)g FE(matc)o(hes)f(a)g(F)l(ortran)75 610 y(v)m(ariable)j(of)e(t)o(yp)q(e)g Fp(REAL)p FE(,)f(and)i(so)f(on.) 26 b(There)18 b(is)f(one)h(exception)g(to)f(this)h(last)f(rule:)25 b(An)18 b(en)o(try)f(with)75 666 y(t)o(yp)q(e)h(name)37 b Fx(MPI)p 408 666 V 15 w(BYTE)19 b FE(can)f(b)q(e)h(used)g(to)e(matc)o (h)h(an)o(y)g(b)o(yte)g(of)g(storage)e(\(on)i(a)g(b)o(yte-addressable) 75 723 y(mac)o(hine\),)d(irresp)q(ectiv)o(e)i(of)d(the)h(datat)o(yp)q (e)g(of)f(the)h(v)m(ariable)i(that)d(con)o(tains)h(this)g(b)o(yte.)20 b(The)15 b(v)m(alue)h(of)75 779 y(the)f(message)g(en)o(try)g(will)i(b)q (e)f(the)f(binary)h(v)m(alue)g(of)f(the)g(corresp)q(onding)h(b)o(yte)f (in)h(memory)l(.)166 835 y(W)l(e)f(th)o(us)g(ha)o(v)o(e)g(t)o(w)o(o)f (cases:)143 925 y FC(\017)23 b FE(Comm)o(unication)11 b(of)f(t)o(yp)q(ed)i(v)m(alues)g(\(e.g.,)e(with)h(datat)o(yp)q(e)g (di\013eren)o(t)g(from)21 b Fx(MPI)p 1586 925 V 16 w(BYTE)p FE(\),)11 b(where)189 981 y(the)k(datat)o(yp)q(es)f(of)g(the)h(corresp) q(onding)h(en)o(tries)f(in)h(the)f(sender)g(program,)e(in)j(the)f(send) g(call,)h(in)189 1037 y(the)f(receiv)o(e)h(call)g(and)g(in)g(the)f (receiv)o(er)h(program)e(should)i(all)g(matc)o(h.)143 1129 y FC(\017)23 b FE(Comm)o(unication)16 b(of)g(un)o(t)o(yp)q(ed)h(v) m(alues)g(\(e.g.,)e(of)g(datat)o(yp)q(e)h Fx(MPI)p 1330 1129 V 16 w(BYTE)p FE(\),)g(where)g(b)q(oth)g(sender)189 1186 y(and)g(receiv)o(er)h(use)f(the)g(datat)o(yp)q(e)g Fx(MPI)p 881 1186 V 15 w(BYTE)p FE(.)h(In)f(this)h(case,)f(there)g(are) g(no)f(requiremen)o(ts)i(on)189 1242 y(the)d(t)o(yp)q(es)h(of)f(the)g (corresp)q(onding)h(en)o(tries)g(in)h(the)e(sender)h(and)g(the)f (receiv)o(er)h(programs,)e(nor)h(is)189 1299 y(it)h(required)h(that)f (they)g(b)q(e)h(the)f(same.)166 1388 y(The)g(follo)o(wing)h(examples)g (illustrate)h(these)e(t)o(w)o(o)f(cases.)166 1444 y(First)h(program:)75 1533 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1590 y(IF\(rank.EQ.0\))123 1646 y(THEN)170 1703 y(CALL)h (MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))123 1759 y(ELSE)170 1815 y(CALL)h(MPI_RECV\(a\(1\),)e(15,)h(MPI_REAL,)g(0,) g(tag,)h(comm,)f(status,)g(ierr\))75 1872 y(END)g(IF)166 1961 y FE(This)c(co)q(de)f(is)h(correct)e(if)i(b)q(oth)f(sender)h(and)f (receiv)o(er)h(programs)d(ha)o(v)o(e)i(allo)q(cated)h(consecutiv)o(e)75 2017 y(storage)14 b(for)h(ten)g(real)g(n)o(um)o(b)q(ers,)g(starting)g (from)f Fp(a\(1\))p FE(.)166 2074 y(Second)i(program:)75 2163 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2219 y(IF\(rank.EQ.0\))123 2276 y(THEN)170 2332 y(CALL)h (MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))123 2389 y(ELSE)170 2445 y(CALL)h(MPI_RECV\(a\(1\),)e(40,)h(MPI_BYTE,)g(0,) g(tag,)h(comm,)f(status,)g(ierr\))75 2502 y(END)g(IF)166 2591 y FE(This)d(co)q(de)g(is)f(erroneous,)h(since)g(sender)g(and)g (receiv)o(er)g(do)f(not)f(pro)o(vide)i(matc)o(hing)f(datat)o(yp)q(e)75 2647 y(parameters.)166 2704 y(Third)d(program:)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 25 30 25 29 bop 75 -100 a Ft(3.6.)34 b(D)o(A)l(T)l(A)15 b(CONVERSION)1197 b FE(25)75 45 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 102 y(IF\(rank.EQ.0\))123 158 y(THEN)170 214 y(CALL)h (MPI_SEND\(a\(1\),)e(40,)h(MPI_BYTE,)g(1,)g(tag,)h(comm,)f(ierr\))123 271 y(ELSE)170 327 y(CALL)h(MPI_RECV\(a\(1\),)e(60,)h(MPI_BYTE,)g(0,)g (tag,)h(comm,)f(status,)g(ierr\))75 384 y(END)g(IF)166 478 y FE(This)16 b(co)q(de)f(is)h(correct,)f(irresp)q(ectiv)o(e)h(of)f (the)g(t)o(yp)q(e)h(of)e Fp(a\(1\))h FE(and)g(follo)o(wing)h(v)m (ariables)h(in)f(store.)166 535 y(Some)e(implemen)o(tations)g(ma)o(y)f (disallo)o(w)i(the)f(transfer)e(of)i(F)l(ortran)e(c)o(haracter)h (strings)h(using)g(the)75 592 y(MPI)h(t)o(yp)q(e)g Fx(MPI)p 366 592 14 2 v 16 w(BYTE)p FE(.)75 736 y Fv(3.6)59 b(Data)19 b(conversion)75 838 y FE(One)d(of)f(the)g(goals)g(of)g(MPI)g(is)h(to)e (supp)q(ort)i(parallel)g(computations)f(across)g(heterogeneous)g(en)o (viron-)75 894 y(men)o(ts.)20 b(Comm)o(unication)15 b(in)h(a)f (heterogeneous)g(en)o(vironmen)o(t)h(ma)o(y)e(require)i(data)f(con)o(v) o(ersions.)166 951 y(W)l(e)g(use)h(the)f(follo)o(wing)h(terminology:)75 1046 y Fz(t)o(yp)q(e)h(con)o(v)o(ersion)22 b FE(c)o(hanges)16 b(the)h(datat)o(yp)q(e)f(of)g(a)g(v)m(alue,)h(e.g.)23 b(b)o(y)17 b(rounding)g(a)f Fx(REAL)h FE(to)f(an)g Fx(INTE-)189 1102 y(GER)p FE(.)75 1197 y Fz(represen)o(tation)h(con)o(v)o(ersion)22 b FE(c)o(hanges)16 b(the)h(binary)g(represen)o(tation)f(of)g(a)g(v)m (alue,)h(e.g.)23 b(from)16 b(Hex)189 1253 y(\015oating)f(p)q(oin)o(t)g (to)g(IEEE)g(\015oating)h(p)q(oin)o(t.)166 1348 y(The)d(t)o(yp)q(e)g (matc)o(hing)g(rules)g(imply)h(that)e(MPI)h(comm)o(unication)h(nev)o (er)f(en)o(tails)g(t)o(yp)q(e)g(con)o(v)o(ersion.)75 1404 y(On)20 b(the)f(other)f(hand,)i(MPI)f(requires)h(that)e(a)h (represen)o(tation)g(con)o(v)o(ersion)g(b)q(e)h(p)q(erformed)f(when)g (a)75 1461 y(t)o(yp)q(ed)i(v)m(alue)h(is)f(transferred)f(across)g(en)o (vironmen)o(ts)h(that)f(use)h(di\013eren)o(t)f(represen)o(tations)h (for)f(the)75 1517 y(datat)o(yp)q(e)g(of)g(this)h(v)m(alue.)38 b(MPI)20 b(do)q(es)h(not)f(sp)q(ecify)i(rules)g(for)d(represen)o (tation)i(con)o(v)o(ersion.)36 b(Suc)o(h)75 1573 y(con)o(v)o(ersion)21 b(is)g(exp)q(ected)i(to)d(preserv)o(e)h(in)o(teger,)h(logical)g(or)e(c) o(haracter)h(v)m(alues,)i(and)e(to)f(con)o(v)o(ert)g(a)75 1630 y(\015oating)15 b(p)q(oin)o(t)h(v)m(alue)g(to)f(the)g(nearest)g(v) m(alue)h(that)f(can)g(represen)o(ted)h(on)f(the)g(target)f(system.)166 1687 y(Ov)o(er\015o)o(w)c(and)h(under\015o)o(w)g(exceptions)h(ma)o(y)e (o)q(ccur)i(during)f(\015oating)g(p)q(oin)o(t)g(con)o(v)o(ersions;)h (con)o(v)o(er-)75 1743 y(sion)i(of)e(in)o(tegers)i(or)e(c)o(haracters)h (ma)o(y)f(also)h(lead)h(to)f(exceptions)h(when)g(a)e(v)m(alue)j(that)d (can)h(represen)o(ted)75 1800 y(in)18 b(one)g(system)f(cannot)g(b)q(e)h (represen)o(ted)g(in)h(the)e(other)g(system.)26 b(An)18 b(exception)h(o)q(ccurring)f(during)75 1856 y(represen)o(tation)f(con)o (v)o(ersion)f(results)h(in)h(a)e(failure)i(of)e(the)g(comm)o (unication;)i(an)e(error)g(o)q(ccurs)h(either)75 1912 y(in)f(the)f(send)h(op)q(eration,)f(or)g(the)g(receiv)o(e)h(op)q (eration,)f(or)g(b)q(oth.)166 1969 y(If)22 b(a)f(v)m(alue)i(sen)o(t)e (in)h(a)f(message)g(is)h(un)o(t)o(yp)q(ed)h(\(i.e.,)f(of)f(t)o(yp)q(e)h Fx(MPI)p 1370 1969 V 15 w(BYTE)p FE(\),)g(then)g(the)f(binary)75 2026 y(represen)o(tation)15 b(of)f(the)h(b)o(yte)g(stored)f(at)g(the)h (receiv)o(er)h(is)f(iden)o(tical)i(to)d(the)h(binary)h(represen)o (tation)e(of)75 2082 y(the)f(b)o(yte)g(loaded)h(at)e(the)h(sender.)20 b(This)14 b(holds)f(true,)g(whether)g(sender)h(and)f(receiv)o(er)h(run) f(in)h(the)f(same)75 2138 y(or)i(in)h(distinct)g(en)o(vironmen)o(ts;)f (no)g(represen)o(tation)h(con)o(v)o(ersion)f(is)h(required.)166 2195 y(Note)d(that)g(no)g(con)o(v)o(ersion)h(ev)o(er)f(o)q(ccurs)h (when)g(an)f(MPI)g(program)g(executes)h(in)g(a)f(homogeneous)75 2252 y(system,)i(where)h(all)g(pro)q(cesses)g(run)g(in)g(the)g(same)f (en)o(vironmen)o(t.)22 b(Also)16 b(note)f(the)h(di\013eren)o(t)f(b)q (eha)o(vior)75 2308 y(of)e Fx(MPI)p 210 2308 V 15 w(BYTE)h FE(and)f(of)g Fx(MPI)p 575 2308 V 15 w(CHAR)p FE(.)h(A)f(bu\013er)g (descriptor)g(en)o(try)g(with)g(datat)o(yp)q(e)g(of)f Fx(MPI)p 1660 2308 V 16 w(CHAR)i FE(can)75 2364 y(only)f(matc)o(h)f(a)g (C)g(v)m(ariable)i(of)e(t)o(yp)q(e)h Fp(CHAR)p FE(;)e(and)i(represen)o (tation)g(con)o(v)o(ersion)f(ma)o(y)g(o)q(ccur)h(when)g(v)m(alues)75 2421 y(of)g(t)o(yp)q(e)h Fx(MPI)p 311 2421 V 15 w(CHAR)h FE(are)e(transferred.,)g(e.g.,)f(from)h(an)g(EBCDIC)h(enco)q(ding)h(to) e(an)g(ASCI)q(I)i(enco)q(ding.)166 2478 y(Consider)h(the)f(previous)h (three)f(examples.)166 2534 y(The)c(\014rst)f(program)f(is)i(correct,)g (assuming)g(b)q(oth)f(sender)i(and)e(receiv)o(er)i(declared)f(ten)g (consecutiv)o(e)75 2591 y(real)19 b(v)m(ariables)h(in)g(storage)e (starting)g(at)g Fp(a\(1\))p FE(.)30 b(If)19 b(the)g(sender)h(and)f (receiv)o(er)g(execute)h(in)g(di\013eren)o(t)75 2647 y(en)o(vironmen)o(ts,)12 b(then)f(the)g(ten)f(real)i(v)m(alues)f(that)g (are)f(fetc)o(hed)h(from)f(the)h(send)h(bu\013er)e(will)j(b)q(e)e(con)o (v)o(erted)75 2704 y(to)g(the)h(represen)o(tation)f(for)g(reals)h(on)f (the)h(receiv)o(er)g(site)g(b)q(efore)g(they)g(are)f(stored)g(in)h(the) g(receiv)o(e)g(bu\013er.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 26 31 26 30 bop 75 -100 a FE(26)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(While)j(the)e(n)o (um)o(b)q(er)h(of)f(real)g(elemen)o(ts)h(fetc)o(hed)g(from)f(the)g (send)h(bu\013er)f(equal)i(the)e(n)o(um)o(b)q(er)h(of)f(real)75 102 y(elemen)o(ts)d(stored)e(in)i(the)f(receiv)o(e)h(bu\013er,)f(the)g (n)o(um)o(b)q(er)g(of)g(b)o(ytes)g(stored)f(need)i(not)f(equal)h(the)f (n)o(um)o(b)q(er)75 158 y(of)f(b)o(ytes)g(loaded:)19 b(e.g.)g(the)12 b(sender)h(ma)o(y)e(use)i(a)f(four)g(b)o(yte)g (represen)o(tation)h(and)f(the)g(receiv)o(er)i(an)e(eigh)o(t)75 214 y(b)o(yte)17 b(represen)o(tation)h(for)f(reals.)27 b(If)18 b(the)g(send)g(or)f(receiv)o(e)h(bu\013er)g(do)f(not)h(con)o (tain)f(ten)h(consecutiv)o(e)75 271 y(real)e(v)m(ariables,)g(then)f (the)g(program)g(is)g(erroneous,)g(and)g(its)h(b)q(eha)o(vior)f(is)h (unde\014ned.)166 331 y(The)f(second)h(program)e(is)i(erroneous,)f(and) g(its)g(b)q(eha)o(vior)h(is)g(unde\014ned.)166 390 y(The)e(third)g (program)e(is)i(correct.)19 b(The)13 b(exact)g(same)g(sequence)i(of)e (fort)o(y)f(b)o(ytes)h(that)g(w)o(ere)g(loaded)75 447 y(from)18 b(the)h(send)h(bu\013er)f(will)i(b)q(e)e(stored)g(in)h(the)f (receiv)o(e)h(bu\013er,)f(ev)o(en)h(if)f(sender)h(and)f(receiv)o(er)h (run)75 503 y(in)e(a)g(di\013eren)o(t)f(en)o(vironmen)o(t.)28 b(The)18 b(message)f(sen)o(t)g(has)h(exactly)f(the)h(same)f(length)i (\(in)f(b)o(ytes\))f(and)75 560 y(the)f(same)f(binary)h(represen)o (tation)f(as)g(the)h(message)f(receiv)o(ed.)22 b(If)16 b(the)f(v)m(ariables)i(in)f(the)g(send)g(bu\013er)75 616 y(are)i(of)h(di\013eren)o(t)g(t)o(yp)q(es)g(as)f(the)h(v)m (ariables)h(in)g(the)f(receiv)o(e)g(bu\013er,)h(or)e(they)h(are)f(of)h (the)f(same)h(t)o(yp)q(e)75 672 y(but)e(di\013eren)o(t)g(data)g (represen)o(tations)f(are)h(used,)h(then)f(the)g(bits)h(stored)e(in)i (the)f(receiv)o(e)h(bu\013er)f(ma)o(y)75 729 y(enco)q(de)f(v)m(alues)h (that)d(are)h(di\013eren)o(t)g(from)g(the)g(v)m(alues)h(they)g(enco)q (ded)g(in)g(the)g(send)f(bu\013er.)166 789 y(Data)f(represen)o(tation)i (con)o(v)o(ersion)f(also)h(applies)h(to)e(the)g(en)o(v)o(elop)q(e)i(of) e(a)g(message:)20 b(source,)15 b(des-)75 845 y(tination)h(and)f(tag)f (are)h(all)h(in)o(tegers)g(that)e(ma)o(y)h(need)h(to)e(b)q(e)i(con)o(v) o(erted.)166 905 y(The)k(curren)o(t)g(draft)f(do)q(es)h(not)g(pro)o (vide)g(a)g(mec)o(hanism)h(for)e(in)o(ter-language)i(comm)o(unication:) 75 961 y(messages)d(sen)o(t)h(b)o(y)f(F)l(ortran)g(calls)i(should)f(b)q (e)h(receiv)o(ed)g(b)o(y)e(F)l(ortran)g(calls)h(and)g(messages)g(sen)o (t)f(b)o(y)75 1018 y(C)e(calls)g(should)h(b)q(e)g(receiv)o(ed)g(b)o(y)e (C)h(calls)h(\(this)f(follo)o(ws)f(from)h(the)f(requiremen)o(ts)i(of)e (t)o(yp)q(e)h(matc)o(hing)75 1074 y(and)g(the)h(fact)f(that)f(F)l (ortran)g(and)i(C)f(datat)o(yp)q(es)g(are)g(distinct\).)24 b(If)16 b(in)o(ter-language)h(comm)o(unication)75 1131 y(is)g(needed)i(then)e(one)g(needs)h(to)e(in)o(v)o(ok)o(e)h(C)f(comm)o (unication)i(routines)f(from)f(a)h(F)l(ortran)f(program)f(or)75 1187 y(F)l(ortran)f(comm)o(unication)i(routines)g(from)e(a)h(C)g (program.)166 1323 y Fo(Implemen)o(tati)o(on)d(note:)166 1376 y Fr(The)k(curren)o(t)i(de\014nition)e(do)q(es)h(not)f(require)h (messages)f(to)g(carry)h(data)f(t)o(yp)q(e)g(information.)22 b(A)16 b(message)75 1425 y(can)c(b)q(e)h(comp)q(osed)f(and)g(sen)o(t)h (using)f(only)g(the)h(information)c(pro)o(vided)j(b)o(y)g(the)h(send)g (call,)e(and)h(can)h(b)q(e)g(receiv)o(ed)75 1475 y(and)k(stored)g (using)g(only)f(the)h(information)d(pro)o(vided)j(b)o(y)g(the)g(receiv) o(e)h(call.)26 b(If)17 b(messages)g(are)g(sen)o(t)h(b)q(et)o(w)o(een)75 1525 y(di\013eren)o(t)c(mac)o(hines)f(then)g(one)h(can)f(either)h(use)g (a)f(\\univ)o(ersal")g(data)g(enco)q(ding)g(for)g(messages,)g(use)h (kno)o(wledge)75 1575 y(of)g(the)i(receiv)o(er)h(en)o(vironmen)o(t)d (in)g(order)i(to)f(con)o(v)o(ert)h(data)e(at)h(the)h(sender,)h(or)e (use)g(kno)o(wledge)g(of)g(the)g(sender)75 1625 y(en)o(vironmen)o(t)f (in)g(order)h(to)g(con)o(v)o(ert)g(data)f(at)h(the)g(receiv)o(er.)22 b(In)15 b(either)g(case)h(the)f(datat)o(yp)q(e)g(parameter)f(in)g(the) 75 1675 y(lo)q(cal)f(call)g(can)h(b)q(e)h(used)g(to)e(deriv)o(e)i(the)f (t)o(yp)q(es)h(of)e(the)i(v)n(alues)e(transferred.)166 1728 y(Additional)f(t)o(yp)q(e)j(information)c(can)j(b)q(e)g(used)h(to) f(pro)o(vide)g(b)q(etter)h(error)g(detection.)75 1972 y Fv(3.7)59 b(Communication)18 b(Mo)r(des)75 2079 y FE(The)13 b(basic)h(send)f(op)q(eration)g(describ)q(ed)i(in)f(section)f(3.2)f (used)h(the)g Fz(standard)g FE(comm)o(unication)h(mo)q(de.)75 2136 y(In)g(suc)o(h)g(a)f(comm)o(unication)h(mo)q(de,)g(a)f(send)h(op)q (eration)g(can)f(b)q(e)h(started)f(whether)h(or)f(not)g(a)g(matc)o (hing)75 2192 y(receiv)o(e)18 b(w)o(as)e(p)q(osted.)24 b(The)17 b(completion)h(of)e(the)h(send)h(op)q(eration)e(indicates)j (that)d(the)g(message)h(and)75 2249 y(its)f(en)o(v)o(elop)q(e)h(ha)o(v) o(e)e(b)q(een)i(safely)f(stored)f(a)o(w)o(a)o(y)f(and)i(that)f(the)h (sender)g(is)g(free)g(to)f(access)h(and)f(mo)q(dify)75 2305 y(the)j(sender)h(bu\013er.)30 b(Th)o(us,)18 b(the)h(op)q(eration)f (is)h Fz(lo)q(cally-blo)r(c)o(king)p FE(:)29 b(it)19 b(do)q(es)f(not)g(return)g(un)o(til)i(the)75 2362 y(send)c(op)q (eration)f(has)g(completed)h Fz(lo)q(cally)p FE(,)i(on)d(the)g(sender)h (side.)166 2421 y(The)f(completion)i(of)e(a)g(send)h(op)q(eration)f (giv)o(es)h(no)f(indication)i(that)e(the)g(message)g(w)o(as)g(receiv)o (ed)75 2478 y(on)i(the)g(receiv)o(er)h(side.)26 b(A)17 b(blo)q(c)o(king)i(send)e(ma)o(y)g(b)q(e)g(implemen)o(ted)i(so)e(that)f (it)h(returns)g(only)h(after)e(a)75 2534 y(matc)o(hing)j(receiv)o(e)h (has)f(b)q(een)h(executed)g(on)f(the)h(receiv)o(er)f(side.)33 b(This)20 b(a)o(v)o(oids)e(the)i(need)g(to)e(bu\013er)75 2591 y(message)e(data)f(out)h(of)f(sender)i(or)e(receiv)o(er)i(memory)l (.)22 b(In)17 b(this)f(case)g(the)g(send)h(op)q(eration)f(completes)75 2647 y(only)e(after)e(the)i(matc)o(hing)f(receiv)o(e)h(has)g(started)e (executing.)21 b(On)13 b(the)h(other)f(hand,)g(it)h(is)g(also)f(p)q (ossible)75 2704 y(for)j(MPI)g(to)f(bu\013er)h(messages,)g(so)g(as)g (to)f(allo)o(w)i(the)f(sender)h(to)e(pro)q(ceed)i(ahead)g(of)e(the)i (receiv)o(er.)23 b(In)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 27 32 27 31 bop 75 -100 a Ft(3.7.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b FE(27)75 45 y(suc)o(h)15 b(a)g(case)g(the)g(send)g(op)q (eration)g(ma)o(y)f(complete)i(successfully)h(b)q(efore)e(the)g (message)f(w)o(as)g(receiv)o(ed.)166 116 y(There)h(are)g(t)o(w)o(o)f (additional)j(comm)o(unication)f(mo)q(des:)166 186 y(A)e(send)h(that)e (uses)h(the)g Fz(ready)g FE(comm)o(unication)g(mo)q(de)h(ma)o(y)e(b)q (e)i(started)e(only)h(if)h(the)f(matc)o(hing)75 242 y(receiv)o(e)20 b(is)f(already)g(p)q(osted;)i(Otherwise)e(the)g(op)q(eration)g(is)g (erroneous)g(and)g(its)g(outcome)g(is)g(unde-)75 299 y(\014ned.)h(In)15 b(some)f(systems,)f(this)i(allo)o(ws)f(the)g(remo)o (v)m(al)h(of)e(a)h(hand-shak)o(e)g(op)q(eration)h(that)e(is)i (otherwise)75 355 y(required,)h(and)f(results)g(in)h(impro)o(v)o(ed)f (p)q(erformance.)20 b(The)15 b(completion)h(of)e(the)h(send)h(op)q (eration)f(do)q(es)75 412 y(not)h(dep)q(end)i(on)e(the)g(status)f(of)h (a)g(matc)o(hing)g(receiv)o(e,)h(and)f(merely)h(indicates)h(that)d(the) h(send)h(bu\013er)75 468 y(can)i(b)q(e)g(reused.)30 b(A)19 b(send)g(op)q(eration)g(that)f(uses)g(the)h(ready)f(mo)q(de)h(has)g (the)f(same)g(seman)o(tics)h(as)f(a)75 525 y(standard)12 b(send)h(op)q(eration;)g(it)g(is)g(merely)g(that)e(it)i(can)g(b)q(e)g (used)g(only)f(in)i(part)e(of)f(the)i(situations)g(where)75 581 y(a)i(regular)g(sen)o(t)g(can)g(b)q(e)h(used.)166 652 y(A)i(send)g(that)f(uses)g(the)h Fz(sync)o(hronous)e FE(mo)q(de)i(can)g(b)q(e)g(started)f(whether)g(or)g(not)h(a)f(matc)o (hing)75 708 y(receiv)o(e)12 b(w)o(as)e(p)q(osted.)19 b(Ho)o(w)o(ev)o(er,)10 b(the)h(send)h(will)g(complete)g(successfully)h (only)f(if)f(a)g(matc)o(hing)g(receiv)o(e)g(is)75 765 y(p)q(osted,)h(and)g(the)g(receiv)o(e)g(op)q(eration)g(has)f(started)g (to)g(receiv)o(e)i(the)e(message)g(sen)o(t)h(b)o(y)f(the)h(sync)o (hronous)75 821 y(send.)29 b(\(I.e.,)19 b(the)f(receiv)o(e)h(has)f(b)q (een)h(p)q(osted,)g(and)g(the)f(incoming)h(message)f(has)g(b)q(een)i (matc)o(hed)e(to)75 878 y(this)g(p)q(osted)g(receiv)o(e.\))28 b(Th)o(us,)18 b(the)g(completion)g(of)g(a)f(sync)o(hronous)h(send)g (not)f(only)i(indicates)g(that)75 934 y(the)g(send)h(bu\013er)f(can)g (b)q(e)h(reused,)g(but)f(also)g(indicates)i(that)d(the)i(receiv)o(er)f (has)g(reac)o(hed)h(a)f(certain)75 990 y(p)q(oin)o(t)e(in)h(its)f (execution,)h(namely)g(that)e(it)h(has)g(started)f(executing)i(the)f (matc)o(hing)g(receiv)o(e.)26 b(If)17 b(b)q(oth)75 1047 y(sends)f(and)h(receiv)o(es)f(are)g(blo)q(c)o(king)h(op)q(erations)f (then)h(the)f(use)g(of)f(the)h(sync)o(hronous)g(mo)q(de)g(pro)o(vides) 75 1103 y(sync)o(hronous)f(comm)o(unication)h(seman)o(tics:)k(a)14 b(comm)o(unication)i(do)q(es)f(not)g(complete)g(at)g(either)g(ends)75 1160 y(b)q(efore)d(b)q(oth)g(pro)q(cesses)g(\\attend")f(to)g(the)h (comm)o(unication;)h(the)f(completion)h(of)e(a)h(sync)o(hronous)f(send) 75 1216 y(is)16 b(a)f Fz(global)i FE(ev)o(en)o(t,)d(and)h(the)h(op)q (eration)f(is)h Fz(globally-blo)q(cking)p FE(.)166 1287 y(Tw)o(o)22 b(additional)j(send)e(functions)h(are)f(pro)o(vided)h(for)f (the)g(t)o(w)o(o)f(additional)i(comm)o(unication)75 1343 y(mo)q(des.)32 b(The)19 b(comm)o(unication)h(mo)q(de)f(is)h(indicated)g (b)o(y)f(a)g(one)g(letter)h(pre\014x:)28 b Fp(R)18 b FE(for)h(ready)g(and)g Fp(S)75 1400 y FE(for)c(sync)o(hronous.)166 1470 y(Send)h(in)g(ready)f(mo)q(de)75 1588 y Fx(MPI)p 160 1588 14 2 v 16 w(RSEND)h(\(buf,)f(count,)h(datat)o(yp)q(e,)h(dest,) f(tag,)f(comm\))117 1679 y Fr(IN)155 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1782 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1885 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)117 1988 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o (teger\))117 2092 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 2195 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))75 2333 y Fp(int)23 b(MPI)p 245 2333 15 2 v 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2333 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2390 y(int)h(tag,)f(MPI)p 683 2390 V 17 w(Comm)g(comm\))75 2490 y(MPI)p 150 2490 V 17 w(RSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 2547 y()f(BUF\(*\))170 2603 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 2704 y FE(Send)16 b(in)g(sync)o(hronous)f(mo)q(de)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 28 33 28 32 bop 75 -100 a FE(28)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(SSEND)g(\(buf,)f(count,)h(datat)o(yp)q(e,)g(dest,)h (tag,)e(comm\))117 123 y Fr(IN)155 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 199 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 276 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h (bu\013er)g(elemen)o(t)117 352 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o(teger\))117 429 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 505 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))75 630 y Fp(int)23 b(MPI)p 245 630 15 2 v 17 w(Ssend\(void*)g(buf,)g(int)g (count,)g(MPI)p 1002 630 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 687 y(int)h(tag,)f(MPI)p 683 687 V 17 w(Comm)g(comm\))75 774 y(MPI)p 150 774 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 830 y()f(BUF\(*\))170 887 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 974 y FE(There)16 b(is)g(only)g(one)g(receiv)o(e)h(mo)q(de,)e(whic)o(h) i(can)f(matc)o(h)f(an)o(y)g(of)h(the)f(send)i(mo)q(des.)k(The)16 b(receiv)o(e)75 1030 y(op)q(eration)g(describ)q(ed)h(in)g(the)e(last)h (section)g(is)g Fz(blo)q(c)o(king)p FE(:)22 b(it)16 b(returns)f(only)h (after)f(the)h(receiv)o(e)g(bu\013er)75 1087 y(con)o(tains)h(the)h (newly)g(receiv)o(ed)g(message.)26 b(It)17 b(is)h Fz(async)o(hronous)p FE(:)23 b(the)17 b(completion)i(of)d(a)h(receiv)o(e)h(is)75 1143 y(a)e(lo)q(cal)i(op)q(eration,)f(and)f(a)g(receiv)o(e)i(can)f (complete)g(b)q(efore)g(the)f(matc)o(hing)h(send)g(has)f(completed)i (\(of)75 1199 y(course,)d(it)g(can)h(complete)g(only)f(after)g(the)g (matc)o(hing)g(send)h(has)f(started\).)166 1257 y(Comm)o(unication)e (imp)q(oses)g(an)f(order)h(on)f(the)h(ev)o(en)o(ts)f(o)q(ccurring)h(at) f(the)h(comm)o(unicating)g(no)q(des.)75 1313 y(It)i(is)g(alw)o(a)o(ys)e (the)i(case)g(that)e(the)i(completion)h(of)e(a)g(receiv)o(e)i(o)q (ccurs)e(after)g(the)h(start)e(of)h(the)h(matc)o(hing)75 1369 y(send.)29 b(If)19 b(the)f(sync)o(hronous)g(send)h(mo)q(de)f(is)h (used,)g(then)g(it)f(is)h(also)f(the)g(case)g(that)g(the)g(completion) 75 1426 y(of)i(a)h(send)g(o)q(ccurs)g(after)f(the)g(start)g(of)g(the)h (matc)o(hing)f(receiv)o(e.)38 b(Of)20 b(course,)i(on)f(eac)o(h)f(pro)q (cess,)i(a)75 1482 y(comm)o(unication)14 b(completes)h(after)d(it)i(is) g(started.)19 b(No)13 b(other)h(order)f(is)h(imp)q(osed)h(b)o(y)e(MPI.) g(E.g.,)g(if)h(the)75 1539 y(standard)h(send)g(mo)q(de)h(is)f(used,)h (the)f(send)h(op)q(eration)f(ma)o(y)f(complete)i(b)q(efore)g(the)f (matc)o(hing)g(receiv)o(e)75 1595 y(w)o(as)f(started.)166 1652 y(In)h(a)f(m)o(ulti-threaded)i(implemen)o(tation)g(of)e(MPI,)h (the)f(system)g(ma)o(y)g(desc)o(hedule)j(a)d(thread)h(that)75 1709 y(is)21 b(blo)q(c)o(k)o(ed)g(on)g(a)f(send)h(or)e(receiv)o(e)j(op) q(eration,)f(and)g(sc)o(hedule)h(another)e(thread)g(for)g(execution)h (in)75 1765 y(the)d(same)g(address)g(space.)28 b(In)19 b(suc)o(h)f(a)f(case)h(it)h(the)f(user)g(resp)q(onsibilit)o(y)i(not)e (to)f(access)h(or)f(mo)q(dify)75 1822 y(a)f(comm)o(unication)h (bu\013er)f(un)o(til)i(the)e(comm)o(unication)h(completes;)g(otherwise) g(the)f(outcome)g(of)g(the)75 1878 y(computation)f(is)h(unde\014ned.) 166 2011 y Fo(Implemen)o(tati)o(on)10 b(note:)34 b Fr(A)13 b(ready)f(send)h(can)f(b)q(e)h(implemen)o(ted)d(as)i(a)g(standard)h (send;)g(in)e(suc)o(h)i(a)f(case)75 2061 y(there)j(will)e(b)q(e)h(no)g (p)q(erformance)g(adv)n(an)o(tage)f(\(or)h(disadv)n(an)o(tage\))f(for)g (the)i(use)f(of)g(ready)g(send.)166 2112 y(A)f(standard)h(send)h(can)e (b)q(e)h(implemen)o(ted)e(as)h(a)g(sync)o(hronous)i(send.)j(In)c(suc)o (h)g(a)f(case,)h(no)f(data)h(bu\013ering)75 2161 y(is)g(needed.)166 2212 y(A)g(p)q(ossible)g(comm)o(unication)c(proto)q(col)k(for)g(the)g (v)n(arious)f(comm)o(unication)e(mo)q(des)i(is)h(outlined)f(b)q(elo)o (w:)166 2262 y Fl(ready)21 b(send)p Fr(:)c(The)d(message)g(is)f(sen)o (t)i(as)f(so)q(on)g(as)g(p)q(ossible.)166 2313 y Fl(synchronous)19 b(send:)29 b Fr(The)20 b(sender)g(sends)h(a)e(request-to-send)j (message.)34 b(The)19 b(receiv)o(er)i(stores)g(this)75 2363 y(request.)f(When)15 b(a)f(matc)o(hing)e(receiv)o(e)j(is)f(p)q (osted,)h(the)g(receiv)o(er)h(sends)f(bac)o(k)f(a)g(p)q (ermission-to-send)g(message,)75 2412 y(and)g(the)g(sender)i(no)o(w)d (sends)i(the)g(message.)166 2463 y Fl(standard)20 b(send:)e Fr(First)d(proto)q(col)f(ma)o(y)e(b)q(e)j(used)h(for)d(short)i (messages,)f(and)g(second)i(proto)q(col)e(for)g(long)75 2513 y(messages.)166 2563 y(Additional)h(con)o(trol)h(messages)h(migh)o (t)e(b)q(e)i(needed)h(for)e(\015o)o(w)g(con)o(trol)h(and)f(error)i (reco)o(v)o(ery)m(.)27 b(Of)16 b(course,)75 2613 y(there)f(are)f(man)o (y)f(other)h(p)q(ossible)g(c)o(hoices.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 29 34 29 33 bop 75 -100 a Ft(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FE(29)75 45 y Fv(3.8)59 b(Nonblo)r(cking)20 b(communication)75 147 y FE(One)e(can)g(impro)o(v)o(e)f(p)q(erformance)g(on)h(man)o(y)e (systems)h(b)o(y)h(o)o(v)o(erlapping)f(comm)o(unication)i(and)e(com-)75 203 y(putation.)35 b(This)21 b(is)g(esp)q(ecially)i(true)d(on)g (systems)g(where)g(comm)o(unication)h(can)g(b)q(e)g(executed)g(au-)75 259 y(tonomously)12 b(b)o(y)h(an)g(in)o(telligen)o(t)h(comm)o (unication)g(con)o(troller.)19 b(Ligh)o(t-w)o(eigh)o(t)13 b(threads)f(are)h(one)g(mec)o(h-)75 316 y(anism)21 b(for)g(ac)o (hieving)h(suc)o(h)g(o)o(v)o(erlap.)37 b(An)21 b(alternativ)o(e)h(mec)o (hanism)g(that)e(often)h(leads)g(to)g(b)q(etter)75 372 y(p)q(erformance)e(is)g(to)f(use)h Fz(non)o(blo)q(c)o(king)j(comm)o (unication)p FE(.)32 b(A)19 b(non)o(blo)q(c)o(king)h(send)f(call)h (initiates)75 429 y(the)15 b(send)h(op)q(eration,)f(but)h(do)q(es)f (not)g(complete)h(it.)k(The)15 b(send)h(call)h(will)f(return)g(b)q (efore)f(the)g(message)75 485 y(w)o(as)d(copied)h(out)f(of)g(the)g (send)h(bu\013er.)19 b(A)13 b(separate)e(call)j(is)f(needed)g(to)f (complete)h(the)g(comm)o(unication,)75 542 y(i.e.)20 b(to)13 b(v)o(erify)i(that)e(the)h(data)g(has)g(b)q(een)h(copied)g(out) f(of)g(the)g(send)h(bu\013er.)k(With)c(suitable)g(hardw)o(are,)75 598 y(the)k(transfer)f(of)h(data)f(out)h(of)g(the)g(sender)g(memory)g (ma)o(y)f(pro)q(ceed)i(concurren)o(tly)g(with)f(computa-)75 655 y(tions)g(done)g(at)f(the)g(sender)h(after)f(the)h(send)g(w)o(as)f (initiated)i(and)e(b)q(efore)h(it)g(completed.)31 b(Similarly)l(,)75 711 y(a)19 b(non)o(blo)q(c)o(king)h(receiv)o(e)g(initiates)h(the)e (receiv)o(e)h(op)q(eration,)g(but)f(do)q(es)g(not)g(complete)h(it.)32 b(The)19 b(call)75 768 y(will)f(return)f(b)q(efore)g(a)f(message)g(is)i (stored)e(in)o(to)g(the)h(receiv)o(e)h(bu\013er.)24 b(A)17 b(separate)f(call)h(is)h(needed)g(to)75 824 y(complete)f(the)f(receiv)o (e)i(op)q(eration)e(and)g(v)o(erify)h(that)e(the)i(data)e(has)i(b)q (een)g(receiv)o(ed)g(in)o(to)g(the)f(receiv)o(e)75 880 y(bu\013er.)k(With)14 b(suitable)h(hardw)o(are,)f(the)g(transfer)f(of)h (data)g(in)o(to)g(the)g(receiv)o(er)h(memory)f(ma)o(y)f(pro)q(ceed)75 937 y(concurren)o(tly)f(with)g(computations)f(done)g(after)g(the)g (receiv)o(e)h(w)o(as)f(initiated)i(and)e(b)q(efore)h(it)f(completed.)75 993 y(The)17 b(use)h(of)f(non)o(blo)q(c)o(king)h(receiv)o(es)g(ma)o(y)f (also)g(a)o(v)o(oid)g(system)f(bu\013ering)i(and)f(memory)g(to)g (memory)75 1050 y(cop)o(ying,)e(as)g(information)g(is)h(pro)o(vided)g (early)g(on)f(the)g(lo)q(cation)h(of)f(the)g(receiv)o(e)h(bu\013er.)166 1106 y(Non)o(blo)q(c)o(king)g(sends)g(can)g(use)f(the)h(same)f(three)g (mo)q(des)h(as)f(blo)q(c)o(king)h(sends:)21 b Fp(standard)p FE(,)14 b Fp(ready)75 1163 y FE(and)21 b Fp(synchronous)p FE(.)35 b(These)21 b(carry)g(the)g(same)f(meaning.)37 b(The)21 b(initiation)i(and)e(completion)h(of)e(a)75 1219 y(standard)e(send)i(do)f(not)f(dep)q(end)j(on)d(the)h(status)f(of) h(a)f(matc)o(hing)h(receiv)o(e.)32 b(A)19 b(ready)g(send)g(can)g(b)q(e) 75 1276 y(initiated)12 b(only)f(if)g(a)g(matc)o(hing)f(receiv)o(e)i (has)e(already)h(b)q(een)h(initiated,)g(otherwise)f(the)g(call)g(is)g (erroneous;)75 1332 y(its)18 b(completion)g(do)q(es)g(not)f(dep)q(end)i (on)f(the)f(status)g(of)g(a)g(matc)o(hing)h(receiv)o(e.)27 b(A)18 b(sync)o(hronous)f(send)75 1389 y(can)f(b)q(e)h(initiated)h(b)q (efore)f(a)f(matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)i(initiated,)g (but)e(will)i(complete)f(only)g(after)75 1445 y(a)f(matc)o(hing)f (receiv)o(e)i(has)f(b)q(een)h(initiated,)g(and)f(has)g(started)f (receiving)j(the)e(message)f(generated)h(b)o(y)75 1501 y(the)f(send)h(op)q(eration.)166 1558 y(In)f(all)h(cases)e(the)h(op)q (eration)g(that)f(initiates)i(the)f(comm)o(unication)g(\(send)g(or)f (receiv)o(e\))i(is)f(a)f(lo)q(cal,)75 1614 y(non)o(blo)q(c)o(king)20 b(call.)31 b(Suc)o(h)19 b(call)g(ma)o(y)f(fail)h(b)q(ecause)h(b)q (ecause)f(the)g(system)f(has)g(exhausted)h(a)o(v)m(ailable)75 1671 y(resources)14 b(\(e.g.,)f(exceeded)j(the)f(limit)h(on)e(n)o(um)o (b)q(er)h(of)f(p)q(ending)i(comm)o(unications)f(p)q(er)g(no)q(de\).)20 b(Go)q(o)q(d)75 1727 y(qualit)o(y)h(implemen)o(tations)g(of)e(MPI)h (will)i(set)e(these)g(limits)h(high)g(enough)f(so)g(that)f(a)h(non)o (blo)q(c)o(king)75 1784 y(comm)o(unication)c(initiation)h(call)f(will)h (fail)f(only)g(in)g(\\pathological")f(cases.)166 1840 y(Non)o(blo)q(c)o(king)h(sends)g(can)f(b)q(e)h(matc)o(hed)f(with)h(blo) q(c)o(king)g(receiv)o(es,)g(and)f(vice-v)o(ersa.)75 1960 y Fq(3.8.1)49 b(Communication)17 b(Objects)75 2046 y FE(Non)o(blo)q(c)o(king)c(comm)o(unications)g(use)f(opaque)g(comm)o (unication)h(ob)s(jects)f(to)f(iden)o(tify)i(comm)o(unication)75 2103 y(op)q(erations)18 b(and)h(matc)o(h)e(the)i(op)q(eration)f(that)f (initiates)j(the)e(comm)o(unication)h(with)f(the)h(op)q(eration)75 2159 y(that)e(terminates)i(it.)29 b(These)18 b(are)g(system)g(ob)s (jects)g(that)f(are)h(accessed)h(via)f(a)g(handle.)30 b(An)19 b(opaque)75 2215 y(comm)o(unication)f(ob)s(ject)e(iden)o (ti\014es)j(v)m(arious)f(prop)q(erties)f(of)g(a)g(comm)o(unication)h (op)q(eration,)f(suc)o(h)g(as)75 2272 y(the)g(\(send)g(or)f(receiv)o (e\))h(bu\013er)g(that)f(is)h(asso)q(ciated)g(with)g(it,)g(its)g(con)o (text,)f(the)g(tag)g(and)h(destination)75 2328 y(parameters)c(to)f(b)q (e)j(used)f(for)e(a)i(send,)g(or)f(the)g(tag)g(and)g(source)h (parameters)e(to)h(b)q(e)h(used)g(for)f(a)g(receiv)o(e.)75 2385 y(In)j(addition,)g(this)g(ob)s(ject)f(stores)f(information)i(ab)q (out)f(the)g(status)g(of)g(the)g(p)q(ending)i(comm)o(unication)75 2441 y(op)q(eration)e(that)g(is)h(p)q(erformed)f(with)h(this)f(ob)s (ject.)75 2561 y Fq(3.8.2)49 b(Communication)17 b(initiation)75 2647 y FE(W)l(e)h(use)g(the)g(same)g(naming)g(con)o(v)o(en)o(tions)g (as)f(for)g(blo)q(c)o(king)j(comm)o(unication:)26 b(a)17 b(pre\014x)h(of)g Fp(R)f FE(\()p Fp(S)p FE(\))g(is)75 2704 y(used)h(for)f Fp(READY)g FE(\()p Fp(SYNCHRONOUS)p FE(\))f(mo)q(de.)27 b(In)19 b(addition)f(a)g(pre\014x)g(of)f Fp(I)h FE(\(for)e Fp(IMMEDIATE)p FE(\))h(indicates)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 30 35 30 34 bop 75 -100 a FE(30)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(that)e(the)i (call)g(is)g(non)o(blo)q(c)o(king.)166 113 y(Initiate)g(a)f(standard)g (mo)q(de)g(non)o(blo)q(c)o(king)i(comm)o(unication.)75 228 y Fx(MPI)p 160 228 14 2 v 16 w(ISEND\(buf,)e(count,)h(datat)o(yp)q (e,)h(dest,)f(tag,)f(comm,)e(request\))117 317 y Fr(IN)155 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 415 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 514 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)117 612 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o (teger\))117 710 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 809 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 907 y(OUT)108 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 1043 y Fp(int)23 b(MPI)p 245 1043 15 2 v 17 w(Isend\(void*)g(buf,)g (int)g(count,)g(MPI)p 1002 1043 V 17 w(Datatype)g(datatype,)g(int)g (dest,)393 1099 y(int)h(tag,)f(MPI)p 683 1099 V 17 w(Comm)g(comm,)g (MPI)p 1034 1099 V 17 w(Comm)p 1147 1099 V 17 w(request)g(*request\))75 1197 y(MPI)p 150 1197 V 17 w(ISEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(REQUEST,)e(IERROR\))170 1254 y()h(BUF\(*\))170 1310 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1408 y FE(Initiate)16 b(a)f(ready)g(mo)q(de)h(non)o(blo)q (c)o(king)g(comm)o(unication.)75 1524 y Fx(MPI)p 160 1524 14 2 v 16 w(IRSEND\(buf,)g(count,)g(datat)o(yp)q(e,)g(dest,)g (tag,)f(comm,)e(request\))117 1612 y Fr(IN)155 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1711 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1809 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)117 1907 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o (teger\))117 2006 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 2104 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 2202 y(OUT)108 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 2338 y Fp(int)23 b(MPI)p 245 2338 15 2 v 17 w(Irsend\(void*)f(buf,)i (int)f(count,)g(MPI)p 1026 2338 V 17 w(Datatype)g(datatype,)f(int)i (dest,)393 2395 y(int)g(tag,)f(MPI)p 683 2395 V 17 w(Comm)g(comm,)g (MPI)p 1034 2395 V 17 w(Comm)p 1147 2395 V 17 w(request)g(*request\))75 2493 y(MPI)p 150 2493 V 17 w(IRSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,) g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2549 y()g(BUF\(*\))170 2606 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 2704 y FE(Initiate)16 b(a)f(sync)o(hronous)g(mo)q(de)h(non) o(blo)q(c)o(king)g(comm)o(unication.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 31 36 31 35 bop 75 -100 a Ft(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FE(31)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(ISSEND\(buf,)15 b(count,)i(datat)o(yp)q(e,)f(dest,)g(tag,)f(comm,)e(request\))117 126 y Fr(IN)155 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 207 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 289 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t) 117 370 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o (teger\))117 452 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 534 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 615 y(OUT)108 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 743 y Fp(int)23 b(MPI)p 245 743 15 2 v 17 w(Issend\(void*)f(buf,)i(int) f(count,)g(MPI)p 1026 743 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 799 y(int)g(tag,)f(MPI)p 683 799 V 17 w(Comm)g(comm,)g(MPI)p 1034 799 V 17 w(Comm)p 1147 799 V 17 w(request)g(*request\))75 889 y(MPI)p 150 889 V 17 w(ISSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 946 y()g(BUF\(*\))170 1002 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1092 y FE(Initiate)16 b(a)f(non)o(blo)q(c)o(king)i(receiv)o (e.)75 1199 y Fx(MPI)p 160 1199 14 2 v 16 w(IRECV)e(\(buf,)h(count,)g (datat)o(yp)q(e,)g(source,)g(tag,)e(comm,)f(request\))117 1279 y Fr(OUT)108 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(receiv) o(e)h(bu\013er)g(\(c)o(hoice\))117 1361 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1442 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)117 1524 y(IN)155 b Fx(source)468 b Fr(rank)14 b(of)f(source)j (\(in)o(teger\))117 1606 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 1687 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 1769 y(OUT)108 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 1897 y Fp(int)23 b(MPI)p 245 1897 15 2 v 17 w(Irecv\(void*)g(buf,)g (int)g(count,)g(MPI)p 1002 1897 V 17 w(Datatype)g(datatype,)g(int)g (source,)393 1953 y(int)h(tag,)f(MPI)p 683 1953 V 17 w(Comm)g(comm,)g(MPI)p 1034 1953 V 17 w(Comm)p 1147 1953 V 17 w(request)g(*request\))75 2043 y(MPI)p 150 2043 V 17 w(IRECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR\))170 2099 y()g(BUF\(*\))170 2156 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR)166 2245 y FE(These)18 b(calls)h(allo)q(cate)g(a)e (comm)o(unication)i(ob)s(ject)e(and)h(asso)q(ciates)g(it)g(with)g(the)g (request.)28 b(The)75 2302 y(request)18 b(can)g(b)q(e)g(used)h(later)e (to)h(query)g(ab)q(out)f(the)h(status)f(of)h(the)f(comm)o(unication)i (or)e(w)o(ait)h(for)f(its)75 2358 y(completion.)166 2418 y(A)i(non)o(blo)q(c)o(king)i(send)f(call)h(indicates)g(that)d(the)i (system)f(ma)o(y)f(buf)i(cop)o(ying)g(data)f(out)g(of)g(the)75 2475 y(send)e(bu\013er.)24 b(The)17 b(sender)g(should)g(not)g(access)f (an)o(y)g(part)g(of)g(the)h(send)g(bu\013er)g(after)e(a)i(non)o(blo)q (c)o(king)75 2531 y(send)f(op)q(eration)f(returns,)g(un)o(til)h(the)g (send)f(completes.)166 2591 y(A)f(non)o(blo)q(c)o(king)h(receiv)o(e)g (call)h(indicates)f(that)f(the)g(sytem)f(ma)o(y)h(start)f(writing)h (data)g(in)o(to)g(the)g(re-)75 2647 y(ceiv)o(e)e(bu\013er.)19 b(The)11 b(receiv)o(er)h(should)h(not)e(access)g(an)o(y)g(part)g(of)g (the)g(receiv)o(e)i(bu\013er)e(after)g(a)g(non)o(blo)q(c)o(king)75 2704 y(receiv)o(e)16 b(op)q(eration)g(returns,)e(un)o(til)j(the)e (receiv)o(e)h(completes.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 32 37 32 36 bop 75 -100 a FE(32)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fq(3.8.3)49 b(Communication)17 b(Completion)75 140 y FE(The)g(functions)g Fx(MPI)p 452 140 14 2 v 16 w(W)l(AIT)f FE(and)g Fx(MPI)p 768 140 V 16 w(TEST)h FE(are)f(used)h(to)f(complete)h(a)f(non)o(blo)q (c)o(king)h(comm)o(unica-)75 196 y(tion.)27 b(The)17 b(completion)i(of)e(a)g(send)h(op)q(eration)f(indicates)i(that)e(the)g (sender)h(is)g(no)o(w)f(free)h(to)e(up)q(date)75 253 y(the)h(lo)q(cations)h(in)g(the)f(send)g(bu\013er,)h(or)e(an)o(y)h (other)f(lo)q(cation)i(that)e(can)i(b)q(e)f(referenced)h(b)o(y)f(the)g (send)75 309 y(op)q(eration)i(\(the)g(send)g(op)q(eration)g(itself)h (lea)o(v)o(es)e(the)h(con)o(ten)o(t)f(of)h(the)g(send)g(bu\013er)g(unc) o(hanged\).)31 b(It)75 366 y(do)q(es)19 b(not)f(indicate)j(that)d(the)h (message)f(has)g(b)q(een)i(receiv)o(ed;)i(rather,)c(it)h(ma)o(y)f(ha)o (v)o(e)h(b)q(een)h(bu\013ered)75 422 y(b)o(y)f(the)g(comm)o(unication)h (subsystem.)32 b(Ho)o(w)o(ev)o(er,)19 b(if)h(a)f Fp(synchronous)f FE(mo)q(de)h(send)h(w)o(as)e(used,)j(the)75 479 y(completion)c(of)e (the)g(send)h(op)q(eration)g(indicates)h(that)e(a)g(matc)o(hing)g (receiv)o(e)i(w)o(as)d(initiated,)j(and)f(that)75 535 y(the)f(message)g(will)i(ev)o(en)o(tually)f(b)q(e)g(receiv)o(ed)g(b)o (y)f(this)h(matc)o(hing)f(receiv)o(e.)166 596 y(The)20 b(completion)h(of)e(a)g(receiv)o(e)i(op)q(eration)f(indicates)h(that)e (the)h(receiv)o(e)g(bu\013er)g(con)o(tains)g(the)75 653 y(receiv)o(ed)f(message,)f(and)g(that)f(the)h(status)f(ob)s(ject)h(is)g (set;)h(the)f(receiv)o(er)h(is)f(no)o(w)f(free)h(to)g(access)g(the)75 709 y(receiv)o(e)c(bu\013er,)f(or)f(an)o(y)g(other)h(lo)q(cation)h (that)e(can)h(b)q(e)g(referenced)h(b)o(y)f(the)g(receiv)o(e)g(op)q (eration.)20 b(It)13 b(do)q(es)75 766 y(not)g(indicate)i(that)d(the)i (matc)o(hing)f(send)h(op)q(eration)g(has)f(completed)h(\(but)g (indicates,)g(of)f(course,)h(that)75 822 y(the)h(send)h(w)o(as)e (initiated\).)75 930 y Fx(MPI)p 160 930 V 16 w(W)l(AIT\(request,)i (status\))117 1012 y Fr(IN)155 b Fx(request)452 b Fr(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject)117 1097 y(OUT)108 b Fx(status)476 b Fr(status)15 b(ob)r(ject)75 1226 y Fp(int)23 b(MPI)p 245 1226 15 2 v 17 w(Wait\(MPI)p 454 1226 V 16 w(Comm)p 566 1226 V 17 w(request)g(request,)g(MPI)p 1061 1226 V 16 w(Status)g(*status\))75 1317 y(MPI)p 150 1317 V 17 w(WAIT\(REQUEST,)f(STATUS,)h(IERROR\))170 1373 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 1373 V 16 w(STATUS)p 979 1373 V 16 w(SIZE\),)g(IERROR)166 1464 y FE(A)14 b(call)h(to)f Fx(MPI)p 436 1464 14 2 v 15 w(W)l(AIT)h FE(returns)e(when)i(the)f(send) h(op)q(eration)f(iden)o(ti\014ed)i(b)o(y)e Fx(request)i FE(is)f(complete.)75 1521 y(If)21 b(the)g(comm)o(unication)h(ob)s(ject) f(asso)q(ciated)g(with)g(this)h(request)f(w)o(as)f(created)h(b)o(y)g(a) g(non)o(blo)q(c)o(king)75 1577 y(send)13 b(or)g(receiv)o(e)h(call,)g (then)f(the)g(ob)s(ject)f(is)h(deallo)q(cated)i(b)o(y)d(the)h(call)h (to)25 b Fx(MPI)p 1431 1577 V 16 w(W)l(AIT)13 b FE(and)g(the)g(request) 75 1634 y(b)q(ecomes)j(n)o(ull.)166 1695 y(The)h(call)g(returns)g(in)g Fx(status)h FE(information)f(on)f(the)h(completed)g(op)q(eration.)24 b(The)17 b(status)f(ob)s(ject)75 1751 y(for)g(a)h(receiv)o(e)h(op)q (eration)g(can)f(b)q(e)h(queried)g(using)g(the)f(functions)h(describ)q (ed)h(in)f(section)g(3.3.1.)24 b(The)75 1808 y(status)15 b(parameter)g(is)i(not)e(used)i(or)e(up)q(dated)i(for)e(a)h(send)g(op)q (eration,)g(and)g(an)g(arbitrary)g(v)m(alue)h(ma)o(y)75 1864 y(b)q(e)f(passed)f(for)g(this)h(parameter)e(in)i(the)f(call.)75 1972 y Fx(MPI)p 160 1972 V 16 w(TEST\(request,)h(\015ag,)f(status\))117 2054 y Fr(IN)155 b Fx(request)452 b Fr(handle)14 b(to)g(comm)o(unicati) o(on)d(ob)r(ject)117 2139 y(OUT)108 b Fx(\015ag)518 b Fr(logical)117 2223 y(OUT)108 b Fx(status)476 b Fr(status)15 b(ob)r(ject)75 2352 y Fp(int)23 b(MPI)p 245 2352 15 2 v 17 w(Test\(MPI)p 454 2352 V 16 w(Comm)p 566 2352 V 17 w(request)g(request,)g(int)g(flag,)g(MPI)p 1299 2352 V 17 w(Status)g(*status\))75 2443 y(MPI)p 150 2443 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g(IERROR\))170 2500 y(INTEGER)g(REQUEST,)g(FLAG,)g(STATUS\(MPI)p 962 2500 V 16 w(STATUS)p 1122 2500 V 16 w(SIZE\),)h(IERROR)166 2591 y FE(A)17 b(call)i(to)e Fx(MPI)p 446 2591 14 2 v 15 w(TEST)h FE(returns)f Fx(\015ag)h(=)f(true)h FE(if)g(the)g(op)q (eration)f(iden)o(ti\014ed)j(b)o(y)d Fx(request)i FE(is)f(com-)75 2647 y(plete.)37 b(In)21 b(suc)o(h)g(a)f(case,)i(the)e(status)g(ob)s (ject)g(is)h(set)f(to)g(con)o(tain)h(information)g(on)f(the)h (completed)75 2704 y(op)q(eration;)15 b(if)h(the)f(comm)o(unication)h (ob)s(ject)e(w)o(as)h(created)g(b)o(y)g(a)g(non)o(blo)q(c)o(king)h (send)g(or)f(receiv)o(e,)h(then)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 33 38 33 37 bop 75 -100 a Ft(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FE(33)75 45 y(it)18 b(is)g(deallo)q(cated)i(and)e(the)g(request)f(b)q (ecomes)i(n)o(ull.)29 b(The)18 b(call)h(returns)f Fx(\015ag=false)p FE(,)g(otherwise.)28 b(In)75 102 y(suc)o(h)16 b(a)e(case,)h(the)h(v)m (alue)g(of)f(the)g(status)f(ob)s(ject)h(is)h(unde\014ned.)166 158 y Fx(MPI)p 251 158 14 2 v 16 w(TEST)f FE(is)h(a)f(lo)q(cal,)h(non)o (blo)q(c)o(king)g(op)q(eration.)166 215 y(The)11 b(return)g(status)f (ob)s(ject)h(for)f(a)h(receiv)o(e)h(op)q(eration)f(carries)g (information)g(that)g(can)g(b)q(e)g(accessed)75 272 y(using)16 b(the)e(functions)i(describ)q(ed)h(in)e(section)h(3.3.1.)i(The)d (status)f(argumen)o(t)g(is)h(not)f(used)i(or)e(up)q(dated)75 328 y(for)h(a)f(send)i(op)q(eration)g(and)f(an)g(arbitrary)g(v)m(alue)h (can)f(b)q(e)h(passed)g(for)e(this)i(parameter)e(in)i(the)g(call.)166 385 y(In)i(m)o(ulti-threaded)h(en)o(vironmen)o(t,)f(the)g(use)g(of)f(a) h(blo)q(c)o(king)h(receiv)o(e)f(op)q(eration)g(\()g Fx(MPI)p 1732 385 V 15 w(W)l(AIT)p FE(\))75 442 y(ma)o(y)g(allo)o(w)i(the)f(op)q (erating)g(system)g(to)g(de-sc)o(hedule)i(the)e(blo)q(c)o(k)o(ed)h (thread)f(and)h(sc)o(hedule)g(another)75 498 y(thread)j(for)g (execution,)j(if)e(suc)o(h)g(is)f(a)o(v)m(ailable.)47 b(The)23 b(use)h(of)f(a)g(non)o(blo)q(c)o(king)h(receiv)o(e)h(op)q (eration)75 554 y(\()p Fx(MPI)p 178 554 V 15 w(TEST)p FE(\))e(allo)o(ws)f(the)g(user)g(to)g(sc)o(hedule)i(alternativ)o(e)e (activities)i(within)f(a)f(single)h(thread)f(of)75 611 y(execution.)166 668 y(Example:)75 764 y Fp(CALL)h (MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 820 y(IF\(rank.EQ.0\))123 877 y(THEN)170 933 y(CALL)h(MPI_ISEND\(request,)d(a\(1\),)i(10,)h (MPI_REAL,)e(1,)i(tag,)f(comm,)h(ierr\))170 989 y(****)g(do)f(some)h (computation)e(to)i(mask)f(latency)g(****)170 1046 y(CALL)h (MPI_WAIT\(request,)d(status,)i(ierr\))123 1102 y(ELSE)170 1159 y(CALL)h(MPI_IRECV\(request,)d(a\(1\),)i(15,)h(MPI_REAL,)e(0,)i (tag,)f(comm,)h(ierr\))170 1215 y(****)g(do)f(some)h(computation)e(to)i (mask)f(latency)g(****)170 1272 y(CALL)h(MPI_WAIT\(request,)d(status,)i (ierr\))75 1328 y(END)g(IF)166 1424 y FE(The)f(functions)h Fx(MPI)p 554 1424 V 15 w(W)l(AIT)f FE(and)g Fx(MPI)p 881 1424 V 16 w(TEST)g FE(can)g(b)q(e)g(used)h(to)e(complete)h(b)q(oth) g(sends)h(and)75 1480 y(receiv)o(es;)j(they)c(will)i(b)q(e)e(also)g (used)h(to)f(complete)g(an)o(y)g(other)g(non)o(blo)q(c)o(king)h(comm)o (unication)g(call)75 1537 y(pro)o(vided)16 b(b)o(y)f(MPI.)75 1661 y Fq(3.8.4)49 b(Semantics)15 b(of)i(Nonblo)q(cking)h (Communications)75 1747 y FE(The)11 b(seman)o(tics)f(of)h(non)o(blo)q (c)o(king)g(comm)o(unication)h(is)f(de\014ned)h(b)o(y)e(suitably)i (extending)g(the)e(de\014nitions)75 1804 y(in)16 b(section)g(3.4)e(for) h(order)g(and)g(progress.)166 1861 y(Non)o(blo)q(c)o(king)k(comm)o (unication)f(op)q(erations)g(are)g(ordered)g(according)g(to)f(the)h (execution)h(order)75 1917 y(of)h(the)g(calls)h(that)e(initiate)j(the)e (comm)o(unication;)i(this)f(de\014nes)g(the)f(meaning)h(of)e(non-o)o(v) o(ertaking)75 1973 y(comm)o(unication.)166 2030 y(Example:)75 2139 y Fp(CALL)k(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2195 y(IF)h(\(RANK.EQ.0\))147 2252 y(THEN)218 2308 y(CALL)f (MPI_ISEND\(h1,)g(a,)g(1,)h(MPI_REAL,)e(1,)i(0,)g(comm,)f(ierr\))218 2365 y(CALL)g(MPI_ISEND\(h2,)g(b,)g(1,)h(MPI_REAL,)e(1,)i(0,)g(comm,)f (ierr\))147 2421 y(ELSE)94 b(!)24 b(rank.EQ.1)218 2478 y(CALL)f(MPI_IRECV\()g(h1,)g(a,)h(1,)g(MPI_REAL,)e(0,)i(MPI_ANY_TAG,)e (comm,)h(ierr\))218 2534 y(CALL)g(MPI_IRECV\()g(h2,)g(b,)h(1,)g (MPI_REAL,)e(0,)i(0,)f(comm,)h(ierr\))75 2591 y(END)f(IF)75 2647 y(CALL)g(MPI_WAIT\(h1,status\))75 2704 y(CALL)g (MPI_WAIT\(h2,status\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 34 39 34 38 bop 75 -100 a FE(34)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FE(The)h(\014rst)f (send)h(of)f(pro)q(cess)h(zero)g(will)h(matc)o(h)e(the)h(\014rst)f (receiv)o(e)i(of)e(pro)q(cess)h(one,)g(ev)o(en)g(if)g(b)q(oth)75 102 y(messages)d(are)g(sen)o(t)g(b)q(efore)g(pro)q(cess)h(one)f (executes)h(either)g(receiv)o(es.)21 b(This)15 b(precludes)h(an)e (implemen-)75 158 y(tation)h(where)g(di\013eren)o(t)h(queues)g(are)f (used)g(to)g(p)q(ost)g(the)g(t)o(w)o(o)f(receiv)o(es.)166 214 y(A)22 b(call)g(to)f Fx(MPI)p 458 214 14 2 v 16 w(W)l(AIT)h FE(that)f(completes)h(a)f(receiv)o(e)i(will)g(ev)o(en)o(tually)g (terminate)e(and)h(return)75 271 y(if)f(a)f(matc)o(hing)g(send)h(has)f (b)q(een)h(started,)g(unless)g(the)g(send)g(is)f(satis\014ed)h(b)o(y)f (another)g(receiv)o(e.)36 b(In)75 327 y(particular,)20 b(if)f(the)g(matc)o(hing)g(send)g(is)g(non)o(blo)q(c)o(king,)i(then)e (the)g(receiv)o(e)g(should)h(complete)f(ev)o(en)g(if)75 384 y(no)c(call)i(is)e(executed)i(b)o(y)e(the)g(sender)h(to)f(complete) h(the)f(send.)20 b(Similarly)l(,)e(a)d(call)h(to)f Fx(MPI)p 1651 384 V 15 w(W)l(AIT)h FE(that)75 440 y(completes)i(a)e(send)i(will) h(ev)o(en)o(tually)f(return)f(if)g(a)g(matc)o(hing)g(receiv)o(e)h(has)f (b)q(een)h(started,)f(unless)h(the)75 497 y(receiv)o(e)e(is)g (satis\014ed)f(b)o(y)g(another)g(send,)g(and)g(ev)o(en)h(if)f(no)g (call)h(is)g(executed)g(to)e(complete)i(the)f(receiv)o(e.)166 553 y(Example:)75 660 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 716 y(IF)h(\(RANK.EQ.0\))147 772 y(THEN)218 829 y(CALL)f(MPI_ISEND\()g(h,)h(a,)f(1,)h(MPI_REAL,)e(1,)i(0,)g(comm,)f (ierr\))218 885 y(CALL)g(MPI_RECV\()g(b,)h(1,)f(MPI_REAL,)g(1,)h(0,)f (comm,)g(ierr\))218 942 y(CALL)g(MPI_WAIT\()g(h,)h(status\))147 998 y(ELSE)94 b(!)24 b(rank.EQ.1)218 1055 y(CALL)f(MPI_RECV\()g(a,)h (1,)f(MPI_REAL,)g(0,)h(0,)f(comm,)g(ierr\))218 1111 y(CALL)g (MPI_SEND\()g(b,)h(1,)f(MPI_REAL,)g(0,)h(0,)f(comm,)g(ierr\))75 1168 y(END)g(IF)166 1274 y FE(This)16 b(co)q(de)f(should)i(not)d (deadlo)q(c)o(k)j(in)f(a)f(correct)f(MPI)h(implemen)o(tation.)166 1330 y(If)i Fx(MPI)p 298 1330 V 15 w(TEST)g FE(that)f(completes)h(a)f (receiv)o(e)i(is)f(rep)q(eatedly)g(called)i(with)d(the)h(same)f (parameters,)75 1387 y(and)e(a)g(matc)o(hing)g(send)g(has)g(b)q(een)h (started,)e(then)h(the)g(call)h(will)g(ev)o(en)o(tually)g(return)f Fx(\015ag=true)p FE(,)h(unless)75 1443 y(the)i(send)g(is)g(satis\014ed) h(b)o(y)f(another)f(receiv)o(e.)25 b(If)17 b Fx(MPI)p 1020 1443 V 16 w(TEST)g FE(that)f(completes)i(a)e(send)i(is)f(rep)q (eatedly)75 1500 y(called)j(with)f(the)g(same)f(parameters,)g(and)h(a)f (matc)o(hing)h(receiv)o(e)g(has)f(b)q(een)i(started,)e(then)h(the)g (call)75 1556 y(will)e(ev)o(en)o(tually)f(return)f Fx(\015ag=true)p FE(,)h(unless)g(the)g(receiv)o(e)g(is)f(satis\014ed)h(b)o(y)f(another)g (send.)166 1613 y(Non)o(blo)q(c)o(king)g(comm)o(unications)g(cannot)f (b)q(e)g(replaced)i(b)o(y)e(blo)q(c)o(king)h(comm)o(unications,)g(ev)o (en)f(in)75 1669 y(the)h Fp(synchronous)f FE(comm)o(unication)i(mo)q (de.)k(Consider)c(the)f(follo)o(wing)h(example:)75 1763 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1819 y(IF\(rank.EQ.0\))123 1876 y(THEN)170 1932 y(CALL)h (MPI_ISEND\(request1,)d(a\(1\),)i(count,)g(MPI_REAL,)g(1,)g(tag1,)h (comm,)f(ierr\))170 1989 y(CALL)h(MPI_ISEND\(request2,)d(a\(2\),)i (count,)g(MPI_REAL,)g(1,)g(tag2,)h(comm,)f(ierr\))170 2045 y(CALL)h(MPI_WAIT\(request1,)d(status,)i(ierr\))170 2102 y(CALL)h(MPI_WAIT\(request2,)d(status,)i(ierr\))123 2158 y(ELSE)170 2215 y(CALL)h(MPI_IRECV\(request2,)d(a\(2\),)i(count,)g (MPI_REAL,)g(0,)g(tag2,)h(comm,)f(ierr\))170 2271 y(CALL)h (MPI_IRECV\(request1,)d(a\(1\),)i(count,)g(MPI_REAL,)g(0,)g(tag1,)h (comm,)f(ierr\))170 2327 y(CALL)h(MPI_WAIT\(request2,)d(status,)i (ierr\))170 2384 y(CALL)h(MPI_WAIT\(request1,)d(status,)i(ierr\))75 2440 y(END)g(IF)166 2534 y FE(The)16 b(co)q(de)g(is)g(guaran)o(teed)f (to)g(execute)h(correctly)l(,)g(ev)o(en)g(in)g(an)g(implemen)o(tation)h (that)d(do)q(es)i(not)75 2591 y(bu\013er)d(messages)f(\(the)h(only)h (resource)f(requiremen)o(t)g(is)h(the)f(abilit)o(y)h(to)e(ha)o(v)o(e)h (t)o(w)o(o)f(p)q(ending)i(comm)o(uni-)75 2647 y(cations\).)20 b(If)15 b(one)h(replaces)g(the)f(non)o(blo)q(c)o(king)i(sends)f (\(receiv)o(es\))f(b)o(y)g(blo)q(c)o(king)i(sends)f(\(receiv)o(es\),)f (and)75 2704 y(deletes)i(the)f(calls)g(to)f Fx(MPI)p 546 2704 V 16 w(W)l(AIT)p FE(,)h(then)g(the)f(co)q(de)i(will)g(deadlo)q (c)o(k.)22 b(The)16 b(co)q(de)h(will)g(also)f(deadlo)q(c)o(k)g(if)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 35 40 35 39 bop 75 -100 a Ft(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FE(35)75 45 y(one)18 b(deletes)i(the)e(calls)i(that)d(initiate)j (comm)o(unication)f(and)f(replaces)h(eac)o(h)g(call)g(to)f Fx(MPI)p 1683 45 14 2 v 16 w(W)l(AIT)g FE(b)o(y)75 102 y(the)d(appropriate)g(blo)q(c)o(king)i(send)f(or)f(receiv)o(e.)75 230 y Fq(3.8.5)49 b(Multiple)17 b(Completions)75 318 y FE(It)23 b(is)h(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)g(to)e(w)o (ait)h(for)g(the)g(completion)i(of)d(an)o(y)h(or)g(all)h(the)g(op)q (erations)f(in)75 375 y(a)g(set,)h(rather)e(than)h(ha)o(ving)g(to)f(w)o (ait)h(for)f(a)g(sp)q(eci\014c)j(message.)43 b(A)23 b(call)h(to)e Fx(MPI)p 1597 375 V 16 w(W)l(AIT)l(ANY)h FE(or)75 431 y Fx(MPI)p 160 431 V 16 w(TEST)l(ANY)e FE(can)f(b)q(e)g(used)g(to)f(w)o (ait)h(for)f(the)h(completion)g(of)g(one)g(out)f(of)g(sev)o(eral)h(op)q (erations;)75 488 y(a)14 b(call)h(to)27 b Fx(MPI)p 347 488 V 15 w(W)l(AIT)l(ALL)14 b FE(or)f Fx(MPI)p 701 488 V 16 w(TEST)l(ALL)h FE(can)g(b)q(e)g(used)h(to)e(w)o(ait)g(for)h(all)g (p)q(ending)i(op)q(erations)e(in)75 544 y(a)h(list.)75 649 y Fx(MPI)p 160 649 V 16 w(W)l(AIT)l(ANY)h(\(count,)g(a)o(rra)o(y)p 640 649 V 14 w(of)p 691 649 V 16 w(requests,)h(index,)f(status\))117 728 y Fr(IN)155 b Fx(count)482 b Fr(list)14 b(length)f(\(in)o(teger\)) 117 805 y(INOUT)62 b Fx(a)o(rra)o(y)p 416 805 V 15 w(of)p 468 805 V 16 w(requests)272 b Fr(arra)o(y)14 b(of)f(handles)h(to)g (comm)o(unication)c(ob)r(jects)117 883 y(OUT)108 b Fx(index)489 b Fr(index)12 b(of)f(handle)g(for)g(op)q(eration)h(that)f(completed)g (\(in)o(teger\))117 960 y(OUT)108 b Fx(status)476 b Fr(status)15 b(ob)r(ject)75 1086 y Fp(int)23 b(MPI)p 245 1086 15 2 v 17 w(Waitany\(int)g(count,)g(MPI)p 788 1086 V 16 w(Comm)p 900 1086 V 17 w(request)g(*array)p 1252 1086 V 16 w(of)p 1316 1086 V 17 w(requests,)g(int)g(*index,)393 1143 y(MPI)p 468 1143 V 17 w(Status)g(*status\))75 1230 y(MPI)p 150 1230 V 17 w(WAITANY\(COUNT,)f(ARRAY)p 645 1230 V 16 w(OF)p 709 1230 V 17 w(REQUESTS,)h(INDEX,)g(STATUS,)g(IERROR\))170 1287 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1287 V 17 w(OF)p 716 1287 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 1287 V 16 w(STATUS)p 1610 1287 V 16 w(SIZE\),)170 1343 y(IERROR)166 1431 y FE(Blo)q(c)o(ks)17 b(un)o(til)h(one)e(of)g (the)h(op)q(erations)f(asso)q(ciated)h(with)g(the)f(comm)o(unication)h (requests)g(in)g(the)75 1487 y(arra)o(y)c(has)g(completed.)21 b(Returns)14 b(the)g(index)h(of)f(that)f(request)h(in)g(the)g(arra)o(y) l(,)f(and)h(returns)g(the)g(status)75 1544 y(of)h(that)f(op)q(eration)i (in)g(the)f(ob)s(ject)g(asso)q(ciated)g(with)h(the)f(status.)166 1601 y(The)k(execution)h(of)e Fx(MPI)p 610 1601 14 2 v 16 w(W)l(AIT)l(ANY\(count,)j(a)o(rra)o(y)p 1079 1601 V 14 w(of)p 1130 1601 V 16 w(requests,)h(index,)e(status\))g FE(has)f(the)f(same)75 1658 y(e\013ect)10 b(as)g(the)g(execution)i(of) 20 b Fx(MPI)p 655 1658 V 16 w(W)l(AIT\(a)o(rra)o(y)p 897 1658 V 14 w(of)p 948 1658 V 16 w(requests[i],)13 b(status\))p FE(,)g(where)e Fx(i)f FE(is)h(the)f(v)m(alue)i(returned)75 1714 y(b)o(y)21 b Fx(index)p FE(.)38 b(In)21 b(particular,)i(the)e (asso)q(ciated)g(comm)o(unication)g(ob)s(ject)g(is)g(is)g(deallo)q (cated,)j(and)d(the)75 1771 y(handle)15 b(to)d(it)i(in)g Fx(a)o(rra)o(y)p 466 1771 V 15 w(of)p 518 1771 V 16 w(requests)h FE(is)f(set)f(to)g(n)o(ull.)20 b(If)14 b(more)f(then)h(one)f(op)q (eration)h(is)f(enabled)i(and)f(can)75 1827 y(terminate,)19 b(one)g(is)g(arbitrarily)g(c)o(hosen.)30 b(There)19 b(is)g(no)g (requiremen)o(t)g(that)f(the)g(c)o(hoice)i(satisfy)e(an)o(y)75 1884 y(fairness)e(criterion.)75 1989 y Fx(MPI)p 160 1989 V 16 w(TEST)l(ANY\(count,)h(a)o(rra)o(y)p 626 1989 V 14 w(of)p 677 1989 V 16 w(requests,)g(index,)f(\015ag,)f(status\))117 2067 y Fr(IN)155 b Fx(count)482 b Fr(list)14 b(length)f(\(in)o(teger\)) 117 2145 y(INOUT)62 b Fx(a)o(rra)o(y)p 416 2145 V 15 w(of)p 468 2145 V 16 w(requests)272 b Fr(arra)o(y)14 b(of)f(handles)h(to)g(comm)o(unication)c(ob)r(jects)117 2222 y(OUT)108 b Fx(index)489 b Fr(index)27 b(of)e(handle)i(for)f(op)q (eration)g(that)g(completed,)j(or)905 2279 y Fm(MPI)p 982 2279 13 2 v 15 w(UNDEFINED)12 b Fr(if)h(none)i(completed)e(\(in)o (teger\))117 2356 y(OUT)108 b Fx(\015ag)518 b Fr(true)15 b(if)e(one)h(of)f(the)i(op)q(erations)f(is)g(complete)117 2434 y(OUT)108 b Fx(status)476 b Fr(status)15 b(ob)r(ject)75 2559 y Fp(int)23 b(MPI)p 245 2559 15 2 v 17 w(Testany\(int)g(count,)g (MPI)p 788 2559 V 16 w(Comm)p 900 2559 V 17 w(request)g(*array)p 1252 2559 V 16 w(of)p 1316 2559 V 17 w(requests,)g(int)g(*index,)393 2616 y(int)h(*flag,)f(MPI)p 731 2616 V 16 w(Status)g(*status\))75 2704 y(MPI)p 150 2704 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 2704 V 16 w(OF)p 709 2704 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g (STATUS,)g(IERROR\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 36 41 36 40 bop 75 -100 a FE(36)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)170 45 y Fp(INTEGER)23 b(COUNT,)g(ARRAY)p 651 45 15 2 v 17 w(OF)p 716 45 V 17 w(REQUESTS\(*\),)f(INDEX,)h(FLAG,)170 102 y(STATUS\(MPI)p 413 102 V 16 w(STATUS)p 573 102 V 17 w(SIZE\),)g(IERROR)166 188 y FE(Causes)11 b(either)h(one)f(or)g(none)h(of)f(the)g(op)q (erations)h(asso)q(ciated)f(with)h(the)f(comm)o(unication)h(requests)75 244 y(to)f(return.)18 b(In)12 b(the)f(former)g(case,)g(it)h(returns)f (in)h Fp(index)f FE(the)g(index)i(of)d(its)i(request)f(in)h(the)f(arra) o(y)l(,)g(returns)75 301 y(in)19 b Fp(status)e FE(the)h(status)f(of)h (that)f(op)q(eration,)i(deallo)q(cates)g(the)f(asso)q(ciated)g(comm)o (unication)h(ob)s(ject,)75 357 y(and)13 b(sets)f(the)h(request)f(to)g (n)o(ull.)21 b(In)13 b(the)f(latter)h(case,)f(it)h(returns)g(a)f(v)m (alue)i(of)e Fm(MPI)p 1469 357 13 2 v 14 w(UNDEFINED)g FE(in)h Fx(index)75 414 y FE(and)i Fx(status)j FE(is)d(unde\014ned.)166 470 y(The)k(execution)g(of)f Fx(MPI)p 609 470 14 2 v 16 w(TEST)l(ANY\(count,)j(a)o(rra)o(y)p 1079 470 V 15 w(of)p 1131 470 V 16 w(requests,)g(index,)f(status\))g FE(has)f(the)f(same)75 527 y(e\013ect)k(as)g(the)g(execution)h(of)f Fx(MPI)p 704 527 V 16 w(TEST\(a)o(rra)o(y)p 947 527 V 14 w(of)p 998 527 V 16 w(requests[i],)k(\015ag,)e(status\))p FE(,)h(for)d Fp(i=0,)h(1)h(,...,)75 583 y(count-1)p FE(,)e(in)g(some)f (arbitrary)g(order,)i(un)o(til)f(one)g(call)h(returns)e Fp(flag=true)p FE(,)h(or)f(all)h(fail.)40 b(In)22 b(the)75 640 y(former)12 b(case,)h Fp(index)f FE(is)h(set)g(to)f(the)h(last)f(v) m(alue)i(of)f Fp(i)p FE(;)g(in)g(the)g(later)g(case,)g(it)g(is)g(set)g (to)f Fm(MPI)p 1618 640 13 2 v 14 w(UNDEFINED)p FE(.)75 743 y Fx(MPI)p 160 743 14 2 v 16 w(W)l(AIT)l(ALL\()i(count,)i(a)o(rra)o (y)p 626 743 V 15 w(of)p 678 743 V 16 w(requests,)h(a)o(rra)o(y)p 971 743 V 15 w(of)p 1023 743 V 15 w(statuses\))117 821 y Fr(IN)155 b Fx(count)482 b Fr(lists)14 b(length)g(\(in)o(teger\))117 895 y(IN)155 b Fx(a)o(rra)o(y)p 416 895 V 15 w(of)p 468 895 V 16 w(requests)272 b Fr(arra)o(y)14 b(of)f(handles)h(to)g(comm)o (unication)c(ob)r(jects)117 970 y(OUT)108 b Fx(a)o(rra)o(y)p 416 970 V 15 w(of)p 468 970 V 16 w(statuses)276 b Fr(arra)o(y)14 b(of)f(status)i(ob)r(jects)75 1094 y Fp(int)23 b(MPI)p 245 1094 15 2 v 17 w(Waitall\(int)g(count,)g(MPI)p 788 1094 V 16 w(Comm)p 900 1094 V 17 w(request)g(*array)p 1252 1094 V 16 w(of)p 1316 1094 V 17 w(requests,)393 1151 y(MPI)p 468 1151 V 17 w(Status)g(*array)p 796 1151 V 16 w(of)p 860 1151 V 17 w(statuses\))75 1237 y(MPI)p 150 1237 V 17 w(WAITALL\(COUNT,)f(ARRAY)p 645 1237 V 16 w(OF)p 709 1237 V 17 w(REQUESTS,)h(ARRAY)p 1085 1237 V 16 w(OF)p 1149 1237 V 17 w(STATUSES,)g(IERROR\))170 1294 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1294 V 17 w(OF)p 716 1294 V 17 w(REQUESTS\(*\),)170 1350 y(ARRAY)p 293 1350 V 17 w(OF)p 358 1350 V 17 w(STATUSES\(MPI)p 663 1350 V 15 w(STATUS)p 822 1350 V 17 w(SIZE,*\),)g(IERROR)166 1437 y FE(Blo)q(c)o(ks)12 b(un)o(til)h(all)g(comm)o(unication)g(op)q (erations)f(asso)q(ciated)g(with)g(requests)f(in)i(the)f(list)h (complete,)75 1493 y(and)k(return)g(the)g(status)f(of)h(all)h(these)f (op)q(erations.)26 b(Both)16 b(arra)o(ys)g(ha)o(v)o(e)h(the)g(same)f(n) o(um)o(b)q(er)i(of)e(v)m(alid)75 1550 y(en)o(tries.)24 b(The)16 b Fx(i)p FE(-th)h(en)o(try)f(in)h Fx(a)o(rra)o(y)p 694 1550 14 2 v 14 w(of)p 745 1550 V 16 w(statuses)i FE(is)e(set)f(to)g(the)h(return)f(status)f(of)h(the)h Fx(i)p FE(-th)f(op)q(eration.)75 1606 y(All)g(comm)o(unication)g(ob)s (jects)f(are)g(deallo)q(cated)h(and)g(all)g(requests)f(are)g(set)g(to)g (n)o(ull.)166 1663 y(The)f(execution)h(of)f Fx(MPI)p 596 1663 V 16 w(W)l(AIT)l(ALL\(count,)g(a)o(rra)o(y)p 1046 1663 V 15 w(of)p 1098 1663 V 16 w(requests,)i(a)o(rra)o(y)p 1390 1663 V 15 w(of)p 1442 1663 V 16 w(statuses)g FE(has)e(the)g(same) 75 1719 y(e\013ect)j(as)g(the)g(execution)i(of)e Fx(MPI)p 680 1719 V 16 w(W)l(AIT\(a)o(rra)o(y)p 922 1719 V 14 w(of)p 973 1719 V 16 w(request[i],)i(a)o(rra)o(y)p 1288 1719 V 15 w(of)p 1340 1719 V 16 w(statuses[i]\))p FE(,)h(for)c Fp(i=0)24 b(,...,)75 1775 y(count-1)p FE(,)14 b(in)i(some)f(arbitrary)f (order.)75 1879 y Fx(MPI)p 160 1879 V 16 w(TEST)l(ALL\(count,)i(a)o (rra)o(y)p 613 1879 V 14 w(of)p 664 1879 V 16 w(requests,)h(\015ag,)e (a)o(rra)o(y)p 1054 1879 V 14 w(of)p 1105 1879 V 16 w(statuses\))117 1956 y Fr(IN)155 b Fx(count)482 b Fr(lists)14 b(length)g(\(in)o (teger\))117 2031 y(INOUT)62 b Fx(a)o(rra)o(y)p 416 2031 V 15 w(of)p 468 2031 V 16 w(requests)272 b Fr(arra)o(y)14 b(of)f(handles)h(to)g(comm)o(unication)c(ob)r(jects)117 2106 y(OUT)108 b Fx(\015ag)518 b Fr(\(logical\))117 2180 y(OUT)108 b Fx(a)o(rra)o(y)p 416 2180 V 15 w(of)p 468 2180 V 16 w(statuses)276 b Fr(arra)o(y)14 b(of)f(status)i(ob)r(jects)75 2305 y Fp(int)23 b(MPI)p 245 2305 15 2 v 17 w(Testall\(int)g(count,)g (MPI)p 788 2305 V 16 w(Comm)p 900 2305 V 17 w(request)g(*array)p 1252 2305 V 16 w(of)p 1316 2305 V 17 w(requests,)g(int)g(*flag,)393 2361 y(MPI)p 468 2361 V 17 w(Status)g(*array)p 796 2361 V 16 w(of)p 860 2361 V 17 w(statuses\))75 2448 y(MPI)p 150 2448 V 17 w(TESTALL\(COUNT,)f(ARRAY)p 645 2448 V 16 w(OF)p 709 2448 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 2448 V 16 w(OF)p 1292 2448 V 17 w(STATUSES,)g(IERROR\))170 2504 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2504 V 17 w(OF)p 716 2504 V 17 w(REQUESTS\(*\),)f(FLAG,)170 2561 y(ARRAY)p 293 2561 V 17 w(OF)p 358 2561 V 17 w(STATUSES\(MPI)p 663 2561 V 15 w(STATUS)p 822 2561 V 17 w(SIZE,*\),)h(IERROR)166 2647 y FE(Causes)13 b(either)g(all)h(or)f(none)g(of)g(the)g(op)q (erations)g(asso)q(ciated)g(with)g(the)g(comm)o(unication)h(requests)75 2704 y(to)k(complete.)30 b(It)18 b(returns)h Fx(\015ag)f(=)h(true)g FE(if)g(all)g(comm)o(unications)g(asso)q(ciated)g(with)f(requests)h(in) g(the)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 37 42 37 41 bop 75 -100 a Ft(3.8.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)868 b FE(37)75 45 y(arra)o(y)10 b(ha)o(v)o(e)h(completed.)19 b(In)12 b(this)g(case,)f(eac)o(h)h(status)e(en)o(try)h(is)h(set)f(to)f (the)i(status)e(of)h(the)g(corresp)q(onding)75 102 y(comm)o(unication.) 20 b(All)14 b(comm)o(unication)g(ob)s(jects)e(are)g(deallo)q(cated,)j (and)e(all)h(requests)e(are)h(set)g(to)f(n)o(ull.)75 158 y(Otherwise,)h Fx(\015ag)f(=)g(false)f FE(is)i(returned,)f(no)g (request)g(is)g(mo)q(di\014ed)h(and)f(the)g(v)m(alues)h(of)e(the)h (status)f(en)o(tries)75 214 y(are)k(unde\014ned.)22 b(This)15 b(is)h(a)f(non)o(blo)q(c)o(king)i(op)q(eration)e(that)f(returns)h (immediately)l(.)166 271 y(Example:)75 419 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 476 y(IF\(rank)g(<)h(2\))123 532 y(THEN)214 b(!)24 b(rank=0)f(or)g(rank=1)170 589 y(CALL)h(MPI_ISEND\(request,)d(a,)j(n,)f(MPI_REAL,)g(2,)h(tag,)f(comm,) g(ierr\))170 645 y(****)h(do)f(some)h(computation)e(to)i(mask)f (latency)g(****)170 702 y(CALL)h(MPI_WAIT\(request,)d(status,)i(ierr\)) 123 758 y(ELSE)214 b(!)24 b(rank=2)170 814 y(CALL)g (MPI_IRECV\(request_list\(0)o(\),)d(a,)j(n,)f(MPI_REAL,)g(0,)g(tag,)h (comm,)f(ierr\))170 871 y(CALL)h(MPI_IRECV\(request_list\(1)o(\),)d(b,) j(n,)f(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))170 927 y(****)h(do)f(some)h(computation)e(to)i(mask)f(latency)g(****)170 984 y(CALL)h(MPI_WAITANY\(2,)e(request_list,)g(index,)h(status,)g (ierr\))170 1040 y(IF\(index.EQ.0\))218 1097 y(THEN)266 1153 y(****)g(handle)g(message)g(from)g(process)g(0)h(****)266 1210 y(CALL)f(MPI_WAIT\(request_list\(1\),)e(status,)i(ierr\))266 1266 y(****)g(handle)g(message)g(from)g(process)g(1)h(****)218 1323 y(ELSE)266 1379 y(****)f(handle)g(message)g(from)g(process)g(1)h (****)266 1435 y(CALL)f(MPI_WAIT\(request_list\(0\),)e(status,)i (ierr\))266 1492 y(****)g(handle)g(message)g(from)g(process)g(0)h(****) 170 1548 y(END)g(IF)75 1605 y(END)f(IF)166 1697 y FE(The)16 b(calls)h(in)o(tro)q(duced)g(in)g(this)g(subsection)f(can)h(b)q(e)f (used)h(to)e(w)o(ait)h(or)f(test)g(for)h(the)g(completion)75 1753 y(of)d(an)h(arbitrary)f(mix)h(of)f(non)o(blo)q(c)o(king)i(comm)o (unication)g(calls,)f(including)j(a)c(mix)h(sends)g(and)g(receiv)o(es) 75 1810 y(and)h(of)g(an)o(y)g(additional)i(non)o(blo)q(c)o(king)f(comm) o(unication)g(calls)g(pro)o(vided)g(b)o(y)f(MPI.)166 1866 y(The)20 b Fx(status)j FE(argumen)o(t)c(in)i Fx(MPI)p 749 1866 14 2 v 16 w(W)l(AIT)l(ANY)h FE(and)e Fx(MPI)p 1163 1866 V 16 w(W)l(AIT)l(ALL)g FE(and)g(the)h Fx(a)o(rra)o(y)p 1660 1866 V 14 w(of)p 1711 1866 V 16 w(statuses)75 1923 y FE(argumen)o(t)f(in)h Fx(MPI)p 427 1923 V 16 w(W)l(AIT)l(ALL)f FE(and)h Fx(MPI)p 828 1923 V 16 w(TEST)l(ALL)f FE(are)h(not)f(used)h (or)f(up)q(dated)i(if)f(the)f(complet-)75 1979 y(ing)h(comm)o (unication)f(is)h(a)f(send.)34 b(If)21 b(all)g(op)q(erations)f(in)h (the)f(list)h(pro)o(vided)f(b)o(y)g(the)h(argumen)o(t)e Fx(a)o(r-)75 2035 y(ra)o(y)p 136 2035 V 15 w(of)p 188 2035 V 16 w(requests)24 b FE(are)c(send)i(op)q(erations,)g(then)f(an)g (arbitrary)f(v)m(alue)j(can)e(b)q(e)g(passed)h(to)e(the)h Fx(status)75 2092 y FE(\(resp.)35 b Fx(a)o(rra)o(y)p 320 2092 V 14 w(of)p 371 2092 V 16 w(statuses)p FE(\))18 b(argumen)o(t.)166 2148 y(An)j Fx(a)o(rra)o(y)p 344 2148 V 15 w(of)p 396 2148 V 16 w(requests)i FE(argumen)o(t)e(that)f(is)i (passed)g(to)e(an)o(y)h(of)g(the)h(last)f(four)g(functions)h(ma)o(y)75 2205 y(con)o(tain)14 b(n)o(ull)i(en)o(tries.)k(These)14 b(en)o(tries)h(are)e(ignored)i(and)f(the)h(call)g(executes)g(as)e(if)i (it)f(w)o(as)g(supplied)i(an)75 2261 y(arra)o(y)g(of)h(requests)h(con)o (taining)g(only)g(the)g(non-n)o(ull)h(en)o(tries.)27 b(As)17 b(a)h(request)f(in)h Fx(a)o(rra)o(y)p 1608 2261 V 15 w(of)p 1660 2261 V 16 w(requests)h FE(is)75 2318 y(set)13 b(to)g(n)o(ull)i(when)f(the)g(corresp)q(onding)g(op)q(eration) g(completes,)g(this)g(allo)o(ws)g(to)f(reuse)g(the)h(same)f(arra)o(y)75 2374 y(for)j(completing)h(in)g(successiv)o(e)g(calls)h(the)e(op)q (erations)g(asso)q(ciated)h(with)f(this)h(arra)o(y)l(.)22 b(Th)o(us,)16 b(the)g(last)75 2431 y(example)g(can)f(b)q(e)h(rewritten) f(as)g(follo)o(ws:)75 2591 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,) h(ierr\))75 2647 y(IF\(rank)g(<)h(2\))123 2704 y(THEN)214 b(!)24 b(rank=0)f(or)g(rank=1)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 38 43 38 42 bop 75 -100 a FE(38)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)170 45 y Fp(CALL)24 b(MPI_ISEND\(request,)d(a,)j(n,)f(MPI_REAL,)g(2,)h(tag,)f(comm,)g (ierr\))170 102 y(CALL)h(MPI_WAIT\(request,)d(status,)i(ierr\))123 158 y(ELSE)214 b(!)24 b(rank=2)170 214 y(CALL)g (MPI_IRECV\(request_list\(0)o(\),)d(a,)j(n,)f(MPI_REAL,)g(0,)g(tag,)h (comm,)f(ierr\))170 271 y(CALL)h(MPI_IRECV\(request_list\(1)o(\),)d(b,) j(n,)f(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))170 327 y(CALL)h(MPI_WAITANY\(2,)e(request_list,)g(index,)h(status,)g(ierr\)) 266 384 y(****)g(handle)g(first)g(message)47 b(****)170 440 y(CALL)24 b(MPI_WAITANY\(2,)e(request_list,)g(index,)h(status,)g (ierr\))266 497 y(****)g(handle)g(second)g(message)47 b(****)166 614 y FE(If)18 b Fx(MPI)p 299 614 14 2 v 16 w(W)l(AIT)l(ANY)i FE(or)d Fx(MPI)p 676 614 V 16 w(TEST)l(ANY)j FE(are)e(called)i(with)e(an)g(empt)o(y)g Fx(a)o(rra)o(y)p 1534 614 V 15 w(of)p 1586 614 V 16 w(requests)i FE(argu-)75 671 y(men)o(t)d(\(i.e.,)g(an)g(arra)o(y)f(of)h(length)h(zero,)f(or)f (an)i(arra)o(y)d(with)j(all)g(en)o(tries)g(n)o(ull\))g(then)f(they)h (return)f(the)75 727 y(v)m(alue)k Fm(MPI)p 274 727 13 2 v 14 w(UNDEFINED)d FE(in)i Fx(index)p FE(;)i Fx(MPI)p 812 727 14 2 v 16 w(W)l(AIT)l(ALL)d FE(with)h(an)f(empt)o(y)g Fx(a)o(rra)o(y)p 1449 727 V 14 w(of)p 1500 727 V 16 w(requests)j FE(argumen)o(t)75 784 y(has)15 b(no)g(e\013ect;)g Fx(MPI)p 437 784 V 16 w(TEST)l(ALL)g FE(with)g(an)g(empt)o(y)g Fx(a)o(rra)o(y)p 1058 784 V 15 w(of)p 1110 784 V 16 w(requests)i FE(argumen)o(t)e(returns)g Fx(\015ag=true)p FE(.)75 940 y Fv(3.9)59 b(Prob)r(e)20 b(and)f(Cancel)75 1046 y FE(The)d Fx(MPI)p 254 1046 V 16 w(PROBE)g FE(and)g Fx(MPI)p 605 1046 V 16 w(IPROBE)g FE(op)q(erations)g(allo)o(w)g(incoming)h(messages) e(to)g(b)q(e)h(c)o(hec)o(k)o(ed)g(for,)75 1102 y(without)g(actually)h (receiving)h(them.)k(The)17 b(user)f(can)g(then)h(decide)g(ho)o(w)f(to) g(receiv)o(e)h(them,)f(based)g(on)75 1159 y(the)i(information)f (returned)h(b)o(y)f(the)h(prob)q(e)g(\(basically)l(,)h(the)e (information)h(returned)g(b)o(y)f Fx(status)p FE(\).)28 b(In)75 1215 y(particular,)16 b(the)g(user)g(ma)o(y)f(allo)q(cate)h (memory)f(for)g(the)h(receiv)o(e)h(bu\013er,)e(according)h(to)f(the)h (length)g(of)75 1272 y(the)f(prob)q(ed)h(message.)166 1331 y(The)h Fx(MPI)p 346 1331 V 16 w(CANCEL)g FE(op)q(eration)g(allo)o (ws)g(p)q(ending)i(comm)o(unications)e(to)f(b)q(e)i(cancelled.)27 b(This)17 b(is)75 1387 y(required)j(for)f(clean)o(up.)33 b(P)o(osting)19 b(a)g(send)h(or)f(a)g(receiv)o(e)h(ties)f(up)h(user)f (resources)h(\(send)f(or)g(receiv)o(e)75 1443 y(bu\013ers\),)14 b(and)i(a)f(cancel)h(ma)o(y)f(b)q(e)g(needed)i(to)d(free)i(these)f (resources)g(gracefully)l(.)75 1549 y Fx(MPI)p 160 1549 V 16 w(IPROBE\(source,)h(tag,)f(comm,)d(\015ag,)j(status\))117 1629 y Fr(IN)155 b Fx(source)468 b Fr(source)15 b(rank,)f(or)27 b Fm(MPI)p 1281 1629 13 2 v 15 w(ANY)p 1381 1629 V 14 w(SOURCE)13 b Fr(\(in)o(teger\))117 1709 y(IN)155 b Fx(tag)526 b Fr(tag)14 b(v)n(alue)f(or)28 b Fm(MPI)p 1226 1709 V 14 w(ANY)p 1325 1709 V 15 w(T)m(A)o(G)12 b Fr(\(in)o(teger\))117 1788 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 1868 y(OUT)108 b Fx(\015ag)518 b Fr(\(logical\))117 1948 y(OUT)108 b Fx(status)476 b Fr(status)15 b(ob)r(ject)75 2074 y Fp(int)23 b(MPI)p 245 2074 15 2 v 17 w(Iprobe\(int)g(source,)g (int)g(tag,)g(MPI)p 1002 2074 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 2131 y(MPI)p 468 2131 V 17 w(Status)g(*status\))75 2220 y(MPI)p 150 2220 V 17 w(IPROBE\(SOURCE,)f(TAG,)h(COMM,)g(FLAG,)g (STATUS,)g(IERROR\))170 2276 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f (FLAG,)g(STATUS\(MPI)p 1201 2276 V 16 w(STATUS)p 1361 2276 V 16 w(SIZE\),)g(IERROR)166 2365 y Fx(MPI)p 251 2365 14 2 v 16 w(IPROBE)c FE(returns)f Fx(\015ag)g(=)h(true)f FE(if)h(there)g(is)f(a)g(message)g(that)g(can)g(b)q(e)h(receiv)o(ed)g (and)g(that)75 2421 y(matc)o(hes)h(the)g(pattern)g(sp)q(eci\014ed)i(b)o (y)e(the)g(parameters)f Fx(source)p FE(,)j Fx(tag)p FE(,)f(and)f Fx(comm)p FE(.)33 b(It)20 b(returns)g Fx(\015ag)75 2478 y(=)j(false)p FE(,)h(otherwise.)41 b(If)23 b Fx(MPI)p 633 2478 V 16 w(IPROBE)g FE(returns)f Fx(\015ag)g(=)h(true)p FE(,)h(then)f(the)f(status)g(ob)s(ject)g(can)g(b)q(e)75 2534 y(subsequen)o(tly)c(queried)g(using)g(the)f(functions)h(describ)q (ed)g(in)g(section)g(3.3.1)d(to)h(\014nd)i(the)f(source,)g(tag)75 2591 y(and)f(length)h(of)f(the)g(message.)23 b(The)16 b(v)m(alues)i(returned)e(b)o(y)h(these)f(queries)h(are)f(the)g(same)g (v)m(alues)i(that)75 2647 y(w)o(ould)12 b(ha)o(v)o(e)e(b)q(een)j (returned)e(b)o(y)h(a)f(call)h(to)e Fx(MPI)p 897 2647 V 16 w(RECV)i FE(executed)g(at)f(the)g(same)g(p)q(oin)o(t)h(in)g(the)f (program.)75 2704 y(The)k(return)h(status)e(ob)s(ject)h(is)g (unde\014ned)i(if)f Fx(\015ag)f(=)h(false)p FE(.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 39 44 39 43 bop 75 -100 a Ft(3.9.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1161 b FE(39)166 45 y(A)13 b(subsequen)o(t)i(receiv)o(e)f(executed)g(with)g (the)g(same)f(con)o(text,)g(and)g(the)h(source)f(and)h(tag)f(returned) 75 102 y(b)o(y)g(querying)h(the)f(status)f(returned)i(b)o(y)f Fx(MPI)p 858 102 14 2 v 16 w(IPROBE)h FE(will)h(receiv)o(e)f(the)f (message)f(that)h(w)o(as)f(matc)o(hed)75 158 y(b)o(y)21 b(the)g(prob)q(e,)h(if)g(no)f(other)f(in)o(terv)o(ening)i(receiv)o(e)g (o)q(ccurred)f(after)g(the)g(prob)q(e.)37 b(If)21 b(the)g(receiving)75 214 y(pro)q(cess)h(is)g(m)o(ulti-threaded,)i(it)e(is)g(the)f(user)h (resp)q(onsibilit)o(y)i(to)d(ensure)h(that)f(the)g(last)h(condition)75 271 y(holds.)75 378 y Fx(MPI)p 160 378 V 16 w(PROBE\(source,)16 b(tag,)f(comm,)e(status\))117 459 y Fr(IN)155 b Fx(source)468 b Fr(source)15 b(rank,)f(or)27 b Fm(MPI)p 1281 459 13 2 v 15 w(ANY)p 1381 459 V 14 w(SOURCE)13 b Fr(\(in)o(teger\))117 541 y(IN)155 b Fx(tag)526 b Fr(tag)14 b(v)n(alue,)e(or)28 b Fm(MPI)p 1237 541 V 14 w(ANY)p 1336 541 V 15 w(T)m(A)o(G)13 b Fr(\(in)o(teger\))117 624 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 706 y(OUT)108 b Fx(status)476 b Fr(status)15 b(ob)r(ject)75 834 y Fp(int)23 b(MPI)p 245 834 15 2 v 17 w(Probe\(int)g(source,)g(int)g(tag,)h(MPI)p 979 834 V 16 w(Comm)g(comm,)f(MPI)p 1330 834 V 17 w(Status)g(*status\)) 75 924 y(MPI)p 150 924 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g (IERROR\))170 981 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 981 V 16 w(STATUS)p 1218 981 V 16 w(SIZE\),)g(IERROR)166 1071 y Fx(MPI)p 251 1071 14 2 v 16 w(PROBE)17 b FE(b)q(eha)o(v)o(es)g (lik)o(e)g Fx(MPI)p 769 1071 V 16 w(IPROBE)g FE(except)g(that)f(it)g (is)h(a)f(blo)q(c)o(king)i(call)f(whic)o(h)g(returns)75 1127 y(only)f(after)e(a)h(matc)o(hing)g(message)g(has)g(b)q(een)h (found.)75 1235 y Fx(MPI)p 160 1235 V 16 w(CANCEL\(request\))117 1315 y Fr(IN)155 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 1444 y Fp(int)23 b(MPI)p 245 1444 15 2 v 17 w(Cancel\(MPI)p 502 1444 V 16 w(Comm)p 614 1444 V 17 w(request)g(request\))75 1534 y(MPI)p 150 1534 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 1590 y(INTEGER)h(REQUEST,)g (IERROR)166 1680 y FE(A)17 b(call)h(to)e Fx(MPI)p 444 1680 14 2 v 15 w(CANCEL)i FE(marks)e(for)g(cancellation)i(a)f(p)q (ending)h(non)o(blo)q(c)o(king)g(comm)o(unication)75 1737 y(op)q(eration)f(\(send)g(or)g(receiv)o(e\).)25 b(The)18 b(cancel)g(call)g(is)f(non)o(blo)q(c)o(king,)h(and)f(lo)q (cal.)27 b(It)17 b(returns)f(immedi-)75 1793 y(ately)l(,)f(p)q(ossibly) i(b)q(efore)e(the)h(comm)o(unication)g(is)f(actually)h(cancelled.)166 1853 y(It)c(is)g(still)h(necessary)f(to)g(complete)g(a)g(comm)o (unication)g(that)f(has)h(b)q(een)h(mark)o(ed)e(for)h(cancellation,)75 1910 y(using)i(a)g(call)h(to)e Fx(MPI)p 450 1910 V 15 w(W)l(AIT)h FE(or)f Fx(MPI)p 728 1910 V 16 w(TEST)h FE(\(or)f(an)o(y)g (of)g(the)h(deriv)o(ed)h(op)q(erations\).)k(If)14 b(the)g(op)q(eration) 75 1966 y(has)h(b)q(een)h(cancelled,)g(then)f(information)g(to)g(that)f (e\013ect)g(will)j(b)q(e)e(returned)g(in)h(the)f(status)f(argumen)o(t) 75 2023 y(of)g(the)g(op)q(eration)g(that)f(completes)i(the)f(comm)o (unication.)20 b(If)14 b(a)g(comm)o(unication)g(is)h(mark)o(ed)e(for)h (can-)75 2079 y(cellation,)f(then)f(a)f Fx(MPI)p 485 2079 V 16 w(W)l(AIT)h FE(call)g(for)f(that)g(comm)o(unication)h(is)g (guaran)o(teed)f(to)g(return,)g(irresp)q(ectiv)o(e)75 2135 y(of)17 b(the)g(activities)i(of)e(other)g(pro)q(cesses)h(\(i.e.,)f Fx(MPI)p 970 2135 V 16 w(W)l(AIT)g FE(b)q(eha)o(v)o(es)h(as)f(a)g(lo)q (cal)i(function\);)f(similarly)75 2192 y(if)f Fx(MPI)p 203 2192 V 16 w(TEST)f FE(is)h(rep)q(eatedly)h(called)g(in)f(a)f(busy)g (w)o(ait)g(lo)q(op)h(for)f(a)g(cancelled)j(comm)o(unication,)e(then)75 2248 y Fx(MPI)p 160 2248 V 16 w(TEST)e FE(will)i(ev)o(en)o(tually)f(b)q (e)g(successful.)166 2308 y(Either)j(the)f(cancellation)i(succeeds,)f (or)f(the)g(comm)o(unication)h(succeeds,)h(but)e(not)g(b)q(oth.)29 b(If)18 b(a)75 2365 y(send)h(is)h(mark)o(ed)e(for)g(cancellation,)j (then)e(it)g(m)o(ust)f(b)q(e)i(the)f(case)g(that)f(either)h(the)g(send) g(completes)75 2421 y(normally)l(,)f(in)h(whic)o(h)f(case)f(the)h (message)f(sen)o(t)g(w)o(as)g(receiv)o(ed)h(at)f(the)h(destination)g (pro)q(cess,)g(or)f(that)75 2478 y(the)e(send)h(is)f(successfully)i (cancelled,)g(in)e(whic)o(h)h(case)f(no)g(part)f(of)h(the)g(message)f (w)o(as)g(receiv)o(ed)j(at)d(the)75 2534 y(destination.)21 b(Then,)15 b(an)o(y)f(matc)o(hing)h(receiv)o(e)h(has)e(to)h(b)q(e)g (satis\014ed)h(b)o(y)e(another)h(send.)20 b(If)15 b(a)g(receiv)o(e)g (is)75 2591 y(mark)o(ed)d(for)f(cancellation,)k(then)d(it)h(m)o(ust)e (b)q(e)i(the)g(case)f(that)f(either)i(the)g(receiv)o(e)g(completes)g (normally)l(,)75 2647 y(or)k(that)g(the)h(receiv)o(e)h(is)f (successfully)i(cancelled,)g(in)f(whic)o(h)f(case)g(no)g(part)f(of)g (the)h(receiv)o(e)h(bu\013er)e(is)75 2704 y(altered.)j(Then,)c(an)o(y)e (matc)o(hing)i(send)g(has)f(to)f(b)q(e)i(satis\014ed)g(b)o(y)f(another) g(receiv)o(e.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 40 45 40 44 bop 75 -100 a FE(40)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(TEST)p 290 45 V 16 w(CANCELLED\(status,)h(\015ag\)) 117 124 y Fr(IN)155 b Fx(status)476 b Fr(status)15 b(ob)r(ject)117 202 y(IN)155 b Fx(\015ag)518 b Fr(\(logical\))75 328 y Fp(int)23 b(MPI)p 245 328 15 2 v 17 w(Test)p 358 328 V 17 w(cancelled\(MPI)p 687 328 V 15 w(Status)g(status,)g(int)h (*flag\))75 416 y(MPI)p 150 416 V 17 w(TEST)p 263 416 V 16 w(CANCELLED\(STATUS,)e(FLAG,)h(IERROR\))170 473 y(INTEGER)g(STATUS\(MPI)p 604 473 V 16 w(STATUS)p 764 473 V 17 w(SIZE\),)g(FLAG,)g(IERROR)166 561 y FE(Returns)11 b Fx(\015ag)f(=)h(true)g FE(if)g(the)f(comm)o(unication)h(asso)q (ciated)g(with)f(the)h(status)e(ob)s(ject)h(w)o(as)g(cancelled)75 617 y(successfully)l(.)27 b(In)18 b(suc)o(h)f(a)g(case,)g(all)g(other)g (\014elds)h(of)e Fx(status)j FE(\(suc)o(h)e(as)g Fx(count)h FE(or)e Fx(tag)p FE(\))h(are)f(unde\014ned.)75 673 y(Returns)h Fx(\015ag)g(=)f(false)p FE(,)h(otherwise.)24 b(If)17 b(a)g(receiv)o(e)g(op)q(eration)g(migh)o(t)f(b)q(e)h(cancelled)i(then)e (one)g(should)75 730 y(call)39 b Fx(MPI)p 266 730 14 2 v 16 w(TEST)p 396 730 V 17 w(CANCELLED)19 b FE(\014rst,)g(to)f(c)o (hec)o(k)i(whether)f(the)g(op)q(eration)g(w)o(as)g(cancelled,)j(b)q (efore)75 786 y(c)o(hec)o(king)16 b(on)f(the)g(other)g(\014elds)i(of)d (the)i(return)f(status.)75 938 y Fv(3.10)59 b(P)n(ersistent)19 b(communication)f(objects)75 1043 y FE(Often)j(a)g(comm)o(unication)g (with)g(the)g(same)g(parameter)f(list)i(is)f(rep)q(eatedly)h(executed)g (within)g(the)75 1099 y(inner)d(lo)q(op)g(of)e(a)h(parallel)i (computation.)28 b(In)18 b(suc)o(h)h(a)f(situation,)g(it)h(ma)o(y)e(b)q (e)i(p)q(ossible)g(to)f(optimize)75 1156 y(the)c(comm)o(unication)g(b)o (y)g(binding)i(the)d(list)i(of)e(comm)o(unication)i(parameters)d(to)h (the)h(comm)o(unication)75 1212 y(ob)s(ject)g(once)g(and,)h(then,)f (rep)q(eatedly)i(using)f(the)f(comm)o(unication)h(ob)s(ject)f(to)g (initiate)i(and)e(complete)75 1269 y(messages.)24 b(The)16 b(comm)o(unication)i(ob)s(ject)e(th)o(us)g(created)h(can)f(b)q(e)i (though)o(t)e(of)g(as)g(a)g(comm)o(unication)75 1325 y(p)q(ort)11 b(or)f(a)h(\\half-c)o(hannel")h(.)18 b(It)12 b(do)q(es)f(not)f(pro)o(vide)i(the)f(full)h(functionalit)o(y)h(of)d(a)h (con)o(v)o(en)o(tional)g(c)o(hannel,)75 1382 y(since)i(there)g(is)g(no) f(binding)i(of)e(the)g(send)h(p)q(ort)f(to)f(the)i(receiv)o(e)g(p)q (ort:)18 b(this)13 b(construct)f(allo)o(ws)g(reduction)75 1438 y(of)18 b(the)g(o)o(v)o(erhead)f(for)h(comm)o(unication)g(b)q(et)o (w)o(een)h(pro)q(cessor)e(and)h(comm)o(unication)h(con)o(troller,)g (but)75 1494 y(not)c(the)g(o)o(v)o(erhead)g(for)f(comm)o(unication)i(b) q(et)o(w)o(een)g(one)f(comm)o(unication)h(con)o(troller)g(and)f (another.)166 1552 y(A)j(comm)o(unication)h(ob)s(ject)f(is)g(created)h (using)f(one)h(of)e(the)i(four)f(follo)o(wing)g(calls.)30 b(These)19 b(calls)75 1609 y(in)o(v)o(olv)o(e)d(no)f(comm)o(unication.) 166 1667 y(The)e(function)g Fx(MPI)p 517 1667 V 16 w(CREA)l(TE)p 702 1667 V 17 w(SEND)g FE(creates)f(a)g(comm)o(unication)h(ob)s(ject)f (for)g(a)g(standard)g(mo)q(de)75 1723 y(send)k(op)q(eration,)f(and)g (binds)i(to)d(it)i(all)g(the)f(parameters)f(of)h(a)g(send)h(op)q (eration.)75 1829 y Fx(MPI)p 160 1829 V 16 w(CREA)l(TE)p 345 1829 V 17 w(SEND\(buf,)g(count,)g(datat)o(yp)q(e,)h(dest,)f(tag,)f (comm,)e(request\))117 1907 y Fr(IN)155 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1986 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 2064 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)117 2142 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o(teger\))117 2220 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 2298 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 2377 y(OUT)108 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 2503 y Fp(int)23 b(MPI)p 245 2503 15 2 v 17 w(Create)p 406 2503 V 17 w(send\(void*)f(buf,)h(int) h(count,)f(MPI)p 1139 2503 V 17 w(Datatype)f(datatype,)h(int)h(dest,) 393 2559 y(int)g(tag,)f(MPI)p 683 2559 V 17 w(Comm)g(comm,)g(MPI)p 1034 2559 V 17 w(Comm)p 1147 2559 V 17 w(request)g(*request\))75 2647 y(MPI)p 150 2647 V 17 w(CREATE)p 311 2647 V 16 w(SEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)i(TAG,)f(COMM,)g(REQUEST,)g(IERROR\))170 2704 y(INTEGER)g(REQUEST,)g(BUF,)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h (COMM,)f(REQUEST,)g(IERROR)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 41 46 41 45 bop 75 -100 a Ft(3.10.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (OBJECTS)673 b FE(41)166 45 y(The)17 b(call)g(allo)q(cates)h(a)e(new)h (comm)o(unication)g(ob)s(ject)f(and)h(asso)q(ciates)f(the)h(comm)o (unication)g(re-)75 102 y(quest)e(with)h(it.)166 166 y(The)g(function)h Fx(MPI)p 524 166 14 2 v 16 w(CREA)l(TE)p 709 166 V 17 w(RSEND)g FE(creates)e(a)h(comm)o(unication)h(ob)s(ject)e (for)g(a)h(ready)g(mo)q(de)75 223 y(send)g(op)q(eration.)75 335 y Fx(MPI)p 160 335 V 16 w(CREA)l(TE)p 345 335 V 17 w(RSEND\(buf,)g(count,)g(datat)o(yp)q(e,)h(dest,)f(tag,)f(comm,)e (request\))117 420 y Fr(IN)155 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 512 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(sen)o(t)i(\(in)o(teger\))117 604 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)117 696 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o (teger\))117 788 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 880 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 971 y(OUT)108 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 1104 y Fp(int)23 b(MPI)p 245 1104 15 2 v 17 w(Create)p 406 1104 V 17 w(rsend\(void*)f(buf,)h(int)h(count,)f(MPI)p 1163 1104 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 1161 y(int)h(tag,)f(MPI)p 683 1161 V 17 w(Comm)g(comm,)g(MPI)p 1034 1161 V 17 w(Comm)p 1147 1161 V 17 w(request)g(*request\))75 1255 y(MPI)p 150 1255 V 17 w(CREATE)p 311 1255 V 16 w(RSEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 1312 y(INTEGER)g(BUF,)h(COUNT,)f(DATATYPE,)f(DEST,)i(TAG,)f(COMM,)g (REQUEST,)g(IERROR)166 1407 y FE(The)15 b(function)i Fx(MPI)p 523 1407 14 2 v 15 w(CREA)l(TE)p 707 1407 V 18 w(SSEND)f FE(creates)f(a)g(comm)o(unication)h(ob)s(ject)e(for)h(a)g (sync)o(hronous)75 1463 y(mo)q(de)g(send)h(op)q(eration.)75 1575 y Fx(MPI)p 160 1575 V 16 w(CREA)l(TE)p 345 1575 V 17 w(SSEND\(buf,)g(count,)g(datat)o(yp)q(e,)h(dest,)f(tag,)f(comm,)e (request\))117 1661 y Fr(IN)155 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1753 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 1844 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)117 1936 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o(teger\))117 2028 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 2120 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 2212 y(OUT)108 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 2345 y Fp(int)23 b(MPI)p 245 2345 15 2 v 17 w(Create)p 406 2345 V 17 w(ssend\(void*)f(buf,)h (int)h(count,)f(MPI)p 1163 2345 V 17 w(Datatype)f(datatype,)h(int)g (dest,)393 2401 y(int)h(tag,)f(MPI)p 683 2401 V 17 w(Comm)g(comm,)g (MPI)p 1034 2401 V 17 w(Comm)p 1147 2401 V 17 w(request)g(*request\))75 2496 y(MPI)p 150 2496 V 17 w(CREATE)p 311 2496 V 16 w(SSEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 2552 y(INTEGER)g(BUF,)h(COUNT,)f(DATATYPE,)f(DEST,)i(TAG,)f(COMM,)g (REQUEST,)g(IERROR)166 2647 y FE(The)15 b(function)h Fx(MPI)p 522 2647 14 2 v 15 w(CREA)l(TE)p 706 2647 V 18 w(RECV)g FE(creates)e(a)h(comm)o(unication)g(ob)s(ject)f(for)h(a)f (receiv)o(e)i(op)q(era-)75 2704 y(tion.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 42 47 42 46 bop 75 -100 a FE(42)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(CREA)l(TE)p 345 45 V 17 w(RECV\(buf,)g(count,)h (datat)o(yp)q(e,)f(source,)f(tag,)g(comm,)e(request\))117 122 y Fr(OUT)108 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(receiv)o (e)h(bu\013er)g(\(c)o(hoice\))117 195 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(receiv)o(ed)j(\(in)o (teger\))117 268 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)117 341 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(source)j(or)d(MPI)p 1309 341 13 2 v 16 w(ANY)p 1418 341 V 15 w(SOUR)o(CE)h(\(in)o(teger\))117 414 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(or)h(MPI)p 1270 414 V 15 w(ANY)p 1378 414 V 16 w(T)m(A)o(G)f(\(in)o(teger\))117 487 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 560 y(OUT)108 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 684 y Fp(int)23 b(MPI)p 245 684 15 2 v 17 w(Create)p 406 684 V 17 w(recv\(void*)f(buf,)h(int)h (count,)f(MPI)p 1139 684 V 17 w(Datatype)f(datatype,)h(int)h(source,) 393 741 y(int)g(tag,)f(MPI)p 683 741 V 17 w(Comm)g(comm,)g(MPI)p 1034 741 V 17 w(Comm)p 1147 741 V 17 w(request)g(*request\))75 827 y(MPI)p 150 827 V 17 w(CREATE)p 311 827 V 16 w(RECV\(BUF,)g(COUNT,) g(DATATYPE,)f(SOURCE,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 884 y(INTEGER)g(BUF,)h(COUNT,)f(DATATYPE,)f(SOURCE,)h(TAG,)h(COMM,)f (REQUEST,)g(IERROR)166 970 y FE(A)e(comm)o(unication)g(\(send)g(or)f (receiv)o(e\))i(that)e(uses)h(a)f(prede\014ned)j(comm)o(unication)e(ob) s(ject)f(is)75 1027 y(initiated)d(b)o(y)e(the)g(function)h Fx(MPI)p 661 1027 14 2 v 16 w(ST)l(ART)p FE(.)75 1130 y Fx(MPI)p 160 1130 V 16 w(ST)l(ART\(request\))117 1208 y Fr(INOUT)62 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 1332 y Fp(int)23 b(MPI)p 245 1332 15 2 v 17 w(Start\(MPI)p 478 1332 V 16 w(Comm)p 590 1332 V 17 w(request)g(request\))75 1418 y(MPI)p 150 1418 V 17 w(START\(REQUEST,)f(IERROR\))170 1475 y(INTEGER)h(REQUEST,)g (IERROR)166 1561 y FE(The)17 b(comm)o(unication)h(ob)s(ject)f(asso)q (ciated)g(with)h Fx(request)g FE(should)g(b)q(e)g(an)f(ob)s(ject)g (that)f(w)o(as)h(cre-)75 1618 y(ated)g(b)o(y)g(one)h(of)f(the)g (previous)h(four)f(functions,)h(so)f(that)f(all)j(the)e(comm)o (unication)h(parameters)e(are)75 1674 y(already)k(de\014ned.)34 b(A)19 b(send)h(can)g(b)q(e)g(started)f(pro)o(vided)h(that)f(the)g (previous)h(send)h(using)f(the)f(same)75 1731 y(ob)s(ject)d(has)h (completed,)h(or)f(as)f(so)q(on)h(as)g(the)g(ob)s(ject)f(is)h(created,) g(if)h(it)f(has)g(not)f(y)o(et)h(b)q(een)h(used)f(in)h(a)75 1787 y(comm)o(unication.)j(In)16 b(addition,)h(if)f(the)f(comm)o (unication)h(mo)q(de)g(is)g Fp(ready)f FE(then)g(a)h(matc)o(hing)f (receiv)o(e)75 1844 y(should)k(b)q(e)f(p)q(osted.)28 b(The)18 b(send)g(bu\013er)g(should)h(not)e(b)q(e)i(up)q(dated)f(after) f(the)h(send)g(is)h(started,)e(un)o(til)75 1900 y(the)e(op)q(eration)h (completes.)166 1957 y(A)e(receiv)o(e)g(can)g(b)q(e)g(started)f(pro)o (vided)i(that)e(the)h(preceding)h(receiv)o(e)f(using)h(the)f(same)f(ob) s(ject)g(has)75 2013 y(completed,)f(or)f(as)f(so)q(on)h(as)f(the)h(ob)s (ject)f(is)i(created,)f(if)g(it)g(has)g(not)g(y)o(et)f(b)q(een)i(used)f (in)h(a)f(comm)o(unication.)75 2069 y(The)18 b(receiv)o(e)h(bu\013er)f (should)h(not)e(b)q(e)i(accessed)g(after)e(the)h(receiv)o(e)h(is)f (started,)g(un)o(til)h(the)f(op)q(eration)75 2126 y(completes.)166 2182 y(The)23 b(call)g(is)g(non)o(blo)q(c)o(king,)i(with)e(similar)g (seman)o(tics)g(as)f(the)h(non)o(blo)q(c)o(king)g(comm)o(unication)75 2239 y(op)q(erations)15 b(describ)q(ed)i(in)g(section)e(3.8.)75 2343 y Fx(MPI)p 160 2343 14 2 v 16 w(ST)l(ART)l(ALL\(count,)h(a)o(rra)o (y)p 641 2343 V 15 w(of)p 693 2343 V 16 w(requestss\))117 2420 y Fr(IN)155 b Fx(count)482 b Fr(list)14 b(length)f(\(in)o(teger\)) 117 2493 y(INOUT)62 b Fx(a)o(rra)o(y)p 416 2493 V 15 w(of)p 468 2493 V 16 w(requests)272 b Fr(arra)o(y)14 b(of)f(comm)o(unication)e(ob)r(jects)75 2617 y Fp(int)23 b(MPI)p 245 2617 15 2 v 17 w(Startall\(int)f(count,)h(MPI)p 811 2617 V 17 w(Comm)p 924 2617 V 17 w(request)g(*array)p 1276 2617 V 16 w(of)p 1340 2617 V 17 w(requests\))75 2704 y(MPI)p 150 2704 V 17 w(STARTALL\(COUNT,)f(ARRAY)p 669 2704 V 16 w(OF)p 733 2704 V 17 w(REQUESTS,)h(IERROR\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 43 48 43 47 bop 75 -100 a Ft(3.11.)34 b(SEND-RECEIVE)1277 b FE(43)170 45 y Fp(INTEGER)23 b(COUNT,)g(ARRAY)p 651 45 15 2 v 17 w(OF)p 716 45 V 17 w(REQUESTS\(*\),)f(IERROR)166 134 y FE(Start)i(all)j(comm)o(unications)f(asso)q(ciated)g(with)f (requests)h(in)g Fx(a)o(rra)o(y)p 1399 134 14 2 v 15 w(of)p 1451 134 V 16 w(requests)p FE(.)52 b(A)26 b(call)h(to)75 190 y Fx(MPI)p 160 190 V 16 w(ST)l(ART)l(ALL\(count,)76 b(a)o(rra)o(y)p 701 190 V 14 w(of)p 752 190 V 16 w(requests\))65 b FE(has)d(the)h(same)f(e\013ect)h(as)f(calls)i(to)75 247 y Fx(MPI)p 160 247 V 16 w(ST)l(ART\(a)o(rra)o(y)p 431 247 V 15 w(of)p 483 247 V 16 w(requests[i]\))p FE(,)23 b(executed)d(for)g Fp(i=0)j(,...,)g(count-1)p FE(,)d(in)h(some)e (arbitrary)g(or-)75 303 y(der.)166 362 y(A)e(comm)o(unication)g (started)f(with)h(a)g(call)h(to)e Fx(MPI)p 1063 362 V 16 w(ST)l(ART)h FE(or)f Fx(MPI)p 1377 362 V 16 w(ST)l(ART)l(ALL)i FE(is)f(completed)75 418 y(b)o(y)24 b(a)h(call)g(to)f Fx(MPI)p 437 418 V 16 w(W)l(AIT)p FE(,)g Fx(MPI)p 685 418 V 16 w(TEST)p FE(,)g(or)g(one)h(of)f(the)g(deriv)o(ed)i(functions)f (describ)q(ed)h(in)g(sec-)75 475 y(tion)21 b(3.8.5.)37 b(These)21 b(comm)o(unication)h(completion)h(functions)f(do)f(not)g (deallo)q(cate)h(the)f(comm)o(uni-)75 531 y(cation)j(ob)s(ject,)g(and)g (this)g(can)g(b)q(e)g(reused)g(anew)g(b)o(y)f(a)g Fx(MPI)p 1213 531 V 16 w(ST)l(ART)h FE(call.)46 b(The)24 b(ob)s(ject)f(needs)75 588 y(to)c(b)q(e)h(explicitly)j(deallo)q(cated)e(b)o(y)e(a)h(call)h(to) e(the)g(function)i Fx(MPI)p 1254 588 V 16 w(COMMOBJ)p 1496 588 V 17 w(FREE)p FE(,)f(b)q(elo)o(w.)34 b(W)l(e)75 644 y(th)o(us)14 b(ha)o(v)o(e)h(t)o(w)o(o)e(t)o(yp)q(es)h(of)g(comm)o (unication)i(ob)s(jects:)j Fz(p)q(ersisten)o(t)14 b FE(ob)s(jects,)g (whic)o(h)h(are)f(allo)q(cated)i(b)o(y)75 700 y(a)i(call)i(to)e Fx(MPI)p 347 700 V 15 w(CREA)l(TE)p 531 700 V 18 w(xxx)p FE(,)h(and)f(are)g(explicitly)k(deallo)q(cated)d(b)o(y)g Fx(MPI)p 1405 700 V 16 w(COMMOBJ)p 1647 700 V 17 w(FREE)p FE(,)g(and)75 757 y Fz(ephemeral)f FE(ob)s(jects)f(that)g(p)q(ersist)h (for)f(one)h(comm)o(unication)h(only:)25 b(they)18 b(are)g(created)f(b) o(y)h(a)g(non-)75 813 y(blo)q(c)o(king)13 b(comm)o(unication)f (initiation)h(function,)g(and)e(are)g(freed)h(b)o(y)f(the)h(comm)o (unication)g(completion)75 870 y(call.)75 976 y Fx(MPI)p 160 976 V 16 w(COMMOBJ)p 402 976 V 17 w(FREE\(request\))117 1055 y Fr(INOUT)62 b Fx(request)452 b Fr(comm)o(unication)11 b(ob)r(ject)j(\(handle\))75 1182 y Fp(int)23 b(MPI)p 245 1182 15 2 v 17 w(Commreq)p 430 1182 V 16 w(free\(MPI)p 638 1182 V 17 w(Comm)p 751 1182 V 16 w(request)g(*request\))75 1270 y(MPI)p 150 1270 V 17 w(COMMOBJ)p 335 1270 V 16 w(FREE\(REQUEST,)f(IERROR\))170 1327 y(INTEGER)h(REQUEST,)g(IERROR)166 1415 y FE(deallo)q(cated)15 b(when)f(there)g(are)g(no)g(p)q(ending)h (comm)o(unications)g(in)o(v)o(olving)g(this)f(ob)s(ject,)f(at)g(whic)o (h)75 1472 y(p)q(oin)o(t)j(the)f(request)g(b)q(ecomes)h(n)o(ull.)166 1530 y(The)h(call)g(is)g(non)o(blo)q(c)o(king.)25 b(It)17 b(is)g(p)q(ermissible)i(to)d(call)h Fx(MPI)p 1228 1530 14 2 v 16 w(COMMOBJ)p 1470 1530 V 18 w(FREE\(request\))h FE(after)75 1587 y(a)h(comm)o(unication)g(that)f(uses)h Fx(request)i FE(has)e(b)q(een)h(initiated,)h(but)e(b)q(efore)g(it)g (has)g(completed.)32 b(The)75 1643 y(ob)s(ject)15 b(will)j(b)q(e)f (deallo)q(cated)g(after)e(the)h(comm)o(unication)h(completes.)22 b(It)16 b(is)h(erroneous)f(to)f(initiate)i(a)75 1700 y(comm)o(unication)f(with)f(a)g(request)h(that)e(has)h(b)q(een)h (freed.)166 1758 y(A)f(correct)f(in)o(v)o(o)q(cation)i(of)e(the)h (functions)h(describ)q(ed)g(in)g(this)f(section)h(will)g(o)q(ccur)g(in) f(a)g(sequence)75 1815 y(of)g(the)g(form)166 1873 y Fz(Create)i FE(\()p Fz(Start)h(Complete)p FE(\))725 1857 y Fk(\003)762 1873 y Fz(F)l(ree)e Fy(;)166 1932 y FE(where)g FC(\003)g FE(indicates)i(zero)e(or)f(more)h(rep)q(etitions.)24 b(If)16 b(the)g(same)g(comm)o(unication)h(ob)s(ject)f(is)g(used)75 1989 y(in)k(sev)o(eral)f(concurren)o(t)g(threads,)g(it)g(is)h(the)f (user)g(resp)q(onsibilit)o(y)i(to)e(co)q(ordinate)g(calls)h(so)f(that)f (the)75 2045 y(correct)d(sequence)h(is)g(ob)q(ey)o(ed.)166 2104 y(A)c(send)g(op)q(eration)f(initiated)j(with)d Fx(MPI)p 874 2104 V 16 w(ST)l(ART)i FE(can)e(b)q(e)i(matc)o(hed)e(with)h(an)o(y) f(receiv)o(e)i(op)q(eration)75 2160 y(and,)g(lik)o(ewise,)h(a)e(receiv) o(e)i(op)q(eration)e(initiated)i(with)f Fx(MPI)p 1098 2160 V 16 w(ST)l(ART)g FE(can)g(receiv)o(e)g(messages)f(generated)75 2217 y(b)o(y)j(an)o(y)g(send)h(op)q(eration.)75 2372 y Fv(3.11)59 b(Send-receive)75 2478 y FE(The)18 b Fz(send-receiv)o(e)g FE(op)q(erations)g(com)o(bine)h(in)g(one)f(call)h(the)f(sending)i(of)d (a)h(message)g(to)f(one)h(desti-)75 2534 y(nation)e(and)g(the)g (receiving)i(of)d(another)h(message,)f(from)g(another)h(destination,)g (p)q(ossibly)i(the)e(same.)75 2591 y(A)k(send-receiv)o(e)j(op)q (eration)d(is)h(v)o(ery)f(useful)i(for)d(executing)j(a)e(shift)h(op)q (eration)f(across)g(a)g(c)o(hain)h(of)75 2647 y(pro)q(cesses.)28 b(If)18 b(blo)q(c)o(king)h(sends)g(and)f(receiv)o(es)h(are)e(used)h (for)g(suc)o(h)g(shift,)g(then)g(one)g(needs)h(to)e(order)75 2704 y(correctly)g(the)g(sends)g(and)g(receiv)o(es)h(\(e.g.,)e(ev)o(en) h(pro)q(cesses)g(send,)h(next)f(receiv)o(e,)g(o)q(dd)h(pro)q(cesses)f (re-)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 44 49 44 48 bop 75 -100 a FE(44)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(ceiv)o(e)h (\014rst,)f(next)g(send\))h(so)f(as)f(to)h(prev)o(en)o(t)g(cyclic)i (dep)q(endencies)i(that)15 b(lead)i(to)f(deadlo)q(c)o(k.)24 b(When)16 b(a)75 102 y(send-receiv)o(e)e(op)q(eration)e(is)g(used,)h (the)f(comm)o(unication)h(subsystem)f(tak)o(es)f(care)g(of)h(these)g (issues.)20 b(The)75 158 y(send-receiv)o(e)14 b(op)q(eration)f(can)f(b) q(e)h(used)g(in)h(conjunction)f(with)g(the)f(functions)i(describ)q(ed)g (in)f(Chapter)f(6)75 214 y(in)19 b(order)f(to)g(p)q(erform)g(shifts)h (on)f(v)m(arious)h(logical)h(top)q(ologies.)30 b(Also,)19 b(a)f(send-receiv)o(e)i(op)q(eration)e(is)75 271 y(useful)e(for)f (implemen)o(ting)i(remote)e(pro)q(cedure)h(calls.)166 343 y(A)g(message)f(sen)o(t)g(b)o(y)h(a)f(send-receiv)o(e)j(op)q (eration)d(can)h(b)q(e)h(receiv)o(ed)f(b)o(y)g(a)f(regular)h(receiv)o (e)h(op)q(er-)75 400 y(ation,)e(and)g(vice)h(v)o(ersa.)75 520 y Fx(MPI)p 160 520 14 2 v 16 w(SENDRECV\(sendbuf,)24 b(sendcount,)f(sendt)o(yp)q(e,)g(dest,)f(sendtag,)g(recvbuf,)g (recvcount,)f(recvt)o(yp)q(e,)75 576 y(source,)15 b(recvtag,)g(comm,)e (status\))117 669 y Fr(IN)155 b Fx(sendbuf)444 b Fr(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 777 y(IN)155 b Fx(sendcount)399 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 884 y(IN)155 b Fx(sendt)o(yp)q(e)424 b Fr(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(bu\013er)117 991 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1099 y(IN)155 b Fx(sendtag)443 b Fr(send)15 b(tag)e(\(in)o(teger\))117 1206 y(OUT)108 b Fx(recvbuf)450 b Fr(initial)12 b(address)j(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1313 y(IN)155 b Fx(recvcount)405 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h (receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1421 y(IN)155 b Fx(recvt)o(yp)q(e)430 b Fr(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f (receiv)o(e)i(bu\013er)117 1528 y(IN)155 b Fx(source)468 b Fr(rank)14 b(of)f(source)j(\(in)o(teger\))117 1635 y(IN)155 b Fx(recvtag)449 b Fr(receiv)o(e)15 b(tag)f(\(in)o(teger\))117 1743 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 1850 y(OUT)108 b Fx(status)476 b Fr(status)15 b(ob)r(ject)75 1990 y Fp(int)23 b(MPI)p 245 1990 15 2 v 17 w(Sendrecv\(void)f (*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1990 V 17 w(Datatype)f(sendtype,)393 2047 y(int)i(dest,)f(int)g(sendtag,)g(void)g (*recvbuf,)g(int)g(recvcount,)393 2103 y(MPI)p 468 2103 V 17 w(Datatype)g(recvtype,)f(int)i(source,)f(MPI)p 1297 2103 V 16 w(Datatype)g(recvtag,)393 2160 y(MPI)p 468 2160 V 17 w(Comm)g(comm,)g(MPI)p 819 2160 V 17 w(Status)g(*status\))75 2262 y(MPI)p 150 2262 V 17 w(SENDRECV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVBUF,)393 2319 y(RECVCOUNT,)h (RECVTYPE,)f(SOURCE,)h(RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))170 2375 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2432 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h (RECVTYPE,)170 2488 y(SOURCE,)g(RECVTAG,)g(COMM,)g(STATUS\(MPI)p 962 2488 V 16 w(STATUS)p 1122 2488 V 16 w(SIZE\),)h(IERROR)166 2591 y FE(Execute)13 b(a)f(blo)q(c)o(king)i(send)g(and)f(receiv)o(e)g (op)q(eration.)19 b(Both)13 b(send)g(and)g(receiv)o(e)h(use)f(the)f (same)h(tag)75 2647 y(v)m(alue)h(and)g(the)f(same)g(comm)o(unicator.)19 b(Ho)o(w)o(ev)o(er,)12 b(the)i(send)g(bu\013er)f(and)g(receiv)o(e)h (bu\013er)g(are)f(disjoin)o(t,)75 2704 y(and)i(ma)o(y)g(ha)o(v)o(e)g (di\013eren)o(t)g(length)h(and)f(di\013eren)o(t)h(datat)o(yp)q(es.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 45 50 45 49 bop 75 -100 a Ft(3.11.)34 b(SEND-RECEIVE)1277 b FE(45)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(SENDRECV)p 408 45 V 17 w(REPLA)o(CE\(buf,)19 b(count,)f(datat)o(yp)q(e,)h(dest,)f (sendtag,)h(source,)f(recvtag,)f(comm,)e(sta-)75 102 y(tus\))117 181 y Fr(INOUT)62 b Fx(buf)527 b Fr(initial)12 b(address)j(of)f(send)g(and)g(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 259 y(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)9 b(of)g(elemen)o(ts)h (in)f(send)i(and)e(receiv)o(e)i(bu\013er)g(\(in)o(teger\))117 338 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(and)f(receiv)o(e)h(bu\013er)117 417 y(IN)155 b Fx(dest)511 b Fr(rank)14 b(of)f(destination)h(\(in)o (teger\))117 495 y(IN)155 b Fx(sendtag)443 b Fr(send)15 b(message)f(tag)f(\(in)o(teger\))117 574 y(IN)155 b Fx(source)468 b Fr(rank)14 b(of)f(source)j(\(in)o(teger\))117 653 y(IN)155 b Fx(recvtag)449 b Fr(receiv)o(e)15 b(message)f(tag)f(\(in)o(teger\)) 117 731 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 810 y(OUT)108 b Fx(status)476 b Fr(status)15 b(ob)r(ject)75 936 y Fp(int)23 b(MPI)p 245 936 15 2 v 17 w(Sendrecv)p 454 936 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g (MPI)p 1258 936 V 17 w(Datatype)g(datatype,)393 993 y(int)h(dest,)f (int)g(sendtag,)g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 993 V 16 w(Comm)h(comm,)393 1049 y(MPI)p 468 1049 V 17 w(Status)f(*status\))75 1138 y(MPI)p 150 1138 V 17 w(SENDRECV)p 359 1138 V 16 w(REPLACE\(BUF,)f(COUNT,)h(DATATYPE,)g (DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)393 1194 y(COMM,)g(STATUS,)g (IERROR\))170 1251 y()g(BUF\(*\))170 1307 y(INTEGER)g(COUNT,)g (DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)g(COMM,)170 1363 y(STATUS\(MPI)p 413 1363 V 16 w(STATUS)p 573 1363 V 17 w(SIZE\),)g(IERROR)166 1452 y FE(Execute)15 b(a)f(blo)q(c)o(king)i (send)f(and)g(receiv)o(e;)g(the)g(same)f(bu\013er)g(is)h(used)h(b)q (oth)e(for)g(the)h(send)g(and)g(for)75 1508 y(the)g(receiv)o(e,)h(so)f (that)f(the)i(message)e(sen)o(t)h(is)h(replaced)g(b)o(y)g(the)f (message)g(receiv)o(ed.)166 1566 y(The)f(seman)o(tics)g(of)f(a)h (send-receiv)o(e)h(op)q(eration)f(is)g(what)g(w)o(ould)g(obtain)g(if)g (the)g(caller)h(fork)o(ed)e(t)o(w)o(o)75 1623 y(concurren)o(t)19 b(threads,)f(one)h(to)f(execute)h(the)g(send,)g(and)g(one)g(to)e (execute)j(the)e(receiv)o(e,)i(follo)o(w)o(ed)f(b)o(y)75 1679 y(a)e(join)g(of)g(these)g(t)o(w)o(o)f(threads.)25 b(Additional)19 b(in)o(termediate)f(bu\013ering)g(is)f(needed)i(for)d (the)i(\\replace")75 1736 y(v)m(arian)o(t.)166 1794 y(A)11 b(send-receiv)o(e)i(cannot)f(b)q(e)g(implemen)o(ted)h(b)o(y)e(a)g(blo)q (c)o(king)i(send)f(follo)o(w)o(ed)g(b)o(y)f(a)g(blo)q(c)o(king)i (receiv)o(e)75 1850 y(or)i(a)g(blo)q(c)o(king)i(receiv)o(e,)f(follo)o (w)o(ed)g(b)o(y)g(a)f(blo)q(c)o(king)i(send.)22 b(Consider)16 b(the)f(follo)o(wing)i(co)q(de,)e(where)h(t)o(w)o(o)75 1907 y(pro)q(cesses)g(exc)o(hange)f(messages:)75 2024 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2080 y(IF)h(\(rank.EQ.0\))123 2137 y(THEN)170 2193 y(CALL)g (MPI_SENDRECV\(send_buff,)c(send_count,)j(MPI_REAL,)f(0,)i(send_tag,) 242 2250 y(recv_buff,)f(recv_count,)f(MPI_REAL,)h(1,)g(recv_tag,)g (comm,)g(status,)g(ierr\))123 2306 y(ELSE)170 2363 y(CALL)h (MPI_SENDRECV\(send_buff,)c(send_count,)j(MPI_REAL,)f(1,)i(send_tag,) 242 2419 y(recv_buff,)f(recv_count,)f(MPI_REAL,)h(0,)g(tag,)g(comm,)h (status,)f(ierr\))75 2475 y(END)g(IF)166 2591 y FE(If)16 b(the)f(send)h(receiv)o(es)h(are)e(replaced)i(either)f(b)o(y)f(blo)q(c) o(king)i(send,)f(follo)o(w)o(ed)g(b)o(y)g(blo)q(c)o(king)g(receiv)o(e,) 75 2647 y(or)g(blo)q(c)o(king)h(receiv)o(e,)g(follo)o(w)o(ed)g(b)o(y)f (blo)q(c)o(king)i(send,)f(then)f(the)h(co)q(de)g(ma)o(y)e(deadlo)q(c)o (k.)24 b(On)17 b(the)f(other)75 2704 y(hand,)f(send-receiv)o(e)i(can)f (b)q(e)f(implemen)o(ted)i(using)f(non)o(blo)q(c)o(king)h(sends)e(and)h (receiv)o(es.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 46 51 46 50 bop 75 -100 a FE(46)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fv(3.12)59 b(Null)21 b(p)n(ro)r(cesses)75 147 y FE(In)15 b(man)o(y)e(instances,)i (it)f(is)h(con)o(v)o(enien)o(t)g(to)e(sp)q(ecify)j(a)d(\\dumm)o(y")h (source)g(or)g(destination)h(for)e(comm)o(u-)75 203 y(nication.)29 b(This)18 b(simpli\014es)i(the)e(co)q(de)g(that)f(is)h(needed)i(for)d (dealing)i(with)f(b)q(oundaries,)h(e.g.,)e(in)i(the)75 259 y(case)c(of)g(a)g(non-circular)i(shift)e(done)h(with)f(calls)i(to)d (send-receiv)o(e.)166 316 y(The)f(sp)q(ecial)j(v)m(alue)26 b Fm(MPI)p 608 316 13 2 v 15 w(PROCNULL)12 b FE(can)i(b)q(e)g(used)g (instead)g(of)f(a)g(rank)g(wherev)o(er)g(a)g(source)g(or)g(a)75 372 y(destination)i(parameter)e(is)i(required)g(in)g(a)e(call.)21 b(A)14 b(comm)o(unication)h(with)f(pro)q(cess)27 b Fm(MPI)p 1649 372 V 15 w(PROCNULL)75 429 y FE(has)13 b(no)h(e\013ect:)k(a)13 b(send)h(to)25 b Fm(MPI)p 633 429 V 14 w(PROCNULL)13 b FE(succeeds)i(and)e(returns)g(as)g(so)q(on)h(as)f(p)q(ossible.)21 b(A)13 b(receiv)o(e)75 485 y(from)26 b Fm(MPI)p 271 485 V 14 w(PROCNULL)13 b FE(succeeds)i(and)f(returns)g(as)f(so)q(on)h(as)f (p)q(ossible)j(with)e(no)g(mo)q(di\014cations)h(to)e(the)75 542 y(receiv)o(e)18 b(bu\013er.)25 b(When)17 b(a)g(receiv)o(e)h(with)f Fx(source)h FE(=)32 b Fm(MPI)p 1090 542 V 15 w(PROCNULL)16 b FE(is)i(executed)f(then)h(the)f(status)75 598 y(ob)s(ject)e(returns) 30 b Fx(source)16 b FE(=)29 b Fm(MPI)p 661 598 V 14 w(PROCNULL)p FE(,)14 b Fx(tag)h FE(=)30 b Fm(MPI)p 1132 598 V 14 w(ANY)p 1231 598 V 15 w(T)m(A)o(G)14 b FE(and)h Fx(count=0)p FE(.)75 741 y Fv(3.13)59 b(Derived)19 b(datat)n(yp)r(es)75 843 y FE(Up)c(to)f(no)o(w,)g(all)i(p)q(oin)o(t)f(to)f(p)q(oin)o(t)h (comm)o(unication)h(in)o(v)o(olv)o(ed)f(only)g(con)o(tiguous)g (bu\013ers)g(con)o(taining)g(a)75 899 y(sequence)g(of)e(elemen)o(ts)h (of)f(the)g(same)g(t)o(yp)q(e.)19 b(This)14 b(is)g(to)q(o)f (constraining)h(on)f(t)o(w)o(o)f(accoun)o(ts:)19 b(One)14 b(often)75 956 y(w)o(an)o(ts)f(to)h(pass)g(messages)g(that)f(con)o (tain)i(v)m(alues)g(with)g(di\013eren)o(t)f(datat)o(yp)q(es)g(\(e.g.,)f (an)h(in)o(teger)h(coun)o(t,)75 1012 y(follo)o(w)o(ed)i(b)o(y)f(a)g (sequence)i(of)e(real)h(n)o(um)o(b)q(ers\);)f(and)h(one)g(often)f(w)o (an)o(ts)f(to)h(send)h(noncon)o(tiguous)g(data)75 1069 y(\(e.g.)h(a)12 b(sub-blo)q(c)o(k)i(of)e(a)g(matrix\).)18 b(One)13 b(solution)g(is)g(to)f(pro)o(vide)h(functions)g(that)e(pac)o (k)i(noncon)o(tiguous)75 1125 y(data)f(in)o(to)h(a)g(con)o(tiguous)g (bu\013er)g(at)f(the)h(sender)h(site)f(and)g(unpac)o(k)h(it)f(bac)o(k)g (at)f(the)h(receiv)o(er)h(site.)19 b(This)75 1182 y(has)f(the)g(disadv) m(an)o(tage)g(of)g(requiring)h(additional)h(memory-to-memory)c(cop)o(y) i(op)q(erations)g(at)g(b)q(oth)75 1238 y(sites,)12 b(ev)o(en)f(when)g (the)g(comm)o(unication)g(subsystem)g(has)f(scatter-gather)g (capabilities.)20 b(Instead,)12 b(MPI)75 1294 y(pro)o(vides)22 b(mec)o(hanisms)f(to)g(sp)q(ecify)h(more)f(general,)i(mixed)f(and)f (noncon)o(tiguous)g(comm)o(unication)75 1351 y(bu\013ers.)i(It)16 b(is)g(up)h(to)f(the)g(implemen)o(tation)h(to)f(decide)i(whether)e (data)f(should)j(b)q(e)e(\014rst)g(pac)o(k)o(ed)g(in)h(a)75 1407 y(con)o(tiguous)h(bu\013er)h(b)q(efore)f(b)q(eeing)i(transmitted,) f(or)e(whether)i(it)g(can)f(b)q(e)h(collected)h(directly)g(from)75 1464 y(where)15 b(it)h(resides.)166 1520 y(The)d(general)g(mec)o (hanisms)g(pro)o(vided)g(here)g(allo)o(ws)g(one)f(to)g(transfer)g (directly)l(,)i(without)f(cop)o(ying,)75 1577 y(ob)s(jects)k(of)g(v)m (arious)h(shap)q(e)h(and)f(size.)28 b(It)17 b(is)i(not)e(assumed)h (that)f(the)g(MPI)h(library)g(is)h(cognizan)o(t)e(of)75 1633 y(the)f(ob)s(jects)g(declared)h(in)g(the)f(host)g(language;)g(th)o (us,)g(if)g(one)g(w)o(an)o(ts)f(to)h(transfer)f(a)h(structure,)f(or)h (an)75 1690 y(arra)o(y)d(section,)i(it)f(will)i(b)q(e)f(necessary)f(to) g(pro)o(vide)h(in)g(MPI)f(a)g(de\014nition)i(of)e(a)g(comm)o(unication) h(bu\013er)75 1746 y(that)e(mimics)j(the)e(de\014nition)i(of)e(the)g (structure)g(or)f(arra)o(y)g(section)i(in)g(question.)20 b(These)15 b(facilities)h(can)75 1803 y(b)q(e)j(used)g(b)o(y)f(library) h(designers)h(to)d(de\014ne)j(comm)o(unication)f(functions)g(that)f (can)g(transfer)g(ob)s(jects)75 1859 y(de\014ned)g(in)f(the)f(host)g (language)g({)g(b)o(y)h(deco)q(ding)g(their)g(de\014nitions)h(as)e(a)o (v)m(ailable)i(in)f(a)f(sym)o(b)q(ol)h(table)75 1915 y(or)e(a)g(dop)q(e)g(v)o(ector.)20 b(Suc)o(h)15 b(higher-lev)o(el)j (comm)o(unication)e(functions)g(are)f(not)g(part)f(of)h(MPI.)166 1972 y(More)d(general)g(comm)o(unication)i(bu\013ers)e(are)g(sp)q (eci\014ed)i(b)o(y)f(replacing)g(the)g(basic)g(datat)o(yp)q(es)f(that) 75 2028 y(ha)o(v)o(e)18 b(used)h(so)f(far)g(with)g(deriv)o(ed)i(datat)o (yp)q(es)e(that)f(are)h(constructed)h(from)e(basic)i(datat)o(yp)q(es)f (using)75 2085 y(the)13 b(constructors)e(describ)q(ed)j(in)g(this)f (section.)19 b(These)13 b(metho)q(ds)f(of)g(constructing)h(deriv)o(ed)h (datat)o(yp)q(es)75 2141 y(can)h(b)q(e)h(applied)h(recursiv)o(ely)l(.) 166 2198 y(A)e Fz(general)j(datat)o(yp)q(e)e FE(is)g(an)f(opaque)h(ob)s (ject)e(that)h(sp)q(eci\014es)i(t)o(w)o(o)d(things:)143 2291 y FC(\017)23 b FE(A)15 b(sequence)h(of)f(basic)h(datat)o(yp)q(es) 143 2385 y FC(\017)23 b FE(A)15 b(sequence)h(of)f(in)o(teger)g(\(b)o (yte\))g(displacemen)o(ts)166 2478 y(The)22 b(displacemen)o(ts)g(are)g (not)f(required)h(to)f(b)q(e)h(p)q(ositiv)o(e,)i(distinct,)g(or)d(in)h (increasing)h(order;)75 2534 y(therefore)17 b(the)g(order)g(of)f(items) i(need)g(not)f(coincide)i(with)e(their)h(order)f(in)g(store,)g(and)g (an)g(item)h(ma)o(y)75 2591 y(app)q(ear)c(more)g(than)g(once.)20 b(W)l(e)14 b(call)h(suc)o(h)g(pair)f(of)g(sequences)h(\(or)e(sequence)j (of)d(pairs\))h(a)g Fz(t)o(yp)q(e)i(map)p FE(.)75 2647 y(The)22 b(sequence)g(of)f(basic)h(datat)o(yp)q(es)f(\(displacemen)o (ts)h(ignored\))g(is)f(the)h Fz(t)o(yp)q(e)i(signature)e FE(of)f(the)75 2704 y(datat)o(yp)q(e.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 47 52 47 51 bop 75 -100 a Ft(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(47)166 45 y(Let)471 115 y Fy(T)6 b(y)r(pemap)12 b FE(=)h FC(f)p FE(\()p Fy(ty)r(pe)844 122 y Fj(0)864 115 y Fy(;)8 b(disp)969 122 y Fj(0)987 115 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1188 122 y Fi(n)p Fk(\000)p Fj(1)1254 115 y Fy(;)g(disp)1359 122 y Fi(n)p Fk(\000)p Fj(1)1426 115 y FE(\))p FC(g)p Fy(;)75 215 y FE(b)q(e)16 b(suc)o(h)f(a)g(t)o(yp)q(e)h(map,)e(where)i Fy(ty)r(pe)709 222 y Fi(i)738 215 y FE(are)f(basic)h(t)o(yp)q(es,)f(and)g Fy(disp)1233 222 y Fi(i)1262 215 y FE(are)g(displacemen)o(ts.)21 b(Let)674 330 y Fy(T)6 b(y)r(pesig)13 b FE(=)g FC(f)p Fy(ty)r(pe)1002 337 y Fj(0)1022 330 y Fy(;)8 b(:::;)g(ty)r(pe)1187 337 y Fi(n)p Fk(\000)p Fj(1)1254 330 y FC(g)75 446 y FE(b)q(e)14 b(the)f(asso)q(ciated)h(t)o(yp)q(e)f(signature.)19 b(This)14 b(t)o(yp)q(e)f(map,)g(together)g(with)g(a)g(base)g(address)h Fs(buf)p FE(,)f(sp)q(eci\014es)75 502 y(a)19 b(comm)o(unication)i (bu\013er:)28 b(the)20 b(comm)o(unication)h(bu\013er)e(that)g(consists) h(of)f Fy(n)h FE(en)o(tries,)h(where)f(the)75 559 y Fy(i)p FE(-th)e(en)o(try)g(is)g(at)g(address)g Fy(buf)f FE(+)12 b Fy(disp)777 566 y Fi(i)809 559 y FE(and)19 b(has)f(t)o(yp)q(e)g Fy(ty)r(pe)1174 566 y Fi(i)1188 559 y FE(.)29 b(A)19 b(message)e(assem)o(bled)i(from)f(suc)o(h)75 615 y(comm)o(unication)e (bu\013er)f(will)i(consist)e(of)g Fy(n)h FE(v)m(alues,)g(of)e(the)i(t)o (yp)q(es)f(de\014ned)h(b)o(y)g Fy(T)6 b(y)r(pesig)r FE(.)166 676 y(W)l(e)11 b(can)f(use)h(a)f(handle)i(to)e(a)g(general)h(datat)o (yp)q(e)f(as)h(an)f(argumen)o(t)g(in)h(a)f(send)i(or)e(receiv)o(e)h(op) q(eration,)75 733 y(in)j(replacemen)o(t)f(of)f(a)h(basic)g(datat)o(yp)q (e)g(argumen)o(t.)18 b(The)13 b(op)q(eration)g Fx(MPI)p 1359 733 14 2 v 16 w(SEND\(buf,)h(1,)e(datat)o(yp)q(e,...\))75 789 y FE(will)j(use)e(the)h(send)g(bu\013er)f(de\014ned)h(b)o(y)g(the)f (base)g(address)h Fx(buf)g FE(and)f(the)g(general)h(datat)o(yp)q(e)f (asso)q(ciated)75 846 y(with)f Fx(datat)o(yp)q(e)p FE(;)i(it)e(will)i (generate)d(a)h(message)f(with)h(the)g(t)o(yp)q(e)g(signature)g (determined)h(b)o(y)e(the)h Fx(datat)o(yp)q(e)75 902 y FE(argumen)o(t.)28 b Fx(MPI)p 390 902 V 16 w(RECV\(buf,)20 b(1,)e(datat)o(yp)q(e,...\))29 b FE(will)20 b(use)e(the)h(receiv)o(e)g (bu\013er)f(de\014ned)h(b)o(y)g(the)f(base)75 959 y(address)d Fx(buf)h FE(and)g(the)f(general)h(datat)o(yp)q(e)e(asso)q(ciated)i (with)f Fx(datat)o(yp)q(e)p FE(.)166 1020 y(General)h(datat)o(yp)q(es)e (can)i(b)q(e)g(used)f(in)h(all)h(send)e(and)h(receiv)o(e)g(op)q (erations.)k(W)l(e)c(address)f(later)g(in)75 1076 y(section)h(3.13.5)d (the)i(case)h(where)f(the)g(second)h(argumen)o(t)e Fx(count)j FE(has)e(v)m(alue)h Fy(>)d FE(1.)166 1137 y(The)e(basic)g(datat)o(yp)q (es)f(presen)o(ted)i(in)f(section)g(3.2.1)f(are)g(particular)h(cases)g (of)f(a)h(general)g(datat)o(yp)q(e,)75 1194 y(and)19 b(are)f(prede\014ned.)32 b(Th)o(us,)19 b Fx(MPI)p 711 1194 V 16 w(INT)f FE(is)h(a)g(prede\014ned)h(handle)g(to)e(a)g(datat)o (yp)q(e)g(with)h(t)o(yp)q(e)g(map)75 1250 y FC(f)p FE(\()p Fp(int)o Fy(;)8 b FE(0\))p FC(g)p FE(,)16 b(with)h(one)h(en)o(try)f(of) g(t)o(yp)q(e)g Fp(int)g FE(and)g(displacemen)o(t)i(zero.)26 b(And)18 b(similarly)h(for)e(all)h(other)75 1306 y(basic)e(datat)o(yp)q (es.)166 1367 y(The)f Fz(exten)o(t)f FE(of)g(a)g(datat)o(yp)q(e)g(is)h (de\014ned)g(to)f(b)q(e)h(the)g(span)f(from)g(the)g(\014rst)g(b)o(yte)h (to)e(the)i(last)f(b)o(yte)75 1424 y(o)q(ccupied)19 b(b)o(y)d(en)o (tries)h(in)h(this)f(datat)o(yp)q(e,)f(rounded)h(up)g(to)f(satisfy)h (alignmen)o(t)g(requiremen)o(ts.)25 b(That)75 1480 y(is,)15 b(if)471 1550 y Fy(T)6 b(y)r(pemap)12 b FE(=)h FC(f)p FE(\()p Fy(ty)r(pe)844 1557 y Fj(0)864 1550 y Fy(;)8 b(disp)969 1557 y Fj(0)987 1550 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1188 1557 y Fi(n)p Fk(\000)p Fj(1)1254 1550 y Fy(;)g(disp)1359 1557 y Fi(n)p Fk(\000)p Fj(1)1426 1550 y FE(\))p FC(g)p Fy(;)75 1650 y FE(then)501 1765 y Fy(l)q(b)p FE(\()p Fy(T)e(y)r(pemap)p FE(\))39 b(=)j(min)906 1794 y Fi(j)959 1765 y Fy(disp)1043 1772 y Fi(j)1061 1765 y Fy(;)489 1856 y(ub)p FE(\()p Fy(T)6 b(y)r(pemap)p FE(\))40 b(=)i(max)910 1885 y Fi(j)968 1856 y Fy(disp)1052 1863 y Fi(j)1080 1856 y FE(+)11 b Fy(siz)r(eof)5 b FE(\()p Fy(ty)r(pe)1358 1863 y Fi(j)1376 1856 y FE(\))p Fy(;)22 b FE(and)406 1946 y Fy(extent)p FE(\()p Fy(T)6 b(y)r(pemap)p FE(\))42 b(=)g Fy(ub)p FE(\()p Fy(T)6 b(y)r(pemap)p FE(\))j FC(\000)h Fy(l)q(b)p FE(\()p Fy(T)c(y)r(pemap)p FE(\))i(+)j Fy(\017:)238 b FE(\(3.1\))75 2062 y(If)17 b Fy(ty)r(pe)206 2069 y Fi(i)237 2062 y FE(requires)g(alignmen)o(t)g(to)f(a)h(b)o(yte)f (address)h(that)f(is)h(is)g(a)f(m)o(ultiple)j(of)d Fy(k)1474 2069 y Fi(i)1488 2062 y FE(,)g(then)h Fy(\017)g FE(is)g(the)g(least)75 2118 y(nonnegativ)o(e)f(incremen)o(t)g(needed)g(to)f(round)g Fy(extent)p FE(\()p Fy(T)6 b(y)r(pemap)p FE(\))16 b(to)e(the)h(next)h (m)o(ultiple)h(of)d(max)1794 2125 y Fi(i)1816 2118 y Fy(k)1840 2125 y Fi(i)1854 2118 y FE(.)166 2179 y(Example:)k(Assume)10 b(that)g Fy(T)c(y)r(pe)13 b FE(=)g FC(f)p FE(\()p Fp(double)n Fy(;)8 b FE(0\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(8\))p FC(g)h FE(\(a)g Fp(double)h FE(at)g(displacemen)o(t)h(zero,)75 2235 y(follo)o(w)o(ed)17 b(b)o(y)g(a)f Fp(char)g FE(at)g(displacemen)o (t)i(eigh)o(t\).)24 b(Assume,)17 b(furthermore,)f(that)g(doubles)i(ha)o (v)o(e)e(to)g(b)q(e)75 2292 y(strictly)f(aligned)g(at)f(addresses)h (that)e(are)h(m)o(ultiple)i(of)e(eigh)o(t.)20 b(Then,)15 b(the)f(exten)o(t)g(of)g(this)h(datat)o(yp)q(e)e(is)75 2348 y(16)d(\(9)g(rounded)h(to)f(the)h(next)g(m)o(ultiple)h(of)f(8\).) 17 b(A)11 b(datat)o(yp)q(e)f(that)g(consists)h(of)f(a)g(c)o(haracter)g (immediately)75 2405 y(follo)o(w)o(ed)16 b(b)o(y)f(a)g(double)h(will)h (also)e(ha)o(v)o(e)g(an)g(exten)o(t)g(of)g(16.)75 2553 y Fq(3.13.1)49 b(Datat)o(yp)q(e)17 b(constructo)o(rs)75 2647 y Fx(Contiguous)47 b FE(The)16 b(simplest)i(datat)o(yp)q(e)e (constructor)f(is)i Fx(MPI)p 1171 2647 V 16 w(TYPE)p 1304 2647 V 17 w(CONTIGUOUS)h FE(whic)o(h)f(allo)o(ws)75 2704 y(replication)g(of)e(a)g(datat)o(yp)q(e)f(in)o(to)h(con)o(tiguous) h(lo)q(cations.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 48 53 48 52 bop 75 -100 a FE(48)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(CONTIGUOUS\(count,)h(oldt)o(yp) q(e,)f(newt)o(yp)q(e\))117 122 y Fr(IN)155 b Fx(count)482 b Fr(replication)14 b(coun)o(t)g(\(nonnegativ)o(e)g(in)o(teger\))117 197 y(IN)155 b Fx(oldt)o(yp)q(e)450 b Fr(old)13 b(datat)o(yp)q(e)i (\(handle\))117 271 y(OUT)108 b Fx(newt)o(yp)q(e)433 b Fr(new)15 b(datat)o(yp)q(e)f(\(handle\))75 396 y Fp(int)23 b(MPI)p 245 396 15 2 v 17 w(Type)p 358 396 V 17 w(contiguous\(int)f (count,)h(MPI)p 972 396 V 17 w(Datatype)f(oldtype,)393 452 y(MPI)p 468 452 V 17 w(Datatype)h(*newtype\))75 539 y(MPI)p 150 539 V 17 w(TYPE)p 263 539 V 16 w(CONTIGUOUS\(COUNT,)f (OLDTYPE,)h(NEWTYPE,)f(IERROR\))170 595 y(INTEGER)h(COUNT,)g(OLDTYPE,)g (NEWTYPE,)g(IERROR)166 682 y Fx(newt)o(yp)q(e)c FE(is)f(the)f(datat)o (yp)q(e)g(obtained)h(b)o(y)f(concatenating)h Fx(count)h FE(copies)f(of)f Fx(oldt)o(yp)q(e)p FE(.)27 b(Concate-)75 738 y(nation)15 b(is)h(de\014ned)h(using)f Fs(extent)e FE(as)h(the)h(size)g(of)e(the)i(concatenated)f(copies.)166 795 y(Example:)26 b(let)18 b Fx(oldt)o(yp)q(e)h FE(ha)o(v)o(e)e(t)o(yp) q(e)h(map)g FC(f)p FE(\()p Fp(double)n Fy(;)8 b FE(0\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(8\))p FC(g)p Fy(;)16 b FE(with)i(exten)o(t)g(16,)f(and)h(let)75 851 y Fx(count)c FE(=)f(3.)19 b(The)d(t)o(yp)q(e)f(map)g(of)g(the)g(datat)o(yp)q(e)g (returned)g(b)o(y)h Fx(newt)o(yp)q(e)h FE(is)257 950 y FC(f)p FE(\()p Fp(double)o Fy(;)8 b FE(0\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(8\))p Fy(;)g FE(\()p Fp(double)l Fy(;)g FE(16\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(24\))p Fy(;)g FE(\()p Fp(double)l Fy(;)g FE(32\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(40\))p FC(g)p FE(;)75 1048 y(i.e.,)15 b(alternating)g Fp(double)g FE(and)g Fp(char)g FE(elemen)o(ts,)g(with)h (displacemen)o(ts)g(0)p Fy(;)8 b FE(8)p Fy(;)g FE(16)p Fy(;)g FE(24)o Fy(;)f FE(32)p Fy(;)g FE(40.)166 1105 y(In)16 b(general,)f(assume)g(that)g(the)g(t)o(yp)q(e)g(map)g(of)g Fx(oldt)o(yp)q(e)h FE(is)595 1203 y FC(f)p FE(\()p Fy(ty)r(pe)720 1210 y Fj(0)740 1203 y Fy(;)8 b(disp)845 1210 y Fj(0)863 1203 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1064 1210 y Fi(n)p Fk(\000)p Fj(1)1130 1203 y Fy(;)g(disp)1235 1210 y Fi(n)p Fk(\000)p Fj(1)1302 1203 y FE(\))p FC(g)p Fy(;)75 1302 y FE(with)16 b(exten)o(t)e Fy(extent)p FE(.)22 b(Then)15 b Fx(newt)o(yp)q(e)i FE(has)e(a)g(t)o(yp)q(e)h(map)f(with)g Fp(count)9 b FC(\001)h Fp(n)15 b FE(en)o(tries)h(de\014ned)g(b)o(y:)75 1400 y FC(f)p FE(\()p Fy(ty)r(pe)200 1407 y Fj(0)219 1400 y Fy(;)8 b(disp)324 1407 y Fj(0)343 1400 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)544 1407 y Fi(n)p Fk(\000)p Fj(1)609 1400 y Fy(;)g(disp)714 1407 y Fi(n)p Fk(\000)p Fj(1)782 1400 y FE(\))p Fy(;)g FE(\()p Fy(ty)r(pe)923 1407 y Fj(0)941 1400 y Fy(;)g(disp)1046 1407 y Fj(0)1068 1400 y FE(+)s Fy(extent)p FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1434 1407 y Fi(n)p Fk(\000)p Fj(1)1502 1400 y Fy(;)g(disp)1607 1407 y Fi(n)p Fk(\000)p Fj(1)1678 1400 y FE(+)s Fy(extent)p FE(\))p Fy(;)128 1499 y(:::;)g FE(\()p Fy(ty)r(pe)290 1506 y Fj(0)308 1499 y Fy(;)g(disp)413 1506 y Fj(0)441 1499 y FE(+)i Fy(extent)i FC(\001)e FE(\()p Fp(count)f FC(\000)h FE(1\)\))p Fy(;)e(:::;)g FE(\()o Fy(ty)r(pe)1081 1506 y Fi(n)p Fk(\000)p Fj(1)1147 1499 y Fy(;)g(disp)1252 1506 y Fi(n)p Fk(\000)p Fj(1)1330 1499 y FE(+)i Fy(extent)i FC(\001)d FE(\()p Fp(count)g FC(\000)i FE(1\)\))p FC(g)p Fy(:)75 1618 y Fx(V)o(ecto)o(r)45 b FE(The)16 b(function)g Fx(MPI)p 600 1618 14 2 v 16 w(TYPE)p 733 1618 V 17 w(VECTOR)g FE(is)g(a)f(more)g(general)h (constructor)e(that)h(allo)o(ws)g(repli-)75 1675 y(cation)20 b(of)g(a)g(datat)o(yp)q(e)f(in)o(to)h(lo)q(cations)h(that)e(consist)i (of)e(equally)j(spaced)f(blo)q(c)o(ks.)35 b(Eac)o(h)20 b(blo)q(c)o(k)g(is)75 1731 y(obtained)g(b)o(y)f(concatenating)h(the)f (same)g(n)o(um)o(b)q(er)h(of)f(copies)h(of)f(the)g(old)h(datat)o(yp)q (e.)32 b(The)19 b(spacing)75 1788 y(b)q(et)o(w)o(een)d(blo)q(c)o(ks)f (is)h(a)f(m)o(ultiple)i(of)e(the)g(exten)o(t)g(of)g(the)g(old)h(datat)o (yp)q(e.)75 1891 y Fx(MPI)p 160 1891 V 16 w(TYPE)p 293 1891 V 17 w(VECTOR\()g(count,)g(oldt)o(yp)q(e,)g(stride,)f(blo)q (cklength,)i(newt)o(yp)q(e\))117 1969 y Fr(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o (teger\))117 2043 y(IN)155 b Fx(oldt)o(yp)q(e)450 b Fr(old)13 b(datat)o(yp)q(e)i(\(handle\))117 2118 y(IN)155 b Fx(stride)484 b Fr(n)o(um)o(b)q(er)13 b(of)f(elemen)o(ts)h(b)q(et)o(w)o(een)h(start)g (of)f(eac)o(h)g(blo)q(c)o(k)g(\(in)o(te-)905 2174 y(ger\))117 2249 y(IN)155 b Fx(blo)q(cklength)371 b Fr(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e(\(nonnegativ)o(e)h (in)o(te-)905 2305 y(ger\))117 2380 y(OUT)108 b Fx(newt)o(yp)q(e)433 b Fr(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2504 y Fp(int)23 b(MPI)p 245 2504 15 2 v 17 w(Type)p 358 2504 V 17 w(vector\(int)f (count,)h(MPI)p 876 2504 V 17 w(Datatype)g(oldtype,)g(int)g(stride,)393 2561 y(int)h(blocklength,)e(MPI)p 874 2561 V 17 w(Datatype)g (*newtype\))75 2647 y(MPI)p 150 2647 V 17 w(TYPE)p 263 2647 V 16 w(VECTOR\(COUNT,)g(OLDTYPE,)h(STRIDE,)g(BLOCKLENGTH,)f (NEWTYPE,)h(IERROR\))170 2704 y(INTEGER)g(COUNT,)g(OLDTYPE,)g(STRIDE,)g (BLOCKLENGTH,)f(NEWTYPE,)h(IERROR)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 49 54 49 53 bop 75 -100 a Ft(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(49)166 45 y(Example:)35 b(Assume,)24 b(again,)f(that)f Fx(oldt)o(yp)q(e)h FE(has)f(t)o(yp)q(e)h(map)f FC(f)p FE(\()p Fp(double)n Fy(;)8 b FE(0\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(8\))p FC(g)p Fy(;)21 b FE(with)75 102 y(exten)o(t)13 b(16.)18 b(A)13 b(call)h(to)e Fx(MPI)p 555 102 14 2 v 16 w(TYPE)p 688 102 V 17 w(VECTOR\()h(2,)g(oldt)o(yp)q (e,)h(4,)f(3,)f(newt)o(yp)q(e\))i FE(will)h(create)e(the)f(datat)o(yp)q (e)75 158 y(with)k(t)o(yp)q(e)f(map)269 242 y FC(f)p FE(\()p Fp(double)n Fy(;)8 b FE(0\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(8\))p Fy(;)g FE(\()p Fp(double)m Fy(;)g FE(16\))p Fy(;)g FE(\()p Fp(char)l Fy(;)g FE(24\))p Fy(;)g FE(\()p Fp(double)m Fy(;)g FE(32\))p Fy(;)g FE(\()p Fp(char)l Fy(;)g FE(40\))p Fy(;)228 327 y FE(\()p Fp(double)o Fy(;)g FE(64\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(72\))p Fy(;)g FE(\()p Fp(dou)o(ble)m Fy(;)g FE(80\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(88\))p Fy(;)g FE(\()p Fp(doub)o(le)m Fy(;)g FE(96\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(104\))p FC(g)i FE(:)75 401 y(t)o(w)o(o)k(blo)q(c)o(ks)i(with)f(three)h(copies)g (eac)o(h)f(of)g(the)g(old)h(t)o(yp)q(e,)f(starting)f(4*16)g(apart.)166 458 y(A)h(call)h(to)f Fx(MPI)p 439 458 V 16 w(TYPE)p 572 458 V 17 w(VECTOR\(3,)g(oldt)o(yp)q(e,)h(-2,)f(1,)f(newt)o(yp)q (e\))j FE(will)g(create)e(the)g(datat)o(yp)q(e)186 542 y FC(f)p FE(\()p Fp(double)o Fy(;)8 b FE(0\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(8\))p Fy(;)g FE(\()p Fp(double)m Fy(;)g FC(\000)p FE(32\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FC(\000)p FE(24\))p Fy(;)g FE(\()p Fp(double)l Fy(;)g FC(\000)p FE(64\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FC(\000)p FE(56\))p FC(g)p Fy(:)166 626 y FE(In)16 b(general,)f(assume)g(that)g Fx(oldt)o(yp)q(e)h FE(has)f(t)o(yp)q(e)g(map)595 711 y FC(f)p FE(\()p Fy(ty)r(pe)720 718 y Fj(0)740 711 y Fy(;)8 b(disp)845 718 y Fj(0)863 711 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1064 718 y Fi(n)p Fk(\000)p Fj(1)1130 711 y Fy(;)g(disp)1235 718 y Fi(n)p Fk(\000)p Fj(1)1302 711 y FE(\))p FC(g)p Fy(;)75 795 y FE(with)k(exten)o(t)e Fy(extent)p FE(.)20 b(The)12 b(newly)g(created)f(datat)o(yp)q(e)g(has)g (a)g(t)o(yp)q(e)g(map)g(with)g Fp(count)r FC(\001)r Fp(blocklength)p FC(\001)r Fy(n)75 852 y FE(en)o(tries:)606 908 y FC(f)p FE(\()p Fy(ty)r(pe)731 915 y Fj(0)751 908 y Fy(;)d(disp)856 915 y Fj(0)874 908 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1075 915 y Fi(n)p Fk(\000)p Fj(1)1141 908 y Fy(;)g(disp)1246 915 y Fi(n)p Fk(\000)p Fj(1)1313 908 y FE(\))p Fy(;)405 983 y FE(\()p Fy(ty)r(pe)507 990 y Fj(0)527 983 y Fy(;)g(disp)632 990 y Fj(0)660 983 y FE(+)j Fy(extent)p FE(\))p Fy(;)d(:::;)g FE(\()p Fy(ty)r(pe)1034 990 y Fi(n)p Fk(\000)p Fj(1)1101 983 y Fy(;)g(disp)1206 990 y Fi(n)p Fk(\000)p Fj(1)1283 983 y FE(+)i Fy(extent)p FE(\))p Fy(;)e(:::;)75 1057 y FE(\()p Fy(ty)r(pe)177 1064 y Fj(0)197 1057 y Fy(;)g(disp)302 1064 y Fj(0)322 1057 y FE(+)r(\()p Fp(blocklength)p FC(\000)r FE(1\))r FC(\001)r Fy(extent)p FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1064 1064 y Fi(n)p Fk(\000)p Fj(1)1130 1057 y Fy(;)g(disp)1235 1064 y Fi(n)p Fk(\000)p Fj(1)1304 1057 y FE(+)r(\()p Fp(blocklength)p FC(\000)r FE(1\))r FC(\001)r Fy(extent)p FE(\))p Fy(;)229 1132 y FE(\()p Fy(ty)r(pe)331 1139 y Fj(0)351 1132 y Fy(;)g(disp)456 1139 y Fj(0)484 1132 y FE(+)j Fp(stride)e FC(\001)h Fy(extent)p FE(\))p Fy(;)e(:::;)g FE(\()p Fy(ty)r(pe)1034 1139 y Fi(n)p Fk(\000)p Fj(1)1101 1132 y Fy(;)g(disp)1206 1139 y Fi(n)p Fk(\000)p Fj(1)1283 1132 y FE(+)i Fp(stride)g FC(\001)f Fy(extent)p FE(\))p Fy(;)f(:::;)412 1206 y FE(\()p Fy(ty)r(pe)514 1213 y Fj(0)534 1206 y Fy(;)g(disp)639 1213 y Fj(0)668 1206 y FE(+)i(\()p Fp(stride)f FE(+)i Fp(blocklength)d FC(\000)j FE(1\))e FC(\001)h Fy(extent)p FE(\))p Fy(;)e(:::;)357 1281 y FE(\()p Fy(ty)r(pe)459 1288 y Fi(n)p Fk(\000)p Fj(1)528 1281 y Fy(;)g(disp)633 1288 y Fi(n)p Fk(\000)p Fj(1)710 1281 y FE(+)j(\()p Fp(stride)e FE(+)h Fp(blocklength)f FC(\000)h FE(1\))g FC(\001)f Fy(extent)p FE(\))p Fy(;)f(::::;)495 1355 y FE(\()p Fy(ty)r(pe)597 1362 y Fj(0)617 1355 y Fy(;)g(disp)722 1362 y Fj(0)751 1355 y FE(+)i Fp(stride)f FC(\001)h FE(\()p Fp(count)f FC(\000)i FE(1\))e FC(\001)h Fy(extent)p FE(\))p Fy(;)e(:::;)446 1430 y FE(\()p Fy(ty)r(pe)548 1437 y Fi(n)p Fk(\000)p Fj(1)617 1430 y Fy(;)g(disp)722 1437 y Fi(n)p Fk(\000)p Fj(1)799 1430 y FE(+)j Fp(stride)e FC(\001)h FE(\()p Fp(count)f FC(\000)h FE(1\))g FC(\001)g Fy(extent)p FE(\))p Fy(;)e(:::;)279 1504 y FE(\()p Fy(ty)r(pe)381 1511 y Fj(0)401 1504 y Fy(;)g(disp)506 1511 y Fj(0)535 1504 y FE(+)i(\()p Fp(stride)f FC(\001)h FE(\()p Fp(count)f FC(\000)h FE(1\))g(+)g Fp(blocklength)f FC(\000)h FE(1\))g FC(\001)g Fy(extent)p FE(\))p Fy(;)e(:::;)255 1579 y FE(\()p Fy(ty)r(pe)357 1586 y Fi(n)p Fk(\000)p Fj(1)425 1579 y Fy(;)g(disp)530 1586 y Fi(n)p Fk(\000)p Fj(1)608 1579 y FE(+)i(\()p Fp(stride)f FC(\001)h FE(\()p Fp(count)f FC(\000)h FE(1\))g(+)g Fp(blocklength)f FC(\000)h FE(1\))g FC(\001)g Fy(extent)p FE(\))p FC(g)166 1654 y FE(A)16 b(call)i(to)e Fx(MPI)p 443 1654 V 16 w(TYPE)p 576 1654 V 16 w(CONTIGUOUS\(count,)k(oldt)o(yp)q(e,)d(newt)o(yp)q(e\))h FE(is)f(equiv)m(alen)o(t)h(to)e(a)g(call)i(to)75 1710 y Fx(MPI)p 160 1710 V 16 w(TYPE)p 293 1710 V 17 w(VECTOR\(count,)g (oldt)o(yp)q(e,)g(1,)e(1,)g(newt)o(yp)q(e\))p FE(,)i(or)e(to)g(a)h (call)g(to)f Fx(MPI)p 1495 1710 V 16 w(TYPE)p 1628 1710 V 17 w(VECTOR\(1,)75 1766 y(oldt)o(yp)q(e,)g(n,)f(count,)h(newt)o(yp)q (e\))p FE(,)h Fp(n)e FE(arbitrary)l(.)75 1884 y Fx(Hvecto)o(r)45 b FE(The)19 b(function)h Fx(MPI)p 631 1884 V 16 w(TYPE)p 764 1884 V 17 w(HVECTOR)g FE(is)f(iden)o(tical)i(to)d Fx(MPI)p 1389 1884 V 16 w(TYPE)p 1522 1884 V 17 w(VECTOR)p FE(,)h(except)75 1940 y(that)h Fx(stride)h FE(is)g(giv)o(en)g(in)g(b)o (ytes,)g(rather)f(then)g(in)i(elemen)o(ts.)36 b(The)20 b(use)h(for)f(b)q(oth)g(t)o(yp)q(es)h(of)f(v)o(ector)75 1997 y(constructors)14 b(is)i(illustrated)h(in)f(section)f(3.13.6.)j (\()p Fp(H)d FE(is)h(used)g(for)e(\\heterogeneous"\).)75 2100 y Fx(MPI)p 160 2100 V 16 w(TYPE)p 293 2100 V 17 w(HVECTOR\()i(count,)g(oldt)o(yp)q(e,)g(stride,)g(blo)q(cklength,)g (newt)o(yp)q(e\))117 2178 y Fr(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o (teger\))117 2250 y(IN)155 b Fx(oldt)o(yp)q(e)450 b Fr(old)13 b(datat)o(yp)q(e)i(\(handle\))117 2322 y(IN)155 b Fx(stride)484 b Fr(n)o(um)o(b)q(er)13 b(of)g(b)o(ytes)h(b)q(et)o(w)o(een)h(start)f (of)f(eac)o(h)h(blo)q(c)o(k)f(\(in)o(teger\))117 2394 y(IN)155 b Fx(blo)q(cklength)371 b Fr(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e(\(nonnegativ)o(e)h (in)o(te-)905 2451 y(ger\))117 2523 y(OUT)108 b Fx(newt)o(yp)q(e)433 b Fr(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2647 y Fp(int)23 b(MPI)p 245 2647 15 2 v 17 w(Type)p 358 2647 V 17 w(hvector\(int)f (count,)h(MPI)p 900 2647 V 17 w(Datatype)g(oldtype,)g(int)g(stride,)393 2704 y(int)h(blocklength,)e(MPI)p 874 2704 V 17 w(Datatype)g (*newtype\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 50 55 50 54 bop 75 -100 a FE(50)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fp(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(HVECTOR\(COUNT,)22 b(OLDTYPE,)h(STRIDE,)g(BLOCKLENGTH,)f(NEWTYPE,)h(IERROR\))170 102 y(INTEGER)g(COUNT,)g(OLDTYPE,)g(STRIDE,)g(BLOCKLENGTH,)f(NEWTYPE,)h (IERROR)166 188 y FE(Assume)15 b(that)g Fx(oldt)o(yp)q(e)h FE(has)f(t)o(yp)q(e)g(map)595 284 y FC(f)p FE(\()p Fy(ty)r(pe)720 291 y Fj(0)740 284 y Fy(;)8 b(disp)845 291 y Fj(0)863 284 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1064 291 y Fi(n)p Fk(\000)p Fj(1)1130 284 y Fy(;)g(disp)1235 291 y Fi(n)p Fk(\000)p Fj(1)1302 284 y FE(\))p FC(g)p Fy(;)75 381 y FE(with)k(exten)o(t)e Fy(extent)p FE(.)20 b(The)12 b(newly)g(created)f(datat)o(yp)q(e)g(has)g(a)g(t)o(yp)q(e)g(map)g(with) g Fp(count)r FC(\001)r Fp(blocklength)p FC(\001)r Fy(n)75 437 y FE(en)o(tries:)606 494 y FC(f)p FE(\()p Fy(ty)r(pe)731 501 y Fj(0)751 494 y Fy(;)d(disp)856 501 y Fj(0)874 494 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1075 501 y Fi(n)p Fk(\000)p Fj(1)1141 494 y Fy(;)g(disp)1246 501 y Fi(n)p Fk(\000)p Fj(1)1313 494 y FE(\))p Fy(;)405 574 y FE(\()p Fy(ty)r(pe)507 581 y Fj(0)527 574 y Fy(;)g(disp)632 581 y Fj(0)660 574 y FE(+)j Fy(extent)p FE(\))p Fy(;)d(:::;)g FE(\()p Fy(ty)r(pe)1034 581 y Fi(n)p Fk(\000)p Fj(1)1101 574 y Fy(;)g(disp)1206 581 y Fi(n)p Fk(\000)p Fj(1)1283 574 y FE(+)i Fy(extent)p FE(\))p Fy(;)e(:::;)75 655 y FE(\()p Fy(ty)r(pe)177 662 y Fj(0)197 655 y Fy(;)g(disp)302 662 y Fj(0)322 655 y FE(+)r(\()p Fp(blocklength)p FC(\000)r FE(1\))r FC(\001)r Fy(extent)p FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1064 662 y Fi(n)p Fk(\000)p Fj(1)1130 655 y Fy(;)g(disp)1235 662 y Fi(n)p Fk(\000)p Fj(1)1304 655 y FE(+)r(\()p Fp(blocklength)p FC(\000)r FE(1\))r FC(\001)r Fy(extent)p FE(\))p Fy(;)390 735 y FE(\()p Fy(ty)r(pe)492 742 y Fj(0)512 735 y Fy(;)g(disp)617 742 y Fj(0)645 735 y FE(+)j Fp(stride)o FE(\))p Fy(;)d(:::;)g FE(\()p Fy(ty)r(pe)1035 742 y Fi(n)p Fk(\000)p Fj(1)1101 735 y Fy(;)g(disp)1206 742 y Fi(n)p Fk(\000)p Fj(1)1283 735 y FE(+)i Fp(stride)p FE(\))p Fy(;)e(:::;)412 816 y FE(\()p Fy(ty)r(pe)514 823 y Fj(0)534 816 y Fy(;)g(disp)639 823 y Fj(0)668 816 y FE(+)i Fp(stride)f FE(+)i(\()p Fp(blocklength)d FC(\000)j FE(1\))e FC(\001)h Fy(extent)p FE(\))p Fy(;)e(:::;)357 896 y FE(\()p Fy(ty)r(pe)459 903 y Fi(n)p Fk(\000)p Fj(1)528 896 y Fy(;)g(disp)633 903 y Fi(n)p Fk(\000)p Fj(1)710 896 y FE(+)j Fp(stride)e FE(+)h(\()p Fp(blocklength)f FC(\000)h FE(1\))g FC(\001)f Fy(extent)p FE(\))p Fy(;)f(::::;)576 977 y FE(\()p Fy(ty)r(pe)678 984 y Fj(0)698 977 y Fy(;)g(disp)803 984 y Fj(0)831 977 y FE(+)j Fp(stride)e FC(\001)h FE(\()p Fp(count)f FC(\000)h FE(1\)\))p Fy(;)e(:::;)527 1057 y FE(\()p Fy(ty)r(pe)629 1064 y Fi(n)p Fk(\000)p Fj(1)698 1057 y Fy(;)g(disp)803 1064 y Fi(n)p Fk(\000)p Fj(1)880 1057 y FE(+)j Fp(stride)e FC(\001)h FE(\()p Fp(count)f FC(\000)h FE(1\)\))p Fy(;)e(:::;)279 1138 y FE(\()p Fy(ty)r(pe)381 1145 y Fj(0)401 1138 y Fy(;)g(disp)506 1145 y Fj(0)535 1138 y FE(+)i Fp(stride)g FC(\001)f FE(\()p Fp(count)g FC(\000)i FE(1\))e(+)i(\()p Fp(blocklength)d FC(\000)i FE(1\))g FC(\001)g Fy(extent)p FE(\))p Fy(;)e(:::;)255 1218 y FE(\()p Fy(ty)r(pe)357 1225 y Fi(n)p Fk(\000)p Fj(1)425 1218 y Fy(;)g(disp)530 1225 y Fi(n)p Fk(\000)p Fj(1)608 1218 y FE(+)i Fp(stride)f FC(\001)h FE(\()p Fp(count)f FC(\000)i FE(1\))e(+)i(\()p Fp(blocklength)d FC(\000)i FE(1\))g FC(\001)g Fy(extent)p FE(\))p FC(g)75 1338 y Fx(Indexed)47 b FE(The)19 b(function)h Fx(MPI)p 629 1338 14 2 v 16 w(TYPE)p 762 1338 V 17 w(INDEXED)f FE(allo)o(ws)h(replication)h(of)d(an)i(old)f(datat)o(yp)q(e)g(in)o(to)g (a)75 1394 y(sequence)c(of)e(con)o(tiguous)h(blo)q(c)o(ks,)g(where)g (eac)o(h)g(blo)q(c)o(k)h(can)f(con)o(tain)g(di\013eren)o(t)g(n)o(um)o (b)q(er)g(of)f(copies)i(and)75 1451 y(ha)o(v)o(e)f(a)g(di\013eren)o(t)g (displacemen)o(t.)21 b(All)16 b(blo)q(c)o(k)f(displacemen)o(ts)g(are)f (m)o(ultiples)i(of)e(the)h(old)f(t)o(yp)q(e)h(exten)o(t.)75 1554 y Fx(MPI)p 160 1554 V 16 w(TYPE)p 293 1554 V 17 w(INDEXED\()d(count,)i(oldt)o(yp)q(e,)g(a)o(rra)o(y)p 926 1554 V 14 w(of)p 977 1554 V 16 w(displacements,)h(a)o(rra)o(y)p 1376 1554 V 14 w(of)p 1427 1554 V 16 w(blo)q(cklengths,)h(newt)o(yp)q (e\))117 1688 y Fr(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f(n)o(um)o(b)q(er)h(of)f(en)o(tries)i (in)905 1744 y Fm(a)o(rra)o(y)p 992 1744 13 2 v 15 w(of)p 1041 1744 V 15 w(displacements)19 b Fr(and)f Fm(a)o(rra)o(y)p 1483 1744 V 15 w(of)p 1532 1744 V 14 w(blo)q(cklengths)i Fr(\(in)o(te-)905 1801 y(ger\))117 1875 y(IN)155 b Fx(oldt)o(yp)q(e)450 b Fr(old)13 b(datat)o(yp)q(e)i(\(handle\))117 1949 y(IN)155 b Fx(a)o(rra)o(y)p 416 1949 14 2 v 15 w(of)p 468 1949 V 16 w(displacements)164 b Fr(displacemen)o(t)19 b(for)h(eac)o(h)g(blo) q(c)o(k,)g(in)g(m)o(ultiples)e(of)h Fm(oldt)o(yp)q(e)905 2006 y Fr(exten)o(t)c(\(arra)o(y)f(of)f(in)o(teger\))117 2080 y(IN)155 b Fx(a)o(rra)o(y)p 416 2080 V 15 w(of)p 468 2080 V 16 w(blo)q(cklengths)191 b Fr(n)o(um)o(b)q(er)18 b(of)f(elemen)o(ts)h(p)q(er)i(blo)q(c)o(k)d(\(arra)o(y)i(of)e (nonnegativ)o(e)905 2136 y(in)o(tegers\))117 2210 y(OUT)108 b Fx(newt)o(yp)q(e)433 b Fr(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2335 y Fp(int)23 b(MPI)p 245 2335 15 2 v 17 w(Type)p 358 2335 V 17 w(indexed\(int)f(count,)h(MPI)p 900 2335 V 17 w(Datatype)g(oldtype,)393 2391 y(int)h(*array)p 636 2391 V 16 w(of)p 700 2391 V 17 w(displacements,)e(int)h(*array)p 1314 2391 V 17 w(of)p 1379 2391 V 17 w(blocklengths,)393 2448 y(MPI)p 468 2448 V 17 w(Datatype)g(*newtype\))75 2534 y(MPI)p 150 2534 V 17 w(TYPE)p 263 2534 V 16 w(INDEXED\(COUNT,)f (OLDTYPE,)h(ARRAY)p 972 2534 V 17 w(OF)p 1037 2534 V 17 w(DISPLACEMENTS,)393 2591 y(ARRAY)p 516 2591 V 17 w(OF)p 581 2591 V 17 w(BLOCKLENGTHS,)f(NEWTYPE,)g(IERROR\))170 2647 y(INTEGER)h(COUNT,)g(OLDTYPE,)g(ARRAY)p 866 2647 V 17 w(OF)p 931 2647 V 17 w(DISPLACEMENTS\(*\),)170 2704 y(ARRAY)p 293 2704 V 17 w(OF)p 358 2704 V 17 w(BLOCKLENGTHS\(*\),)f (NEWTYPE,)g(IERROR)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 51 56 51 55 bop 75 -100 a Ft(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(51)166 45 y(Example:)29 b(let)20 b Fx(oldt)o(yp)q(e)h FE(ha)o(v)o(e)e(t)o(yp)q(e)g(map)h FC(f)p FE(\()p Fp(double)n Fy(;)8 b FE(0\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(8\))p FC(g)p Fy(;)18 b FE(with)i(exten)o(t)f(16.)32 b(Let)20 b Fp(D)75 102 y(=)k(\(4,)f(0\))17 b FE(and)g(let)h Fp(B)24 b(=)g(\(3,)f(1\))p FE(.)j(A)17 b(call)h(to)f Fx(MPI)p 998 102 14 2 v 16 w(TYPE)p 1131 102 V 17 w(INDEXED\(2,)f(oldt)o(yp)q (e,)j(D,)d(B,)i(newt)o(yp)q(e\))75 158 y FE(returns)d(a)g(data)o(yp)q (e)g(with)g(t)o(yp)q(e)h(map)241 247 y FC(f)p FE(\()p Fp(double)n Fy(;)8 b FE(64\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(72\))p Fy(;)g FE(\()p Fp(double)l Fy(;)g FE(80\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(88\))p Fy(;)g FE(\()p Fp(double)l Fy(;)g FE(96\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(104\))743 336 y(\()p Fp(double)o Fy(;)g FE(0\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(8\))p FC(g)j FE(:)75 413 y(three)k(copies)h(of)e(the)i(old)f(t)o(yp)q(e)g(starting)f(at)h (displacemen)o(t)h(64,)e(and)h(one)h(cop)o(y)e(starting)h(at)f (displace-)75 469 y(men)o(t)h(0.)166 526 y(In)h(general,)f(assume)g (that)g Fx(oldt)o(yp)q(e)h FE(has)f(t)o(yp)q(e)g(map)595 615 y FC(f)p FE(\()p Fy(ty)r(pe)720 622 y Fj(0)740 615 y Fy(;)8 b(disp)845 622 y Fj(0)863 615 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1064 622 y Fi(n)p Fk(\000)p Fj(1)1130 615 y Fy(;)g(disp)1235 622 y Fi(n)p Fk(\000)p Fj(1)1302 615 y FE(\))p FC(g)p Fy(;)75 704 y FE(with)34 b(exten)o(t)g Fs(extent)p FE(.)76 b(Let)35 b Fx(B)f FE(b)q(e)h(the)f Fx(a)o(rra)o(y)p 1004 704 V 15 w(of)p 1056 704 V 16 w(blo)q(cklength)i FE(parameter)d(and)h Fx(D)g FE(b)q(e)h(the)75 760 y Fx(a)o(rra)o(y)p 173 760 V 15 w(of)p 225 760 V 16 w(displacements)16 b FE(parameter.)j(The)14 b(newly)h(created)f(datat)o(yp)q(e)g(has)g Fy(n)9 b FC(\001)1455 728 y Fh(P)1499 741 y Fg(count)n Fk(\000)p Fj(1)1499 773 y Fi(i)p Fj(=0)1651 760 y Fx(B)p FE([)p Fx(i)p FE(])14 b(en)o(tries:)280 849 y FC(f)p FE(\()p Fy(ty)r(pe)405 856 y Fj(0)424 849 y Fy(;)8 b(disp)529 856 y Fj(0)558 849 y FE(+)i Fx(D)p FE([)p Fx(0)p FE(])f FC(\001)h Fy(extent)p FE(\))p Fy(;)e(:::;)g FE(\()p Fy(ty)r(pe)1045 856 y Fi(n)p Fk(\000)p Fj(1)1112 849 y Fy(;)g(disp)1217 856 y Fi(n)p Fk(\000)p Fj(1)1294 849 y FE(+)j Fx(D)p FE([)p Fx(0)p FE(])e FC(\001)g Fy(extent)p FE(\))p Fy(;)f(:::;)75 938 y FE(\()p Fy(ty)r(pe)177 945 y Fj(0)197 938 y Fy(;)g(disp)302 945 y Fj(0)326 938 y FE(+)e(\()p Fx(D)p FE([)p Fx(0)p FE(])g(+)g Fx(B)p FE([)p Fx(0)p FE(])g FC(\000)g FE(1\))g FC(\001)g Fy(extent)p FE(\))p Fy(;)i(:::;)g FE(\()p Fy(ty)r(pe)1034 945 y Fi(n)p Fk(\000)p Fj(1)1101 938 y Fy(;)g(disp)1206 945 y Fi(n)p Fk(\000)p Fj(1)1279 938 y FE(+)e(\()p Fx(D)p FE([)p Fx(0)p FE(])f(+)h Fx(B)p FE([)p Fx(0)p FE(])g FC(\000)g FE(1\))g FC(\001)g Fy(extent)p FE(\))p Fy(;)i(:::;)129 1015 y FE(\()p Fy(ty)r(pe)231 1022 y Fj(0)251 1015 y Fy(;)g(disp)356 1022 y Fj(0)385 1015 y FE(+)i Fx(D)p FE([)p Fx(count)h FC(\000)f Fx(1)p FE(])g FC(\001)g Fy(extent)p FE(\))p Fy(;)e(:::;)g FE(\()p Fy(ty)r(pe)1034 1022 y Fi(n)p Fk(\000)p Fj(1)1101 1015 y Fy(;)g(disp)1206 1022 y Fi(n)p Fk(\000)p Fj(1)1283 1015 y FE(+)i Fx(D)p FE([)p Fx(count)h FC(\000)g Fx(1)p FE(])e FC(\001)h Fy(extent)p FE(\))p Fy(;)e(:::;)374 1092 y FE(\()p Fy(ty)r(pe)476 1099 y Fj(0)496 1092 y Fy(;)g(disp)601 1099 y Fj(0)629 1092 y FE(+)j(\()p Fx(D)p FE([)p Fx(count)f FC(\000)h Fx(1)p FE(])e(+)h Fx(B)p FE([)p Fx(count)i FC(\000)e Fx(1)p FE(])g FC(\000)g FE(1\))g FC(\001)f Fy(extent)p FE(\))p Fy(;)f(:::;)343 1169 y FE(\()p Fy(ty)r(pe)445 1176 y Fi(n)p Fk(\000)p Fj(1)513 1169 y Fy(;)g(disp)618 1176 y Fi(n)p Fk(\000)p Fj(1)696 1169 y FE(+)i(\()p Fx(D)p FE([)p Fx(count)h FC(\000)f Fx(1)p FE(])g(+)g Fx(B)p FE([)p Fx(count)h FC(\000)g Fx(1)p FE(])e FC(\000)i FE(1\))e FC(\001)h Fy(extent)p FE(\))p FC(g)p Fy(:)166 1246 y FE(A)j(call)g(to)f Fx(MPI)p 431 1246 V 16 w(TYPE)p 564 1246 V 17 w(VECTOR\(count,)j(oldt)o(yp)q(e,)f(stride,)f(blo)q (cklength,)i(newt)o(yp)q(e\))f FE(is)f(equiv)m(alen)o(t)75 1302 y(to)i(a)f(call)j(to)d Fx(MPI)p 392 1302 V 16 w(TYPE)p 525 1302 V 17 w(INDEX\(count,)i(oldt)o(yp)q(e,)g(D,)e(B,)h(newt)o(yp)q (e\))i FE(where)616 1391 y Fx(D)p FE([)p Fx(j)p FE(])12 b(=)h Fy(j)g FC(\001)c Fx(stride)16 b Fy(;)8 b(j)14 b FE(=)f(0)p Fy(;)8 b(:::;)g Fp(count)f FC(\000)j FE(1)p Fy(;)75 1480 y FE(and)588 1537 y Fx(B)p FE([)p Fx(j)p FE(])i(=)h Fx(blo)q(cklength)k Fy(;)8 b(j)14 b FE(=)f(0)p Fy(;)8 b(:::;)g Fp(count)f FC(\000)j FE(1)p Fy(:)75 1655 y Fx(Hindexed)47 b FE(The)11 b(function)h Fx(MPI)p 643 1655 V 15 w(TYPE)p 775 1655 V 17 w(HINDEXED)f FE(is)g(iden)o(tical)i (to)d Fx(MPI)p 1383 1655 V 15 w(TYPE)p 1515 1655 V 17 w(INDEXED)p FE(,)g(except)75 1711 y(that)19 b(blo)q(c)o(k)i (displacemen)o(ts)h(in)41 b Fx(a)o(rra)o(y)p 774 1711 V 14 w(of)p 825 1711 V 16 w(displacements)22 b FE(are)e(sp)q(eci\014ed) i(in)f(b)o(ytes,)g(rather)e(than)h(in)75 1768 y(m)o(ultiples)d(of)e (the)g Fx(oldt)o(yp)q(e)h FE(exten)o(t.)75 1871 y Fx(MPI)p 160 1871 V 16 w(TYPE)p 293 1871 V 17 w(HINDEXED\()21 b(count,)k(oldt)o(yp)q(e,)f(a)o(rra)o(y)p 988 1871 V 15 w(of)p 1040 1871 V 16 w(displacements,)h(a)o(rra)o(y)p 1449 1871 V 15 w(of)p 1501 1871 V 16 w(blo)q(cklengths,)g(new-)75 1928 y(t)o(yp)q(e\))117 2005 y Fr(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f(n)o(um)o(b)q (er)h(of)f(en)o(tries)i(in)905 2062 y Fm(a)o(rra)o(y)p 992 2062 13 2 v 15 w(of)p 1041 2062 V 15 w(displacements)19 b Fr(and)f Fm(a)o(rra)o(y)p 1483 2062 V 15 w(of)p 1532 2062 V 14 w(blo)q(cklengths)i Fr(\(in)o(te-)905 2118 y(ger\))117 2191 y(IN)155 b Fx(oldt)o(yp)q(e)450 b Fr(old)13 b(datat)o(yp)q(e)i(\(handle\))117 2264 y(IN)155 b Fx(a)o(rra)o(y)p 416 2264 14 2 v 15 w(of)p 468 2264 V 16 w(displacements)164 b Fr(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o(h)h(blo)q(c)o(k)g (\(arra)o(y)g(of)f(in)o(teger\))117 2337 y(IN)155 b Fx(a)o(rra)o(y)p 416 2337 V 15 w(of)p 468 2337 V 16 w(blo)q(cklengths)191 b Fr(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)h(in)f(eac)o(h)h(blo)q(c)o (k)g(\(arra)o(y)f(of)g(nonnega-)905 2393 y(tiv)o(e)f(in)o(tegers\))117 2466 y(OUT)108 b Fx(newt)o(yp)q(e)433 b Fr(new)15 b(datat)o(yp)q(e)f (\(handle\))75 2591 y Fp(int)23 b(MPI)p 245 2591 15 2 v 17 w(Type)p 358 2591 V 17 w(hindexed\(int)f(count,)h(MPI)p 924 2591 V 17 w(Datatype)g(oldtype,)393 2647 y(int)h(*array)p 636 2647 V 16 w(of)p 700 2647 V 17 w(displacements,)e(int)h(*array)p 1314 2647 V 17 w(of)p 1379 2647 V 17 w(blocklengths,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)g(*newtype\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 52 57 52 56 bop 75 -100 a FE(52)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fp(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(HINDEXED\(COUNT,)22 b(OLDTYPE,)h(ARRAY)p 996 45 V 16 w(OF)p 1060 45 V 17 w(DISPLACEMENTS,)393 102 y(ARRAY)p 516 102 V 17 w(OF)p 581 102 V 17 w(BLOCKLENGTHS,)f(NEWTYPE,)g(IERROR\))170 158 y(INTEGER)h(COUNT,)g(OLDTYPE,)g(ARRAY)p 866 158 V 17 w(OF)p 931 158 V 17 w(DISPLACEMENTS\(*\),)170 214 y(ARRAY)p 293 214 V 17 w(OF)p 358 214 V 17 w(BLOCKLENGTHS\(*\),)f (NEWTYPE,)g(IERROR)166 302 y FE(Assume)15 b(that)g Fx(oldt)o(yp)q(e)h FE(has)f(t)o(yp)q(e)g(map)595 408 y FC(f)p FE(\()p Fy(ty)r(pe)720 415 y Fj(0)740 408 y Fy(;)8 b(disp)845 415 y Fj(0)863 408 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1064 415 y Fi(n)p Fk(\000)p Fj(1)1130 408 y Fy(;)g(disp)1235 415 y Fi(n)p Fk(\000)p Fj(1)1302 408 y FE(\))p FC(g)p Fy(;)75 513 y FE(with)34 b(exten)o(t)g Fs(extent)p FE(.)76 b(Let)35 b Fx(B)f FE(b)q(e)h(the)f Fx(a)o(rra)o(y)p 1004 513 14 2 v 15 w(of)p 1056 513 V 16 w(blo)q(cklength)i FE(parameter)d(and)h Fx(D)g FE(b)q(e)h(the)75 570 y Fx(a)o(rra)o(y)p 173 570 V 15 w(of)p 225 570 V 16 w(displacements)22 b FE(parameter.)36 b(The)21 b(newly)h(created)e(datat)o(yp)q(e)h(has)f(a)h(t)o(yp)q(e)g (map)f(with)h Fy(n)14 b FC(\001)75 594 y Fh(P)119 607 y Fg(count)n Fk(\000)p Fj(1)119 639 y Fi(i)p Fj(=0)272 626 y Fx(B)p FE([)p Fx(i)p FE(])g(en)o(tries:)441 732 y FC(f)p FE(\()p Fy(ty)r(pe)566 739 y Fj(0)586 732 y Fy(;)8 b(disp)691 739 y Fj(0)719 732 y FE(+)j Fx(D)p FE([)p Fx(0)p FE(])n(\))p Fy(;)d(:::;)g FE(\()p Fy(ty)r(pe)1046 739 y Fi(n)p Fk(\000)p Fj(1)1112 732 y Fy(;)g(disp)1217 739 y Fi(n)p Fk(\000)p Fj(1)1294 732 y FE(+)j Fx(D)p FE([)p Fx(0)p FE(])o(\))p Fy(;)d(:::;)75 841 y FE(\()p Fy(ty)r(pe)177 848 y Fj(0)197 841 y Fy(;)g(disp)302 848 y Fj(0)326 841 y FE(+)e Fx(D)p FE([)p Fx(0)p FE(])g(+)g(\()p Fx(B)p FE([)p Fx(0)p FE(])g FC(\000)g FE(1\))g FC(\001)g Fy(extent)p FE(\))p Fy(;)i(:::;)g FE(\()p Fy(ty)r(pe)1034 848 y Fi(n)p Fk(\000)p Fj(1)1101 841 y Fy(;)g(disp)1206 848 y Fi(n)p Fk(\000)p Fj(1)1279 841 y FE(+)e Fx(D)p FE([)p Fx(0)p FE(])g(+)g(\()p Fx(B)p FE([)p Fx(0)p FE(])f FC(\000)h FE(1\))g FC(\001)g Fy(extent)p FE(\))p Fy(;)i(:::;)291 931 y FE(\()p Fy(ty)r(pe)393 938 y Fj(0)412 931 y Fy(;)g(disp)517 938 y Fj(0)546 931 y FE(+)i Fx(D)p FE([)p Fx(count)h FC(\000)g Fx(1)p FE(])o(\))p Fy(;)d(:::;)g FE(\()p Fy(ty)r(pe)1035 938 y Fi(n)p Fk(\000)p Fj(1)1101 931 y Fy(;)g(disp)1206 938 y Fi(n)p Fk(\000)p Fj(1)1283 931 y FE(+)i Fx(D)p FE([)p Fx(count)h FC(\000)g Fx(1)p FE(])o(\))p Fy(;)d(:::;)374 1022 y FE(\()p Fy(ty)r(pe)476 1029 y Fj(0)496 1022 y Fy(;)g(disp)601 1029 y Fj(0)629 1022 y FE(+)j Fx(D)p FE([)p Fx(count)f FC(\000)h Fx(1)p FE(])e(+)i(\()p Fx(B)p FE([)p Fx(count)g FC(\000)f Fx(1)p FE(])g FC(\000)g FE(1\))g FC(\001)f Fy(extent)p FE(\))p Fy(;)f(:::;)343 1113 y FE(\()p Fy(ty)r(pe)445 1120 y Fi(n)p Fk(\000)p Fj(1)513 1113 y Fy(;)g(disp)618 1120 y Fi(n)p Fk(\000)p Fj(1)696 1113 y FE(+)i Fx(D)p FE([)p Fx(count)h FC(\000)f Fx(1)p FE(])g(+)g(\()p Fx(B)p FE([)p Fx(count)h FC(\000)g Fx(1)p FE(])e FC(\000)i FE(1\))e FC(\001)h Fy(extent)p FE(\))p FC(g)p Fy(:)75 1240 y Fx(Struct)49 b Fp(MPI)p 315 1240 15 2 v 17 w(TYPE)p 428 1240 V 16 w(STRUCT)15 b FE(is)h(the)f(most)g (general)h(constructor:)j(It)c(further)h(generalizes)h(the)e(previ-)75 1296 y(ous)g(one)g(in)h(that)f(it)g(allo)o(ws)h(eac)o(h)f(blo)q(c)o(k)h (to)e(consist)i(of)f(replications)i(of)d(a)h(di\013eren)o(t)h(datat)o (yp)q(e.)75 1401 y Fx(MPI)p 160 1401 14 2 v 16 w(TYPE)p 293 1401 V 17 w(STRUCT\(count,)c(a)o(rra)o(y)p 729 1401 V 15 w(of)p 781 1401 V 16 w(t)o(yp)q(es,)g(a)o(rra)o(y)p 1014 1401 V 14 w(of)p 1065 1401 V 16 w(displacements,)h(a)o(rra)o(y)p 1462 1401 V 15 w(of)p 1514 1401 V 16 w(blo)q(cklengths,)g(new-)75 1458 y(t)o(yp)q(e\))117 1536 y Fr(IN)155 b Fx(count)482 b Fr(n)o(um)o(b)q(er)18 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))g({)g (also)f(n)o(um)o(b)q(er)h(of)f(en)o(tries)905 1592 y(in)c(arra)o(ys)h Fm(a)o(rra)o(y)p 1167 1592 13 2 v 15 w(of)p 1216 1592 V 14 w(t)o(yp)q(es)p Fr(,)h Fm(a)o(rra)o(y)p 1432 1592 V 15 w(of)p 1481 1592 V 14 w(displacements)g Fr(and)e Fm(a)o(r-)905 1649 y(ra)o(y)p 959 1649 V 15 w(of)p 1008 1649 V 15 w(blo)q(cklengths)117 1726 y Fr(IN)155 b Fx(a)o(rra)o(y)p 416 1726 14 2 v 15 w(of)p 468 1726 V 16 w(t)o(yp)q(es)327 b Fr(t)o(yp)q(e)20 b(of)f(elemen)o(ts)g(in)g(eac)o(h)h(blo)q(c)o(k)f (\(arra)o(y)g(of)g(handles)g(to)905 1783 y(datat)o(yp)q(e)14 b(ob)r(jects\))117 1860 y(IN)155 b Fx(a)o(rra)o(y)p 416 1860 V 15 w(of)p 468 1860 V 16 w(displacements)164 b Fr(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o(h)h(blo)q(c)o(k)g(\(arra)o (y)g(of)f(in)o(teger\))117 1938 y(IN)155 b Fx(a)o(rra)o(y)p 416 1938 V 15 w(of)p 468 1938 V 16 w(blo)q(cklength)208 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o (k)g(\(arra)o(y)g(of)f(in)o(teger\))117 2015 y(OUT)108 b Fx(newt)o(yp)q(e)433 b Fr(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2141 y Fp(int)23 b(MPI)p 245 2141 15 2 v 17 w(Type)p 358 2141 V 17 w(struct\(int)f(count,)h(MPI)p 876 2141 V 17 w(Datatype)g(*array)p 1252 2141 V 16 w(of)p 1316 2141 V 17 w(types,)393 2197 y(int)h(*array)p 636 2197 V 16 w(of)p 700 2197 V 17 w(displacements,)e(int)h(*array)p 1314 2197 V 17 w(of)p 1379 2197 V 17 w(blocklengths,)393 2253 y(MPI)p 468 2253 V 17 w(Datatype)g(*newtype\))75 2341 y(MPI)p 150 2341 V 17 w(TYPE)p 263 2341 V 16 w(STRUCT\(COUNT,)f (ARRAY)p 733 2341 V 17 w(OF)p 798 2341 V 17 w(TYPES,)h(ARRAY)p 1102 2341 V 16 w(OF)p 1166 2341 V 17 w(DISPLACEMENTS,)393 2398 y(ARRAY)p 516 2398 V 17 w(OF)p 581 2398 V 17 w(BLOCKLENGTHS,)f (NEWTYPE,)g(IERROR\))170 2454 y(INTEGER)h(COUNT,)g(ARRAY)p 651 2454 V 17 w(OF)p 716 2454 V 17 w(TYPES,)g(ARRAY)p 1020 2454 V 16 w(OF)p 1084 2454 V 17 w(DISPLACEMENTS\(*\),)170 2510 y(ARRAY)p 293 2510 V 17 w(OF)p 358 2510 V 17 w(BLOCKLENGTHS\(*\),) f(NEWTYPE,)g(IERROR)166 2598 y FE(Example:)e(Let)c Fx(t)o(yp)q(e)g FE(ha)o(v)o(e)f(t)o(yp)q(e)g(map)738 2704 y FC(f)p FE(\()p Fp(double)o Fy(;)8 b FE(0\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(8\))p FC(g)p Fy(;)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 53 58 53 57 bop 75 -100 a Ft(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FE(53)75 45 y(with)24 b(exten)o(t)g(16.)45 b(Let)24 b Fp(D)g(=)g(\(0,)f(16,)g(26\))p FE(,)j Fp(B)d(=)h(\(2,)g(1,)f(3\))h FE(and)g Fp(T)f(=)h(\(MPI)p 1551 45 15 2 v 17 w(FLOAT,)f(type1,)75 102 y(MPI)p 150 102 V 17 w(CHAR\))p FE(.)13 b(Then)i(a)f(call)h(to)f Fx(MPI)p 687 102 14 2 v 16 w(TYPE)p 820 102 V 17 w(STRUCT\(3,)g(T,)g (D,)g(B,)g(newt)o(yp)q(e\))i FE(returns)e(datat)o(yp)q(e)g(with)75 158 y(t)o(yp)q(e)h(map,)166 262 y FC(f)p FE(\()p Fp(float)o Fy(;)8 b FE(0\))p Fy(;)g FE(\()p Fp(float)m Fy(;)g FE(4\))p Fy(;)g FE(\()p Fp(double)m Fy(;)g FE(16\))p Fy(;)g FE(\()p Fp(cha)o(r)m Fy(;)g FE(24\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(26\))p Fy(;)g FE(\()p Fp(char)m Fy(;)g FE(27\))p Fy(;)g FE(\()p Fp(cha)o(r)m Fy(;)g FE(28\))p FC(g)j FE(:)75 365 y(t)o(w)o(o)f(copies)j(of)e Fx(MPI)p 421 365 V 15 w(FLO)o(A)l(T)h FE(starting)f(at)g(0,)g(follo)o(w)o(ed)h(b)o(y)f(one)h (cop)o(y)f(of)g Fx(t)o(yp)q(e1)i FE(starting)e(at)f(16,)i(follo)o(w)o (ed)75 422 y(b)o(y)i(three)g(copies)h(of)e Fx(MPI)p 520 422 V 16 w(CHAR)p FE(,)i(starting)e(at)g(26.)19 b(\(W)l(e)14 b(assume)g(that)f(a)g(\015oat)h(o)q(ccupies)h(four)f(b)o(ytes.\))166 479 y(In)i(general,)f(let)h Fx(T)f FE(b)q(e)h(the)f Fx(a)o(rra)o(y)p 742 479 V 14 w(of)p 793 479 V 16 w(t)o(yp)q(es)i FE(parameter,)d(where) 31 b Fx(T[i])15 b FE(is)h(a)f(handle)h(to)547 582 y Fy(ty)r(pe)631 589 y Fi(i)659 582 y FE(=)d FC(f)p FE(\()p Fy(ty)r(pe)832 563 y Fi(i)832 593 y Fj(0)851 582 y Fy(;)8 b(disp)956 563 y Fi(i)956 593 y Fj(0)974 582 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1175 563 y Fi(i)1175 593 y(n)1196 598 y Ff(i)1209 582 y Fy(;)g(disp)1314 563 y Fi(i)1314 593 y(n)1335 598 y Ff(i)1349 582 y FE(\))p FC(g)p Fy(;)75 686 y FE(with)i(exten)o(t)g Fy(extent)434 693 y Fi(i)450 686 y FE(.)18 b(Let)10 b Fx(B)h FE(b)q(e)g(the)f Fx(a)o(rra)o(y)p 826 686 V 14 w(of)p 877 686 V 16 w(blo)q(cklength)i FE(parameter)e(and) g Fx(D)g FE(b)q(e)h(the)f Fx(a)o(rra)o(y)p 1685 686 V 15 w(of)p 1737 686 V 16 w(indices)75 742 y FE(parameter.)19 b(Then)14 b(the)g(newly)h(created)f(datat)o(yp)q(e)f(has)h(a)f(t)o(yp)q (e)h(map)g(with)1399 710 y Fh(P)1443 723 y Fg(count)n Fk(\000)p Fj(1)1443 755 y Fi(i)p Fj(=0)1596 742 y Fx(B)p FE([)p Fx(i)p FE(])6 b FC(\001)h Fy(n)1716 749 y Fi(i)1745 742 y FE(en)o(tries)469 846 y FC(f)p FE(\()p Fy(ty)r(pe)594 827 y Fj(0)594 857 y(0)613 846 y Fy(;)h(disp)718 827 y Fj(0)718 857 y(0)747 846 y FE(+)i Fx(D)p FE([)p Fx(0)p FE(])o(\))p Fy(;)e(:::;)g FE(\()p Fy(ty)r(pe)1074 827 y Fj(0)1074 857 y Fi(n)1095 862 y FB(0)1112 846 y Fy(;)g(disp)1217 827 y Fj(0)1217 857 y Fi(n)1238 862 y FB(0)1266 846 y FE(+)j Fx(D)p FE([)p Fx(0)p FE(])o(\))p Fy(;)d(:::;)75 951 y FE(\()p Fy(ty)r(pe)177 932 y Fj(0)177 962 y(0)197 951 y Fy(;)g(disp)302 932 y Fj(0)302 962 y(0)327 951 y FE(+)f Fx(D)p FE([)p Fx(0)p FE(])f(+)h(\()p Fx(B)p FE([)p Fx(0)p FE(])f FC(\000)h FE(1\))g FC(\001)g Fy(extent)839 958 y Fj(0)858 951 y FE(\))p Fy(;)h(:::;)g FE(\()p Fy(ty)r(pe)1059 932 y Fj(0)1059 962 y Fi(n)1080 967 y FB(0)1097 951 y Fy(;)g(disp)1202 932 y Fj(0)1202 962 y Fi(n)1223 967 y FB(0)1248 951 y FE(+)f Fx(D)p FE([)p Fx(0)p FE(])f(+)h(\()p Fx(B)p FE([)p Fx(0)p FE(])i FC(\000)i Fx(1)p FE(\))6 b FC(\001)h Fy(extent)1766 958 y Fj(0)1787 951 y FE(\))p Fy(;)h(:::;)75 1037 y FE(\()p Fy(ty)r(pe)177 1018 y Fg(count)n Fk(\000)p Fj(1)177 1050 y(0)322 1037 y Fy(;)g(disp)427 1018 y Fg(count)m Fk(\000)p Fj(1)427 1050 y(0)579 1037 y FE(+)h Fx(D)p FE([)p Fx(count)h FC(\000)h Fx(1)p FE(])o(\))p Fy(;)d(:::;)g FE(\()p Fy(ty)r(pe)1066 1018 y Fg(co)o(unt)l Fk(\000)p Fj(1)1066 1048 y Fi(n)1087 1058 y Fg(cou)o(nt)m Fe(\000)p FB(1)1226 1037 y Fy(;)g(disp)1331 1018 y Fg(count)m Fk(\000)p Fj(1)1331 1048 y Fi(n)1352 1058 y Fg(count)m Fe(\000)p FB(1)1501 1037 y FE(+)g Fx(D)p FE([)p Fx(count)j FC(\000)f Fx(1)p FE(]\))p Fy(;)e(:::;)176 1136 y FE(\()p Fy(ty)r(pe)278 1117 y Fg(count)n Fk(\000)p Fj(1)278 1148 y(0)423 1136 y Fy(;)g(disp)528 1117 y Fg(count)m Fk(\000)p Fj(1)528 1148 y(0)682 1136 y FE(+)i Fx(D)p FE([)p Fx(count)h FC(\000)g Fx(1)p FE(])e(+)i(\()p Fx(B)p FE([)p Fx(count)g FC(\000)f Fx(1)p FE(])g FC(\000)g FE(1\))g FC(\001)f Fy(extent)1539 1143 y Fg(count)p Fk(\000)p Fj(1)1686 1136 y FE(\))p Fy(;)f(:::;)182 1222 y FE(\()p Fy(ty)r(pe)284 1204 y Fg(count)n Fk(\000)p Fj(1)284 1234 y Fi(n)305 1243 y Fg(count)o Fe(\000)p FB(1)447 1222 y Fy(;)g(disp)552 1204 y Fg(count)m Fk(\000)p Fj(1)552 1234 y Fi(n)573 1243 y Fg(count)m Fe(\000)p FB(1)724 1222 y FE(+)i Fx(D)p FE([)p Fx(count)h FC(\000)f Fx(1)p FE(])g(+)g(\()p Fx(B)p FE([)p Fx(count)h FC(\000)g Fx(1)p FE(])e FC(\000)i Fx(1)p FE(\))e FC(\001)h Fy(extent)1581 1229 y Fg(count)o Fk(\000)p Fj(1)1728 1222 y FE(\))p FC(g)166 1314 y FE(A)15 b(call)h(to)f Fx(MPI)p 439 1314 V 16 w(TYPE)p 572 1314 V 16 w(HINDEXED\()g(count,)h (oldt)o(yp)q(e,)g(D,)f(B,)g(newt)o(yp)q(e\))h FE(is)g(equiv)m(alen)o(t) h(to)d(a)h(call)75 1370 y(to)d Fx(MPI)p 213 1370 V 16 w(TYPE)p 346 1370 V 17 w(STRUCT\()h(count,)i(T,)d(D,)h(B,)g(newt)o(yp)q (e\))p FE(,)h(where)f(eac)o(h)g(en)o(try)g(of)g Fx(T)g FE(is)g(equal)h(to)e Fx(oldt)o(yp)q(e)p FE(.)75 1495 y Fq(3.13.2)49 b(Address)16 b(and)g(extent)f(functions)75 1581 y FE(The)i(displacemen)o(ts)h(in)f(a)f(general)h(datat)o(yp)q(e)f (are)g(relativ)o(e)h(to)f(some)g(buf)h(address.)24 b Fz(Absolute)19 b(ad-)75 1638 y(dresses)10 b FE(can)i(b)q(e)h (substituted)f(for)g(these)g(displacemen)o(ts:)19 b(w)o(e)12 b(treat)f(them)h(as)f(displacemen)o(ts)i(relativ)o(e)75 1694 y(to)j(\\address)g(zero",)h(the)f(start)g(of)g(the)h(address)g (space.)24 b(This)17 b(initial)i(address)e(zero)f(is)i(indicated)g(b)o (y)75 1751 y(the)g(constan)o(t)f Fm(MPI)p 419 1751 13 2 v 15 w(BOTTOM)p FE(.)g(Th)o(us,)i(a)e(datat)o(yp)q(e)h(can)g(sp)q (ecify)i(the)e(absolute)g(address)g(of)g(the)g(en-)75 1807 y(tries)j(in)h(the)g(comm)o(unication)g(bu\013er,)g(in)g(whic)o(h) g(case)f(the)h Fx(buf)g FE(parameter)e(is)i(passed)f(the)h(v)m(alue)75 1864 y Fm(MPI)p 152 1864 V 14 w(BOTTOM)p FE(.)166 1921 y(The)31 b(address)g(of)f(a)h(lo)q(cation)g(in)h(memory)e(can)h(b)q(e)h (found)f(b)o(y)g(in)o(v)o(oking)g(the)g(function)75 1977 y Fx(MPI)p 160 1977 14 2 v 16 w(ADDRESS)p FE(.)75 2081 y Fx(MPI)p 160 2081 V 16 w(ADDRESS\(lo)q(cation,)16 b(address\))117 2159 y Fr(IN)155 b Fx(lo)q(cation)437 b Fr(lo)q(cation)13 b(in)h(caller)f(memory)f(\(c)o(hoice\))117 2235 y(OUT)108 b Fx(address)449 b Fr(address)15 b(of)f(lo)q(cation)f(\(in)o(teger\))75 2360 y Fp(int)23 b(MPI)p 245 2360 15 2 v 17 w(Address\(void*)f (location,)h(MPI)p 907 2360 V 17 w(ADDR)g(address\))75 2447 y(MPI)p 150 2447 V 17 w(ADDRESS\(LOCATION,)e(ADDRESS,)i(IERROR\)) 170 2503 y()g(LOCATION\(*\))170 2560 y(INTEGER)g(ADDRESS,)g (IERROR)166 2647 y FE(Returns)16 b(the)f(\(b)o(yte\))f(address)i(of)e Fx(lo)q(cation)p FE(.)166 2704 y(Example:)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 54 59 54 58 bop 75 -100 a FE(54)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)147 45 y Fp(REAL)23 b(A\(100,100\))147 102 y(INTEGER)f(I1,)i(I2,)f(DIFF)147 158 y(CALL)g(MPI_ADDRESS\(A\(1,1\),)e(I1\))147 214 y(CALL)i (MPI_ADDRESS\(A\(10,10\),)e(I2\))147 271 y(DIFF)i(=)h(I2)f(-)h(I1)75 327 y(!)g(The)f(value)g(of)h(DIFF)f(is)h(1009*sizeofreal;)e(the)h (values)g(of)h(I1)f(and)h(I2)f(are)75 384 y(!)h(implementation)e (dependent.)166 530 y FE(Some)15 b(implemen)o(tations)i(ma)o(y)d(not)h (supp)q(ort)g(calls)h(to)f Fx(MPI)p 1219 530 14 2 v 16 w(ADDRESS)h FE(in)g(F)l(ortran,)d(where)j(the)75 586 y Fx(lo)q(cation)g FE(argumen)o(t)e(is)i(of)e(t)o(yp)q(e)i Fx(CHARA)o(CTER)p FE(.)166 651 y(The)f(follo)o(wing)h(auxiliary)h (functions)f(pro)o(vide)f(useful)i(information)e(on)g(deriv)o(ed)h (datat)o(yp)q(es.)75 762 y Fx(MPI)p 160 762 V 16 w(TYPE)p 293 762 V 17 w(EXTENT\(datat)o(yp)q(e,)g(extent\))117 847 y Fr(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(\(handle\))117 939 y(OUT)108 b Fx(extent)471 b Fr(datat)o(yp)q(e)14 b(exten)o(t)h(\(in)o(teger\))75 1071 y Fp(int)23 b(MPI)p 245 1071 15 2 v 17 w(Type)p 358 1071 V 17 w(extent\(MPI)p 615 1071 V 16 w(Datatype)f(datatype,)h(int)h(*extent\))75 1165 y(MPI)p 150 1165 V 17 w(TYPE)p 263 1165 V 16 w(EXTENT\(DATATYPE,)e (EXTENT,)h(IERROR\))170 1222 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR) 166 1316 y FE(Returns)16 b(the)f(exten)o(t)g(of)g(a)f(datat)o(yp)q(e)h ({)g(where)g(exten)o(t)g(is)h(as)f(de\014ned)h(in)g(Eq.)f(3.1.)75 1428 y Fx(MPI)p 160 1428 14 2 v 16 w(TYPE)p 293 1428 V 17 w(SIZE\(datat)o(yp)q(e,)g(size\))117 1513 y Fr(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(\(handle\))117 1604 y(OUT)108 b Fx(size)519 b Fr(datat)o(yp)q(e)14 b(size)h(\(in)o (teger\))75 1736 y Fp(int)23 b(MPI)p 245 1736 15 2 v 17 w(Type)p 358 1736 V 17 w(size\(MPI)p 567 1736 V 16 w(Datatype)g(datatype,)f(int)i(*size\))75 1831 y(MPI)p 150 1831 V 17 w(TYPE)p 263 1831 V 16 w(SIZE\(DATATYPE,)e(SIZE,)i (IERROR\))170 1887 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)166 1982 y FE(Returns)15 b(the)h(n)o(um)o(b)q(er)f(of)g(b)o(ytes)f(o)q (ccupied)j(b)o(y)e(en)o(tries)h(in)g(the)f(datat)o(yp)q(e)f({)h(i.e.)20 b(the)15 b(sum)g(of)g(the)75 2038 y(sizes)h(of)f(the)g(datat)o(yp)q(e)g (elemen)o(ts.)75 2150 y Fx(MPI)p 160 2150 14 2 v 16 w(TYPE)p 293 2150 V 17 w(COUNT\(datat)o(yp)q(e,)i(count\))117 2235 y Fr(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(\(handle\))117 2326 y(OUT)108 b Fx(count)482 b Fr(datat)o(yp)q(e)14 b(coun)o(t)g(\(in)o(teger\))75 2458 y Fp(int)23 b(MPI)p 245 2458 15 2 v 17 w(Type)p 358 2458 V 17 w(count\(MPI)p 591 2458 V 16 w(Datatype)g(datatype,)f(int)i(*count\))75 2553 y(MPI)p 150 2553 V 17 w(TYPE)p 263 2553 V 16 w(COUNT\(DATATYPE,)e (COUNT,)h(IERROR\))170 2609 y(INTEGER)g(DATATYPE,)g(COUNT,)g(IERROR)166 2704 y FE(Returns)16 b(the)f(n)o(um)o(b)q(er)g(of)g(en)o(tries)h(in)g (the)f(datat)o(yp)q(e.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 55 60 55 59 bop 75 -100 a Ft(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(55)75 45 y Fq(3.13.3)49 b(Lo)o(w)o(er-b)q(ound)17 b(and)g(upp)q(er-b)q(ound)g(ma)o(rk)o(ers)75 132 y FE(It)e(is)g(often)g (con)o(v)o(enien)o(t)g(to)g(de\014ne)h(explicitly)h(the)e(lo)o(w)o(er)g (b)q(ound)h(and)f(upp)q(er)h(b)q(ound)f(of)g(a)g(t)o(yp)q(e)g(map,)75 189 y(and)d(o)o(v)o(erwrite)e(the)i(de\014nition)h(giv)o(en)f(b)o(y)f (Equation)g(3.1.)18 b(This)12 b(allo)o(ws)f(to)g(de\014ne)i(a)e(datat)o (yp)q(e)f(that)h(has)75 245 y(\\holes")k(at)f(its)h(b)q(eginning)i(or)d (its)h(end;)h(or)e(a)h(datat)o(yp)q(e)f(with)h(en)o(tries)g(that)f (extend)i(ab)q(o)o(v)o(e)e(the)h(upp)q(er)75 301 y(b)q(ound)h(or)f(b)q (elo)o(w)h(the)f(lo)o(w)o(er)g(b)q(ound.)21 b(Examples)16 b(of)f(suc)o(h)g(a)g(usage)g(are)g(pro)o(vided)h(in)g(section)g (3.13.6.)75 358 y(T)l(o)f(ac)o(hiev)o(e)h(this,)g(w)o(e)f(add)g(t)o(w)o (o)f(additional)j(\\pseudo-datat)o(yp)q(es")e Fx(MPI)p 1349 358 14 2 v 16 w(LB)p FE(,)g(and)g Fx(MPI)p 1618 358 V 16 w(UB)h FE(that)f(can)75 414 y(b)q(e)j(used,)g(resp)q(ectiv)o (ely)l(,)h(to)e(mark)f(the)i(lo)o(w)o(er)f(b)q(ound)h(or)f(the)g(upp)q (er)h(b)q(ound)g(of)f(a)g(datat)o(yp)q(e.)26 b(These)75 471 y(pseudo-datat)o(yp)q(es)16 b(o)q(ccup)o(y)g(no)f(space)h(\()p Fy(extent)p FE(\()p Fx(MPI)p 1022 471 V 16 w(LB)p FE(\))d(=)h Fy(extent)p FE(\()p Fx(MPI)p 1400 471 V 16 w(UB)q FE(\))f(=)g(0\).)20 b(They)c(do)f(not)75 527 y(a\013ect)d(the)h(size)h(or)e(coun)o(t)h(of)f (a)h(datat)o(yp)q(e,)f(and)h(do)g(not)f(a\013ect)g(the)h(the)g(con)o (ten)o(t)g(of)f(a)h(message)f(created)75 584 y(with)k(this)f(datat)o (yp)q(e.)20 b(Ho)o(w)o(ev)o(er,)13 b(they)j(do)f(a\013ect)f(the)i (de\014nition)h(of)e(the)g(exten)o(t)g(of)g(a)f(datat)o(yp)q(e)h(and,) 75 640 y(therefore,)g(a\013ect)f(the)h(outcome)g(of)g(a)g(replication)i (of)d(this)i(datat)o(yp)q(e)f(b)o(y)g(a)g(datat)o(yp)q(e)f (constructor.)166 697 y(Example:)30 b(Let)21 b Fp(D)j(=)f(\(-3,)h(0,)f (6\))p FE(;)f Fp(T)i(=)g(\(MPI)p 1033 697 15 2 v 16 w(LB,)g(MPI)p 1217 697 V 17 w(INT,)f(MPI)p 1425 697 V 17 w(UB\))p FE(,)c(and)h Fp(B)k(=)g(\(1,)f(1,)75 754 y(1\))p FE(.)c(Then)d(a)e(call)i(to)f Fx(MPI)p 535 754 14 2 v 15 w(TYPE)p 667 754 V 17 w(STRUCT\(3,)g(T,)f (D,)g(B,)h(t)o(yp)q(e1\))g FE(creates)g(a)g(new)g(datat)o(yp)q(e)f (that)g(has)75 810 y(an)e(exten)o(t)f(of)g(9)h(\(from)e(-3)i(to)f(5\),) g(and)h(con)o(tains)g(an)g(in)o(teger)g(at)f(displacemen)o(t)i(0.)18 b(This)12 b(is)h(the)e(datat)o(yp)q(e)75 867 y(de\014ned)j(b)o(y)e(the) g(sequence)i FC(f)p Fp(\(lb,)23 b(-3\),)g(\(int,)g(0\),)h(\(ub,)f(6\))p FC(g)g FE(.)c(If)13 b(this)g(t)o(yp)q(e)f(is)h(replicated)h(t)o(wice)75 923 y(b)o(y)h(a)g(call)h(to)f Fx(MPI)p 400 923 V 15 w(TYPE)p 532 923 V 17 w(CONTIGUOUS\(2,)h(t)o(yp)q(e1,)g(t)o(yp)q(e2\))f FE(then)h(the)f(newly)h(created)f(t)o(yp)q(e)g(can)g(b)q(e)75 980 y(describ)q(ed)e(b)o(y)d(the)h(sequence)h FC(f)p Fp(\(lb,)23 b(-3\),)g(\(int,)g(0\),)h(\(int,9\),)e(\(ub,)i(15\))p FC(g)f FE(.)18 b(\(En)o(tries)11 b(of)f(t)o(yp)q(e)75 1036 y Fp(lb)15 b FE(or)g Fp(ub)f FE(can)i(b)q(e)g(deleted)g(if)g(they) f(are)g(not)g(at)f(the)i(endp)q(oin)o(ts)g(of)f(the)g(datat)o(yp)q (e.\))166 1093 y(In)h(general,)f(if)471 1197 y Fy(T)6 b(y)r(pemap)12 b FE(=)h FC(f)p FE(\()p Fy(ty)r(pe)844 1204 y Fj(0)864 1197 y Fy(;)8 b(disp)969 1204 y Fj(0)987 1197 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1188 1204 y Fi(n)p Fk(\000)p Fj(1)1254 1197 y Fy(;)g(disp)1359 1204 y Fi(n)p Fk(\000)p Fj(1)1426 1197 y FE(\))p FC(g)p Fy(;)75 1301 y FE(then)16 b(the)f Fz(lo)o(w)o(er)h(b)q(ound)g FE(of)f Fy(T)6 b(y)r(pemap)15 b FE(is)g(de\014ned)i(to)e(b)q(e)213 1435 y Fy(l)q(b)p FE(\()p Fy(T)6 b(y)r(pemap)p FE(\))k(=)530 1363 y Fh(\()584 1407 y FE(min)660 1414 y Fi(j)686 1407 y Fy(disp)770 1414 y Fi(j)1143 1407 y FE(if)16 b(no)f(en)o(try)g(has)g (basic)h(t)o(yp)q(e)f Fp(lb)584 1463 y FE(min)q FC(f)p Fy(disp)767 1470 y Fi(j)812 1463 y FE(:)28 b Fy(ty)r(pe)937 1470 y Fi(j)968 1463 y FE(=)13 b Fp(lb)p FC(g)41 b FE(otherwise)166 1570 y(Similarly)l(,)17 b(the)e Fz(upp)q(er)j(b)q(ound)e FE(of)e Fy(T)6 b(y)r(pemap)15 b FE(is)h(de\014ned)h(to)d(b)q(e)190 1704 y Fy(ub)p FE(\()p Fy(T)6 b(y)r(pemap)p FE(\))11 b(=)519 1632 y Fh(\()573 1675 y FE(max)658 1682 y Fi(j)684 1675 y Fy(disp)768 1682 y Fi(j)795 1675 y FE(+)g Fy(siz)r(eof)5 b FE(\()p Fy(ty)r(pe)1073 1682 y Fi(j)1092 1675 y FE(\))56 b(if)16 b(no)f(en)o(try)g(has)g(basic)g(t)o(yp)q(e)h Fp(ub)573 1732 y FE(max)p FC(f)p Fy(disp)765 1739 y Fi(j)810 1732 y FE(:)27 b Fy(ty)r(pe)934 1739 y Fi(j)966 1732 y FE(=)13 b Fp(ub)o FC(g)67 b FE(otherwise)166 1838 y(Then)441 1897 y Fy(extent)p FE(\()p Fy(T)6 b(y)r(pemap)p FE(\))14 b(=)e Fy(ub)p FE(\()p Fy(T)6 b(y)r(pemap)p FE(\))j FC(\000)i Fy(l)q(b)p FE(\()p Fy(T)6 b(y)r(pemap)p FE(\))i(+)i Fy(\017)75 1982 y FE(If)17 b Fy(ty)r(pe)206 1989 y Fi(i)237 1982 y FE(requires)g(alignmen)o(t)g(to)f(a)h(b)o(yte)f(address)h(that)f(is)h (is)g(a)f(m)o(ultiple)j(of)d Fy(k)1474 1989 y Fi(i)1488 1982 y FE(,)g(then)h Fy(\017)g FE(is)g(the)g(least)75 2039 y(nonnegativ)o(e)f(incremen)o(t)g(needed)g(to)f(round)g Fy(extent)p FE(\()p Fy(T)6 b(y)r(pemap)p FE(\))16 b(to)e(the)h(next)h (m)o(ultiple)h(of)d(max)1794 2046 y Fi(i)1816 2039 y Fy(k)1840 2046 y Fi(i)1854 2039 y FE(.)166 2096 y(The)g(formal)f (de\014nitions)j(giv)o(en)e(for)f(the)h(v)m(arious)g(datat)o(yp)q(e)f (constructors)g(apply)i(no)o(w,)e(with)h(the)75 2152 y(amended)i(de\014nition)h(of)e Fz(exten)o(t)p FE(.)75 2277 y Fq(3.13.4)49 b(Commit)16 b(and)h(free)75 2364 y FE(A)g(datat)o(yp)q(e)f(ob)s(ject)h(has)g(to)f(b)q(e)h Fz(committed)h FE(b)q(efore)f(it)g(can)g(b)q(e)h(used)g(in)f(a)g(comm)o (unication.)26 b(The)75 2421 y(system)20 b(ma)o(y)g(\\compile")i(at)e (commit)g(time)h(an)g(in)o(ternal)g(represen)o(tation)g(for)f(the)h (datat)o(yp)q(e)f(that)75 2477 y(facilitates)c(comm)o(unication,)g (e.g.)k(c)o(hange)15 b(from)g(a)g(compacted)g(represen)o(tation)h(to)e (a)h(\015at)g(represen-)75 2534 y(tation)j(of)g(the)g(datat)o(yp)q(e,)g (and)g(select)h(the)f(most)g(con)o(v)o(enien)o(t)g(transfer)g(mec)o (hanism.)29 b(A)19 b(commited)75 2590 y(datat)o(yp)q(e)c(can)g(still)i (b)q(e)e(used)h(as)f(a)g(parameter)f(in)i(datat)o(yp)q(e)f (constructors.)166 2647 y(Note:)28 b(the)20 b(commit)f(op)q(eration)h (commits)f(the)h(datat)o(yp)q(e,)f(i.e.,)i(the)e(formal)g(description)i (of)e(a)75 2704 y(comm)o(unication)i(bu\013er,)h(not)e(the)h(con)o(ten) o(t)f(of)g(that)g(bu\013er.)36 b(Th)o(us,)22 b(after)e(a)g(datat)o(yp)q (e)g(has)h(b)q(een)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 56 61 56 60 bop 75 -100 a FE(56)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(committed,)i(it)h (can)f(b)q(e)h(rep)q(eatedly)g(reused)g(to)e(comm)o(unicate)h(the)h(c)o (hanging)f(con)o(ten)o(t)g(of)f(a)h(bu\013er)75 102 y(or,)c(indeed,)j (the)e(con)o(ten)o(t)g(of)g(di\013eren)o(t)g(bu\013ers,)g(with)g (di\013eren)o(t)h(starting)e(addresses.)75 206 y Fx(MPI)p 160 206 14 2 v 16 w(TYPE)p 293 206 V 17 w(COMMIT\(datat)o(yp)q(e\))117 284 y Fr(INOUT)62 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(that)g(is)g(committed)e(\(handle\))75 410 y Fp(int)23 b(MPI)p 245 410 15 2 v 17 w(Type)p 358 410 V 17 w(commit\(MPI)p 615 410 V 16 w(Datatype)f(datatype\))75 497 y(MPI)p 150 497 V 17 w(TYPE)p 263 497 V 16 w(COMMIT\(DATATYPE,)g(IERROR\))170 553 y(INTEGER)h(DATATYPE,)g(IERROR)166 641 y FE(There)15 b(is)h(no)f(need)h(to)f(commit)g(basic)h(datat)o(yp)q(es;)e(they)h(are) g(\\pre-committed".)75 745 y Fx(MPI)p 160 745 14 2 v 16 w(TYPE)p 293 745 V 17 w(FREE\(datat)o(yp)q(e\))117 824 y Fr(INOUT)62 b Fx(datat)o(yp)q(e)424 b Fr(handle)14 b(to)g(datat)o(yp)q(e)g(ob)r(ject)75 949 y Fp(int)23 b(MPI)p 245 949 15 2 v 17 w(Type)p 358 949 V 17 w(free\(MPI)p 567 949 V 16 w(Datatype)g(datatype\))75 1036 y(MPI)p 150 1036 V 17 w(TYPE)p 263 1036 V 16 w(FREE\(DATATYPE,)f(IERROR\))170 1093 y(INTEGER)h(DATATYPE,)g(IERROR)166 1180 y FE(Marks)16 b(the)h(datat)o(yp)q(e)f(ob)s(ject)g(asso)q(ciated)h(with)g Fx(datat)o(yp)q(e)i FE(for)d(deallo)q(cation.)26 b(The)17 b(ob)s(ject)f(will)75 1237 y(b)q(e)g(deallo)q(cated)h(after)d(an)o(y)i (p)q(ending)h(comm)o(unication)f(that)e(uses)i(this)g(ob)s(ject)f (completes,)g(at)g(whic)o(h)75 1293 y(p)q(oin)o(t)21 b Fx(datat)o(yp)q(e)g FE(b)q(ecomes)g(n)o(ull.)36 b(It)20 b(is)h(erroneous)f(to)f(use)i(a)f(datat)o(yp)q(e)f(as)h(argumen)o(t)f (to)h(an)g(MPI)75 1350 y(function)c(after)e(it)i(w)o(as)e(freed.)166 1407 y(Example:)75 1575 y Fp(INTEGER)23 b(type1,)g(type2)75 1632 y(CALL)g(MPI_TYPE_CONTIGUOUS\(5,)e(MPI_REAL,)i(type1,)g(ierr\))409 1688 y(!)h(new)f(type)h(object)f(created)75 1744 y(CALL)g (MPI_TYPE_COMMIT\(type1,)e(ierr\))409 1801 y(!)j(now)f(type1)g(can)h (be)f(used)h(for)f(communication)75 1857 y(type2)g(=)h(type1)409 1914 y(!)g(type2)f(can)g(be)h(used)f(for)h(communication)409 1970 y(!)g(\(it)f(is)h(a)g(handle)f(to)g(same)h(object)f(as)g(type1\)) 75 2027 y(CALL)g(MPI_TYPE_VECTOR\(3,)f(MPI_REAL,)g(4,)i(5,)g(type1,)f (ierr\))409 2083 y(!)h(new)f(uncommitted)g(type)g(object)g(created)75 2140 y(CALL)g(MPI_TYPE_COMMIT\(type1,)e(ierr\))409 2196 y(!)j(now)f(type1)g(can)h(be)f(used)h(anew)f(for)h(communication)166 2307 y FE(F)l(reeing)19 b(a)f(datat)o(yp)q(e)f(do)q(es)i(not)f (a\013ect)f(an)o(y)h(other)g(datat)o(yp)q(e)f(that)h(w)o(as)f(built)j (from)d(the)i(freed)75 2363 y(datat)o(yp)q(e.)34 b(The)20 b(system)g(should)h(b)q(eha)o(v)o(e)g(as)e(if)i(a)f(separate)f(cop)o(y) h(of)g(the)g(datat)o(yp)q(e)g(w)o(as)f(created)75 2420 y(whenev)o(er)d(this)f(datat)o(yp)q(e)g(w)o(as)f(incorp)q(orated)i(in)g (a)f(new)g(deriv)o(ed)i(datat)o(yp)q(e.)166 2553 y Fo(Implemen)o(tati)o (on)12 b(note:)166 2604 y Fr(The)e(implemen)o(tati)o(on)d(ma)o(y)g(k)o (eep)j(a)f(reference)j(coun)o(t)e(of)f(activ)o(e)g(comm)o(unications)d (that)k(use)g(the)g(datat)o(yp)q(e,)75 2654 y(in)15 b(order)i(to)f (decide)h(when)f(to)g(free)h(it.)24 b(Also,)15 b(one)h(ma)o(y)e (implemen)o(t)g(constructors)k(of)d(deriv)o(ed)h(datat)o(yp)q(es)h(so) 75 2704 y(that)e(they)h(k)o(eep)g(p)q(oin)o(ters)g(to)f(their)h(datat)o (yp)q(e)f(argumen)o(ts,)f(rather)j(then)e(cop)o(ying)g(them.)21 b(In)16 b(suc)o(h)g(case,)g(one)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 57 62 57 61 bop 75 -100 a Ft(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FE(57)75 45 y Fr(needs)14 b(to)e(k)o(eep)h(trac)o(k)f(of)g (activ)o(e)g(datat)o(yp)q(e)g(de\014nition)g(references)j(in)d(order)h (to)f(kno)o(w)g(when)h(a)f(datat)o(yp)q(e)g(ob)r(ject)75 95 y(can)i(b)q(e)h(freed.)75 312 y Fq(3.13.5)49 b(Use)16 b(of)h(general)f(datat)o(yp)q(es)g(in)h(communication)75 402 y FE(Handles)g(to)f(deriv)o(ed)h(datat)o(yp)q(es)f(can)h(b)q(e)f (passed)h(to)f(a)g(comm)o(unication)h(call)g(wherev)o(er)f(a)g(datat)o (yp)q(e)75 459 y(argumen)o(t)10 b(is)g(required.)20 b(A)10 b(call)i(of)e(the)g(form)g Fx(MPI)p 940 459 14 2 v 16 w(SEND\(buf,)h(count,)i(datat)o(yp)q(e)f(,)f(...\))p FE(,)e(where)i Fp(count)h Fy(>)75 515 y Fp(1)p FE(,)g(is)h(in)o (terpreted)g(as)f(if)h(the)g(call)g(w)o(as)f(passed)h(a)f(new)g(datat)o (yp)q(e)g(whic)o(h)i(is)f(the)f(concatenation)h(of)f Fx(count)75 572 y FE(copies)j(of)f Fx(datat)o(yp)q(e)p FE(.)21 b(The)15 b(same)f(holds)h(true)f(for)g(the)h(other)e(comm)o (unication)j(functions)f(that)e(ha)o(v)o(e)h(a)75 628 y(datat)o(yp)q(e)h(argumen)o(t.)k(Th)o(us,)166 687 y Fx(MPI)p 251 687 V 16 w(SEND\(buf,)c(count,)i(datat)o(yp)q(e,)f(dest,)g (tag,)f(comm\))e FE(is)j(equiv)m(alen)o(t)h(to)75 807 y Fp(MPI_TYPE_CONTIGUOUS\(count,)j(datatype,)j(newtype\))75 863 y(MPI_TYPE_COMMIT\(newtype\))75 919 y(MPI_SEND\(buf,)f(1,)i (newtype,)e(dest,)i(tag,)f(comm\))166 1037 y FE(The)14 b(same)g(applies)h(to)f(all)h(other)e(comm)o(unication)i(functions)g (that)e(ha)o(v)o(e)g(a)h Fx(count)h FE(and)f Fx(datat)o(yp)q(e)75 1093 y FE(argumen)o(t.)166 1152 y(Let)h Fx(datat)o(yp)q(e)i FE(ha)o(v)o(e)e(t)o(yp)q(e)g(map)595 1272 y FC(f)p FE(\()p Fy(ty)r(pe)720 1279 y Fj(0)740 1272 y Fy(;)8 b(disp)845 1279 y Fj(0)863 1272 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1064 1279 y Fi(n)p Fk(\000)p Fj(1)1130 1272 y Fy(;)g(disp)1235 1279 y Fi(n)p Fk(\000)p Fj(1)1302 1272 y FE(\))p FC(g)p Fy(;)166 1383 y FE(with)20 b(exten)o(t)f Fy(extent)i FE(\(empt)o(y)e(en)o(tries)g(deleted\).)34 b(Then)20 b(the)g(send)g(bu\013er)f(used)h(b)o(y)g(the)f(send)75 1440 y(op)q(eration)c(consists)h(of)f Fy(n)10 b FC(\002)g Fp(count)15 b FE(en)o(tries,)g(where)h(en)o(try)e Fy(i)c FC(\002)h Fy(n)f FE(+)g Fy(j)18 b FE(is)e(at)e(lo)q(cation)637 1560 y Fy(addr)730 1567 y Fi(i;j)781 1560 y FE(=)f Fp(buf)d FE(+)g Fy(extent)i FC(\002)e Fy(i)g FE(+)h Fy(disp)1296 1567 y Fi(j)166 1671 y FE(and)k(has)f(t)o(yp)q(e)h Fy(ty)r(pe)520 1678 y Fi(j)538 1671 y FE(,)g(for)f Fy(i)e FE(=)h(0)p Fy(;)8 b(:::;)g Fp(cou)o(nt)e FC(\000)j FE(1)15 b(and)f Fy(j)h FE(=)e(0)p Fy(;)8 b(:::;)g(n)e FC(\000)k FE(1.)19 b(These)c(en)o(tries)g(need)g(not)75 1727 y(b)q(e)h(con)o(tiguous,)f (nor)g(distinct;)h(their)f(order)g(can)h(b)q(e)g(arbitrary)l(.)166 1786 y(The)g(v)m(ariable)h(stored)e(at)g(address)g Fy(addr)879 1793 y Fi(i;j)933 1786 y FE(in)i(the)e(calling)j(program)c(should)j(b)q (e)f(of)f(a)g(t)o(yp)q(e)h(that)75 1842 y(matc)o(hes)d Fy(ty)r(pe)333 1849 y Fi(j)352 1842 y FE(,)h(where)g(t)o(yp)q(e)g(matc) o(hing)f(is)i(de\014ned)g(as)e(in)i(section)f(3.5.)19 b(The)14 b(message)f(sen)o(t)h(con)o(tains)75 1899 y Fy(n)c FC(\002)h Fp(count)j FE(en)o(tries,)i(where)f(en)o(try)g Fy(i)10 b FC(\002)g Fy(n)g FE(+)h Fy(j)17 b FE(has)e(t)o(yp)q(e)h Fy(ty)r(pe)1159 1906 y Fi(j)1177 1899 y FE(.)166 1957 y(Similarly)l(,)f(supp)q(ose)f(that)e(a)h(receiv)o(e)h(op)q(eration)f Fx(MPI)p 1100 1957 V 16 w(RECV\(buf,)h(count,)g(datat)o(yp)q(e,)h (source,)e(tag,)75 2014 y(comm,)g(status\))k FE(is)e(executed,)h(where) g Fx(datat)o(yp)q(e)g FE(has)g(t)o(yp)q(e)f(map)595 2136 y FC(f)p FE(\()p Fy(ty)r(pe)720 2143 y Fj(0)740 2136 y Fy(;)8 b(disp)845 2143 y Fj(0)863 2136 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1064 2143 y Fi(n)p Fk(\000)p Fj(1)1130 2136 y Fy(;)g(disp)1235 2143 y Fi(n)p Fk(\000)p Fj(1)1302 2136 y FE(\))p FC(g)p Fy(;)166 2247 y FE(with)23 b(exten)o(t)f Fy(extent)j FE(\(empt)o(y)d(en)o(tries)h(deleted\).)43 b(Then)24 b(the)f(receiv)o(e)g(bu\013er)g(used)g(for)g(this)75 2304 y(receiv)o(e)e(op)q(eration)f(consists)g(of)f(the)h Fy(n)13 b FC(\002)h Fp(count)19 b FE(en)o(tries,)i(where)f(en)o(try)f Fy(i)13 b FC(\002)h Fy(n)f FE(+)h Fy(j)22 b FE(is)e(at)f(lo)q(cation)75 2360 y Fp(buf)6 b FE(+)g Fy(extent)g FC(\002)h Fy(i)f FE(+)h Fy(disp)517 2367 y Fi(j)551 2360 y FE(and)13 b(has)h(t)o(yp)q(e) f Fy(ty)r(pe)901 2367 y Fi(j)920 2360 y FE(.)19 b(If)14 b(the)f(incoming)i(message)e(consists)g(of)g Fy(k)i FE(elemen)o(ts,)75 2417 y(then)i(w)o(e)f(m)o(ust)g(ha)o(v)o(e)g Fy(k)f FC(\024)g Fy(n)d FC(\002)f Fp(count)p FE(;)16 b(the)g Fy(i)11 b FC(\002)g Fy(n)g FE(+)h Fy(j)s FE(-th)j(elemen)o(t)i(of)f(the)h (message)f(should)h(ha)o(v)o(e)f(a)75 2473 y(t)o(yp)q(e)f(that)g(matc)o (hes)g Fy(ty)r(pe)536 2480 y Fi(j)554 2473 y FE(.)166 2532 y(Note)c(that)g(t)o(yp)q(e)h(matc)o(hing)f(is)h(de\014ned)h (according)f(to)f(the)h(t)o(yp)q(e)f(signature)h(of)f(the)h(corresp)q (onding)75 2588 y(datat)o(yp)q(es,)f(i.e.,)h(the)f(sequence)i(of)e (basic)h(t)o(yp)q(e)g(comp)q(onen)o(ts,)f(and)h(do)q(es)g(not)f(dep)q (end)i(on)e(the)g(datat)o(yp)q(e)75 2645 y(de\014nition.)166 2704 y(Example:)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 58 63 58 62 bop 75 -100 a FE(58)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fp(...)75 102 y(CALL)23 b(MPI_TYPE_CONTIGUOUS\()e(MPI_REAL,)i(2,)h(type2,)f (...\))75 158 y(CALL)g(MPI_TYPE_CONTIGUOUS\()e(MPI_REAL,)i(4,)h(type4,) f(...\))75 214 y(CALL)g(MPI_TYPE_CONTIGUOUS\()e(type2,)i(2,)h(type22,)f (...\))75 271 y(...)75 327 y(CALL)g(MPI_SEND\()g(a,)h(4,)f(MPI_REAL,)g (...\))75 384 y(CALL)g(MPI_SEND\()g(a,)h(2,)f(type2,)g(...\))75 440 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type22,)g(...\))75 497 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type4,)g(...\))75 553 y(...)75 610 y(CALL)g(MPI_RECV\()g(a,)h(4,)f(MPI_REAL,)g(...\))75 666 y(CALL)g(MPI_RECV\()g(a,)h(2,)f(type2,)g(...\))75 723 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type22,)g(...\))75 779 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type4,)g(...\))166 878 y FE(Eac)o(h)15 b(of)g(the)g(sends)h(matc)o(hes)e(eac)o(h)i(of)e (the)i(receiv)o(es.)166 935 y(A)c(datat)o(yp)q(e)g(used)h(in)g(a)f (receiv)o(e)h(op)q(eration)g(ma)o(y)e(not)h(ha)o(v)o(e)g(o)o(v)o (erlaping)h(en)o(tries)g(in)g(its)f(t)o(yp)q(e)h(map:)75 992 y(the)i(same)g(lo)q(cation)h(in)g(the)f(receiv)o(e)i(bu\013er)e (cannot)g(b)q(e)h(written)f(t)o(wice)g(during)h(a)f(receiv)o(e.)166 1049 y(T)o(yp)q(e)23 b(matc)o(hing)g(requiremen)o(ts)h(imply)g(that)f (\(nonempt)o(y\))f(en)o(tries)i(in)f(the)h(t)o(yp)q(e)f(map)g(of)f(a)75 1106 y(datat)o(yp)q(e)f(cannot)g(o)o(v)o(erlap,)h(unless)h(they)e (coincide)j(and)d(ha)o(v)o(e)g(the)g(same)g(basic)h(t)o(yp)q(e,)h(or)e (one)g(of)75 1162 y(them)d(ha)o(v)o(e)f(basic)h(t)o(yp)q(e)g Fx(b)o(yte)p FE(:)25 b(otherwise,)18 b(when)g(the)g(datat)o(yp)q(e)f (is)h(used)g(in)h(a)e(comm)o(unication,)i(at)75 1219 y(least)e(on)g(of)g(the)g(t)o(w)o(o)f(o)o(v)o(erlaping)h(en)o(tries)h (will)h(not)d(b)q(e)i(t)o(yp)q(e-consisten)o(t)g(with)f(the)g(comm)o (unication)75 1275 y(bu\013er.)166 1333 y(Supp)q(ose,)i(again,)f(that)f Fx(MPI)p 683 1333 14 2 v 15 w(RECV\(buf,)j(count,)f(datat)o(yp)q(e,)g (dest,)g(tag,)f(comm,)e(status\))j FE(is)f(exe-)75 1389 y(cuted,)d(where)h Fx(datat)o(yp)q(e)h FE(has)e(t)o(yp)q(e)g(map)595 1505 y FC(f)p FE(\()p Fy(ty)r(pe)720 1512 y Fj(0)740 1505 y Fy(;)8 b(disp)845 1512 y Fj(0)863 1505 y FE(\))p Fy(;)g(:::;)g FE(\()p Fy(ty)r(pe)1064 1512 y Fi(n)p Fk(\000)p Fj(1)1130 1505 y Fy(;)g(disp)1235 1512 y Fi(n)p Fk(\000)p Fj(1)1302 1505 y FE(\))p FC(g)p Fy(:)166 1612 y FE(The)k(receiv)o(ed)g (message)f(need)h(not)f(\014ll)i(all)g(the)e(receiv)o(e)h(bu\013er,)g (nor)f(do)q(es)h(it)g(need)g(to)f(\014ll)i(a)e(n)o(um)o(b)q(er)75 1668 y(of)i(lo)q(cations)h(whic)o(h)g(is)f(a)g(m)o(ultiple)i(of)e Fy(n)p FE(;)h(an)o(y)f(n)o(um)o(b)q(er)g Fy(k)i FE(of)d(basic)i(elemen) o(ts)g(can)f(b)q(e)h(receiv)o(ed,)h(where)75 1725 y(0)d FC(\024)h Fy(k)h FC(\024)f Fp(count)c FC(\002)i Fy(n)p FE(.)166 1782 y(The)j(n)o(um)o(b)q(er)g(of)f(elemen)o(ts)i(receiv)o(ed) g(can)f(b)q(e)g(retriev)o(ed)g(from)f Fx(status)j FE(using)e(the)g (query)g(function)75 1839 y Fx(MPI)p 160 1839 V 16 w(GET)p 264 1839 V 17 w(ELEMENTS)p FE(:)75 1944 y Fx(MPI)p 160 1944 V 16 w(GET)p 264 1944 V 17 w(ELEMENTS\()h(status,)h(datat)o(yp)q (e,)h(count\))117 2022 y Fr(IN)155 b Fx(status)476 b Fr(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 2099 y(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(used)h(b)o(y)f(receiv)o(e)h(op)q(eration)117 2177 y(OUT)108 b Fx(count)482 b Fr(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(basic)f (elemen)o(ts)g(\(in)o(teger\))75 2303 y Fp(int)23 b(MPI)p 245 2303 15 2 v 17 w(Get)p 334 2303 V 17 w(elements\(MPI)p 639 2303 V 16 w(Status)g(status,)f(MPI)p 1084 2303 V 17 w(Datatype)h(datatype,)g(int)g(*count\))75 2390 y(MPI)p 150 2390 V 17 w(GET)p 239 2390 V 17 w(ELEMENTS\(STATUS,)e(DATATYPE,)i (COUNT,)g(IERROR\))170 2447 y(INTEGER)g(STATUS\(MPI)p 604 2447 V 16 w(STATUS)p 764 2447 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 2534 y FE(The)13 b(previously)h(used)f(function)h Fx(MPI)p 834 2534 14 2 v 16 w(GET)p 938 2534 V 16 w(COUNT)g FE(has)f(a)f(di\013eren)o(t)h(b)q(eha)o(vior:)19 b(it)13 b(returns)g(the)75 2591 y(n)o(um)o(b)q(er)g(of)f(\\top)g(elemen)o(ts")h (receiv)o(ed.)21 b(In)13 b(the)g(previous)g(example,)h Fx(MPI)p 1368 2591 V 16 w(GET)p 1472 2591 V 16 w(COUNT)g FE(ma)o(y)e(return)75 2647 y(an)o(y)h(in)o(teger)g(v)m(alue)i Fy(k)q FE(,)e(where)h(0)e FC(\024)h Fy(k)h FC(\024)f Fp(count)o FE(.)19 b(If)14 b Fx(MPI)p 1053 2647 V 15 w(GET)p 1156 2647 V 17 w(COUNT)g FE(returns)f Fy(k)q FE(,)h(then)f(the)h(n)o(um)o(b)q(er)75 2704 y(of)j(basic)i(elemen)o(ts) f(receiv)o(ed)h(\(and)e(the)h(v)m(alue)h(returned)f(b)o(y)g Fx(MPI)p 1258 2704 V 16 w(GET)p 1362 2704 V 16 w(ELEMENTS)p FE(\))g(is)g Fy(n)12 b FC(\002)g Fy(k)q FE(.)28 b(If)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 59 64 59 63 bop 75 -100 a Ft(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FE(59)75 45 y(the)18 b(n)o(um)o(b)q(er)g(of)f(basic)h(elemen)o (ts)h(receiv)o(ed)g(is)f(not)f(a)h(m)o(ultiple)h(of)e Fy(n)p FE(,)h(i.e.)28 b(if)18 b(the)g(receiv)o(e)h(op)q(eration)75 102 y(has)e(not)g(receiv)o(ed)i(an)f(in)o(tegral)f(n)o(um)o(b)q(er)h (of)f Fx(datat)o(yp)q(e)i FE(\\copies",)f(then)g Fx(MPI)p 1441 102 14 2 v 16 w(GET)p 1545 102 V 17 w(COUNT)g FE(returns)75 158 y(the)d(v)m(alue)i Fm(MPI)p 348 158 13 2 v 14 w(UNDEFINED)p FE(.)166 215 y(Example:)75 327 y Fp(...)75 384 y(CALL)23 b(MPI_TYPE_CONTIGUOUS\(2,)e(MPI_REAL,)i(Type2,)g(ierr\))75 440 y(CALL)g(MPI_COMMIT\(Type2,)f(ierr\))75 497 y(...)75 553 y(CALL)h(MPI_RANK\(comm,)f(rank,)h(ierr\))75 610 y(IF\(rank.EQ.0\))147 666 y(THEN)218 722 y(CALL)g(MPI_SEND\(a,)g(2,)g (MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))218 779 y(CALL)g(MPI_SEND\(a,)g (3,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))147 835 y(ELSE)218 892 y(CALL)g(MPI_RECV\(a,)g(2,)g(Type2,)g(0,)h(0,)g(comm,)f(stat,)g (ierr\))218 948 y(CALL)g(MPI_GET_COUNT\(stat,)f(i,)h(ierr\))191 b(!)23 b(returns)g(i=1)218 1005 y(CALL)g(MPI_GET_ELEMENTS\(stat,)e(i,)j (ierr\))119 b(!)23 b(returns)g(i=2)218 1061 y(CALL)g(MPI_RECV\(a,)g(2,) g(Type2,)g(0,)h(0,)g(comm,)f(stat,)g(ierr\))218 1118 y(CALL)g(MPI_GET_COUNT\(stat,)f(i,)h(ierr\))191 b(!)23 b(returns)g(i=MPI_UNDEFINED)218 1174 y(CALL)g(MPI_GET_ELEMENTS\(stat,)e (i,)j(ierr\))119 b(!)23 b(returns)g(i=3)75 1231 y(END)g(IF)166 1341 y FE(The)15 b(function)h Fx(MPI)p 522 1341 14 2 v 16 w(GET)p 626 1341 V 17 w(ELEMENTS)f FE(can)g(also)g(b)q(e)g(used)h (after)e(a)h(prob)q(e)g(to)g(\014nd)h(the)f(n)o(um)o(b)q(er)75 1398 y(of)20 b(elemen)o(ts)h(in)g(the)f(prob)q(ed)h(message.)34 b(Note)20 b(that)f(the)h(t)o(w)o(o)f(functions)i Fx(MPI)p 1507 1398 V 16 w(GET)p 1611 1398 V 17 w(COUNT)g FE(and)75 1454 y Fx(MPI)p 160 1454 V 16 w(GET)p 264 1454 V 17 w(ELEMENTS)15 b FE(return)g(the)g(same)g(v)m(alues)i(when)e(they)h(are)e(used)i(with) g(basic)g(datat)o(yp)q(es.)166 1588 y Fo(Implemen)o(tati)o(on)i(note:) 58 b Fr(The)19 b(curren)o(t)i(de\014nition)e(implies)e(that)i(a)g (receiv)o(e)h(cannot)g(c)o(hange)f(the)75 1638 y(v)n(alue)d(of)g (storage)h(b)q(et)o(w)o(een)h(the)f(en)o(tries)g(de\014ned)h(to)e(comp) q(ose)g(the)h(comm)o(unication)c(bu\013er.)27 b(In)17 b(particular,)75 1687 y(the)c(de\014nition)g(implies)e(that)i(padding)f (space)i(in)e(a)h(structure)i(should)d(not)h(b)q(e)g(mo)q(di\014ed)f (when)h(suc)o(h)h(structure)75 1737 y(is)f(copied)g(from)f(one)h(pro)q (cess)i(to)e(another.)18 b(This)13 b(w)o(ould)f(prev)o(en)o(ts)j(the)f (ob)o(vious)e(optimization)e(of)j(cop)o(ying)f(the)75 1787 y(structure,)i(together)f(with)e(the)i(padding,)e(as)h(one)g(con)o (tiguous)f(blo)q(c)o(k.)18 b(The)12 b(implemen)o(tatio)o(n)d(is)j(free) h(to)e(do)h(this)75 1837 y(optimization)h(when)j(it)f(do)q(es)i(not)e (impact)f(the)i(outcome)f(of)g(the)h(computation.)22 b(The)16 b(user)g(can)g(\\force")g(this)75 1887 y(optimization)11 b(b)o(y)j(explicitly)f(including)g(padding)g(as)h(part)g(of)f(the)h (message.)75 2096 y Fq(3.13.6)49 b(Examples)75 2184 y FE(The)15 b(follo)o(wing)h(examples)g(illustrate)h(the)e(use)g(of)g (deriv)o(ed)h(datat)o(yp)q(es.)75 2309 y Fx(First)f(example:)45 b FE(Send)16 b(and)g(receiv)o(e)g(a)e(section)i(of)f(a)g(3D)g(arra)o(y) l(.)218 2421 y Fp(REAL)23 b(a\(100,100,100\),)f(e\(9,9,9\))218 2478 y(INTEGER)h(oneslice,)g(twoslice,)f(threeslice,)h(sizeofreal,)f (myrank,)h(ierr)218 2534 y(MPI_STATUS)g(status)75 2647 y(C)143 b(extract)23 b(the)g(section)g(a\(1:17:2,)g(3:11,)g(2:10\))75 2704 y(C)143 b(and)23 b(store)h(it)f(in)h(e\(*,*,*\).)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 60 65 60 64 bop 75 -100 a FE(60)613 b Ft(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)218 102 y Fp(CALL)23 b(MPI_COMM_RANK\(MPI_ALL,)e(myrank\))218 214 y(CALL)i(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 327 y(C)119 b(create)23 b(datatype)g(for)g(a)h(1D)g(section)218 384 y(CALL)f(MPI_TYPE_VECTOR\()f(9,)i(MPI_REAL,)46 b(2,)24 b(1,)f(oneslice,)g(ierr\))75 497 y(C)119 b(create)23 b(datatype)g(for)g(a)h(2D)g(section)218 553 y(CALL)f (MPI_TYPE_HVECTOR\(9,oneslice,10)o(0*sizeo)o(freal,1)o(,twoslic)o (e,ierr\))75 666 y(C)119 b(create)23 b(datatype)g(for)g(the)h(entire)f (section)218 723 y(CALL)g(MPI_TYPE_HVECTOR\()f(9,)i(twoslice,)46 b(100*100*sizeofreal,)21 b(1,)767 779 y(threeslice,)h(ierr\))218 892 y(CALL)h(MPI_TYPE_COMMIT\()f(threeslice,)h(ierr\))218 948 y(CALL)g(MPI_SENDRECV\(a\(1,3,2\),)e(1,)j(threeslice,)e(myrank,)h (0,)h(e,)f(9*9*9,)648 1005 y(MPI_REAL,)f(myrank,)h(0,)h(MPI_ALL,)f (status,)g(ierr\))75 1126 y Fx(Second)17 b(example:)44 b FE(Cop)o(y)15 b(the)g(\(strictly\))g(lo)o(w)o(er)g(triangular)g(part) g(of)g(a)g(matrix.)218 1233 y Fp(REAL)23 b(a\(100,100\),)g (b\(100,100\))218 1290 y(INTEGER)47 b(disp\(100\),)22 b(blocklen\(100\),)g(ltype,)h(myrank,)g(ierr)218 1346 y(MPI_STATUS)g(status)75 1459 y(C)119 b(copy)23 b(lower)h(triangular)e (part)h(of)h(array)f(a)75 1515 y(C)119 b(onto)23 b(lower)h(triangular)e (part)h(of)h(array)f(b)218 1628 y(CALL)g(MPI_COMM_RANK\(MPI_ALL,)e (myrank\))75 1741 y(C)119 b(compute)23 b(start)g(and)h(size)f(of)h (each)f(column)218 1798 y(DO)h(i=1,)f(100)266 1854 y(disp\(i\))g(=)h (100*\(i-1\))e(+)i(i)266 1911 y(block\(i\))f(=)g(100-i)218 1967 y(END)h(DO)75 2080 y(C)119 b(create)23 b(datatype)g(for)g(lower)h (triangular)e(part)218 2136 y(CALL)h(MPI_TYPE_INDEX\()f(100,)i (MPI_REAL,)e(disp,)h(block,)g(ltype,)h(ierr\))218 2249 y(CALL)f(MPI_TYPE_COMMIT\(ltype,)e(ierr\))218 2306 y(CALL)i (MPI_SENDRECV\()g(a,)g(1,)h(ltype,)f(myrank,)g(0,)g(b,)h(1,)552 2362 y(ltype,)f(myrank,)g(0,)h(MPI_ALL,)f(status,)f(ierr\))75 2483 y Fx(Third)15 b(example:)45 b FE(T)l(ransp)q(ose)15 b(a)g(matrix)218 2591 y Fp(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 2647 y(INTEGER)g(row,)g(xpose,)g(sizeofreal,)g(myrank,)g(ierr)218 2704 y(MPI_STATUS)g(status)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 61 66 61 65 bop 75 -100 a Ft(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(61)75 102 y Fp(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b) 218 214 y(CALL)g(MPI_COMM_RANK\(MPI_ALL,)e(myrank\))218 327 y(CALL)i(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 440 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 497 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(MPI_REAL,)g(100,)g(1,)h(row,)f (ierr\))75 610 y(C)119 b(create)23 b(datatype)g(for)g(matrix)g(in)h (row-major)f(order)218 666 y(CALL)g(MPI_TYPE_HVECTOR\()f(100,)h(row,)h (sizeofreal,)e(1,)i(xpose,)f(ierr\))218 779 y(CALL)g(MPI_TYPE_COMMIT\() f(xpose,)h(ierr\))75 892 y(C)119 b(send)23 b(matrix)h(in)f(row-major)g (order)g(and)g(receive)g(in)h(column)f(major)g(order)218 948 y(CALL)g(MPI_SENDRECV\()g(a,)g(1,)h(xpose,)f(myrank,)g(0,)g(b,)h (100*100,)457 1005 y(MPI_REAL,)e(myrank,)h(0,)h(MPI_ALL,)f(status,)g (ierr\))166 1090 y FE(Another)15 b(approac)o(h)g(to)f(the)i(transp)q (ose)f(problem:)218 1169 y Fp(REAL)23 b(a\(100,100\),)g(b\(100,100\)) 218 1226 y(INTEGER)47 b(disp\(2\),)23 b(blocklen\(2\),)f(type\(2\),)h (row,)g(row1,)g(sizeofreal)218 1282 y(INTEGER)47 b(myrank,)23 b(ierr)218 1339 y(MPI_STATUS)g(status)218 1451 y(CALL)g (MPI_COMM_RANK\(MPI_ALL,)e(myrank\))75 1564 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 1677 y(CALL)g(MPI_TYPE_EXTENT\()f (MPI_REAL,)h(sizeofreal,)f(ierr\))75 1790 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 1847 y(CALL)f(MPI_TYPE_VECTOR\()f (MPI_REAL,)h(100,)g(100,)g(1,)h(row,)f(ierr\))75 1960 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row,)f(with)h(the)f (extent)g(of)h(one)f(real)g(number)218 2016 y(disp\(1\))g(=)h(0)218 2072 y(disp\(2\))f(=)h(sizeofreal)218 2129 y(type\(1\))47 b(=)24 b(row)218 2185 y(type\(2\))47 b(=)24 b(MPI_UB)218 2242 y(blocklen\(1\))46 b(=)24 b(1)218 2298 y(blocklen\(2\))46 b(=)24 b(1)218 2355 y(CALL)f(MPI_TYPE_STRUCT\()f(2,)i(type,)f(disp,)g (blocklen,)g(row1\))218 2468 y(CALL)g(MPI_TYPE_COMMIT\()f(row1,)h (ierr\))75 2581 y(C)119 b(send)23 b(100)h(rows)f(and)h(receive)f(in)g (column)g(major)g(order)218 2637 y(CALL)g(MPI_SENDRECV\()g(a,)g(100,)g (row1,)h(myrank,)f(0,)g(b,)h(100*100,)457 2693 y(MPI_REAL,)e(myrank,)h (0,)h(MPI_ALL,)f(status,)g(ierr\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 62 67 62 66 bop 75 -100 a FE(62)613 b Ft(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fx(F)o(ourth)f (example:)45 b FE(manipulate)17 b(an)e(arra)o(y)f(of)g(structures.)75 163 y Fp(struct)23 b(Partstruct)147 220 y({)147 276 y(int)95 b(class;)47 b(/*)23 b(particle)g(class)g(*/)147 332 y(double)g(d[6];)71 b(/*)23 b(particle)g(coordinates)f(*/)147 389 y(char)71 b(b[7];)g(/*)23 b(some)g(additional)g(information)f(*/)147 445 y(};)75 558 y(struct)h(Partstruct)94 b(particle[1000];)75 671 y(int)429 b(i,)24 b(dest,)f(rank;)75 728 y(MPI_Comm)118 b(comm;)75 897 y(/*)24 b(build)f(datatype)g(describing)f(structure)h (*/)75 1010 y(MPI_Datatype)f(Particletype;)75 1066 y(MPI_Datatype)g (type[3])h(=)h({MPI_int,)e(MPI_double,)h(MPI_char};)75 1123 y(int)238 b(blocklen[3])23 b(=)g({1,)h(6,)f(7};)75 1179 y(int)238 b(disp[3];)75 1349 y(/*)24 b(compute)e(displacements)h (of)g(structure)g(components)f(*/)75 1462 y(MPI_Address\()g(\(void)h (*\)particle,)g(disp\);)75 1518 y(MPI_Address\()f(\(void)h (*\)particle[0].d,)f(disp+1\);)75 1574 y(MPI_Address\()g(\(void)h (*\)particle[0].b,)f(disp+2\);)75 1631 y(for)h(\(i=0;)h(i)f(<3;)h (i++\))f(disp[i])g(-=)h(disp[0];)75 1744 y(MPI_Type_struct\()e(3,)h (type,)g(disp,)h(blocklen,)e(Particletype\);)147 1857 y(/*)h(If)h(compiler)f(does)g(padding)g(in)g(mysterious)g(ways,)147 1913 y(the)g(following)g(may)g(be)h(safer)f(*/)75 2026 y(MPI_Datatype)f(type1[4])h(=)h({MPI_int,)e(MPI_double,)h(MPI_char,)f (MPI_UB};)75 2083 y(int)238 b(blocklen1[4])22 b(=)i({1,)g(6,)f(7,)h (1};)75 2139 y(int)238 b(disp1[4];)75 2252 y(/*)24 b(compute)e (displacements)h(of)g(structure)g(components)f(*/)75 2365 y(MPI_Address\()g(\(void)h(*\)particle,)g(disp1\);)75 2421 y(MPI_Address\()f(\(void)h(*\)particle[0].d,)f(disp1+1\);)75 2478 y(MPI_Address\()g(\(void)h(*\)particle[0].b,)f(disp1+2\);)75 2534 y(MPI_Address\()g(\(void)h(*\)\(particle+1\),)f(disp1\);)75 2591 y(for)h(\(i=0;)h(i)f(<4;)h(i++\))f(disp1[i])g(-=)g(disp1[0];)75 2704 y(/*)h(build)f(datatype)g(describing)f(structure)h(*/)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 63 68 63 67 bop 75 -100 a Ft(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FE(63)75 102 y Fp(MPI_Type_struct\()22 b(4,)h(type1,)g(disp1,)g (blocklen1,)g(Particletype\);)409 271 y(/*)h(4.1:)266 327 y(send)f(the)h(entire)f(array)g(*/)75 440 y(MPI_Commit\()f (Particletype\);)75 497 y(MPI_Send\()h(particle,)f(1000,)h (Particletype,)g(dest,)g(tag,)g(comm\);)409 666 y(/*)h(4.2:)266 723 y(send)f(only)h(the)f(entries)g(of)g(class)h(zero)f(particles,)266 779 y(preceded)g(by)g(the)h(number)f(of)g(such)h(entries)f(*/)75 892 y(MPI_Datatype)f(Zparticles;)70 b(/*)24 b(datatype)f(describing)f (all)i(particles)791 948 y(with)f(index)g(zero)h(\(needs)f(to)g(be)h (recomputed)791 1005 y(if)f(classes)g(change\))g(*/)75 1061 y(MPI_Datatype)f(Ztype;)75 1174 y(int)h(zdisp[1000],)g (zblock[1000],)f(j,)h(k;)75 1231 y(int)g(zzblock[2])g(=)h({1,1};)75 1287 y(int)f(zztype[2],)g(zzindex[2];)75 1456 y(/*)h(compute)e(indices) h(of)h(class)f(zero)g(particles)g(*/)75 1513 y(j)h(=)f(0;)75 1569 y(for\(i=0;)g(i)g(<)h(1000;)f(i++\))123 1626 y(if)g (\(particle[i].class==0\))194 1682 y({)194 1739 y(zindex[j])g(=)h(i;) 194 1795 y(zblock[j])f(=)h(1;)194 1852 y(j++;)194 1908 y(})75 2021 y(/*)g(create)f(datatype)f(for)i(class)f(zero)g(particles) 47 b(*/)75 2077 y(MPI_Type_indexed\()22 b(j,)h(Particletype,)46 b(zindex,)23 b(zblock,)g(Zparticles\);)75 2190 y(/*)h(prepend)e (particle)h(count)g(*/)75 2247 y(MPI_Address\(\(void)f(*\)&j,)h (zzindex\);)75 2303 y(zzindex[1])f(=)i(particle;)75 2360 y(zztype[0])f(=)g(MPI_INT;)75 2416 y(zztype[1])g(=)g(Zparticles;)75 2473 y(MPI_Type_struct\(2,)e(zztype,)i(zzindex,)g(zzblock,)g(Ztype\);) 75 2586 y(MPI_Type_commit\()f(Ztype\);)75 2642 y(MPI_Send\()h (MPI_BOTTOM,)f(1,)i(Ztype,)f(dest,)g(tag,)g(comm\);)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 64 69 64 68 bop 75 -100 a FE(64)613 b Ft(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)242 102 y Fp(/*)24 b(A)f(probably)g(more)g(efficient)g(way)h(of)f(defining)g (Zparticles)f(*/)75 214 y(/*)i(consecutive)e(particles)h(with)g(index)g (zero)g(are)h(handled)f(as)g(one)h(block)f(*/)75 271 y(j=0;)75 327 y(for)g(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 384 y(if)g(\(particle[i].index==0\))170 440 y({)170 497 y(for)h(\(k=i+1;)f(\(k)g(<)h(1000\)&&\(particle[k].index)d(==)i(0\))h (;)f(k++\);)170 553 y(zindex[j])g(=)h(i;)170 610 y(zblock[j])f(=)h (k-i;)170 666 y(j++;)170 723 y(i)g(=)g(k;)170 779 y(})75 835 y(MPI_Type_indexed\()e(j,)h(Particletype,)f(zindex,)h(zblock,)g (Zparticles\);)457 1005 y(/*)g(4.3:)314 1061 y(send)g(the)g(first)h (two)f(coordinates)f(of)i(all)f(entries)g(*/)75 1174 y(MPI_Datatype)f(Allpairs;)142 b(/*)24 b(datatype)f(for)g(all)g(pairs)h (of)f(coordinates)g(*/)75 1287 y(int)238 b(sizeofentry;)75 1400 y(MPI_Extent\()22 b(Particletype,)g(sizeofentry\);)194 1513 y(/*)i(sizeofentry)e(can)i(also)f(be)h(computed)e(by)i (subtracting)e(the)i(address)266 1569 y(of)f(particle[0])g(from)g(the)h (address)f(of)g(particle[1])f(*/)75 1682 y(MPI_Type_hvector\()g(1000,)h (MPI_REAL,)f(sizeofentry,)h(2,)g(Allpairs\);)75 1739 y(MPI_Type_commit\()f(Allpairs\);)75 1795 y(MPI_Send\()h(particle.d,)f (1,)i(Allpairs,)e(dest,)h(tag,)h(comm\);)218 1908 y(/*)g(an)f (alternative)g(solution)f(to)i(4.3)g(*/)75 2021 y(MPI_Datatype)e (Onepair;)71 b(/*)23 b(datatype)g(for)h(one)f(pair)g(of)h(coordinates,) e(with)695 2077 y(the)i(extent)f(of)h(one)f(particle)g(entry)g(*/)75 2134 y(int)g(disp2[3];)75 2190 y(int)g(type2[3])g(=)h({MPI_LB,)f (MPI_DOUBLE,)f(MPI_UB};)75 2247 y(int)h(blocklen2[3])g(=)g({1,)h(2,)f (1};)75 2360 y(MPI_Address\()f(\(void)h(*\)particle,)g(disp2\);)75 2416 y(MPI_Address\()f(\(void)h(*\)particle[0].d,)f(disp2+1\);)75 2473 y(MPI_Address\()g(\(void)h(*\)\(particle+1\),)f(disp2+2\);)75 2529 y(for)h(\(i=0;)h(i<2;)f(i++\))g(disp2[i])g(-=)h(disp2[0];)75 2642 y(MPI_Type_struct\()e(3,)h(type2,)g(disp2,)g(blocklen2,)g (Onepair\);)75 2698 y(MPI_Commit\()f(Onepair\);)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 65 70 65 69 bop 75 -100 a Ft(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FE(65)75 45 y Fp(MPI_Send\()23 b(particle[0].d,)f(1000,)h (Onepair,)g(dest,)g(tag,)g(comm\);)75 224 y Fx(Fifth)18 b(example:)45 b FE(same)17 b(arra)o(y)g(manipulations)i(as)f(in)g(the)g (previous)g(example,)h(but)f(using)h(absolute)75 280 y(addresses)c(in)h(datat)o(yp)q(es.)75 389 y Fp(struct)23 b(Partstruct)147 445 y({)147 502 y(int)g(class;)147 558 y(double)g(d[6];)147 615 y(char)g(b[7];)147 671 y(};)75 784 y(struct)g(Partstruct)g(particle[1000];)337 897 y(/*)h(build)f (datatype)g(describing)g(first)g(array)g(entry)g(*/)75 1010 y(MPI_Datatype)f(Particletype;)75 1066 y(MPI_Datatype)g(type[3])h (=)h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR};)75 1123 y(int)238 b(block[3])23 b(=)h({1,)f(6,)h(7};)75 1179 y(int)238 b(disp[3];)75 1292 y(MPI_Address\()22 b(\(void)h(*\)particle,)g (disp\);)75 1349 y(MPI_Address\()f(\(void)h(*\)particle[0].d,)f (disp\);)75 1405 y(MPI_Address\()g(\(void)h(*\)particle[0].b,)f (disp\);)75 1462 y(MPI_Type_struct\()g(3,)h(stype,)g(sindex,)g(sblock,) g(Particletype\);)75 1518 y(/*)h(Particletype)e(describes)g(first)i (array)f(entry)g(--)h(using)f(absolute)147 1574 y(addresses)f(*/)505 1687 y(/*)h(5.1:)361 1744 y(send)h(the)f(entire)g(array)g(*/)75 1857 y(MPI_Commit\()f(Particletype\);)75 1913 y(MPI_Send\()h (MPI_BOTTOM,)f(1000,)h(Particletype,)f(dest,)h(tag,)h(comm\);)481 2083 y(/*)f(5.2:)290 2139 y(send)g(the)h(entries)e(of)i(class)f(zero,) 290 2195 y(preceded)g(by)g(the)h(number)f(of)g(such)h(entries)e(*/)75 2308 y(MPI_Datatype)g(Zparticles,)h(Ztype;)75 2421 y(int)g (zindex[1000],)g(zblock[1000],)f(i,)h(j,)h(k;)75 2478 y(int)f(zzblock[2])g(=)h({1,1};)75 2534 y(int)f(zztype[2],)g (zzindex[2];)75 2704 y(j=0;)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 66 71 66 70 bop 75 -100 a FE(66)613 b Ft(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fp(for)23 b(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 102 y(if)g (\(particle[i].index==0\))170 158 y({)170 214 y(for)h(\(k=i+1;)f(\(k)g (<)h(1000\)&&\(particle[k].index)d(==)i(0\))h(;)f(k++\);)170 271 y(zindex[j])g(=)h(i;)170 327 y(zblock[j])f(=)h(k-i;)170 384 y(j++;)170 440 y(i)g(=)g(k;)170 497 y(})75 553 y (MPI_Type_indexed\()e(j,)h(Particletype,)46 b(zindex,)23 b(zblock,)g(Zparticles\);)75 610 y(/*)h(Zparticles)e(describe)h (particles)f(with)i(class)f(zero,)g(using)147 666 y(their)g(absolute)g (addresses*/)75 779 y(/*)h(prepend)e(particle)h(count)g(*/)75 835 y(MPI_Address\(\(void)f(*\)&j,)h(zzindex\);)75 892 y(zzindex[1])f(=)i(MPI_BOTTOM;)75 948 y(zztype[0])f(=)g(MPI_INT;)75 1005 y(zztype[1])g(=)g(Zparticles;)75 1061 y(MPI_Type_struct\(2,)e (zztype,)i(zzindex,)g(zzblock,)g(Ztype\);)75 1174 y(MPI_Type_commit\()f (Ztype\);)75 1231 y(MPI_Send\()h(MPI_bottom,)f(1,)i(Ztype,)f(dest,)g (tag,)g(comm\);)75 1409 y Fx(Sixth)16 b(example:)45 b FE(handling)17 b(of)e(unions.)75 1518 y Fp(union)23 b({)147 1574 y(int)118 b(ival;)147 1631 y(float)70 b(fval;)218 1687 y(})24 b(u[1000])75 1800 y(int)f(utype;)75 1913 y(/*)h(All)f(entries)g(of)h(u)f(have)h(identical)e(type;)h(variable)147 1970 y(utype)g(keeps)g(track)g(of)h(their)f(current)g(type)g(*/)75 2083 y(MPI_Datatype)70 b(type[2];)75 2139 y(int)286 b(blocklen[2])22 b(=)i({1,1};)75 2195 y(int)286 b(disp[2];)75 2252 y(MPI_Datatype)70 b(mpi_utype[2];)75 2308 y(int)286 b(i,j;)75 2421 y(/*)24 b(compute)e(an)i(MPI)g(datatype)e(for)i(each)f(possible)g(union)g (type;)147 2478 y(assume)g(values)g(are)g(left-aligned)f(in)i(union)f (storage.)g(*/)75 2591 y(MPI_Address\()f(\(void)h(*\)u,)h(&i\);)75 2647 y(MPI_Address\()e(\(void)h(*\)\(u+1\),)g(&j\);)75 2704 y(disp[0])g(=)h(0;)f(disp[1])g(=)h(j-i)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 67 72 67 71 bop 75 -100 a Ft(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(67)75 45 y Fp(type[1])23 b(=)h(MPI_UB;)75 158 y(type[0])f(=)h(MPI_INT;)75 214 y(MPI_type_struct\(2,)d(type,)j(disp,)f (blocklen,)f(mpi_utype[0]\);)75 327 y(type[0])h(=)h(MPI_FLOAT;)75 384 y(MPI_Type_struct\(2,)d(type,)j(disp,)f(blocklen,)f (mpi_utype[1]\);)75 497 y(for\(i=0;)h(i<2;)g(i++\))g (MPI_Type_commit\(mpi_utype[i]\))o(;)75 610 y(/*)h(actual)f (communication)f(*/)75 723 y(MPI_Send\(u,)g(1000,)i(mpi_utype[utype],)d (dest,)i(tag,)h(comm\);)75 851 y Fq(3.13.7)49 b(Co)o(rrect)15 b(use)h(of)g(addresses)75 939 y FE(Successiv)o(ely)23 b(declared)f(v)m(ariables)h(in)e(C)g(or)g(F)l(ortran)e(are)i(not)g (necessarily)h(stored)f(at)f(con)o(tiguous)75 996 y(lo)q(cations.)i(Th) o(us,)16 b(care)g(m)o(ust)f(b)q(e)h(exercised)i(that)d(displacemen)o (ts)i(do)f(not)f(cross)h(from)f(one)h(v)m(ariable)75 1052 y(to)e(another.)20 b(Also,)15 b(in)g(mac)o(hines)h(with)f(segmen)o (ted)g(address)g(space,)g(addresses)g(are)g(not)f(unique)j(and)75 1108 y(address)11 b(arithmetic)h(has)g(some)f(p)q(eculiar)i(prop)q (erties.)19 b(Th)o(us,)12 b(use)g(of)f Fz(addresses)p FE(,)f(i.e.)20 b(displacemen)o(ts)75 1165 y(relativ)o(e)c(to)e(the)i (start)e(address)29 b Fm(MPI)p 733 1165 13 2 v 14 w(BOTTOM)p FE(,)14 b(has)h(to)g(b)q(e)h(restricted.)166 1223 y(V)l(ariables)k(b)q (elong)f(to)f(the)h(same)f Fz(sequen)o(tial)k(storage)d FE(if)g(they)g(b)q(elong)g(to)f(the)h(same)f(arra)o(y)l(,)75 1279 y(to)g(the)g(same)g Fp(COMMON)g FE(blo)q(c)o(k)h(in)g(F)l(ortran,) f(or)g(to)f(the)i(same)f(structure)g(in)h(C.)f(V)l(alid)i(addresses)e (are)75 1335 y(de\014ned)f(recursiv)o(ely)f(as)f(follo)o(ws:)131 1448 y(1.)22 b(The)g(function)g Fx(MPI)p 558 1448 14 2 v 16 w(ADDRESS)h FE(returns)e(a)h(v)m(alid)h(address,)h(when)e (passed)g(as)f(argumen)o(t)g(a)189 1504 y(v)m(ariable)16 b(of)f(the)g(calling)i(program.)131 1603 y(2.)22 b(The)c Fx(buf)h FE(parameter)f(of)f(a)h(comm)o(unication)h(function)g(ev)m (aluates)g(to)f(a)g(v)m(alid)i(address,)e(when)189 1659 y(passed)d(as)g(argumen)o(t)f(a)h(v)m(ariable)i(of)e(the)g(calling)i (program.)131 1757 y(3.)22 b(If)15 b Fx(v)g FE(is)h(a)f(v)m(alid)i (address,)d(and)i Fx(i)f FE(is)g(an)h(in)o(teger,)f(then)g Fx(v+i)g FE(is)h(a)f(v)m(alid)i(address,)e(pro)o(vided)h Fx(v)f FE(and)189 1814 y Fx(v+i)g FE(are)g(in)h(the)f(same)g(sequen)o (tial)h(storage.)131 1912 y(4.)22 b(If)15 b Fx(v)g FE(is)h(a)f(v)m (alid)i(address)e(then)29 b Fm(MPI)p 820 1912 13 2 v 15 w(BOTTOM)13 b(+)h(v)h FE(is)h(a)f(v)m(alid)i(address.)166 2025 y(A)22 b(correct)g(program)g(uses)g(only)h(v)m(alid)i(addresses)d (to)g(iden)o(tify)i(the)e(lo)q(cations)h(of)f(en)o(tries)h(in)75 2081 y(comm)o(unication)16 b(bu\013ers.)k(F)l(urthermore,)14 b(if)h Fx(u)h FE(and)f Fx(v)g FE(are)g(t)o(w)o(o)f(v)m(alid)j (addresses,)e(then)g(the)g(\(in)o(teger\))75 2137 y(di\013erence)j Fx(u)g(-)f(v)g FE(can)g(b)q(e)h(computed)g(only)f(if)h(b)q(oth)f Fx(u)g FE(and)h Fp(v)f FE(are)f(in)i(the)g(same)e(sequen)o(tial)j (storage;)75 2194 y(no)c(other)g(arithmetic)h(op)q(erations)f(can)g(b)q (e)h(meaningfully)i(executed)e(on)f(addresses.)166 2252 y(W)l(e)f(sa)o(y)g(that)g(a)g(datat)o(yp)q(e)g(is)h Fz(absolute)g FE(if)g(all)h(displacemen)o(ts)g(within)f(this)g(datat)o(yp)q(e)f(are)g (v)m(alid)75 2308 y(\(absolute\))h(addresses;)g(it)g(is)h Fz(relativ)o(e)g FE(otherwise.)166 2366 y(A)f(correct)g(program)f(ob)q (eys)i(the)f(follo)o(wing)h(constrain)o(ts)e(in)i(the)g(use)f(of)g (datat)o(yp)q(es:)143 2478 y FC(\017)23 b FE(If)j(the)g Fx(oldt)o(yp)q(e)h FE(argumen)o(t)e(used)i(in)g Fx(MPI)p 976 2478 14 2 v 15 w(TYPE)p 1108 2478 V 17 w(CONTIGUOUS)p FE(,)h Fx(MPI)p 1536 2478 V 15 w(TYPE)p 1668 2478 V 17 w(VECTOR)p FE(,)189 2534 y Fx(MPI)p 274 2534 V 15 w(TYPE)p 406 2534 V 17 w(HVECTOR)p FE(,)20 b Fx(MPI)p 749 2534 V 16 w(TYPE)p 882 2534 V 17 w(INDEX)p FE(,)f(or)f Fx(MPI)p 1207 2534 V 16 w(TYPE)p 1340 2534 V 17 w(HINDEX)h FE(is)h(absolute,)g (then)189 2591 y(all)15 b(addresses)g(within)g(it)g(m)o(ust)f(b)q(e)h (to)f(v)m(ariables)i(con)o(tained)f(within)g(the)g(same)f(arra)o(y)f (or)h(struc-)189 2647 y(ture;)j(the)g(result)h(datat)o(yp)q(e)e(is)i (also)f(absolute,)h(and)f(all)h(computed)f(addresses)h(m)o(ust)e(also)h (fall)189 2704 y(within)f(the)f(same)g(arra)o(y)f(or)h(structure.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 68 73 68 72 bop 75 -100 a FE(68)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)143 45 y FC(\017)23 b FE(If)18 b(all)g(en)o(tries)h(of)e(the)h Fx(a)o(rra)o(y)p 684 45 14 2 v 14 w(of)p 735 45 V 16 w(t)o(yp)q(es)i FE(argumen)o(t)d (of)g Fx(MPI)p 1210 45 V 16 w(TYPE)p 1343 45 V 17 w(STRUCT)h FE(are)g(absolute)g(ad-)189 102 y(dresses)12 b(\(computed)g(b)o(y)g Fx(MPI)p 707 102 V 15 w(ADDRESS)p FE(\),)h(then)f(the)g(result)g(datat) o(yp)q(e)g(is)g(also)g(absolute.)19 b(Eac)o(h)189 158 y(new)14 b(address)h(computed)g(from)f(an)g(old)h(address)g(m)o(ust)f (fall)h(within)h(the)f(same)f(arra)o(y)f(or)h(record)189 214 y(as)h(the)i(old)f(address.)23 b(\(Addresses)16 b(in)h(the)f(old)h (t)o(yp)q(e)f(ma)o(y)f(not)h(b)q(e)h(all)g(within)g(the)f(same)g(arra)o (y)189 271 y(or)e(record.\))143 362 y FC(\017)23 b FE(If)14 b(a)g(comm)o(unication)h(call)h(is)e(in)o(v)o(ok)o(ed)h(with)g (parameters)e Fx(buf)i FE(and)29 b Fx(datat)o(yp)q(e)p FE(,)16 b(then)e(either)h Fx(buf)189 419 y FE(=)25 b Fm(MPI)p 326 419 13 2 v 15 w(BOTTOM)13 b FE(and)g Fx(datat)o(yp)q(e)i FE(is)f(a)f(handle)i(to)d(an)i(absolute)f(datat)o(yp)q(e,)g(or)g Fx(buf)h FE(is)g(set)f(to)g(the)189 475 y(address)h(of)g(a)g(program)f (v)m(ariable,)j(and)e(all)h(displacemen)o(ts)h(in)30 b Fx(datat)o(yp)q(e)16 b FE(relativ)o(e)e(to)g(this)h(base)189 532 y(yield)h(addresses)g(that)e(are)h(within)i(the)e(same)g(sequen)o (tial)h(storage)e(as)h(this)h(v)m(ariable.)166 629 y(In)21 b(summary)l(,)g(the)f(t)o(yp)q(e)h(constructors)e Fx(MPI)p 981 629 14 2 v 16 w(TYPE)p 1114 629 V 17 w(CONTIGUOUS,)j(MPI)p 1536 629 V 15 w(TYPE)p 1668 629 V 17 w(VECTOR,)75 685 y(MPI)p 160 685 V 16 w(TYPE)p 293 685 V 17 w(HVECTOR,)15 b(MPI)p 631 685 V 16 w(TYPE)p 764 685 V 17 w(INDEX)f FE(and)29 b Fx(MPI)p 1114 685 V 16 w(TYPE)p 1247 685 V 17 w(HINDEX)14 b FE(can)h(b)q(e)g(recursiv)o(ely)h(ap-)75 742 y(plied)22 b(to)e(build)j(datat)o(yp)q(es)d(that)f(will)k(com)o (bine)e(v)m(ariables)h(that)e(b)q(elong)h(to)f(the)h(same)f(sequen)o (tial)75 798 y(storage;)d(v)m(ariables)i(that)e(do)g(not)g(b)q(elong)i (to)d(the)i(same)f(sequen)o(tial)i(storage)d(can)i(b)q(e)g(com)o(bined) g(to-)75 855 y(gether)d(using)h(one)f(application)i(of)e Fx(MPI)p 788 855 V 16 w(TYPE)p 921 855 V 16 w(STRUCT)p FE(.)166 987 y Fo(Implemen)o(tati)o(on)d(note:)166 1037 y Fr(There)i(is)f(no)f(need)i(to)f(distinguish)f(\(absolute\))h (addresses)i(and)e(\(relativ)o(e\))g(displacemen)o(ts)f(on)h(a)f(mac)o (hine)75 1087 y(with)k(con)o(tiguous)h(address)g(space:)25 b Fm(MPI)p 742 1087 13 2 v 14 w(BOTTOM)16 b Fr(is)h(zero,)g(and)g(b)q (oth)f(addresses)j(and)d(displacemen)o(ts)h(are)75 1137 y(in)o(tegers.)h(On)12 b(mac)o(hines)e(where)j(the)f(distinction)f(is)g (required,)h(addresses)i(are)e(recognized)g(as)g(expressions)h(that)75 1186 y(in)o(v)o(olv)o(e)g Fm(MPI)p 292 1186 V 14 w(BOTTOM)p Fr(.)75 1411 y Fv(3.14)59 b(Universal)20 b(communication)d(functions)75 1513 y FE(This)g(section)f(is)g(not)g(part)f(of)h(the)g(MPI)g (de\014nition.)24 b(Its)15 b(goal)h(is)h(to)e(indicate)i(ho)o(w)f(the)g (de\014nition)h(or)75 1569 y(implemen)o(tation)i(of)e(the)g(p)q(oin)o (t)h(to)f(p)q(oin)o(t)g(comm)o(unication)h(functions)h(can)e(b)q(e)h (reduced)g(to)f(a)g(small)75 1625 y(n)o(um)o(b)q(er)i(of)e(primitiv)o (es.)31 b(W)l(e)18 b(assume)g(the)h(a)o(v)m(ailabilit)o(y)h(of)e(the)g (primitiv)o(e)i(functions)f(listed)g(at)f(the)75 1682 y(end)e(of)f(the)g(section,)g(and)h(the)f(abilit)o(y)h(to)f(allo)q (cate)h(memory)f(for)f(new)i(v)m(ariables.)166 1738 y(W)l(e)g(assume)g (that)f(the)i(function)g Fx(MPI)p 846 1738 14 2 v 15 w(TYPE)p 978 1738 V 17 w(COMMIT)f FE(is)g(not)g(c)o(hanging)h(the)f (represen)o(tation)75 1795 y(of)f(a)g(datat)o(yp)q(e,)f(so)h(that)f (this)i(function)g(is)g(redundan)o(t.)166 1851 y(Comm)o(unication)g (with)h(a)e Fx(count)j FE(argumen)o(t)d(that)g(is)i Fy(>)d FE(1)i(can)g(b)q(e)h(reduced)g(to)e(comm)o(unication)75 1908 y(with)k Fx(count)g(=1)p FE(,)g(b)o(y)g(replicating)h(the)37 b Fx(datat)o(yp)q(e)20 b FE(argumen)o(t)d Fx(count)j FE(times.)30 b(Comm)o(unication)19 b(with)75 1964 y(a)d Fx(a)g(buf)h FE(argumen)o(t)f(that)g(is)h(not)31 b Fm(MPI)p 755 1964 13 2 v 14 w(BOTTOM)16 b FE(can)g(b)q(e)h(reduced)h(to)e(comm)o (unication)h(with)f Fx(buf)i FE(=)75 2021 y Fm(MPI)p 152 2021 V 14 w(BOTTOM)d FE(b)o(y)g(creating)h(a)e(suitable)j (displaced)g(new)f(datat)o(yp)q(e.)j(Th)o(us:)166 2077 y Fx(MPI)p 251 2077 14 2 v 16 w(SEND\(buf,)c(count,)i(datat)o(yp)q(e,)f (dest,)g(tag,)f(comm\))e FE(is)75 2165 y Fp(Type[1])23 b(=)h(datatype)75 2221 y(MPI_ADDRESS\()e(buf,)i(Disp[1]\);)75 2277 y(Blocklen[1])e(=)i(count;)75 2334 y(MPI_TYPE_STRUCT\()e(1,)h (Type,)g(Disp,)h(Blocklen,)e(Newtype\))75 2390 y(MPI_SEND\()h (MPI_BOTTOM,)f(1,)i(Newtype,)e(dest,)i(tag,)f(comm\))166 2478 y FE(The)18 b(same)g(construction)h(applies)g(to)f(all)h(other)f (comm)o(unication)g(functions)h(with)g(argumen)o(ts)75 2534 y Fx(count)e FE(and)f Fx(datat)o(yp)q(e)p FE(.)24 b(W)l(e)15 b(shall)i(henceforth)g(restrict)e(ourselv)o(es)i(to)e(comm)o (unication)h(that)f(in)o(v)o(olv)o(es)75 2591 y(only)h(one)f(elemen)o (t)h(with)g(an)f(absolute)g(address)g(\()p Fx(count)i(=1)p FE(,)d(and)i Fx(buf)g FE(=)29 b Fm(MPI)p 1465 2591 13 2 v 15 w(BOTTOM)p FE(\).)166 2647 y(It)10 b(is)h(con)o(v)o(enien)o(t)g (to)e(to)h(mak)o(e)f(explicit)k(the)d Fz(p)q(ersistence)g FE(attribute)g(of)g(comm)o(unication)h(ob)s(jects.)75 2704 y(A)f Fx(p)q(ersistent)j FE(comm)o(unication)e(ob)s(ject)e(need)i (b)q(e)g(explicitly)i(deallo)q(cated)f(b)o(y)e(a)g Fx(MPI)p 1516 2704 14 2 v 15 w(COMMOBJ)p 1757 2704 V 18 w(FREE)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 69 74 69 73 bop 75 -100 a Ft(3.14.)34 b(UNIVERSAL)17 b(COMMUNICA)l(TION)f (FUNCTIONS)633 b FE(69)75 45 y(op)q(eration.)19 b(On)14 b(the)e(other)h(hand,)g(an)g Fx(ephemeral)g FE(comm)o(unication)g(ob)s (ject)f(is)i(go)q(o)q(d)e(for)g(a)h(single)h(com-)75 102 y(m)o(unication.)33 b(It)19 b(is)h(deallo)q(cated)g(b)o(y)f(the)g (system)g(when)h(the)f(\014rst)g(comm)o(unication)h(it)f(is)h(used)g (for)75 158 y(completes.)166 215 y(The)15 b(function)h Fx(MPI)p 522 215 14 2 v 16 w(COMM)p 680 215 V 16 w(INIT)f FE(is)g(a)g(new)g(univ)o(ersal)i(function)f(for)e(the)h(creation)h(of)e (comm)o(u-)75 272 y(nication)i(ob)s(jects:)75 377 y Fx(MPI)p 160 377 V 16 w(COMM)p 318 377 V 16 w(INIT\(datat)o(yp)q(e,)g (source-dest,)h(tag,)d(comm,)f(op-mo)q(de,)i(p)q(ersistence,)j (request\))117 455 y Fr(IN)155 b Fx(datat)o(yp)q(e)424 b Fr(datat)o(yp)q(e)14 b(of)g(elemen)o(t)f(sen)o(t)i(or)f(receiv)o(ed) 117 532 y(IN)155 b Fx(source-dest)377 b Fr(rank)14 b(of)f(destination)h (or)g(source)h(\(in)o(teger\))117 609 y(IN)155 b Fx(tag)526 b Fr(message)14 b(tag)f(\(in)o(teger\))117 686 y(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 764 y(IN)155 b Fx(op-mo)q(de)422 b Fr(one)64 b(of)127 b Fm(MPI)p 1269 764 13 2 v 14 w(ST)m(AND)o(ARD)p Fr(,)f Fm(MPI)p 1715 764 V 15 w(READ)o(Y)p Fr(,)905 820 y Fm(MPI)p 982 820 V 15 w(SYNCHRONOUS)12 b Fr(or)28 b Fm(MPI)p 1455 820 V 14 w(RECV)117 897 y Fr(IN)155 b Fx(p)q(ersistence)383 b Fr(one)14 b(of)27 b Fm(MPI)p 1119 897 V 15 w(PERSISTENT)13 b Fr(or)28 b Fm(MPI)p 1534 897 V 14 w(EPHEMERAL)117 974 y Fr(OUT)108 b Fx(request)452 b Fr(new)15 b(comm)o(uni)o(cation)c(ob)r (ject)k(\(handle\))75 1170 y Fq(3.14.1)49 b(P)o(ersistent)14 b(communication)j(objects)75 1258 y Fx(MPI)p 160 1258 14 2 v 16 w(CREA)l(TE)p 345 1258 V 17 w(SEND\(MPI)p 579 1258 V 16 w(BOTTOM,)f(1,)e(datat)o(yp)q(e,)j(dest,)f(tag,)f(comm,)e (request\))j FE(is)75 1370 y Fp(MPI_COMM_INIT\(datatype,)21 b(dest,)i(tag,)g(comm,)g(MPI_STANDARD,)170 1427 y(MPI_PERSISTENT,)f (request\))166 1538 y FE(The)f(functions)g Fx(MPI)p 551 1538 V 16 w(CREA)l(TE)p 736 1538 V 17 w(RSEND)p FE(,)g Fx(MPI)p 1015 1538 V 16 w(CREA)l(TE)p 1200 1538 V 17 w(RSEND)g FE(and)g Fx(MPI)p 1560 1538 V 16 w(CREA)l(TE)p 1745 1538 V 17 w(RECV)75 1594 y FE(are)15 b(dealt)h(with)f(in)h(a)f (similar)h(manner.)166 1652 y(The)f(functions)h Fx(MPI)p 540 1652 V 16 w(ST)l(ART)g FE(and)g Fx(MPI)p 885 1652 V 15 w(COMMOBJ)p 1126 1652 V 18 w(FREE)g FE(are)f(primitiv)o(e.)75 1779 y Fq(3.14.2)49 b(Nonblo)q(cking)19 b(communication)d(initiation)75 1867 y Fx(MPI)p 160 1867 V 16 w(ISEND\(MPI)p 406 1867 V 15 w(BOTTOM,)g(1,)e(datat)o(yp)q(e,)j(dest,)f(tag,)f(comm,)e (request\))j FE(is)75 1980 y Fp(MPI_COMM_INIT\(datatype,)21 b(dest,)i(tag,)g(comm,)g(MPI_STANDARD,)194 2036 y(MPI_EPHEMERAL,)f (request\))75 2093 y(MPI_START\(request\))166 2204 y FE(The)g(functions)g Fx(MPI)p 553 2204 V 16 w(IRSEND)p FE(,)f Fx(MPI)p 844 2204 V 16 w(ISSEND)h FE(and)g Fx(MPI)p 1214 2204 V 16 w(IRECV)g FE(are)f(handled)i(in)g(a)e(similar)75 2260 y(manner.)75 2388 y Fq(3.14.3)49 b(Communication)17 b(completion)75 2476 y FE(The)e(t)o(w)o(o)f(primitiv)o(e)j(completion)f (op)q(erations)g(are)e Fx(MPI)p 1063 2476 V 16 w(W)l(AIT)l(ANY)i FE(and)g Fx(MPI)p 1467 2476 V 16 w(TEST)l(ALL)p FE(.)166 2533 y(The)c(function)h Fx(MPI)p 516 2533 V 15 w(W)l(AIT)f FE(can)g(b)q(e)h(implemen)o(ted)g(b)o(y)f(a)f(call)i(to)e Fx(MPI)p 1368 2533 V 16 w(W)l(AIT)l(ANY)i FE(with)f(an)24 b Fx(a)o(rra)o(y-)75 2590 y(of-requests)17 b FE(argumen)o(t)d(of)h (length)h(one.)166 2647 y(The)f(function)g Fx(MPI)p 521 2647 V 16 w(W)l(AIT)l(ALL)g FE(can)g(b)q(e)g(implemen)o(ted)h(as)f(a)f (lo)q(op)h(where)g(an)g Fx(MPI)p 1621 2647 V 16 w(W)l(AIT)g FE(call)g(is)75 2704 y(executed)h(for)f(eac)o(h)g(successiv)o(e)h (request)g(in)g(the)f Fx(a)o(rra)o(y-of-requests)p FE(.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 70 75 70 74 bop 75 -100 a FE(70)608 b Ft(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FE(The)e(function)g Fx(MPI)p 519 45 14 2 v 16 w(TEST)g FE(can)f(b)q(e)h(implemen)o(ted)i(b) o(y)d(a)g(call)i(to)e Fx(MPI)p 1387 45 V 16 w(TEST)l(ALL)g FE(with)h(an)g Fx(a)o(rra)o(y-)75 102 y(ofrequests)j FE(of)e(length)h(one.)166 158 y(The)h(function)h Fx(MPI)p 526 158 V 16 w(TEST)l(ANY)h FE(can)e(b)q(e)h(implemen)o(ted)h(as)d(a)h (lo)q(op)h(where)f(an)h Fx(MPI)p 1663 158 V 15 w(TEST)g FE(call)75 214 y(is)e(executed)g(for)e(eac)o(h)i(successiv)o(e)g (request)f(in)h(the)g Fx(a)o(rra)o(y-of-request)p FE(.)75 334 y Fq(3.14.4)49 b(Blo)q(cking)18 b(communication)75 420 y Fx(MPI)p 160 420 V 16 w(SEND\(MPI)p 393 420 V 16 w(BOTTOM,)d(1,)g(datat)o(yp)q(e,)h(dest,)h(tag,)d(comm\))f FE(is)75 507 y Fp(MPI_COMM_INIT\(datatype,)21 b(dest,)i(tag,)g(comm,)g (MPI_STANDARD,)218 564 y(MPI_EPHEMERAL,)f(request\))75 620 y(MPI_START\(request\))75 677 y(MPI_WAIT\(request,)g (dontcarestatus\))166 764 y FE(The)15 b(functions)h Fx(MPI)p 540 764 V 16 w(RSEND)g FE(and)f Fx(MPI)p 888 764 V 16 w(SSEND)h FE(are)f(handled)i(in)f(a)f(similar)h(manner.)166 821 y Fx(MPI)p 251 821 V 16 w(RECV\(MPI)p 482 821 V 16 w(BOTTOM,)g(1,)e(datat)o(yp)q(e,)j(source,)e(tag,)g(comm\))e FE(is)75 909 y Fp(MPI_COMM_INIT\(datatype,)21 b(dest,)i(tag,)g(comm,)g (MPI_RECV,)g(MPI_EPHEMERAL,)f(request\))75 965 y(MPI_START\(request\)) 75 1022 y(MPI_WAIT\(request,)g(status\))75 1141 y Fq(3.14.5)49 b(Prob)q(e)17 b(and)f(cancel)75 1227 y FE(The)f(functions)h Fx(MPI)p 449 1227 V 16 w(PROBE)h FE(and)e Fx(MPI)p 800 1227 V 16 w(CANCEL)g FE(are)g(primitiv)o(e.)75 1346 y Fq(3.14.6)49 b(Return)16 b(status)75 1432 y FE(The)k(functions)g Fx(MPI)p 458 1432 V 16 w(GET)p 562 1432 V 16 w(SOURCE)p FE(,)h Fx(MPI)p 868 1432 V 16 w(GET)p 972 1432 V 16 w(T)l(A)o(G)p FE(,)f Fx(MPI)p 1189 1432 V 16 w(GET)p 1293 1432 V 16 w(LEN)p FE(,)f Fx(MPI)p 1507 1432 V 16 w(PROBE)p 1671 1432 V 18 w(LEN)p FE(,)g(and)75 1488 y Fx(MPI)p 160 1488 V 16 w(TEST)p 290 1488 V 16 w(CANCELLED)d FE(are)e(primitiv)o(e.)166 1545 y(These)h(functions)h(are)f(simple)i(macros)d(that)h(access)g (records)g(in)h(a)f(structure.)75 1664 y Fq(3.14.7)49 b(send-receive)15 b(and)i(exchange)75 1750 y Fx(MPI)p 160 1750 V 16 w(SENDRECV\()11 b(send)p 520 1750 V 18 w(buf,)g(send)p 705 1750 V 18 w(count,)h(send)p 936 1750 V 18 w(t)o(yp)q(e,)g(dest,)g(recv)p 1237 1750 V 17 w(buf,)f(recv)p 1415 1750 V 17 w(count,)h(recv)p 1639 1750 V 16 w(t)o(yp)q(e,)g (source,)75 1806 y(tag,)j(comm,)e(status\))k FE(is)75 1894 y Fp(MPI_ISEND\(send_buf,)k(send_count,)i(send_type,)f(dest,)h (tag,)h(comm,)f(request[0]\))75 1950 y(MPI_IRECV\(recv_buf,)e (recv_count,)i(recv_type,)f(source,)h(tag,)g(comm,)h(request[1]\))75 2007 y(MPI_WAITALL\(2,)e(request,)h(status_array\))75 2063 y(status)g(=)h(status_array[1])166 2151 y FE(The)13 b(non)o(blo)q(c)o(king)h(sends)f(and)g(receiv)o(es)h(can)f(b)q(e)g (replaced)h(b)o(y)f(their)g(primitiv)o(e)h(implemen)o(tation.)166 2207 y(The)j(function)g Fx(MPI)p 525 2207 V 16 w(SENDRECV)p 773 2207 V 18 w(REPLA)o(CE)g FE(can)g(b)q(e)g(handled)h(in)g(a)e (similar)i(manner;)f(a)f(tem-)75 2264 y(p)q(orary)j(bu\013er)h(need)g (b)q(e)h(allo)q(cated)f(to)f(replicate)i(the)f(send)g(and)g(receiv)o(e) h(bu\013er.)33 b(A)20 b(size)g(for)g(this)75 2320 y(bu\013er)15 b(can)h(b)q(e)f(computed)h(using)g(the)f(function)h Fx(MPI)p 1020 2320 V 16 w(TYPE)p 1153 2320 V 17 w(SIZE)p FE(.)75 2440 y Fq(3.14.8)49 b(Derived)16 b(datat)o(yp)q(es)75 2525 y FE(W)l(e)22 b(ha)o(v)o(e)g(outlined)h(in)g(section)g(3.13.1)d (ho)o(w)i(eac)o(h)g(datat)o(yp)q(e)f(constructor)g(can)i(b)q(e)f (expressed)h(in)75 2582 y(terms)h(of)g(the)g(next)h(one.)47 b(Th)o(us)25 b(all)g(datat)o(yp)q(e)f(constructors)g(can)g(b)q(e)h (expressed)g(in)h(terms)d(of)75 2638 y(the)12 b(constructor)f Fx(MPI)p 472 2638 V 16 w(TYPE)p 605 2638 V 16 w(STRUCT)p FE(;)h(the)g(functions)h Fx(MPI)p 1172 2638 V 15 w(TYPE)p 1304 2638 V 17 w(FREE)p FE(,)f Fx(MPI)p 1537 2638 V 16 w(TYPE)p 1670 2638 V 17 w(EXTENT)p FE(,)75 2695 y Fx(MPI)p 160 2695 V 16 w(TYPE)p 293 2695 V 17 w(SIZE)p FE(,)i Fx(MPI)p 512 2695 V 16 w(TYPE)p 645 2695 V 17 w(COUNT)i FE(and)f Fx(MPI)p 1004 2695 V 16 w(ADDRESS)h FE(are)f(primitiv)o(e.) 1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 71 76 71 75 bop 75 357 a Fw(Chapter)34 b(4)75 565 y FA(Collecti)q(v)m(e)42 b(Comm)m(unication)75 806 y Fv(4.1)59 b(Intro)r(duction)75 909 y FE(Collectiv)o(e)13 b(comm)o(unication)f(is)g(de\014ned)h(to)e(b) q(e)h(comm)o(unication)g(that)f(in)o(v)o(olv)o(es)h(a)f(group)g(of)g (pro)q(cesses.)75 965 y(The)k(functions)h(pro)o(vided)g(b)o(y)g(the)f (MPI)g(collectiv)o(e)i(comm)o(unication)f(include:)143 1061 y FC(\017)23 b FE(Broadcast)14 b(from)g(one)i(mem)o(b)q(er)f(to)g (all)h(mem)o(b)q(ers)f(of)g(a)g(group.)143 1157 y FC(\017)23 b FE(Barrier)15 b(sync)o(hronization)h(across)f(all)h(group)f(mem)o(b)q (ers)143 1253 y FC(\017)23 b FE(Gather)14 b(data)h(from)f(all)i(group)f (mem)o(b)q(ers)g(to)g(one)g(mem)o(b)q(er.)143 1349 y FC(\017)23 b FE(Scatter)14 b(data)h(from)f(one)i(mem)o(b)q(er)f(to)g (all)h(mem)o(b)q(ers)f(of)g(a)g(group.)143 1445 y FC(\017)23 b FE(Global)15 b(op)q(erations)g(suc)o(h)h(as)e(sum,)h(max,)f(min,)i (etc.,)e(w)o(ere)h(the)g(result)g(is)h(kno)o(wn)e(b)o(y)h(all)h(group) 189 1501 y(mem)o(b)q(ers)e(and)h(a)g(v)m(ariation)g(where)g(the)g (result)g(is)g(kno)o(wn)f(b)o(y)h(only)g(one)g(mem)o(b)q(er.)20 b(The)15 b(abilit)o(y)189 1558 y(to)f(ha)o(v)o(e)h(user)g(de\014ned)i (global)f(op)q(erations.)143 1654 y FC(\017)23 b FE(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g(called)h (parallel)h(pre\014x\).)143 1749 y FC(\017)23 b FE(Broadcast)14 b(from)g(all)j(mem)o(b)q(ers)e(to)f(all)j(mem)o(b)q(ers)e(of)g(a)g (group.)143 1845 y FC(\017)23 b FE(Scatter/Gather)10 b(data)h(from)h(all)g(mem)o(b)q(ers)g(to)g(all)g(mem)o(b)q(ers)g(of)g (a)g(group)f(\(also)h(called)h(complete)189 1902 y(exc)o(hange)i(or)g (all-to-all\).)75 1998 y(While)23 b(v)o(endors)f(ma)o(y)f(optimize)i (some)e(of)h(these)g(collectiv)o(e)h(routines)f(for)g(their)g(arc)o (hitectures,)h(a)75 2054 y(complete)15 b(library)g(of)f(the)h (collectiv)o(e)h(comm)o(unication)f(routines)g(can)g(b)q(e)g(written)f (en)o(tirely)i(using)f(the)75 2111 y(MPI)g(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unication)g(functions.)166 2168 y(A)i(collectiv)o(e)i(op)q (eration)e(is)g(executed)h(b)o(y)f(ha)o(ving)g(all)h(pro)q(cesses)f(in) h(the)f(group)g(call)h(the)f(com-)75 2224 y(m)o(unication)f(routine,)f (with)g(matc)o(hing)g(argumen)o(ts.)21 b(The)16 b(syn)o(tax)f(and)h (seman)o(tics)g(of)f(the)h(collectiv)o(e)75 2280 y(op)q(erations)e(are) g(de\014ned)i(to)e(b)q(e)g(consisten)o(t)h(with)f(the)h(syn)o(tax)e (and)i(seman)o(tics)f(of)g(the)g(p)q(oin)o(t-to-p)q(oin)o(t)75 2337 y(op)q(erations.)25 b(Th)o(us)17 b(general)g(datat)o(yp)q(es)f (are)h(allo)o(w)o(ed)g(and)g(m)o(ust)f(matc)o(h)h(b)q(et)o(w)o(een)g (sending)h(and)f(re-)75 2393 y(ceiving)j(pro)q(cesses)g(as)e(sp)q (eci\014ed)j(in)f(the)f(c)o(hapter)f(on)h(p)q(oin)o(t-to-p)q(oin)o(t)h (functions.)31 b(One)20 b(of)e(the)h(k)o(ey)75 2450 y(argumen)o(ts)12 b(is)h(a)g(comm)o(unicator)f(that)g(de\014nes)i(the)f(group)g(of)f (participating)i(pro)q(cesses)f(and)g(pro)o(vides)75 2506 y(a)j(con)o(text)g(for)g(the)g(op)q(eration.)24 b(Sev)o(eral)17 b(collectiv)o(e)h(routines)f(suc)o(h)f(as)g(broadcast)g (and)h(gather)e(ha)o(v)o(e)75 2563 y(a)g(single)i(originating)f(or)f (receiving)i(pro)q(cess.)k(In)16 b(this)g(c)o(hapter)f(suc)o(h)h(pro)q (cesses)g(are)f(called)i(the)e Fs(r)n(o)n(ot)p FE(.)75 2619 y(Some)e(argumen)o(ts)f(in)h(the)g(collectiv)o(e)i(functions)e (are)g(sp)q(eci\014ed)h(as)f(\\signi\014can)o(t)g(only)h(at)e(ro)q (ot".)18 b(These)p 75 2661 720 2 v 127 2688 a FB(1)144 2704 y Fu(V)m(ersion)c(of)f(Oct)f(18,)h(1993)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 72 77 72 76 bop 75 -100 a FE(72)703 b Ft(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y FE(argumen)o(ts)c(are)h (ignored)g(for)g(all)h(participan)o(ts)f(except)g(the)h(ro)q(ot,)e(and) h(can)g(b)q(e)h(set)e(to)h(an)o(y)f(v)m(alue.)21 b(The)75 102 y(reader)d(is)g(referred)g(to)f(c)o(hapter)g(3)g(for)h(information) f(concerning)i(comm)o(unication)g(bu\013ers,)e(general)75 158 y(datat)o(yp)q(es)11 b(and)g(t)o(yp)q(e)h(matc)o(hing)f(rules;)i (and)f(to)e(c)o(hapter)i(5)f(for)f(information)i(on)f(ho)o(w)g(to)g (de\014ne)h(groups)75 214 y(and)j(create)g(comm)o(unicators.)166 280 y(Collectiv)o(e)i(routines)g(can)f(\(but)g(are)g(not)g(required)h (to\))e(return)i(as)e(so)q(on)h(as)g(their)h(participation)75 336 y(in)i(the)g(collectiv)o(e)h(comm)o(unication)f(is)g(complete.)31 b(The)19 b(completion)g(of)f(a)h(call)g(indicates)h(that)e(the)75 393 y(caller)d(is)g(no)o(w)e(free)h(to)g(access)g(the)g(lo)q(cations)h (in)g(the)f(comm)o(unication)g(bu\013er,)g(or)g(an)o(y)g(other)f(lo)q (cation)75 449 y(that)g(can)g(b)q(e)i(referenced)f(b)o(y)g(the)f (collectiv)o(e)i(op)q(eration.)20 b(It)14 b(do)q(es)f(not)h(indicate)h (that)d(other)i(pro)q(cesses)75 506 y(in)g(the)f(group)f(ha)o(v)o(e)h (started)f(the)h(op)q(eration)g(\(unless)h(otherwise)f(indicated)i(in)f (the)f(description)h(of)f(the)75 562 y(op)q(eration\).)19 b(The)11 b(successful)i(completion)g(of)e(a)h(collectiv)o(e)h(comm)o (unication)f(call)h(ma)o(y)e(dep)q(end)i(on)f(the)75 619 y(execution)i(of)e(a)h(matc)o(hing)f(call)i(at)e(all)i(pro)q (cesses)f(in)h(the)f(group.)18 b(Th)o(us,)13 b(a)g(collectiv)o(e)h (comm)o(unication)75 675 y(call)23 b(ma)o(y)l(,)g(or)e(ma)o(y)h(not,)h (ha)o(v)o(e)e(the)h(e\013ect)g(of)g(sync)o(hronizing)h(all)g(calling)h (pro)q(cesses.)41 b(Collectiv)o(e)75 732 y(comm)o(unication)19 b(calls)g(ma)o(y)f(use)h(the)f(same)g(comm)o(unicators)g(as)g(p)q(oin)o (t)h(to)f(p)q(oin)o(t)h(comm)o(unication;)75 788 y(MPI)f(guarnatees)g (that)f(messages)h(generated)g(on)g(b)q(ehalf)h(of)f(collectiv)o(e)i (comm)o(unication)f(calls)g(will)75 844 y(not)c(b)q(e)h(confused)g (with)f(messages)g(generated)g(b)o(y)g(p)q(oin)o(t)h(to)e(p)q(oin)o(t)i (comm)o(unication.)166 910 y(A)e(more)g(detailed)i(discussion)g(of)e (the)g(correct)g(use)h(of)e(the)i(collectiv)o(e)h(routines)f(can)f(b)q (e)h(found)g(at)75 966 y(the)g(end)h(of)f(this)h(c)o(hapter.)75 1161 y Fv(4.2)59 b(Communication)18 b(F)n(unctions)75 1280 y FE(The)d(k)o(ey)h(concept)f(of)g(the)g(collectiv)o(e)i (functions)f(is)g(to)f(ha)o(v)o(e)f(a)h(\\group")g(of)g(participating)h (pro)q(cesses.)75 1336 y(The)d(routines)f(do)h(not)e(ha)o(v)o(e)h(a)g (group)g(iden)o(ti\014er)i(as)e(an)g(explicit)j(argumen)o(t.)j (Instead,)13 b(there)f(is)h(a)f(com-)75 1393 y(m)o(unicator)g(argumen)o (t.)18 b(In)12 b(this)g(c)o(hapter)g(a)f(comm)o(unicator)g(can)h(b)q(e) h(though)o(t)e(of)g(as)g(a)h(group)f(iden)o(ti\014er)75 1449 y(link)o(ed)20 b(with)e(a)g(con)o(text.)27 b(\(In)o(ter-comm)o (unicators,)18 b(that)f(is)i(b)q(et)o(w)o(een)f(groups)f(comm)o (unicators,)h(are)75 1506 y(not)d(allo)o(w)o(ed)g(in)h(the)g(collectiv) o(e)h(functions.\))75 1700 y Fv(4.3)59 b(Ba)n(rrier)21 b(synchronization)75 1867 y Fx(MPI)p 160 1867 14 2 v 16 w(BARRIER\()16 b(comm)d(\))117 1953 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))75 2086 y Fp(int)23 b(MPI)p 245 2086 15 2 v 17 w(BARRIER\(MPI)p 526 2086 V 16 w(Comm)g(comm)h(\))75 2182 y(MPI)p 150 2182 V 17 w(BARRIER\(COMM\))170 2238 y(INTEGER)f(COMM,)h(IERROR)166 2334 y Fx(MPI)p 251 2334 14 2 v 16 w(BARRIER)15 b FE(blo)q(c)o(ks)g (the)f(caller)h(un)o(til)g(all)f(group)g(mem)o(b)q(ers)g(ha)o(v)o(e)f (called)j(it;)e(the)g(call)h(returns)75 2390 y(at)g(an)o(y)f(pro)q (cess)i(only)g(after)e(all)i(group)f(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o (tered)h(the)f(call.)75 2585 y Fv(4.4)59 b(Data)19 b(move)g(functions) 75 2704 y FE(Figure)c(4.1)g(illustrates)h(the)f(the)g(di\013eren)o(t)h (collectiv)o(e)h(mo)o(v)o(e)d(functions)i(supp)q(orted)g(b)o(y)f(MPI.) 1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 73 78 73 77 bop 75 -100 a Ft(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FE(73)75 2420 y @beginspecial @setspecial %%BeginDocument: coll-fig1.ps /arrowdict 13 dict def % Local storage for the procedure % ``arrow.'' /arrow % The procedure ``arrow'' adds an { arrowdict begin % arrow shape to the current path. /headlength exch def % It takes seven arguments: the x /halfheadthickness exch 2 div def % and y coordinates of the tail /halfthickness exch 2 div def % (imagine that a line has been /tipy exch def /tipx exch def % drawn down the center of the /taily exch def /tailx exch def % arrow from the tip to the tail, % then x and y lie on this line), % the x and y coordinates of the % tip of the arrow, the thickness % of the arrow in the tail % portion, the thickness of the % arrow at the widest part of the % arrowhead and the length of the % arrowhead. /dx tipx tailx sub def % Compute the differences in x and /dy tipy taily sub def % y for the tip and tail. These /arrowlength dx dx mul dy dy mul add % will be used to compute the sqrt def % length of the arrow and to /angle dy dx atan def % compute the angle of direction % that the arrow is facing with % respect to the current user % coordinate system origin. /base arrowlength headlength sub def % Compute where the base of the % arrowhead will be. /savematrix matrix currentmatrix def % Save the current user coordinate % system. We are using the same % strategy to localize the effect % of transformations as was used % in the program to draw an % ellipse. tailx taily translate % Translate to the starting point % of the tail. angle rotate % Rotate the x-axis to correspond % with the center line of the % arrow. 0 halfthickness neg moveto % Add the arrow shape to the % current path. base halfthickness neg lineto base halfheadthickness neg lineto arrowlength 0 lineto base halfheadthickness lineto base halfthickness lineto 0 halfthickness lineto closepath savematrix setmatrix % Restore the current user % coordinate system. end } def /Box { /height exch def /length exch def length 0 rlineto 0 height rlineto length neg 0 rlineto closepath } def /Gdict 200 dict def /Grid { Gdict begin /ny exch def /nx exch def /dely exch def /delx exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def 0 1 ny { pop x y moveto lenx 0 rlineto stroke /y y dely add def} for /y ypos def /x xpos def 0 1 nx { pop x y moveto 0 leny rlineto stroke /x x delx add def} for end } def /GLdict 300 dict def /GridLabels { GLdict begin /shift exch def /raise exch def /yoff exch def /xoff exch def /p1 exch def /p2 exch def /ny exch def /nx exch def /dely exch def /delx exch def /Darray exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def /dx3 delx 3 div def /dy3 dely 3 div def /ix -1 def /iy ny 1 sub def Darray{ aload pop /Subc exch def /Text exch def /ix ix 1 add def ix nx ge { /ix 0 def /iy iy 1 sub def} if /x xpos delx ix 0.5 add mul add /Helvetica findfont p1 scalefont setfont Text stringwidth pop /Helvetica findfont p2 scalefont setfont Subc stringwidth pop add xoff add 2 div sub shift add def /y ypos dely iy 0.5 add mul add raise add def x y moveto /Helvetica findfont p1 scalefont Text show xoff yoff rmoveto /Helvetica findfont p2 scalefont Subc show } forall end clear } def 2 setlinecap 6.5 72 mul 320 sub 2 div 0 translate 0 150 moveto 20 20 6 6 Grid 0 150 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 225 200 225 12 24 18 arrow stroke 200 195 140 195 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (one-all scatter) dup stringwidth pop 170 exch 2 div sub 242 moveto show (one-all gather) dup stringwidth pop 170 exch 2 div sub 170 moveto show 220 150 moveto 20 20 6 6 Grid 220 150 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(1)] [()()] [()()] [()()] [()()] [()()] [(A)(2)] [()()] [()()] [()()] [()()] [()()] [(A)(3)] [()()] [()()] [()()] [()()] [()()] [(A)(4)] [()()] [()()] [()()] [()()] [()()] [(A)(5)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 0 moveto 20 20 6 6 Grid 0 0 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [(B)(0)] [(B)(1)] [(B)(2)] [(B)(3)] [(B)(4)] [(B)(5)] [(C)(0)] [(C)(1)] [(C)(2)] [(C)(3)] [(C)(4)] [(C)(5)] [(D)(0)] [(D)(1)] [(D)(2)] [(D)(3)] [(D)(4)] [(D)(5)] [(E)(0)] [(E)(1)] [(E)(2)] [(E)(3)] [(E)(4)] [(E)(5)] [(F)(0)] [(F)(1)] [(F)(2)] [(F)(3)] [(F)(4)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 220 0 moveto 20 20 6 6 Grid 220 0 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(1)] [(B)(1)] [(C)(1)] [(D)(1)] [(E)(1)] [(F)(1)] [(A)(2)] [(B)(2)] [(C)(2)] [(D)(2)] [(E)(2)] [(F)(2)] [(A)(3)] [(B)(3)] [(C)(3)] [(D)(3)] [(E)(3)] [(F)(3)] [(A)(4)] [(B)(4)] [(C)(4)] [(D)(4)] [(E)(4)] [(F)(4)] [(A)(5)] [(B)(5)] [(C)(5)] [(D)(5)] [(E)(5)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 60 200 60 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (all-all scatter) dup stringwidth pop 170 exch 2 div sub 77 moveto show 0 300 moveto 20 20 6 6 Grid 0 300 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(B)(0)] [()()] [()()] [()()] [()()] [()()] [(C)(0)] [()()] [()()] [()()] [()()] [()()] [(D)(0)] [()()] [()()] [()()] [()()] [()()] [(E)(0)] [()()] [()()] [()()] [()()] [()()] [(F)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 360 200 360 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (all gather) dup stringwidth pop 170 exch 2 div sub 377 moveto show 220 300 moveto 20 20 6 6 Grid 220 300 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 450 moveto 20 20 6 6 Grid 0 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 510 200 510 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont 0 575 moveto (data) show (data) stringwidth pop 4 add 575 4 add (data) stringwidth pop 33 add 575 4 add 1 4 5 arrow fill (one-all broadcast) dup stringwidth pop 170 exch 2 div sub 527 moveto show gsave 0 570 (processes) stringwidth pop sub translate 90 rotate 0 5 moveto (processes) show -4 8 -33 8 1 4 5 arrow fill grestore 220 450 moveto 20 20 6 6 Grid 220 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels showpage %%EndDocument @endspecial 98 x(Figure)15 b(4.1:)k(Collectiv)o(e)e(mo)o(v)o(e)d (functions)i(illustrated)h(for)e(a)f(group)h(of)g(six)h(pro)q(cesses.)k (In)c(eac)o(h)f(case,)75 2574 y(eac)o(h)g(ro)o(w)g(of)g(b)q(o)o(xes)g (represen)o(ts)g(data)g(lo)q(cations)h(in)g(one)g(pro)q(cess.)k(Th)o (us,)15 b(in)h(the)f(one-all)i(broadcast,)75 2631 y(initially)k(just)e (the)f(\014rst)h(pro)q(cess)g(con)o(tains)f(the)h(data)f Fy(A)1085 2638 y Fj(0)1105 2631 y FE(,)h(but)g(after)e(the)i(broadcast) f(all)i(pro)q(cesses)75 2687 y(con)o(tain)15 b(it.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 74 79 74 78 bop 75 -100 a FE(74)703 b Ft(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fq(4.4.1)49 b(Broadcast)90 181 y Fx(MPI)p 175 181 14 2 v 16 w(BCAST\()16 b(bu\013er,)f(count,)h(datat)o(yp)q(e,)g(ro)q(ot,)f(comm)e(\))117 259 y Fr(INOUT)62 b Fx(bu\013er)478 b Fr(starting)14 b(address)h(of)f(bu\013er)g(\(c)o(hoice\))117 337 y(IN)171 b Fx(count)466 b Fr(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e (bu\013er)i(\(in)o(teger\))117 414 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(bu\013er)117 492 y(IN)171 b Fx(ro)q(ot)492 b Fr(rank)14 b(of)f(broadcast)i(ro)q(ot)f(\(in)o (teger\))117 569 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 695 y Fp(int)47 b(MPI)p 269 695 15 2 v 17 w(BCAST\(void*)22 b(buf,)i(int)f(count,)g(MPI)p 1026 695 V 17 w(Datatype)g(datatype,)f(int)i(root,)393 752 y(MPI)p 468 752 V 17 w(Comm)f(comm)h(\))75 839 y(MPI)p 150 839 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g(ROOT,)g(COMM\))170 896 y(INTEGER)g(BUFFER,)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR) 166 983 y Fx(MPI)p 251 983 14 2 v 16 w(BCAST)11 b FE(broadcasts)e(a)h (message)g(from)g(the)g(pro)q(cess)h(with)f(rank)21 b Fx(ro)q(ot)10 b FE(to)f(all)j(other)e(pro)q(cesses)75 1040 y(of)18 b(the)g(group.)28 b(It)18 b(is)h(called)h(b)o(y)e(all)h (mem)o(b)q(ers)f(of)g(group)g(using)g(the)h(same)e(argumen)o(ts)h(for) 35 b Fx(comm,)75 1096 y(ro)q(ot)16 b FE(and)g(matc)o(hing)g(argumen)o (ts)f(for)31 b Fx(count,)17 b(datat)o(yp)q(e)p FE(.)23 b(On)17 b(return)f(the)g(con)o(ten)o(ts)f(of)g(the)h(bu\013er)g(of)75 1153 y(the)f(pro)q(cess)h(with)f(rank)30 b Fx(ro)q(ot)15 b FE(is)h(con)o(tained)g(in)g(the)f(bu\013er)g(of)g(the)g(calling)i (pro)q(cess.)75 1281 y Fq(4.4.2)49 b(Gather)75 1417 y Fx(MPI)p 160 1417 V 16 w(GA)l(THER\()16 b(sendbuf,)h(sendcount,)h (sendt)o(yp)q(e,)f(recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e (comm\))117 1495 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1573 y(IN)171 b Fx(sendcount)383 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1651 y(IN)171 b Fx(sendt)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)117 1728 y(OUT)124 b Fx(recvbuf)434 b Fr(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i (signi\014can)o(t)d(only)g(at)905 1785 y(ro)q(ot\))117 1862 y(IN)171 b Fx(recvcount)389 b Fr(n)o(um)o(b)q(er)11 b(of)g(elemen)o(ts)g(for)g(an)o(y)g(single)g(receiv)o(e)i(\(in)o (teger,)f(sig-)905 1919 y(ni\014can)o(t)i(only)f(at)h(ro)q(ot\))117 1996 y(IN)171 b Fx(recvt)o(yp)q(e)414 b Fr(data)16 b(t)o(yp)q(e)g(of)f (recv)i(bu\013er)g(elemen)o(ts)f(\(signi\014can)o(t)f(only)g(at)905 2053 y(ro)q(ot\))117 2130 y(IN)171 b Fx(ro)q(ot)492 b Fr(rank)14 b(of)f(receiving)i(pro)q(cess)g(\(in)o(teger\))117 2208 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 2334 y Fp(int)23 b(MPI)p 245 2334 15 2 v 17 w(GATHER\(void*)f(sendbuf,) h(int)h(sendcnt,)e(MPI)p 1169 2334 V 17 w(Datatype)h(sendtype,)393 2390 y(void*)g(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 2390 V 17 w(Datatype,)g(int)i(root,)393 2447 y(MPI)p 468 2447 V 17 w(Comm)f(comm\))75 2534 y(MPI)p 150 2534 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 2591 y(ROOT,)g(COMM\))170 2647 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2704 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 75 80 75 79 bop 75 -100 a Ft(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FE(75)166 45 y(Eac)o(h)17 b(pro)q(cess)g(\(including) i(the)e(ro)q(ot)f(pro)q(cess\))h(sends)g(the)g(con)o(ten)o(ts)g(of)f (its)h(send)h(bu\013er)f(to)f(the)75 102 y(ro)q(ot)i(pro)q(cess.)31 b(The)19 b(ro)q(ot)f(pro)q(cess)h(receiv)o(es)h(the)f(messages)f(at)g (stores)g(them)h(in)h(rank)e(order.)31 b(The)75 158 y(outcome)15 b(is)h(as)e(if)i(eac)o(h)f(of)g(the)g Fp(n)g FE(pro)q(cesses)h(in)g (the)f(group)g(had)h(executed)g(a)f(call)h(to,)465 244 y Fp(MPI)p 540 244 15 2 v 17 w(SEND)p FE(\()p Fp(sendbuf)p Fy(;)8 b Fp(sendcount)o Fy(;)g Fp(s)o(endtype)o Fy(;)g Fp(ro)o(ot)p Fy(;)f(:::)o FE(\))p Fy(;)75 330 y FE(and)15 b(the)h(ro)q(ot)e(had)h(executed)i Fp(n)e FE(calls)h(to,)477 416 y Fp(MPI)p 552 416 V 16 w(RECV)p FE(\()p Fp(recvbuf)p FE([)p Fp(i)p FE(])p Fy(;)8 b Fp(recv)o(count)p Fy(;)f Fp(recv)o(type)p Fy(;)g Fp(i)p Fy(;)g(::)o(:)p FE(\))p Fy(;)75 501 y FE(where,)495 558 y Fp(recvbuf)p FE([)p Fp(i)p FE(])j(=)j Fp(recvbuf)c FE(+)i Fp(i)f FC(\002)g Fp(extent)p FE(\()p Fp(recvtype)p FE(\))p Fy(:)166 633 y FE(An)18 b(alternativ)o(e)h(description)h(is)e(that)g(the)g Fp(n)g FE(messages)g(sen)o(t)g(b)o(y)g(the)g(pro)q(cesses)h(in)g(the)f (group)75 690 y(are)c(concatenated)h(in)g(rank)f(order,)g(and)h(the)g (resulting)g(message)f(is)h(receiv)o(ed)h(b)o(y)e(the)h(ro)q(ot)f(as)g (if)h(b)o(y)f(a)75 746 y(call)i(to)f Fx(MPI)p 299 746 14 2 v 16 w(RECV\(recvbuf,)h(recvcount)g FC(\002)p Fx(n,)g(rect)o(yp)q (e,)f(...\))p FE(.)166 803 y(The)g(receiv)o(e)h(bu\013er)g(is)f (ignored)h(for)f(all)h(non-ro)q(ot)f(pro)q(cesses.)166 859 y(General)21 b(deriv)o(ed)h(datat)o(yp)q(es)f(are)f(allo)o(w)o(ed)i (for)e(b)q(oth)42 b Fx(sendt)o(yp)q(e)24 b FE(and)42 b Fx(recvt)o(yp)q(e)p FE(.)c(The)21 b(t)o(yp)q(e)75 916 y(signature)f(of)40 b Fx(sendcount,)24 b(sendt)o(yp)q(e)f FE(on)d(pro)q(cess)h(i)g(m)o(ust)f Fs(exactly)g FE(matc)o(h)f(the)i(t)o (yp)q(e)f(signature)g(of)75 972 y Fx(recvcount,)c(recvt)o(yp)q(e)p FE(,)f(and)g(the)g(t)o(yp)q(e)f(signature)h(of)29 b Fx(sendcount,)17 b(sendt)o(yp)q(e)h FE(m)o(ust)c(b)q(e)h(the)g(same)f(on)h(all)75 1028 y(pro)q(cesses.)23 b(Exact)15 b(matc)o(hing)h(means)g(the)g(basic) h(t)o(yp)q(es)f(in)h(the)f(t)o(yp)q(e)g(signature)g(m)o(ust)f(matc)o(h) h(\(as)f(in)75 1085 y(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unications\)) g(and)f(the)h(n)o(um)o(b)q(er)f(of)g(basic)h(t)o(yp)q(es)f(m)o(ust)g(b) q(e)h(the)g(same)f(on)g(b)q(oth)75 1141 y(sides)g(\(in)g(con)o(trast)e (with)h(p)q(oin)o(t-to-p)q(oin)o(t\).)166 1198 y(All)g(argumen)o(ts)f (to)f(the)h(function)h(are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fx(ro)q(ot)p FE(,)e(while)j(on)e(other)g(pro)q(cesses,)75 1254 y(only)19 b(argumen)o(ts)f Fx(sendbuf,)k(sendcount,)g(sendt)o(yp)q (e,)g(ro)q(ot,)d(comm)e FE(are)i(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 1311 y Fx(ro)q(ot)c FE(and)g Fx(comm)f FE(m)o(ust)g(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)f(on)f (all)i(pro)q(cesses.)75 1414 y Fx(MPI)p 160 1414 V 16 w(GA)l(THERV\()24 b(sendbuf,)j(sendcount,)g(sendt)o(yp)q(e,)g(recvbuf,) e(displs,)h(recvcounts,)g(recvt)o(yp)q(e,)g(ro)q(ot,)75 1471 y(comm\))117 1548 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1621 y(IN)171 b Fx(sendcount)383 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1693 y(IN)171 b Fx(sendt)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)117 1765 y(OUT)124 b Fx(recvbuf)434 b Fr(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i (signi\014can)o(t)d(only)g(at)905 1822 y(ro)q(ot\))117 1894 y(IN)171 b Fx(displs)469 b Fr(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fl(i)f Fr(sp)q(eci\014es) 905 1951 y(the)f(displacemen)o(t)e(\(relativ)o(e)g(to)h Fm(recvbuf)i Fr(at)d(whic)o(h)h(to)g(place)905 2007 y(the)j(incoming)e (data)h(from)f(pro)q(cess)k Fl(i)d Fr(\(signi\014can)o(t)g(only)g(at) 905 2064 y(ro)q(ot\))117 2136 y(IN)171 b Fx(recvcounts)372 b Fr(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(con)o (taining)e(the)i(n)o(um-)905 2192 y(b)q(er)20 b(of)e(elemen)o(ts)h (that)f(are)i(receiv)o(ed)g(from)d(eac)o(h)i(pro)q(cess)905 2249 y(\(signi\014can)o(t)14 b(only)f(at)h(ro)q(ot\))117 2321 y(IN)171 b Fx(recvt)o(yp)q(e)414 b Fr(data)16 b(t)o(yp)q(e)g(of)f (recv)i(bu\013er)g(elemen)o(ts)f(\(signi\014can)o(t)f(only)g(at)905 2378 y(ro)q(ot\))117 2450 y(IN)171 b Fx(ro)q(ot)492 b Fr(rank)14 b(of)f(receiving)i(pro)q(cess)g(\(in)o(teger\))117 2523 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 2647 y Fp(int)23 b(MPI)p 245 2647 15 2 v 17 w(GATHERV\(void*)f (sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 2647 V 17 w(Datatype)h (sendtype,)393 2704 y(void*)g(recvbuf,)g(int)h(*displs,)e(int)i (*recvcounts,)e(MPI)p 1542 2704 V 17 w(Datatype,)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 76 81 76 80 bop 75 -100 a FE(76)703 b Ft(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)393 45 y Fp(int)24 b(root,)f(MPI)p 707 45 15 2 v 17 w(Comm)g(comm\))75 132 y(MPI)p 150 132 V 17 w(GATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)g (RECVBUF,)f(DISPLS,)h(RECVCOUNTS,)393 188 y(RECVTYPE,)g(ROOT,)g(COMM\)) 170 244 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 301 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(DISPLS,)h(RECVCOUNTS,)g(RECVTYPE,) f(ROOT,)i(COMM,)170 357 y(IERROR)166 444 y Fx(MPI)p 251 444 14 2 v 16 w(GA)l(THERV)15 b FE(extends)g(the)f(functionalit)o(y)h (of)e Fx(MPI)p 1126 444 V 16 w(GA)l(THER)i FE(b)o(y)f(allo)o(wing)h(a)f (v)m(arying)g(coun)o(t)75 500 y(of)e(data)h(from)f(eac)o(h)g(pro)q (cess)i(\(since)f Fx(recvcounts)i FE(is)e(no)o(w)f(an)h(arra)o(y\),)e (and)i(also)g(allo)o(ws)g(more)f(\015exibili)q(t)o(y)75 557 y(as)j(to)f(where)i(the)f(data)g(is)g(placed)i(on)e(the)g(ro)q(ot)g (\(b)o(y)f(pro)o(viding)i(the)g(new)f(argumen)o(t,)f Fx(displs)p FE(\).)166 613 y(The)h(outcome)g(is)h(as)f(if)g(eac)o(h)h (pro)q(cess)f(has)g(sen)o(t)g(a)g(message)g(to)f(the)i(ro)q(ot,)465 712 y Fp(MPI)p 540 712 15 2 v 17 w(SEND)p FE(\()p Fp(sendbuf)p Fy(;)8 b Fp(sendcount)o Fy(;)g Fp(s)o(endtype)o Fy(;)g Fp(ro)o(ot)p Fy(;)f(:::)o FE(\))p Fy(;)75 811 y FE(and)15 b(the)h(ro)q(ot)e(executed)i Fp(n)f FE(receiv)o(es,)164 910 y Fp(MPI)p 239 910 V 17 w(RECV)p FE(\()p Fp(recvbuf)8 b FE(+)j Fp(disp)p FE([)p Fp(i)p FE(])d FC(\002)j Fp(extent)p FE(\()p Fp(recvtype)p FE(\))p Fy(;)c Fp(recvcou)o(nt)p FE([)p Fp(i)p FE(])o Fy(;)h Fp(r)o(ecvtype)o Fy(;)g Fp(i)o Fy(;)g(:)o(::)p FE(\))o Fy(:)166 1010 y FE(Messages)14 b(are)g(placed)i(in)f(the)g(receiv)o(e)g(bu\013er)g(of)f(the)g(ro)q(ot) g(pro)q(cess)h(in)g(rank)f(order,)g(that)g(is,)h(the)75 1066 y(data)d(sen)o(t)g(from)f(pro)q(cess)i Fp(j)f FE(is)h(placed)h(in) f(the)f Fp(j)p FE(-th)h(p)q(ortion)f(of)g(the)h(receiv)o(e)g(bu\013er)f Fx(recvbuf)h FE(on)g(pro)q(cess)75 1122 y Fx(ro)q(ot)p FE(.)29 b(The)18 b Fp(j)p FE(-th)g(p)q(ortion)h(of)f Fx(recvbuf)h FE(b)q(egins)g(at)f(o\013set)f Fx(displs[j])j FE(elemen)o(ts)f(\(in)g(terms)f(of)f Fx(recvt)o(yp)q(e)p FE(\))75 1179 y(in)o(to)e Fx(recvbuf)p FE(.)166 1235 y(The)d(t)o(yp)q(emap)f(implied)j(b)o(y)e Fx(sendcount,)j(sendt)o(yp)q (e)f FE(need)e(not)g(b)q(e)g(the)g(same)f(for)g(eac)o(h)h(pro)q(cess,)g (but)75 1292 y(it)k(still)h(m)o(ust)d(exactly)i(matc)o(h)f(the)h(t)o (yp)q(emap)f(implied)j(b)o(y)d Fx(recvcounts,)i(recvt)o(yp)q(e)p FE(.)22 b(That)14 b(is,)i(t)o(yp)q(emap)75 1348 y(of)g Fx(sendcount,)k(sendt)o(yp)q(e)f FE(on)d(pro)q(cess)h Fp(i)g FE(m)o(ust)f(exactly)h(matc)o(h)f(t)o(yp)q(emap)h(of)f Fx(recvcounts[i],)i(recvt)o(yp)q(e)75 1405 y FE(on)d(pro)q(cess)h Fx(ro)q(ot)p FE(.)166 1461 y(All)f(argumen)o(ts)f(to)f(the)h(function)h (are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fx(ro)q(ot)p FE(,)e(while)j(on)e(other)g(pro)q(cesses,)75 1518 y(only)19 b(argumen)o(ts)f Fx(sendbuf,)k(sendcount,)g(sendt)o(yp)q(e,)g(ro)q(ot,) d(comm)e FE(are)i(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 1574 y Fx(ro)q(ot)c FE(and)g Fx(comm)f FE(m)o(ust)g(ha)o(v)o(e)h(iden)o (tical)i(v)m(alues)f(on)f(all)i(pro)q(cesses.)166 1631 y(F)l(or)f(b)q(oth)h(functions,)g(the)g(sp)q(eci\014cation)h(of)f(coun) o(t\(s\),)e(t)o(yp)q(e\(s\),)h(\(displacemen)o(ts\),)i(should)f(not)75 1687 y(cause)e(an)o(y)g(lo)q(cation)h(on)f(the)h(ro)q(ot)e(to)h(b)q(e)g (written)h(more)e(than)i(once.)k(Suc)o(h)c(a)e(call)j(is)f(erroneous.) 166 1743 y(W)l(e)f(illustrate)h(the)g(matc)o(hing)f(conditions)h(with)g (the)f(follo)o(wing)h(examples.)75 1865 y Fq(4.4.3)49 b(Examples)16 b(of)h(Usage)g(of)f(MPI)p 768 1865 V 18 w(GA)l(THER,)h(MPI)p 1091 1865 V 18 w(GA)l(THERV)75 1951 y Fx(Example)e(1)75 2036 y FE(Gather)f(100)h(in)o(ts)g(from)g(ev)o(ery) g(pro)q(cess)g(in)h(group)f(to)g(ro)q(ot.)k(See)d(\014gure)f(4.2.)170 2139 y Fp(MPI_Comm)23 b(comm;)170 2195 y(MPI_Group)g(mygroup;)170 2252 y(int)h(gsize,sendarray[100];)170 2308 y(int)g(root,)f(*rbuf;)170 2421 y(...)170 2534 y(/*)h(The)f(variable)g(comm)h(is)f(set)h (elsewhere)e(in)i(the)f(program)194 2591 y(*/)170 2647 y(MPI_Comm_group\()f(comm,)h(&mygroup)g(\);)170 2704 y(MPI_Group_size\()f(mygroup,)h(&gsize)g(\);)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 77 82 77 81 bop 75 -100 a Ft(4.4.)29 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1082 b FE(77)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig2.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr n 189 244 m 189 294 l gs 0.50 setgray fill gr gs col-1 s gr n 191.000 286.000 m 189.000 294.000 l 187.000 286.000 l gs 2 setlinejoin col-1 s gr n 289 244 m 259 294 l gs 0.50 setgray fill gr gs col-1 s gr n 264.831 288.169 m 259.000 294.000 l 261.401 286.111 l gs 2 setlinejoin col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr $F2psEnd %%EndDocument endTexFig 186 511 a FE(Figure)16 b(4.2:)j(The)c(ro)q(ot)g(pro)q(cess)g(gathers)f (100)h Fp(int)p FE(s)f(from)h(eac)o(h)g(pro)q(cess)h(in)g(the)f(group.) 170 646 y Fp(rbuf)24 b(=)f(\(int)h(*\)malloc\(gsize*100*sizeof)o (\(int\)\);)170 703 y(MPI_Gather\()f(sendarray,)f(100,)i(MPI_INT,)e (rbuf,)i(100,)f(MPI_INT,)g(root,)g(comm\);)75 881 y Fx(Example)15 b(2)75 969 y FE(Do)d(the)h(same)g(as)f(in)i(previous)g(example,)f(but)g (use)h(a)e(deriv)o(ed)i(datat)o(yp)q(e.)19 b(Note)12 b(that)g(the)h(t)o(yp)q(e)g(cannot)75 1025 y(b)q(e)19 b(the)f(en)o(tire)g(set)g(of)f Fp(gsize*100)23 b(int)p FE(s)18 b(since)h(t)o(yp)q(e)f(matc)o(hing)g(is)g(de\014ned)i(pairwise) f(b)q(et)o(w)o(een)f(the)75 1082 y(ro)q(ot)c(and)i(eac)o(h)f(pro)q (cess)g(in)h(the)g(gather.)170 1194 y Fp(MPI_Comm)23 b(comm;)170 1250 y(MPI_Group)g(mygroup;)170 1306 y(int)h (gsize,sendarray[100];)170 1363 y(int)g(root,)f(*rbuf;)170 1419 y(MPI_Datatype)g(rtype;)170 1532 y(...)170 1645 y(/*)h(The)f(variable)g(comm)h(is)f(set)h(elsewhere)e(in)i(the)f (program)194 1702 y(*/)170 1758 y(MPI_Comm_group\()f(comm,)h(&mygroup)g (\);)170 1815 y(MPI_Group_size\()f(mygroup,)h(&gsize)g(\);)170 1871 y(rbuf)h(=)f(\(int)h(*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 1927 y(MPI_Type_contiguous\()e(100,)h(MPI_INT,)g(&rtype)g(\);)170 1984 y(MPI_Type_commit\()f(rtype)h(\);)170 2040 y(MPI_Gather\()g (sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(1,)f(rtype,)g(root,)g(comm\);) 75 2166 y Fx(Example)15 b(3)75 2253 y FE(No)o(w)e(ha)o(v)o(e)g(eac)o(h) h(pro)q(cess)g(send)g(100)f(in)o(ts)h(to)f(ro)q(ot,)g(but)g(place)i (eac)o(h)f(set)f(\(of)g(100\))f Fs(stride)i FE(in)o(ts)g(apart)f(at)75 2310 y(receiving)18 b(end.)24 b(Use)16 b Fx(MPI)p 547 2310 14 2 v 16 w(GA)l(THERV)i FE(and)e(the)h Fx(displs)g FE(argumen)o(t)f(to)f(ac)o(hiev)o(e)i(this)g(e\013ect.)23 b(Assume)75 2366 y Fy(str)q(ide)13 b FC(\025)g FE(100.)19 b(See)c(\014gure)h(4.3.)170 2478 y Fp(MPI_Comm)23 b(comm;)170 2534 y(MPI_Group)g(mygroup;)170 2591 y(int)h(gsize,sendarray[100];)170 2647 y(int)g(root,)f(*rbuf,)g(stride;)170 2704 y(int)h (*displs,i,*rcounts;)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 78 83 78 82 bop 75 -100 a FE(78)708 b Ft(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig3.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 189 244 m 204 294 l gs col-1 s gr n 203.617 285.763 m 204.000 294.000 l 199.786 286.912 l gs 2 setlinejoin col-1 s gr n 289 244 m 289 294 l gs col-1 s gr n 291.000 286.000 m 289.000 294.000 l 287.000 286.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 511 a FE(Figure)h(4.3:)22 b(The)16 b(ro)q(ot)g(pro)q(cess)h (gathers)f(100)g Fp(int)p FE(s)g(from)f(eac)o(h)i(pro)q(cess)g(in)g (the)g(group,)f(eac)o(h)h(set)f(is)75 568 y(placed)g Fp(stride)f FE(in)o(ts)g(apart.)170 758 y Fp(...)170 871 y(MPI_Comm_group\()22 b(comm,)h(&mygroup)g(\);)170 927 y(MPI_Group_size\()f(mygroup,)h(&gsize)g(\);)170 984 y(rbuf)h(=)f(\(int)h(*\)malloc\(gsize*stride*siz)o(eof\(int)o (\)\);)170 1040 y(displs)f(=)h(\(int)g(*\)malloc\(gsize*sizeof\(i)o (nt\)\);)170 1096 y(rcounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)170 1153 y(for)h(\(i=0;)f(i)g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2497 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)166 2587 y Fx(MPI)p 251 2587 14 2 v 16 w(SCA)l(TTER)16 b FE(is)g(the)f(in)o(v)o(erse)h(op)q(eration)f(to)g Fx(MPI)p 1097 2587 V 15 w(GA)l(THER)p FE(.)166 2647 y(The)e(ro)q(ot)g (pro)q(cess)g(sends)h(a)f(message)g(to)g(eac)o(h)g(pro)q(cess)g (\(including)j(itself)t(\);)e(eac)o(h)f(pro)q(cess)h(stores)75 2704 y(the)i(incoming)h(message)e(in)i(its)f(receiv)o(e)h(bu\013er.)22 b(The)16 b(outcome)f(is)i(as)e(if)h(the)g(ro)q(ot)f(exectuted)i Fp(n)e FE(send)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 84 89 84 88 bop 75 -100 a FE(84)703 b Ft(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y FE(op)q(erations,)249 144 y Fp(MPI)p 324 144 15 2 v 17 w(SEND)p FE(\()p Fp(sendbuf)8 b FE(+)j Fp(i)f FC(\002)g Fp(extent)p FE(\()p Fp(sendtype)p FE(\))p Fy(;)e Fp(s)o(endcount)o Fy(;)g Fp(s)o(endtype)o Fy(;)g Fp(i)p Fy(;)f(:)o(::)p FE(\))o Fy(;)75 244 y FE(and)15 b(eac)o(h)h(pro)q(cess)f(executed)h(a)f(receiv)o(e,)501 343 y Fp(MPI)p 576 343 V 17 w(RECV)p FE(\()p Fp(recvbuf)p Fy(;)8 b Fp(recvcou)o(nt)p Fy(;)f Fp(recvtyp)o(e)p Fy(;)g Fp(i)p Fy(;)h(:)o(::)o FE(\))p Fy(:)166 442 y FE(An)19 b(alternativ)o(e)h(description)g(is)g(that)f(the)g(ro)q(ot)f(send)i(a)f (message)g(with)g Fx(MPI)p 1569 442 14 2 v 16 w(SEND\(sendbuf,)75 499 y(sendcount)p FC(\002)p Fx(n)q(,)25 b(sendt)o(yp)q(e,)h(...\))p FE(;)c(this)g(message)f(is)h(split)h(in)o(to)e Fp(n)g FE(equal)i(segmen)o(ts,)f(and)f(the)h Fy(i)14 b FC(\000)g Fy(th)75 555 y FE(segmen)o(t)h(is)i(sen)o(t)e(to)g(the)h Fy(i)p FE(-th)g(pro)q(cess)g(in)g(the)g(group;)f(and)h(eac)o(h)g(pro)q (cess)g(receiv)o(es)h(this)f(message)f(as)75 612 y(ab)q(o)o(v)o(e.)166 668 y(The)j(t)o(yp)q(e)g(signature)g(of)35 b Fx(sendcount,)21 b(sendt)o(yp)q(e)f FE(m)o(ust)e Fs(exactly)f FE(matc)o(h)h(the)g(t)o (yp)q(e)f(signature)h(of)75 725 y Fx(recvcount,)e(recvt)o(yp)q(e)p FE(,)f(and)g(the)g(t)o(yp)q(e)f(signature)h(of)29 b Fx(sendcount,)17 b(sendt)o(yp)q(e)h FE(m)o(ust)c(b)q(e)h(the)g(same)f(on)h(all)75 781 y(pro)q(cesses.)166 838 y(All)g(argumen)o(ts)f(to)f(the)h(function) h(are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fx(ro)q(ot)p FE(,)e(while)j(on)e(other)g(pro)q(cesses,)75 894 y(only)f(argumen)o(ts) f Fx(recvbuf,)i(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)f FE(are)h(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fx(ro)q(ot)75 950 y FE(and)j Fx(comm)f FE(m)o(ust)g(ha)o(v)o(e)h(iden)o (tical)i(v)m(alues)g(on)e(all)h(pro)q(cesses.)75 1054 y Fx(MPI)p 160 1054 V 16 w(SCA)l(TTERV\()21 b(sendbuf,)j(displs,)f (sendcounts)q(,)h(sendt)o(yp)q(e,)g(recvbuf,)f(recvcount,)f(recvt)o(yp) q(e,)h(ro)q(ot,)75 1111 y(comm\))117 1188 y Fr(IN)171 b Fx(sendbuf)428 b Fr(address)13 b(of)d(send)i(bu\013er)h(\(c)o(hoice,) f(signi\014can)o(t)e(only)h(at)g(ro)q(ot\))117 1263 y(IN)171 b Fx(displs)469 b Fr(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g (size\).)20 b(En)o(try)15 b Fl(i)f Fr(sp)q(eci\014es)905 1319 y(the)20 b(displacemen)o(t)e(\(relativ)o(e)h(to)f Fm(sendbuf)j Fr(from)c(whic)o(h)i(to)905 1375 y(tak)o(e)14 b(the)h(outgoing)d(data)i(to)g(pro)q(cess)h Fl(i)117 1450 y Fr(IN)171 b Fx(sendcounts)366 b Fr(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(sp)q(ecifying)g(the)g(n)o (um-)905 1507 y(b)q(er)15 b(of)e(elemen)o(ts)h(to)g(send)h(to)e(eac)o (h)i(pro)q(cessor)117 1581 y(IN)171 b Fx(sendt)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)117 1656 y(OUT)124 b Fx(recvbuf)434 b Fr(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 1731 y(IN)171 b Fx(recvcount)389 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1805 y(IN)171 b Fx(recvt)o(yp)q(e)414 b Fr(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts) 117 1880 y(IN)171 b Fx(ro)q(ot)492 b Fr(rank)14 b(of)f(sending)h(pro)q (cess)i(\(in)o(teger\))117 1955 y(IN)171 b Fx(group)463 b Fr(comm)o(unicator)11 b(\(handle\))75 2079 y Fp(int)23 b(MPI)p 245 2079 15 2 v 17 w(SCATTERV\(void*)f(sendbuf,)h(int)g (*displs,)g(int)h(*sendcounts,)393 2135 y(MPI)p 468 2135 V 17 w(Datatype)f(sendtype,)f(void*)i(recvbuf,)e(int)i(recvcount,)393 2192 y(MPI)p 468 2192 V 17 w(Datatype,)f(int)g(root,)g(MPI)p 1034 2192 V 17 w(Comm)g(comm\))75 2278 y(MPI)p 150 2278 V 17 w(SCATTERV\(SENDBUF,)e(DISPLS,)i(SENDCOUNTS,)g(SENDTYPE,)f (RECVBUF,)h(RECVCOUNT,)393 2335 y(RECVTYPE,)g(ROOT,)g(COMM\))170 2391 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2448 y(INTEGER)g(DISPLS,)g(SENDCOUNTS,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,) f(ROOT,)i(COMM,)170 2504 y(IERROR)166 2591 y Fx(MPI)p 251 2591 14 2 v 16 w(SCA)l(TTERV)16 b FE(is)g(the)f(in)o(v)o(erse)h(op) q(eration)f(to)g Fx(MPI)p 1127 2591 V 16 w(GA)l(THERV)p FE(.)166 2647 y Fx(MPI)p 251 2647 V 16 w(SCA)l(TTERV)24 b FE(extends)f(the)g(functionalit)o(y)g(of)g Fx(MPI)p 1193 2647 V 15 w(SCA)l(TTER)h FE(b)o(y)f(allo)o(wing)g(a)f(v)m(arying) 75 2704 y(coun)o(t)14 b(of)g(data)g(to)g(b)q(e)h(sen)o(t)f(to)f(eac)o (h)i(pro)q(cess)f(\(since)i Fx(sendcounts)h FE(is)e(no)o(w)f(an)g(arra) o(y\),)f(and)h(also)h(allo)o(ws)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 85 90 85 89 bop 75 -100 a Ft(4.4.)29 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1082 b FE(85)75 45 y(more)21 b(\015exibili)q(t)o(y)j(as)d (to)g(where)h(the)g(data)f(is)i(tak)o(en)e(from)g(on)h(the)g(ro)q(ot)f (\(b)o(y)g(pro)o(viding)i(the)f(new)75 102 y(argumen)o(t,)14 b Fx(displs)p FE(\).)166 158 y(The)h(outcome)g(is)h(as)f(if)g(the)h(ro) q(ot)e(excuted)i Fp(n)f FE(send)h(op)q(erations,)140 262 y Fp(MPI)p 215 262 15 2 v 17 w(SEND)p FE(\()p Fp(sendbuf)8 b FE(+)j Fp(displs)p FE([)p Fp(i)p FE(])d FC(\002)j Fp(extent)p FE(\()p Fp(sendtype)p FE(\))p Fy(;)c Fp(sendco)o(unt)p FE([)p Fp(i)p FE(])o Fy(;)g Fp(sendtyp)o(e)p Fy(;)g Fp(i)p Fy(;)h(:)o(::)o FE(\))p Fy(;)75 365 y FE(and)15 b(eac)o(h)h(pro)q(cess) f(executed)h(a)f(receiv)o(e,)501 469 y Fp(MPI)p 576 469 V 17 w(RECV)p FE(\()p Fp(recvbuf)p Fy(;)8 b Fp(recvcou)o(nt)p Fy(;)f Fp(recvtyp)o(e)p Fy(;)g Fp(i)p Fy(;)h(:)o(::)o FE(\))p Fy(:)166 572 y FE(The)k(t)o(yp)q(emap)f(implied)j(b)o(y)e Fx(sendcount,)j(sendt)o(yp)q(e)f FE(need)e(not)g(b)q(e)g(the)g(same)f (for)g(eac)o(h)h(pro)q(cess,)g(but)75 629 y(it)k(still)h(m)o(ust)d (exactly)i(matc)o(h)f(the)h(t)o(yp)q(emap)f(implied)j(b)o(y)d Fx(recvcounts,)i(recvt)o(yp)q(e)p FE(.)22 b(That)14 b(is,)i(t)o(yp)q (emap)75 685 y(of)d Fx(sendcount[i],)k(sendt)o(yp)q(e)f FE(on)d(pro)q(cess)h Fx(ro)q(ot)f FE(m)o(ust)g(exactly)h(matc)o(h)f(t)o (yp)q(emap)g(of)g Fx(recvcount,)i(recvt)o(yp)q(e)75 742 y FE(on)g(pro)q(cess)h Fp(i)p FE(.)166 799 y(All)f(argumen)o(ts)f(to)f (the)h(function)h(are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fx(ro)q(ot)p FE(,)e(while)j(on)e(other)g(pro)q(cesses,)75 855 y(only)f(argumen)o(ts)f Fx(recvbuf,)i(recvcount,)g(recvt)o(yp)q(e,) g(ro)q(ot,)e(comm)f FE(are)h(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fx(ro)q(ot)75 911 y FE(and)j Fx(comm)f FE(m)o(ust)g(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)g(on)e(all)h(pro)q (cesses.)166 968 y(F)l(or)g(b)q(oth)h(functions,)g(the)g(sp)q (eci\014cation)h(of)f(coun)o(t\(s\),)e(t)o(yp)q(e\(s\),)h (\(displacemen)o(ts\),)i(should)f(not)75 1025 y(cause)e(an)o(y)g(lo)q (cation)h(on)f(the)h(ro)q(ot)e(to)h(b)q(e)g(read)h(more)e(than)h(once.) 75 1149 y Fq(4.4.5)49 b(Examples)16 b(of)h(Usage)g(of)f(MPI)p 768 1149 V 18 w(SCA)l(TTER,)h(MPI)p 1115 1149 V 19 w(SCA)l(TTERV)75 1236 y Fx(Example)e(9)75 1322 y FE(The)k(rev)o(erse)g(of)g(Example)h (1.)31 b(Scatter)19 b(sets)g(of)f(100)h(in)o(ts)g(from)f(the)h(ro)q(ot) g(to)f(eac)o(h)h(pro)q(cess)h(in)g(the)75 1379 y(group.)g(See)15 b(\014gure)h(4.7.)170 1487 y Fp(MPI_Comm)23 b(comm;)170 1544 y(MPI_Group)g(mygroup;)170 1600 y(int)h(gsize,*sendbuf;)170 1657 y(int)g(root,)f(rbuf[100];)170 1770 y(...)170 1883 y(/*)h(The)f(variable)g(comm)h(is)f(set)h(elsewhere)e(in)i(the)f (program)194 1939 y(*/)170 1996 y(MPI_Comm_group\()f(comm,)h(&mygroup)g (\);)170 2052 y(MPI_Group_size\()f(mygroup,)h(&gsize)g(\);)170 2108 y(sendbuf)g(=)h(\(int)f(*\)malloc\(gsize*100*sizeof\(int)o(\)\);) 170 2165 y(MPI_Scatter\()g(sendbuf,)f(100,)i(MPI_INT,)f(rbuf,)g(100,)g (MPI_INT,)g(root,)g(comm\);)75 2339 y Fx(Example)15 b(10)75 2425 y FE(The)21 b(rev)o(erse)f(of)g(Example)i(3.)35 b(The)21 b(ro)q(ot)f(pro)q(cess)h(scatters)e(sets)i(of)f(100)f(in)o(ts) i(to)f(the)h(other)f(pro-)75 2482 y(cesses,)j(but)f(the)f(sets)h(of)f (100)g(are)g Fs(stride)g FE(in)o(ts)h(apart)f(in)h(the)g(sending)h (bu\013er.)38 b(Requires)23 b(use)f(of)75 2538 y Fx(MPI)p 160 2538 14 2 v 16 w(SCA)l(TTERV)p FE(.)16 b(Assume)f Fy(str)q(ide)e FC(\025)g FE(100.)19 b(See)d(\014gure)f(4.8.)170 2647 y Fp(MPI_Comm)23 b(comm;)170 2704 y(MPI_Group)g(mygroup;)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 86 91 86 90 bop 75 -100 a FE(86)708 b Ft(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6630800 0 0 33943388 13682606 startTexFig 458 4 a %%BeginDocument: mycoll-fig7.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 373.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 414 m gs 1 -1 scale (sendbuf) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 119 299 m 89 244 l gs col-1 s gr n 91.075 251.981 m 89.000 244.000 l 94.587 250.065 l gs 2 setlinejoin col-1 s gr n 189 299 m 189 244 l gs col-1 s gr n 187.000 252.000 m 189.000 244.000 l 191.000 252.000 l gs 2 setlinejoin col-1 s gr n 249 299 m 289 244 l gs col-1 s gr n 282.677 249.294 m 289.000 244.000 l 285.912 251.646 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 138 522 a FE(Figure)g(4.7:)i(The)e(ro)q(ot)e(pro)q(cess)i(scatters)e (sets)h(of)g(100)f Fp(int)p FE(s)h(to)f(eac)o(h)i(pro)q(cess)f(in)h (the)f(group.)458 578 y 16577003 6796570 0 0 33943388 14011514 startTexFig 458 578 a %%BeginDocument: mycoll-fig8.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 378.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 99 419 m gs 1 -1 scale (sendbuf) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr n 119 299 m 84 244 l gs col-1 s gr n 86.608 251.823 m 84.000 244.000 l 89.982 249.676 l gs 2 setlinejoin col-1 s gr n 204 304 m 189 244 l gs col-1 s gr n 189.000 252.246 m 189.000 244.000 l 192.881 251.276 l gs 2 setlinejoin col-1 s gr n 284 304 m 289 244 l gs col-1 s gr n 286.343 251.806 m 289.000 244.000 l 290.329 252.138 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 1107 a FE(Figure)g(4.8:)j(The)d(ro)q(ot)e(pro)q(cess)i(scatters)f (sets)g(of)g(100)f Fp(int)p FE(s,)h(mo)o(ving)g(b)o(y)h Fp(stride)e FE(in)o(ts)i(from)f(send)h(to)75 1163 y(send)h(in)g(the)f (scatter.)170 1284 y Fp(int)24 b(gsize,*sendbuf;)170 1340 y(int)g(root,)f(rbuf[100],)g(i,)g(*displs,)g(*scounts;)170 1453 y(...)170 1566 y(MPI_Comm_group\()f(comm,)h(&mygroup)g(\);)170 1622 y(MPI_Group_size\()f(mygroup,)h(&gsize)g(\);)170 1679 y(sendbuf)g(=)h(\(int)f(*\)malloc\(gsize*stride*sizeof\()o (int\)\);)170 1735 y(displs)g(=)h(\(int)g(*\)malloc\(gsize*sizeof\(i)o (nt\)\);)170 1792 y(scounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)170 1848 y(for)h(\(i=0;)f(i)g (SENDBUF\(*\),)g(RECVBUF\(*\))170 1898 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g(IERROR)166 2026 y Fx(MPI)p 251 2026 14 2 v 16 w(ALLGA)l(THER)16 b FE(can)g(b)q(e)h(though)o(t)e(of)g(as)h Fx(MPI)p 1047 2026 V 15 w(GA)l(THER)p FE(,)h(but)f(where)g(all)h(pro)q(cesses)f (receiv)o(e)75 2082 y(the)22 b(result,)j(instead)e(of)f(just)g(the)g (ro)q(ot.)41 b(The)22 b Fp(j)p FE(-th)h(blo)q(c)o(k)g(of)f(data)g(sen)o (t)g(from)f(eac)o(h)i(pro)q(cess)f(is)75 2139 y(receiv)o(ed)17 b(b)o(y)f(ev)o(ery)f(pro)q(cess)h(and)g(placed)h(in)f(the)g Fp(j)p FE(-th)g(blo)q(c)o(k)g(of)g(the)f(bu\013er)h Fx(recvbuf)p FE(.)22 b(The)16 b(t)o(yp)q(emap)75 2195 y(of)h Fx(sendcount,)j(sendt)o (yp)q(e)g FE(m)o(ust)d(exactly)h(matc)o(h)e(the)i(t)o(yp)q(emap)f(of)g Fx(recvcount,)h(recvt)o(yp)q(e)p FE(.)27 b(Th)o(us,)18 b(the)75 2252 y(outcome)d(of)g(a)f(call)j(to)d Fx(MPI)p 570 2252 V 16 w(ALLGA)l(THER\(...\))20 b FE(is)15 b(as)g(if)h(all)g (pro)q(cesses)f(executed)i Fp(n)e FE(calls)h(to,)96 2478 y Fp(MPI)p 171 2478 15 2 v 17 w(GATHER)p FE(\()p Fp(sendbuf)p Fy(;)8 b Fp(send)o(count)p Fy(;)f Fp(send)o(type)p Fy(;)g Fp(recvb)o(uf)p Fy(;)g Fp(recvcou)o(nt)p Fy(;)g Fp(recvtyp)o(e)p Fy(;)g Fp(root)p Fy(;)g Fp(comm)p FE(\))o Fy(;)75 2704 y FE(for)15 b Fp(root)d FE(=)h Fp(0)p Fy(;)8 b FC(\001)g(\001)g(\001)t Fy(;)g Fp(n)h FC(\000)i Fp(1)p FE(.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 89 94 89 93 bop 75 -100 a Ft(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FE(89)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(ALLGA)l(THERV\()11 b(sendbuf,)i(sendcount,)h(sendt)o(yp)q(e,)f (recvbuf,)f(displs,)g(recvcounts,)g(recvt)o(yp)q(e,)g(comm\))117 197 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 308 y(IN)171 b Fx(sendcount)383 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 419 y(IN)171 b Fx(sendt)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)117 530 y(OUT)124 b Fx(recvbuf)434 b Fr(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 641 y(IN)171 b Fx(displs)469 b Fr(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fl(i)f Fr(sp)q(eci\014es)905 698 y(the)f(displacemen)o (t)e(\(relativ)o(e)g(to)h Fm(recvbuf)i Fr(at)d(whic)o(h)h(to)g(place) 905 754 y(the)j(incoming)c(data)j(from)e(pro)q(cess)k Fl(i)117 865 y Fr(IN)171 b Fx(recvcounts)372 b Fr(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(con)o(taining)e(the)i(n)o (um-)905 922 y(b)q(er)15 b(of)e(elemen)o(ts)h(that)g(are)g(receiv)o(ed) h(from)e(eac)o(h)h(pro)q(cess)117 1033 y(IN)171 b Fx(recvt)o(yp)q(e)414 b Fr(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts) 117 1144 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 1287 y Fp(int)23 b(MPI)p 245 1287 15 2 v 17 w(ALLGATHERV\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1287 V 16 w(Datatype)g(sendtype,)393 1343 y(void*)g(recvbuf,)g (int)h(*displs,)e(int)i(*recvcounts,)e(MPI)p 1542 1343 V 17 w(Datatype,)393 1399 y(MPI)p 468 1399 V 17 w(Comm)h(comm\))75 1504 y(MPI)p 150 1504 V 17 w(ALLGATHERV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)f(RECVBUF,)h(DISPLS,)g(RECVCOUNTS,)393 1560 y(RECVTYPE,)g(COMM\))170 1617 y()g(SENDBUF\(*\),)g(RECVBUF\(*\)) 170 1673 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(DISPLS,)h(RECVCOUNTS,)g (RECVTYPE,)f(COMM,)i(IERROR)166 1778 y Fx(MPI)p 251 1778 14 2 v 16 w(ALLGA)l(THERV)11 b FE(can)g(b)q(e)g(though)o(t)e(of)h(as)g Fx(MPI)p 1044 1778 V 16 w(GA)l(THERV)p FE(,)h(but)g(where)f(all)h(pro)q (cesses)g(receiv)o(e)75 1834 y(the)i(result,)g(instead)g(of)f(just)g (the)h(ro)q(ot.)18 b(The)13 b Fp(j)p FE(-th)f(blo)q(c)o(k)h(of)f(data)g (sen)o(t)h(from)e(eac)o(h)i(pro)q(cess)g(is)g(receiv)o(ed)75 1891 y(b)o(y)j(ev)o(ery)f(pro)q(cess)h(and)g(placed)h(in)f(the)g Fp(j)p FE(-th)g(blo)q(c)o(k)g(of)f(the)h(bu\013er)g Fx(recvbuf)p FE(.)21 b(These)16 b(blo)q(c)o(ks)h(need)f(not)75 1947 y(all)g(b)q(e)f(the)g(same)f(size.)21 b(The)15 b(t)o(yp)q(emap)g(of)f Fx(sendcount,)j(sendt)o(yp)q(e)h FE(on)d(pro)q(cess)g Fp(j)f FE(m)o(ust)g(exactly)i(matc)o(h)75 2004 y(the)f(t)o(yp)q(emap)f (of)g Fx(recvcounts[j],)i(recvt)o(yp)q(e)p FE(,)f(\(on)f(all)i(pro)q (cesses\).)j(The)c(outcome)f(is)h(as)f(if)h(all)h(pro)q(cesses)75 2060 y(executed)g(calls)g(to,)115 2216 y Fp(MPI)p 190 2216 15 2 v 17 w(GATHERV)p FE(\()p Fp(sendbuf)p Fy(;)8 b Fp(sen)o(dcount)p Fy(;)f Fp(sen)o(dtype)p Fy(;)g Fp(recv)o(buf)p Fy(;)g Fp(displ)o(s)p Fy(;)h Fp(r)o(ecvcoun)o(ts)p Fy(;)f Fp(recvtyp)o(e)p Fy(;)854 2426 y Fp(root)p Fy(;)h Fp(comm)p FE(\))p Fy(;)75 2573 y FE(for)15 b Fp(root)d FE(=)h Fp(0)p Fy(;)8 b FC(\001)g(\001)g(\001)t Fy(;)g Fp(n)h FC(\000)i Fp(1)p FE(.)166 2647 y(F)l(or)i(b)q(oth)g Fx(MPI)p 436 2647 14 2 v 16 w(ALLGA)l(THER)i FE(and)e Fx(MPI)p 890 2647 V 16 w(ALLGA)l(THERV)p FE(,)h(all)h(argumen)o(ts)e(on)g(all)h(pro) q(cesses)g(are)75 2704 y(signi\014can)o(t.)21 b(The)15 b(argumen)o(t)g Fx(comm)e FE(m)o(ust)i(ha)o(v)o(e)f(iden)o(tical)j(v)m (alues)g(on)e(all)h(pro)q(cesses.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 90 95 90 94 bop 75 -100 a FE(90)703 b Ft(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fq(4.4.7)49 b(All-to-All)19 b(Scatter/Gather)75 236 y Fx(MPI)p 160 236 14 2 v 16 w(ALL)l(TO)o(ALL\(sendbuf,)d(sendcount,)i(sendt)o(yp)q (e,)f(recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(comm\))117 343 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 477 y(IN)171 b Fx(sendcount)383 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 612 y(IN)171 b Fx(sendt)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)117 746 y(OUT)124 b Fx(recvbuf)434 b Fr(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 881 y(IN)171 b Fx(recvcount)389 b Fr(n)o(um)o(b)q(er)17 b(of)f(elemen)o(ts)h(receiv)o(ed)i(from)c(an)o (y)i(pro)q(cess)i(\(in)o(te-)905 937 y(ger\))117 1072 y(IN)171 b Fx(recvt)o(yp)q(e)414 b Fr(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)117 1206 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 1360 y Fp(int)23 b(MPI)p 245 1360 15 2 v 17 w(ALLTOALL\(void*)f (sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1360 V 17 w(Datatype)f (sendtype,)393 1417 y(void*)h(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 1417 V 17 w(Datatype,)g(MPI)p 1511 1417 V 17 w(Comm)i(comm\))75 1533 y(MPI)p 150 1533 V 17 w(ALLTOALL\(SENDBUF,)d(SENDCOUNT,)i (SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)g(RECVTYPE,)393 1589 y(COMM\))170 1646 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1702 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g (COMM,)g(IERROR)166 1818 y Fx(MPI)p 251 1818 14 2 v 16 w(ALL)l(TO)o(ALL)18 b FE(is)h(an)g(extension)h(of)e Fx(MPI)p 962 1818 V 16 w(ALLGA)l(THER)i FE(to)e(the)h(case)g(where)g(eac)o(h)g (pro)q(cess)75 1875 y(sends)d(distinct)h(data)d(to)h(eac)o(h)h(of)f (the)g(receiv)o(ers.)22 b(The)15 b Fp(j)p FE(-th)h(blo)q(c)o(k)g(sen)o (t)f(from)g(pro)q(cess)h Fp(i)f FE(is)h(receiv)o(ed)75 1931 y(b)o(y)e(pro)q(cess)f Fp(j)h FE(and)g(is)g(placed)g(in)h(the)e Fp(i)p FE(-th)h(blo)q(c)o(k)g(of)f Fx(recvbuf)p FE(.)20 b(The)14 b(t)o(yp)q(emap)g(of)f Fx(sendcount,)j(sendt)o(yp)q(e)75 1988 y FE(m)o(ust)f(exactly)g(matc)o(h)g(the)g(t)o(yp)q(emap)g(of)g Fx(recvcount,)h(recvt)o(yp)q(e)p FE(.)166 2074 y(The)d(outcome)g(is)h (as)e(if)i(eac)o(h)f(pro)q(cess)g(executed)h(a)f(send)h(to)e(eac)o(h)h (pro)q(cess)h(\(itself)f(included\))j(with)75 2130 y(a)f(call)h(to,)249 2321 y Fp(MPI)p 324 2321 15 2 v 17 w(SEND)p FE(\()p Fp(sendbuf)8 b FE(+)j Fp(i)f FC(\002)g Fp(extent)p FE(\()p Fp(sendtype)p FE(\))p Fy(;)e Fp(s)o(endcount)o Fy(;)g Fp(s)o(endtype)o Fy(;)g Fp(i)p Fy(;)f(:)o(::)p FE(\))o Fy(;)75 2512 y FE(and)15 b(a)g(receiv)o(e)h(from)f(ev)o(ery)g(other)g(pro)q(cess)g (with)h(a)f(call)h(to,)355 2704 y Fp(MPI)p 430 2704 V 16 w(RECV)p FE(\()p Fp(recvbuf)9 b FE(+)h Fp(i)g FC(\002)h Fp(extent)p FE(\()p Fp(recvtype)p FE(\))p Fy(;)c Fp(recvc)o(ount)p Fy(;)g Fp(i)p Fy(;)g(::)o(:)p FE(\))p Fy(:)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 91 96 91 95 bop 75 -100 a Ft(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FE(91)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V\(sendbuf,)13 b(sdispls,)h(sendcounts,)h (sendt)o(yp)q(e,)f(recvbuf,)e(rdispls,)h(recvcounts,)g(recvt)o(yp)q(e,) 75 102 y(comm\))117 180 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 257 y(IN)171 b Fx(sdispls)452 b Fr(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fl(j)f Fr(sp)q(eci\014es) 905 313 y(the)20 b(displacemen)o(t)e(\(relativ)o(e)h(to)f Fm(sendbuf)j Fr(from)c(whic)o(h)i(to)905 370 y(tak)o(e)14 b(the)h(outgoing)d(data)i(destined)h(for)e(pro)q(cess)j Fl(j)117 447 y Fr(IN)171 b Fx(sendcounts)366 b Fr(in)o(teger)20 b(arra)o(y)f(equal)h(to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 503 y(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(to)h(send)h(to)f(eac)o(h) g(pro)q(cessor)117 580 y(IN)171 b Fx(sendt)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)117 657 y(OUT)124 b Fx(recvbuf)434 b Fr(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 734 y(IN)171 b Fx(rdispls)453 b Fr(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fl(i)f Fr(sp)q(eci\014es)905 790 y(the)f(displacemen)o (t)e(\(relativ)o(e)g(to)h Fm(recvbuf)i Fr(at)d(whic)o(h)h(to)g(place) 905 847 y(the)j(incoming)c(data)j(from)e(pro)q(cess)k Fl(i)117 924 y Fr(IN)171 b Fx(recvcounts)372 b Fr(in)o(teger)20 b(arra)o(y)f(equal)h(to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 980 y(maxim)n(um)d(n)o(um)o(b)q(er)j(of)f(elemen)o(ts)i(that)g(can)f(b) q(e)i(receiv)o(ed)905 1037 y(from)12 b(eac)o(h)j(pro)q(cessor)117 1114 y(IN)171 b Fx(recvt)o(yp)q(e)414 b Fr(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)117 1191 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 1316 y Fp(int)23 b(MPI)p 245 1316 15 2 v 17 w(ALLTOALLV\(void*)f (sendbuf,)h(int)g(*sdispls,)g(int)g(*sendcounts,)393 1373 y(MPI)p 468 1373 V 17 w(Datatype)g(sendtype,)f(void*)i(recvbuf,)e (int)i(*rdispls,)393 1429 y(int)g(*recvcounts,)e(MPI)p 874 1429 V 17 w(Datatype,)g(MPI)p 1201 1429 V 17 w(Comm)h(comm\))75 1516 y(MPI)p 150 1516 V 17 w(ALLTOALLV\(SENDBUF,)e(SDISPLS,)i (SENDCOUNTS,)f(SENDTYPE,)h(RECVBUF,)g(RDISPLS,)393 1573 y(RECVCOUNTS,)g(RECVTYPE,)f(COMM\))170 1629 y()h(SENDBUF\(*\),)g (RECVBUF\(*\))170 1686 y(INTEGER)g(SDISPLS,)g(SENDCOUNTS,)f(SENDTYPE,)h (RDISPLS,)g(RECVCOUNTS,)f(RECVTYPE,)170 1742 y(COMM,)i(IERROR)166 1830 y Fx(MPI)p 251 1830 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V)12 b FE(adds)g(\015exibili)q(t)o(y)j(to)c Fx(MPI)p 959 1830 V 16 w(ALL)l(TO)o(ALL)h FE(in)h(that)f(the)h(lo)q(cation)g(of) f(data)g(for)g(the)75 1886 y(send)j(is)f(sp)q(eci\014ed)j(b)o(y)d Fx(sdispls)i FE(and)e(the)g(lo)q(cation)h(of)f(the)g(placemen)o(t)h(of) f(the)g(data)g(on)g(the)g(receiv)o(e)h(side)75 1943 y(is)h(sp)q (eci\014ed)h(b)o(y)e Fx(rdispls)p FE(.)166 2000 y(The)10 b Fp(j)p FE(-th)g(blo)q(c)o(k)h(sen)o(t)f(from)g(pro)q(cess)g Fp(i)g FE(is)h(receiv)o(ed)g(b)o(y)f(pro)q(cess)h Fp(j)f FE(and)g(is)h(placed)g(in)g(the)f Fp(i)p FE(-th)g(blo)q(c)o(k)75 2056 y(of)15 b Fx(recvbuf)p FE(.)21 b(These)15 b(blo)q(c)o(ks)h(need)h (not)d(all)j(ha)o(v)o(e)e(the)g(same)g(size,)h(but)f(the)h(t)o(yp)q (emap)f(of)g Fx(sendcounts[j],)75 2113 y(sendt)o(yp)q(e)i FE(on)d(pro)q(cess)h Fp(i)f FE(m)o(ust)g(exactly)g(matc)o(h)g(the)g(t)o (yp)q(emap)h(of)e Fx(recvcounts[i],)j(recvt)o(yp)q(e)f FE(on)g(pro)q(cess)75 2169 y Fp(j)p FE(.)166 2227 y(The)g(outcome)g(is) h(as)f(if)g(eac)o(h)h(pro)q(cess)f(sen)o(t)g(a)g(message)g(to)f(eac)o (h)h(other)g(pro)q(cess)h(with,)353 2332 y Fp(MPI)p 428 2332 15 2 v 17 w(SEND)p FE(\()p Fp(sendbuf)8 b FE(+)i Fp(displs)p FE([)p Fp(i)p FE(])p Fy(;)e Fp(sendcount)p FE([)o Fp(i)p FE(])p Fy(;)f Fp(sen)o(dtype)p Fy(;)g Fp(i)p Fy(;)g(::)o(:)p FE(\))p Fy(;)75 2436 y FE(and)15 b(receiv)o(ed)i(a)e (message)f(from)h(ev)o(ery)g(other)g(pro)q(cess)g(with)h(a)f(call)h (to,)353 2541 y Fp(MPI)p 428 2541 V 17 w(RECV)p FE(\()p Fp(recvbuf)8 b FE(+)i Fp(displs)p FE([)p Fp(i)p FE(])p Fy(;)e Fp(recvcount)p FE([)o Fp(i)p FE(])p Fy(;)f Fp(rec)o(vtype)p Fy(;)g Fp(i)p Fy(;)g(::)o(:)p FE(\))p Fy(:)166 2647 y FE(F)l(or)21 b(b)q(oth)g Fx(MPI)p 452 2647 14 2 v 16 w(ALL)l(TO)o(ALL)g FE(and)h Fx(MPI)p 885 2647 V 16 w(ALL)l(TO)o(ALL)-5 b(V)p FE(,)20 b(all)j(argumen)o(ts)d(on)i(all)g(pro)q(cesses)g(are)75 2704 y(signi\014can)o(t.)f(The)15 b(argumen)o(t)g Fx(comm)e FE(m)o(ust)i(ha)o(v)o(e)f(iden)o(tical)j(v)m(alues)g(on)e(all)h(pro)q (cesses.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 92 97 92 96 bop 75 -100 a FE(92)703 b Ft(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y Fo(Implemen)o(tati)o(on)h (note:)54 b Fr(Although)18 b(the)h(discussion)g(of)e(collectiv)o(e)i (comm)o(unicati)o(on)d(in)h(terms)i(of)75 102 y(p)q(oin)o(t)e(to)g(p)q (oin)o(t)f(op)q(eration)h(implies)f(that)h(eac)o(h)h(message)e(is)i (transfered)g(directly)g(from)d(sender)k(to)e(receiv)o(er,)75 158 y(implemen)o(tatio)o(ns)11 b(ma)o(y)g(use)j(a)f(tree)h(comm)o (unication)c(pattern,)k(where)g(messages)f(are)h(forw)o(arded)f(b)o(y)g (in)o(terme-)75 214 y(diate)i(no)q(des)h(where)g(they)f(are)h(split)e (\(for)h(scatter\))i(or)d(concatenated)j(\(for)e(gather\),)g(if)f(this) h(is)g(more)f(e\016cien)o(t.)75 497 y Fv(4.5)59 b(Global)20 b(Compute)e(Op)r(erations)75 598 y FE(The)13 b(functions)g(in)h(this)f (section)h(p)q(erform)e(one)h(of)f(the)h(follo)o(wing)g(op)q(erations)g (across)f(all)i(the)f(mem)o(b)q(ers)75 655 y(of)i(a)g(group:)189 748 y(global)g(max)g(on)g(in)o(teger)h(and)f(\015oating)g(p)q(oin)o(t)h (data)e(t)o(yp)q(es)189 842 y(global)h(min)h(on)g(in)o(teger)f(and)g (\015oating)g(p)q(oin)o(t)h(data)f(t)o(yp)q(es)189 935 y(global)g(sum)h(on)f(in)o(teger)g(and)h(\015oating)f(p)q(oin)o(t)g (data)g(t)o(yp)q(es)189 1029 y(global)g(pro)q(duct)h(on)f(in)o(teger)h (and)f(\015oating)g(p)q(oin)o(t)h(data)e(t)o(yp)q(es)189 1122 y(global)h(AND)h(on)f(logical)h(and)f(in)o(teger)h(data)e(t)o(yp)q (es)189 1216 y(global)h(OR)h(on)g(logical)g(and)f(in)o(teger)h(data)e (t)o(yp)q(es)189 1310 y(global)h(X)o(OR)h(on)f(logical)i(and)e(in)o (teger)h(data)e(t)o(yp)q(es)189 1403 y(rank)h(of)f(pro)q(cess)i(with)f (maxim)o(um)g(v)m(alue)189 1497 y(rank)g(of)f(pro)q(cess)i(with)f (minim)o(um)i(v)m(alue)189 1591 y(user)e(de\014ned)i(\(asso)q(ciativ)o (e\))d(op)q(eration)189 1684 y(user)h(de\014ned)i(\(asso)q(ciativ)o(e)e (and)g(comm)o(utativ)o(e\))f(op)q(eration)75 1806 y Fq(4.5.1)49 b(Reduce)75 1939 y Fx(MPI)p 160 1939 14 2 v 16 w(REDUCE\()15 b(sendbuf,)j(recvbuf,)d(count,)h(datat)o(yp)q(e,)h(op,)e(ro)q(ot,)f (comm\))117 2016 y Fr(IN)171 b Fx(sendbuf)428 b Fr(address)15 b(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 2091 y(OUT)124 b Fx(recvbuf)434 b Fr(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o (hoice,)i(signi\014can)o(t)d(only)g(at)905 2148 y(ro)q(ot\))117 2223 y(IN)171 b Fx(count)466 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 2298 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)12 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)h(bu\013er)h(\(basic)f(t)o(yp)q(es)h(only\)) 117 2373 y(IN)171 b Fx(op)525 b Fr(op)q(eration)14 b(\(MPI-de\014ned)h (constan)o(t\))117 2448 y(IN)171 b Fx(ro)q(ot)492 b Fr(rank)14 b(of)f(ro)q(ot)h(pro)q(cess)i(\(in)o(teger\))117 2523 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 2647 y Fp(int)23 b(MPI)p 245 2647 15 2 v 17 w(REDUCE\(void*)f(sendbuf,) h(void*)g(recvbuf,)g(int)h(count,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)f(datatype,)f(void)i(op,)f(int)h(root,)f(MPI)p 1464 2704 V 17 w(Comm)g(comm\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 93 98 93 97 bop 75 -100 a Ft(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FE(93)75 45 y Fp(MPI)p 150 45 15 2 v 17 w(REDUCE\(SENDBUF,)22 b(RECVBUF,)g(COUNT,)h(DATATYPE,)g(OP,)g(ROOT,)h(COMM\))170 102 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 158 y(INTEGER)g(COUNT,) g(DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR)166 246 y FE(Com)o(bines)c(the)f(v)m(alues)i(pro)o(vided)f(in)g(the)g(send)g (bu\013er)f(of)g(eac)o(h)g(pro)q(cess)h(in)g(the)g(group,)f(using)75 302 y(the)d(op)q(eration)30 b Fx(op)p FE(,)15 b(and)g(returns)g(the)g (com)o(bined)h(v)m(alue)h(in)f(the)f(receiv)o(e)h(bu\013er)f(of)f(the)h (pro)q(cess)h(with)75 359 y(rank)32 b Fx(ro)q(ot)p FE(.)21 b(Eac)o(h)16 b(pro)q(cess)g(can)g(pro)o(vide)h(one)f(v)m(alue,)h(or)e (a)h(sequence)h(of)e(v)m(alues,)i(in)g(whic)o(h)g(case)f(the)75 415 y(com)o(bine)j(op)q(eration)f(is)h(executed)g(elemen)o(t-wise)g(on) g(eac)o(h)f(en)o(try)f(of)h(the)g(sequence.)30 b(F)l(or)17 b(example,)75 472 y(if)f(the)f(op)q(eration)h(is)g Fm(MPI)p 522 472 13 2 v 14 w(MAX)f FE(and)h(the)f(send)h(bu\013er)g(con)o(tains) f(t)o(w)o(o)f(\015oating)h(p)q(oin)o(t)h(n)o(um)o(b)q(ers,)g(then)75 528 y(recvbuf\(1\))h(=)h(global)h(max\(sendbuf\(1\)\))d(and)i (recvbuf\(2\))g(=)g(global)g(max\(sendbuf\(2\)\).)26 b(All)19 b(send)75 584 y(bu\013ers)d(should)i(de\014ne)g(sequences)f (of)f(equal)i(length)f(of)f(en)o(tries)h(all)g(of)f(the)h(same)f(data)g (t)o(yp)q(e,)h(where)75 641 y(the)d(t)o(yp)q(e)h(is)f(a)g Fz(basic)h FE(MPI)f(datat)o(yp)q(e)g(and)h(one)f(of)g(those)g(allo)o(w) o(ed)g(for)g(op)q(erands)h(of)27 b Fx(op)15 b FE(\(see)f(b)q(elo)o (w\).)75 697 y(F)l(or)f(all)j(op)q(erations)e(the)g(n)o(um)o(b)q(er)g (of)g(elemen)o(ts)h(in)g(the)f(send)h(bu\013er)f(are)g(the)g(same)g(as) f(for)h(the)g(receiv)o(e)75 754 y(bu\013ers.)22 b(F)l(or)16 b(all)h(op)q(erations,)e(the)i(t)o(yp)q(e)f(of)f(elemen)o(ts)i(in)g (the)f(send)g(bu\013er)g(are)g(the)g(same)g(as)f(for)h(the)75 810 y(receiv)o(e)g(bu\013ers.)166 868 y(The)f(op)q(eration)g(de\014ned) h(b)o(y)30 b Fx(op)15 b FE(is)g(asso)q(ciativ)o(e)g(and)g(comm)o (utativ)o(e,)f(and)h(the)g(implemen)o(tation)75 925 y(can)j(tak)o(e)f (adv)m(an)o(tage)g(of)g(asso)q(ciativit)o(y)h(and)g(comm)o(utativit)o (y)f(in)h(order)f(to)g(c)o(hange)h(order)f(of)h(ev)m(alu-)75 981 y(ation.)30 b(The)19 b(routine)g(is)g(called)h(b)o(y)f(all)g(group) g(mem)o(b)q(ers)f(using)i(the)e(same)h(argumen)o(ts)f(for)36 b Fx(count,)75 1037 y(datat)o(yp)q(e,)16 b(op,)f(ro)q(ot)g FE(and)31 b Fx(comm)p FE(.)166 1095 y(W)l(e)15 b(list)h(b)q(elo)o(w)g (the)f(supp)q(orted)h(options)f(for)30 b Fx(op)p FE(.)117 1199 y Fm(MPI)p 194 1199 V 14 w(MAX)649 b FE(maxim)o(um)117 1256 y Fm(MPI)p 194 1256 V 14 w(MIN)664 b FE(minim)o(um)117 1313 y Fm(MPI)p 194 1313 V 14 w(SUM)653 b FE(sum)117 1370 y Fm(MPI)p 194 1370 V 14 w(PROD)626 b FE(pro)q(duct)117 1427 y Fm(MPI)p 194 1427 V 14 w(LAND)632 b FE(logical)16 b(and)117 1484 y Fm(MPI)p 194 1484 V 14 w(BAND)626 b FE(bit-wise)16 b(and)117 1541 y Fm(MPI)p 194 1541 V 14 w(LOR)661 b FE(logical)16 b(or)117 1598 y Fm(MPI)p 194 1598 V 14 w(BOR)655 b FE(bit-wise)16 b(or)117 1654 y Fm(MPI)p 194 1654 V 14 w(LX)o(OR)634 b FE(logical)16 b(xor)117 1711 y Fm(MPI)p 194 1711 V 14 w(BX)o(OR)628 b FE(bit-wise)16 b(xor)117 1768 y Fm(MPI)p 194 1768 V 14 w(MAXLOC)569 b FE(maxim)o(um)15 b(v)m(alue)h(and)g(rank)f(of)f(pro)q (cess)i(with)f(it)117 1825 y Fm(MPI)p 194 1825 V 14 w(MINLOC)584 b FE(minim)o(um)16 b(v)m(alue)g(and)g(rank)f(of)g(pro)q(cess)g(with)h (it)166 1910 y(The)25 b Fm(MPI)p 346 1910 V 14 w(MINLOC)13 b FE(\()e Fm(MPI)p 634 1910 V 14 w(MAXLOC)p FE(\))h(op)q(erations)h (return)g(b)q(oth)g(minim)o(um)h(\(maxim)o(um\))e(v)m(alues)75 1966 y(and)17 b(the)h(ranks)f(of)g(pro)q(cesses)g(con)o(taining)h (those)f(v)m(alues.)28 b(The)17 b(p)q(oten)o(tially)i(mixed-t)o(yp)q(e) f(nature)f(of)75 2022 y(the)g(output)g(bu\013er)g(is)g(a)g(concern,)g (so)g(MPI)g(treats)f(the)h(bu\013ers)g(uniformly)h(and)f(co)q(erces)g (the)g(ranks)75 2079 y(to)j(the)h(same)f(t)o(yp)q(e)h(as)f(the)h(v)m (alues.)37 b(When)j Fm(MPI)p 995 2079 V 14 w(MINLOC)21 b FE(or)39 b Fm(MPI)p 1342 2079 V 14 w(MAXLOC)20 b FE(are)g(in)o(v)o (ok)o(ed,)i(the)75 2135 y(input)e(bu\013er)f(should)i(con)o(tain)e Fy(m)g FE(elemen)o(ts)h(of)f(a)g(data)f(t)o(yp)q(e)i(to)e(whic)o(h)i (the)f(op)q(eration)37 b Fm(MPI)p 1785 2135 V 15 w(MIN)75 2192 y FE(or)c Fm(MPI)p 226 2192 V 14 w(MAX)17 b FE(can)h(b)q(e)g (applied,)h(follo)o(w)o(ed)f(b)o(y)g(space)f(for)g(another)g Fy(m)g FE(elemen)o(ts)i(of)e(the)g(same)g(t)o(yp)q(e.)75 2248 y(In)o(ternally)l(,)h(the)e(function)h(will)h(co)q(erce)f(the)g (rank)f(of)g(the)g(calling)i(pro)q(cess)f(\(an)f(in)o(teger)g(t)o(yp)q (e\))g(to)g(the)75 2305 y(t)o(yp)q(e)g(of)g(the)g(data)f(v)m(alues,)i (and)f(carry)g(these)g(v)m(alues)h(along)f(during)h(the)f(reduction.)23 b(The)17 b(op)q(eration)75 2361 y(returns)f(at)f(the)i(ro)q(ot)e(the)h Fy(m)g FE(minim)o(um)h(\(or)e(maxim)o(um\))h(v)m(alues,)h(follo)o(w)o (ed)f(b)o(y)g(the)h Fy(m)e FE(ranks)h(of)g(the)75 2418 y(pro)q(cesses)k(con)o(taining)g(these)f(v)m(alues.)34 b(T)l(o)19 b(reco)o(v)o(er)f(the)i(ranks)f(of)g(the)g(pro)q(cesses)h (as)f(in)o(tegers,)h(the)75 2474 y(second)c(set)f(of)f(v)m(alues)j(can) e(no)o(w)g(b)q(e)h(copied)g(out)f(of)g(the)g(bu\013er)g(and)h(co)q (erced)g(to)e(in)o(teger)h(v)m(alues.)166 2532 y(The)g(op)q(eration)h (that)e(de\014nes)30 b Fm(MPI)p 801 2532 V 15 w(MAXLOC)14 b FE(is)721 2597 y Fh( )774 2641 y Fy(u)779 2697 y(i)821 2597 y Fh(!)864 2669 y FC(\016)897 2597 y Fh( )950 2641 y Fy(v)952 2697 y(j)995 2597 y Fh(!)1040 2669 y FE(=)1088 2597 y Fh( )1142 2641 y Fy(w)1146 2697 y(k)1196 2597 y Fh(!)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 94 99 94 98 bop 75 -100 a FE(94)708 b Ft(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y FE(where)833 104 y Fy(w)d FE(=)g(max\()p Fy(u;)8 b(v)r FE(\))75 191 y(and)710 297 y Fy(k)13 b FE(=)795 198 y Fh(8)795 235 y(>)795 248 y(<)795 323 y(>)795 335 y(:)853 240 y Fy(i)194 b FE(if)15 b Fy(u)e(>)g(v)853 297 y FE(min)q(\()p Fy(i;)8 b(j)s FE(\))39 b(if)15 b Fy(u)e FE(=)g Fy(v)853 353 y(j)191 b FE(if)15 b Fy(u)e(<)g(v)166 439 y FE(Note)21 b(that)g(ties)h(are)f (resolv)o(ed)h(in)h(fa)o(v)o(or)d(of)h(the)h(pro)q(cess)g(with)g(lo)o (w)o(er)f(rank)g(and)h(hence)h(this)75 496 y(op)q(eration)15 b(is)h(asso)q(ciativ)o(e)g(and)f(comm)o(utativ)o(e.)166 553 y Fm(MPI)p 243 553 13 2 v 14 w(MINLOC)g FE(is)h(de\014ned)g (similarly:)721 617 y Fh( )774 661 y Fy(u)779 717 y(i)821 617 y Fh(!)864 689 y FC(\016)897 617 y Fh( )950 661 y Fy(v)952 717 y(j)995 617 y Fh(!)1040 689 y FE(=)1088 617 y Fh( )1142 661 y Fy(w)1146 717 y(k)1196 617 y Fh(!)75 824 y FE(where)837 884 y Fy(w)e FE(=)f(min\()p Fy(u;)8 b(v)r FE(\))75 970 y(and)710 1076 y Fy(k)13 b FE(=)795 977 y Fh(8)795 1015 y(>)795 1027 y(<)795 1102 y(>)795 1114 y(:)853 1019 y Fy(i)194 b FE(if)15 b Fy(u)e(<)g(v)853 1076 y FE(min)q(\()p Fy(i;)8 b(j)s FE(\))39 b(if)15 b Fy(u)e FE(=)g Fy(v)853 1132 y(j)191 b FE(if)15 b Fy(u)e(>)g(v)75 1259 y Fx(Op)q(eration)j(/)f(T)l(yp)q(e)i(Compatibilit)o(y)75 1346 y FE(All)h(of)d(the)i(options)f(for)g Fx(op)g FE(in)h Fx(MPI)p 715 1346 14 2 v 16 w(REDUCE)g FE(do)f(not)g(apply)h(for)f(eac) o(h)g(of)g(the)g(p)q(ossible)i(MPI)e(basic)75 1403 y(datat)o(yp)q(es.)j (W)l(e)14 b(en)o(umerate)f(the)h(allo)o(w)o(ed)g(com)o(binations)h (here.)20 b(First,)13 b(de\014ne)i(groups)e(of)h(MPI)f(basic)75 1459 y(datat)o(yp)q(es)i(in)h(the)f(follo)o(wing)h(w)o(a)o(y)l(.)75 1570 y Fz(C)h(in)o(teger:)22 b Fp(MPI)p 400 1570 15 2 v 17 w(CHAR,)h(MPI)p 632 1570 V 17 w(INT,)g(MPI)p 840 1570 V 17 w(LONG,)g(MPI)p 1072 1570 V 17 w(SHORT,)189 1627 y(MPI)p 264 1627 V 16 w(UNSIGNED)p 472 1627 V 16 w(SHORT,)h(MPI)p 728 1627 V 16 w(UNSIGNED,)f(MPI)p 1055 1627 V 17 w(UNSIGNED)p 1264 1627 V 16 w(LONG)75 1724 y Fz(F)l(ortran)17 b(in)o(teger:)22 b Fp(MPI)p 535 1724 V 17 w(INTEGER)75 1822 y Fz(Floating)d(p)q(oin)o(t:)24 b Fp(MPI)p 513 1822 V 16 w(FLOAT,)g(MPI)p 769 1822 V 16 w(DOUBLE,)f(MPI)p 1048 1822 V 17 w(LONG)p 1161 1822 V 17 w(DOUBLE,)g(MPI)p 1441 1822 V 16 w(REAL,)189 1878 y(MPI)p 264 1878 V 16 w(DOUBLE)p 424 1878 V 17 w(PRECISION)75 1976 y Fz(Logical:)i Fp(MPI)p 349 1976 V 16 w(LOGICAL)75 2073 y Fz(Complex:)d Fp(MPI)p 390 2073 V 17 w(COMPLEX)75 2171 y Fz(Byte:)g Fp(MPI)p 296 2171 V 17 w(BYTE)166 2282 y FE(No)o(w,)14 b(the)h(v)m(alid)i(datat)o(yp)q(es)e(for)f(eac)o(h)i (option)f(is)h(sp)q(eci\014ed)h(b)q(elo)o(w.)117 2443 y Fm(MPI)p 194 2443 13 2 v 14 w(MAX,)c(MPI)p 400 2443 V 14 w(MIN,)h(MPI)p 592 2443 V 14 w(MAXLOC,)f(MPI)p 878 2443 V 14 w(MINLOC)41 b FE(C)10 b(in)o(teger,)h(F)l(ortran)e(in)o (teger,)i(Floating)f(p)q(oin)o(t)117 2499 y Fm(MPI)p 194 2499 V 14 w(SUM,)j(MPI)p 396 2499 V 14 w(PROD)424 b FE(C)10 b(in)o(teger,)h(F)l(ortran)e(in)o(teger,)i(Floating)f(p)q (oin)o(t,)h(Com-)905 2556 y(plex)117 2612 y Fm(MPI)p 194 2612 V 14 w(LAND,)j(MPI)p 418 2612 V 14 w(LOR,)f(MPI)p 612 2612 V 15 w(LX)o(OR)215 b FE(C)15 b(in)o(teger,)g(Logical)117 2668 y Fm(MPI)p 194 2668 V 14 w(BAND,)e(MPI)p 423 2668 V 14 w(BOR,)g(MPI)p 623 2668 V 14 w(BX)o(OR)199 b FE(C)15 b(in)o(teger,)g(Byte)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 95 100 95 99 bop 75 -100 a Ft(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FE(95)75 45 y Fx(Example)15 b(of)g(Reduce)75 135 y FE(Eac)o(h)h(pro)q(cess)h(has)g(an)f(arra)o(y)g(of)g(30)g Fp(double)p FE(s,)g(in)h(C.)f(F)l(or)g(eac)o(h)h(of)f(the)h(30)f(lo)q (cations,)h(compute)g(the)75 191 y(v)m(alue)f(and)g(rank)f(of)g(the)g (pro)q(cess)g(con)o(taining)h(the)f(largest)g(v)m(alue.)170 310 y Fp(...)170 366 y(/*)24 b(each)f(process)g(has)h(an)f(array)g(of)h (30)g(double:)f(a[])194 422 y(*/)170 479 y(double)g(a[30];)170 535 y(double)g(in[60],out[60];)170 592 y(int)h(i,outranks[30];)170 705 y(for)g(\(i=0;)f(i<30;)g(++i\))h({)75 761 y(in[i])f(=)h(a[i];)170 818 y(})170 874 y(MPI_Reduce\()f(in,)g(out,)h(30,)f(MPI_DOUBLE,)f (MPI_MAXLOC,)h(root,)g(comm)g(\);)170 931 y(/*)h(At)g(this)f(point,)g (the)g(answer)g(resides)g(on)h(process)f(root)194 987 y(*/)170 1043 y(if)h(\(myrank)f(==)h(root\))f({)75 1100 y(/*)h(read)f(ranks)g(out)h(and)f(coerce)g(back)g(to)h(ints)99 1156 y(*/)266 1213 y(for)f(\(i=0;)h(i<30;)f(++i\))g({)170 1269 y(outranks[i])g(=)h(out[30+i];)266 1326 y(})170 1382 y(})75 1515 y Fq(4.5.2)49 b(User-Reduce)75 1652 y Fx(MPI)p 160 1652 14 2 v 16 w(USER)p 288 1652 V 17 w(REDUCE\()16 b(sendbuf,)h(recvbuf,)f(count,)g(datat)o(yp)q(e,)g (function,)g(ro)q(ot,)f(comm\))117 1732 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c) o(hoice\))117 1811 y(OUT)124 b Fx(recvbuf)434 b Fr(starting)19 b(address)h(of)e(receiv)o(e)i(bu\013er)f(\(c)o(hoice,)h(signi\014can)o (t)905 1867 y(only)13 b(at)h(ro)q(ot\))117 1946 y(IN)171 b Fx(count)466 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input) g(bu\013er)g(\(in)o(teger\))117 2025 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er) 117 2105 y(IN)171 b Fx(function)418 b Fr(user)15 b(de\014ned)g (function)117 2184 y(IN)171 b Fx(ro)q(ot)492 b Fr(rank)14 b(of)f(ro)q(ot)h(pro)q(cess)i(\(in)o(teger\))117 2263 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 2389 y Fp(int)23 b(MPI)p 245 2389 15 2 v 17 w(USER)p 358 2389 V 17 w(REDUCE\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g (count,)393 2446 y(MPI)p 468 2446 V 17 w(Datatype)g(datatype,)f(void)i (function,)e(int)i(root,)f(MPI)p 1607 2446 V 17 w(Comm)g(comm\))75 2534 y(MPI)p 150 2534 V 17 w(USER)p 263 2534 V 16 w(REDUCE\(SENDBUF,)f (RECVBUF,)h(COUNT,)g(DATATYPE,)g(FUNCTION,)f(ROOT,)h(COMM\))170 2591 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2647 y(EXTERNAL)g(FUNCTION)170 2704 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g (COMM,)g(IERROR)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 96 101 96 100 bop 75 -100 a FE(96)708 b Ft(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y FE(Similar)h(to)d(the)i (reduce)g(op)q(eration)f(function)h(ab)q(o)o(v)o(e)f(except)h(that)e(a) h(user)h(supplied)h(function)f(is)75 102 y(used.)39 b Fx(function)15 b FE(is)f(a)e(function)i(with)g(three)f(argumen)o(ts.)18 b(A)c(C)f(protot)o(yp)q(e)f(for)g(suc)o(h)i(function)g(is)g Fp(void)75 158 y(f\()24 b(invec,)f(inoutvec,)f(*len\))p FE(.)d(Both)28 b Fx(invec)15 b FE(and)29 b Fx(inoutvec)16 b FE(are)e(arra)o(ys)e(with)29 b Fx(*len)15 b FE(en)o(tries.)20 b(The)75 214 y(function)f(computes)f(elemen)o(t-wise)h(a)f(comm)o (utativ)o(e)f(and)h(asso)q(ciativ)o(e)g(op)q(eration)g(on)g(eac)o(h)g (pair)g(of)75 271 y(en)o(tries)13 b(and)g(returns)g(the)f(result)i(in) 26 b Fx(inoutvec)p FE(.)20 b(A)13 b(pseudo-co)q(de)h(for)25 b Fx(function)14 b FE(is)f(giv)o(en)g(b)q(elo)o(w,)h(where)75 327 y Fp(op)h FE(is)h(the)f(comm)o(utativ)o(e)f(and)i(asso)q(ciativ)o (e)f(op)q(eration)h(de\014ned)g(b)o(y)30 b Fx(function)p FE(.)361 430 y Fp(for\(i=0;)23 b(i)h(<)g(*len;)f(i++\))g({)552 486 y(inoutvec[i])g(op=)g(invec[i])361 543 y(})166 646 y FE(The)16 b(t)o(yp)q(e)h(of)f(the)g(elemen)o(ts)h(of)32 b Fx(invec)17 b FE(and)g(of)32 b Fx(inoutvec)18 b FE(matc)o(h)d(the)i (t)o(yp)q(e)f(of)g(the)g(elemen)o(ts)h(of)75 702 y(the)i(send)g (bu\013ers)f(and)h(the)f(receiv)o(e)i(bu\013er.)30 b(No)18 b(MPI)g(functions)i(ma)o(y)e(b)q(e)h(called)h(inside)g(the)f(user)75 758 y(de\014ned)e(function.)166 815 y(The)k(addition)i(of)d(the)h (third)h(argumen)o(t,)44 b Fx(*len)p FE(,)23 b(in)43 b Fx(function)22 b FE(allo)o(ws)g(the)f(system)g(to)f(a)o(v)o(oid)75 871 y(calling)40 b Fx(function)21 b FE(for)d(eac)o(h)i(elemen)o(t)g(in) g(the)f(input)h(bu\013er.)32 b(Rather,)20 b(the)f(system)g(can)g(c)o (ho)q(ose)h(to)75 928 y(apply)31 b Fx(function)17 b FE(to)d(c)o(h)o (unks)i(of)f(input.)75 1047 y Fx(Example)g(of)g(User-Reduce)75 1133 y FE(Compute)g(the)g(pro)q(duct)h(of)f(an)g(arra)o(y)f(of)g (complex)i(n)o(um)o(b)q(ers,)g(in)g(C.)75 1236 y Fp(typedef)23 b(struct)g(_complex)g({)170 1292 y(double)g(real,imag;)75 1349 y(})h(Complex;)75 1462 y(/*)g(the)f(user-defined)f(function)99 1518 y(*/)75 1631 y(void)h(myProd\()g(Complex)g(invec[],)g(Complex)g (inoutvec[],)f(int)i(*len)f(\))75 1687 y({)170 1744 y(int)h(i;)170 1800 y(Complex)f(c;)170 1913 y(for)h(\(i=0;)f(i<)h(*len;)f(++i\))g({) 266 1970 y(c.real)g(=)h(inoutvec[i].real*invec[i)o(].real)c(-)528 2026 y(inoutvec[i].imag*invec[i].ima)o(g;)266 2083 y(c.imag)j(=)h (inoutvec[i].real*invec[i)o(].imag)c(+)528 2139 y (inoutvec[i].imag*invec[i].rea)o(l;)266 2195 y(inoutvec[i])i(=)i(c;)170 2252 y(})75 2308 y(})75 2421 y(/*)g(and,)f(to)g(call)h(it...)99 2478 y(*/)75 2534 y(...)170 2647 y(/*)g(each)f(process)g(has)h(an)f (array)g(of)h(100)g(Complexes)194 2704 y(*/)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 97 102 97 101 bop 75 -100 a Ft(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FE(97)170 45 y Fp(Complex)23 b(a[100],)g(answer[100];)170 102 y(MPI_Datatype)g(ctype;)170 214 y(/*)h(explain)f(to)h(MPI)f(what)g (a)h("complex")f(is)g(\(remember,)g(this)g(is)h(C!\))194 271 y(*)g(Note)f(we)h(are)f(assuming)g(compiler)g(implements)f(type)i (Complex)f(as)194 327 y(*)h(two)f(contiguous)g(doubles)194 384 y(*/)170 440 y(MPI_Type_contiguous\()f(2,)h(MPI_DOUBLE,)g(&ctype)g (\);)170 497 y(MPI_Type_commit\()f(ctype)h(\);)170 610 y(MPI_User_reduce\()f(a,)i(answer,)f(100,)g(ctype,)g(myProd,)g(root,)g (comm)g(\);)170 723 y(/*)h(At)g(this)f(point,)g(the)g(answer,)g(which)h (consists)e(of)i(100)f(Complexes,)194 779 y(*)h(resides)f(on)h(process) e(root)194 835 y(*/)75 1031 y Fx(MPI)p 160 1031 14 2 v 16 w(USER)p 288 1031 V 17 w(REDUCEA\()16 b(sendbuf,)h(recvbuf,)f (count,)g(datat)o(yp)q(e,)g(function,)h(ro)q(ot,)d(comm\))117 1116 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 1208 y(OUT)124 b Fx(recvbuf)434 b Fr(starting)19 b(address)h(of)e(receiv)o(e)i(bu\013er) f(\(c)o(hoice,)h(signi\014can)o(t)905 1265 y(only)13 b(at)h(ro)q(ot\))117 1356 y(IN)171 b Fx(count)466 b Fr(n)o(um)o(b)q(er) 13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g(\(in)o(teger\))117 1448 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(input)h(bu\013er)117 1540 y(IN)171 b Fx(function)418 b Fr(user)15 b(de\014ned)g(function)117 1632 y(IN)171 b Fx(ro)q(ot)492 b Fr(rank)14 b(of)f(ro)q(ot)h(pro)q (cess)i(\(in)o(teger\))117 1724 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 1857 y Fp(int)23 b(MPI)p 245 1857 15 2 v 17 w(USER)p 358 1857 V 17 w(REDUCEA\(void*)f (sendbuf,)h(void*)g(recvbuf,)g(int)g(count,)393 1913 y(MPI)p 468 1913 V 17 w(Datatype)g(datatype,)f(void)i(function,)e(int)i (root,)f(MPI)p 1607 1913 V 17 w(Comm)g(comm\))75 2008 y(MPI)p 150 2008 V 17 w(USER)p 263 2008 V 16 w(REDUCEA\(SENDBUF,)f (RECVBUF,)h(COUNT,)g(DATATYPE,)g(FUNCTION,)f(ROOT,)h(COMM\))170 2064 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2121 y(EXTERNAL)g(FUNCTION)170 2177 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g (COMM,)g(IERROR)166 2272 y FE(Iden)o(tical)16 b(to)f Fx(MPI)p 493 2272 14 2 v 16 w(USER)p 621 2272 V 17 w(REDUCE)p FE(,)g(except)g(that)g(the)g(op)q(eration)g(de\014ned)h(b)o(y)30 b Fx(function)16 b FE(is)f(not)75 2329 y(required)22 b(to)e(b)q(e)i(comm)o(utativ)o(e,)f(but)g(only)h(asso)q(ciativ)o(e.)37 b(Th)o(us,)22 b(the)f(order)f(of)h(computation)g(can)75 2385 y(b)q(e)d(mo)q(di\014ed)h(only)e(using)h(asso)q(ciativit)o(y)l(.) 27 b(Use)17 b(of)g(this)h(function)g(means)f(that)f(the)i(implemen)o (tation)75 2441 y Fs(c)n(annot)c FE(try)g(and)h(c)o(hange)f(the)h (order)f(of)g(computation)h(of)f(the)h(reduce)g(b)o(y)g(assuming)g (comm)o(utativit)o(y)l(.)166 2582 y Fo(Implemen)o(tati)o(on)d(note:)166 2647 y Fm(MPI)p 243 2647 13 2 v 14 w(USER)p 361 2647 V 14 w(REDUCEA)e Fr(and)h Fm(MPI)p 729 2647 V 14 w(USER)p 847 2647 V 15 w(REDUCE)e Fr(can)j(ha)o(v)o(e)f(iden)o(tical)g(implemen) o(tati)o(ons,)e(if)i(one)g(do)q(es)75 2704 y(not)j(wish)g(to)f(tak)o(e) h(adv)n(an)o(tage)f(of)h(comm)o(utati)o(vit)o(y)l(.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 98 103 98 102 bop 75 -100 a FE(98)703 b Ft(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y Fo(Implemen)o(tati)o(on)h (note:)52 b Fr(The)19 b(addition)e(of)g(the)i(third)f(argumen)o(t,)36 b Fm(*len)18 b Fr(in)36 b Fm(function)19 b Fr(allo)o(w)d(the)75 102 y(system)d(to)g(a)o(v)o(oid)f(calling)26 b Fm(function)14 b Fr(for)f(eac)o(h)h(elemen)o(t)f(in)g(the)g(input)h(bu\013er;)g (rather,)g(the)f(system)h(can)f(c)o(ho)q(ose)75 158 y(to)j(apply)33 b Fm(function)17 b Fr(to)f(c)o(h)o(unks)i(of)d(inputs,)i(where)h(the)f (size)g(of)f(the)h(c)o(h)o(unk)g(is)f(c)o(hosen)i(b)o(y)e(the)h(system) f(so)h(as)75 214 y(to)f(optimize)e(comm)o(unicati)o(on)f(and)i (computation)f(pip)q(elining.)22 b(E.g.,)31 b Fm(*len)16 b Fr(could)g(b)q(e)g(set)h(to)e(b)q(e)h(the)h(t)o(ypical)75 271 y(pac)o(k)o(et)d(size)h(in)e(the)i(comm)o(unicatio)o(n)c (subsystem.)166 411 y FE(MPI)j(includes)i(v)m(arian)o(ts)e(of)f(eac)o (h)h(of)f(the)h(reduce)h(op)q(erations)f(where)g(the)g(result)g(is)g (kno)o(wn)g(to)f(all)75 467 y(pro)q(cesses)j(in)g(the)f(group)g(on)g (return.)75 572 y Fx(MPI)p 160 572 14 2 v 16 w(ALLREDUCE\()g(sendbuf,)i (recvbuf,)f(count,)g(datat)o(yp)q(e,)g(op,)f(comm\))117 649 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 726 y(OUT)124 b Fx(recvbuf)434 b Fr(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er) g(\(c)o(hoice\))117 802 y(IN)171 b Fx(count)466 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 879 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)i(bu\013er)117 955 y(IN)171 b Fx(op)525 b Fr(op)q(eration)117 1032 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 1157 y Fp(int)23 b(MPI)p 245 1157 15 2 v 17 w(ALLREDUCE\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)g(count,)393 1214 y(MPI)p 468 1214 V 17 w(Datatype)g(datatype,)f(void)i(op,)f(MPI)p 1225 1214 V 17 w(Comm)g(comm\))75 1301 y(MPI)p 150 1301 V 17 w (ALLREDUCE\(SENDBUF,)e(RECVBUF,)i(COUNT,)g(DATATYPE,)g(OP,)g(COMM\))170 1357 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1414 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h(IERROR)166 1501 y FE(Same)19 b(as)h(the)f Fx(MPI)p 519 1501 14 2 v 16 w(REDUCE)h FE(op)q(eration)g(function)g(except)g(that)f(the)h (result)g(app)q(ears)f(in)i(the)75 1557 y(receiv)o(e)16 b(bu\013er)f(of)g(all)h(the)f(group)g(mem)o(b)q(ers.)75 1662 y Fx(MPI)p 160 1662 V 16 w(USER)p 288 1662 V 17 w(ALLREDUCE\()g(sendbuf,)j(recvbuf,)d(count,)h(datat)o(yp)q(e,)h (function,)f(comm\))117 1739 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1816 y(OUT)124 b Fx(recvbuf)434 b Fr(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1892 y(IN)171 b Fx(count)466 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h (bu\013er)f(\(in)o(teger\))117 1969 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(send)i(bu\013er)117 2045 y(IN)171 b Fx(function)418 b Fr(user)15 b(de\014ned)g(function)117 2122 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 2247 y Fp(int)23 b(MPI)p 245 2247 15 2 v 17 w(USER)p 358 2247 V 17 w(ALLREDUCE\(void*)f(sendbuf,)g(void*)i(recvbuf,)e(int)i (count,)393 2303 y(MPI)p 468 2303 V 17 w(Datatype)f(datatype,)f(void)i (function,)e(MPI)p 1368 2303 V 17 w(Comm)h(comm\))75 2391 y(MPI)p 150 2391 V 17 w(USER)p 263 2391 V 16 w (ALLREDUCE\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)f(FUNCTION,)h (COMM\))170 2447 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2504 y(EXTERNAL)g(FUNCTION)170 2560 y(INTEGER)g(COUNT,)g(DATATYPE,)g (COMM,)g(IERROR)166 2647 y FE(Same)15 b(as)g(the)g Fx(MPI)p 506 2647 14 2 v 16 w(USER)p 634 2647 V 18 w(REDUCE)h FE(op)q(eration)f(function)h(except)g(that)f(the)g(result)h(app)q(ears) f(in)75 2704 y(the)g(receiv)o(e)h(bu\013er)g(of)e(all)i(the)g(group)f (mem)o(b)q(ers.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 99 104 99 103 bop 75 -100 a Ft(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FE(99)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(USER)p 288 45 V 17 w(ALLREDUCEA\()16 b(sendbuf,)h(recvbuf,)e(count,)h(datat)o (yp)q(e,)h(function,)f(comm\))117 123 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 200 y(OUT)124 b Fx(recvbuf)434 b Fr(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 277 y(IN)171 b Fx(count)466 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h (bu\013er)f(\(in)o(teger\))117 354 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(send)i(bu\013er)117 431 y(IN)171 b Fx(function)418 b Fr(user)15 b(de\014ned)g(function)117 508 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))166 634 y FE(e)p Fp(int)24 b(MPI)p 357 634 15 2 v 16 w(USER)p 469 634 V 17 w(ALLREDUCEA\(void*)e(sendbuf,)g(void*)i(recvbuf,)e(int)i (count,)393 690 y(MPI)p 468 690 V 17 w(Datatype)f(datatype,)f(void)i (function,)e(MPI)p 1368 690 V 17 w(Comm)h(comm\))75 777 y(MPI)p 150 777 V 17 w(USER)p 263 777 V 16 w(ALLREDUCEA\(SENDBUF,)f (RECVBUF,)g(COUNT,)h(DATATYPE,)g(FUNCTION,)g(COMM\))170 834 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 890 y(EXTERNAL)g (FUNCTION)170 947 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)166 1034 y FE(Same)16 b(as)f Fx(MPI)p 429 1034 14 2 v 16 w(USER)p 557 1034 V 18 w(REDUCEA)p FE(,)h(except)g(that)g(the)g(result) g(app)q(ears)g(in)h(the)f(receiv)o(e)h(bu\013er)e(of)75 1091 y(all)h(the)f(group)g(mem)o(b)q(ers.)166 1231 y Fo(Implemen)o(tati)o(on)c(note:)36 b Fr(The)13 b(all-reduce)h(op)q (erations)f(can)g(b)q(e)h(implemen)o(ted)d(as)i(a)g(reduce,)i(follo)o (w)o(ed)75 1287 y(b)o(y)f(a)f(broadcast.)19 b(Ho)o(w)o(ev)o(er,)14 b(a)f(direct)i(implemen)o(tatio)o(n)c(can)j(lead)g(to)g(b)q(etter)h(p)q (erformance.)75 1497 y Fq(4.5.3)49 b(Reduce-Scatter)75 1584 y FE(MPI)15 b(also)f(includes)j(v)m(arian)o(ts)e(of)f(eac)o(h)g (of)h(the)f(reduce)i(op)q(erations)f(where)f(the)h(result)g(is)g (scattered)g(to)75 1641 y(all)h(pro)q(cesses)g(in)g(the)f(group)g(on)g (return.)75 1745 y Fx(MPI)p 160 1745 V 16 w(REDUCE)p 352 1745 V 17 w(SCA)l(TTER\()h(sendbuf,)h(recvbuf,)f(recvcounts,)g (datat)o(yp)q(e,)h(op,)e(comm\))117 1824 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 1901 y(OUT)124 b Fx(recvbuf)434 b Fr(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\)) 117 1978 y(IN)171 b Fx(recvcounts)372 b Fr(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g(the)h(n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g (in)f(re-)905 2034 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 2091 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 2167 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er) 117 2244 y(IN)171 b Fx(op)525 b Fr(op)q(eration)117 2321 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 2447 y Fp(int)23 b(MPI)p 245 2447 15 2 v 17 w(REDUCE)p 406 2447 V 17 w(SCATTER\(void*)f(sendbuf,)g(void*)i(recvbuf,)e(int)i (*recvcounts,)393 2503 y(MPI)p 468 2503 V 17 w(Datatype)f(datatype,)f (void)i(op,)f(MPI)p 1225 2503 V 17 w(Comm)g(comm\))75 2591 y(MPI)p 150 2591 V 17 w(REDUCE)p 311 2591 V 16 w (SCATTER\(SENDBUF,)f(RECVBUF,)h(RECVCOUNTS,)f(DATATYPE,)h(OP,)g(COMM\)) 170 2647 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2704 y(INTEGER)g(RECVCOUNTS,)g(DATATYPE,)f(OP,)i(COMM,)f(IERROR)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 100 105 100 104 bop 75 -100 a FE(100)680 b Ft(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y Fx(MPI)p 251 45 14 2 v 16 w(REDUCE)p 443 45 V 17 w(SCA)l(TTER)g FE(\014rst)f(do)q(es)h(a)f(comp)q(onen)o(t)o(wise)g(reduction)h(on)g(v) o(ectors)e(pro)o(vided)i(b)o(y)75 102 y(the)h(pro)q(cesses.)24 b(Next,)17 b(the)f(resulting)i(v)o(ector)e(of)g(results)h(is)g(split)g (in)o(to)g Fp(n)f FE(disjoin)o(t)i(segmen)o(ts,)e(where)75 158 y Fp(n)h FE(is)g(the)g(n)o(um)o(b)q(er)g(of)f(mem)o(b)q(ers)h(in)h (the)f(group;)g(segmen)o(t)f Fp(i)h FE(con)o(tains)g Fx(recvcounts[i])h FE(elemen)o(ts.)26 b(The)75 214 y Fp(i)p FE(-th)15 b(segmen)o(t)g(is)h(sen)o(t)f(to)f(pro)q(cess)i(with)f (rank)g Fp(i)p FE(.)166 384 y Fo(Implemen)o(tati)o(on)10 b(note:)17 b Fr(The)12 b Fm(MPI)p 792 384 13 2 v 15 w(REDUCE)p 970 384 V 13 w(SCA)m(TTER)f Fr(routine)h(is)g(functionally)f(equiv)n (alen)o(t)g(to:)17 b(A)75 441 y Fm(MPI)p 152 441 V 14 w(REDUCE)28 b Fr(op)q(eration)g(function)h(with)f Fm(count)i Fr(equal)f(to)f(the)i(sum)e(of)g Fm(recvcounts[i])j Fr(follo)o(w)o(ed)c (b)o(y)75 497 y Fm(MPI)p 152 497 V 14 w(SCA)m(TTERV)15 b Fr(with)i Fm(sendcounts)h Fr(equal)f(to)f Fm(recvcounts)p Fr(.)28 b(Ho)o(w)o(ev)o(er,)17 b(a)g(direct)g(implemen)o(tatio)o(n)d (ma)o(y)h(run)75 554 y(faster.)75 934 y Fq(4.5.4)49 b(Scan)75 1127 y Fx(MPI)p 160 1127 14 2 v 16 w(SCAN\()15 b(sendbuf,)j(recvbuf,)d (count,)h(datat)o(yp)q(e,)h(op,)e(comm)e(\))117 1235 y Fr(IN)171 b Fx(sendbuf)428 b Fr(starting)14 b(address)h(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 1372 y(IN)171 b Fx(recvbuf)434 b Fr(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er)g(\(c)o (hoice\))117 1508 y(IN)171 b Fx(count)466 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g(\(in)o(teger\))117 1645 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(input)h(bu\013er)117 1782 y(IN)171 b Fx(op)525 b Fr(op)q(eration)117 1919 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 2074 y Fp(int)23 b(MPI)p 245 2074 15 2 v 17 w(SCAN\(void*)g(sendbuf,)f(void*)i(recvbuf,) e(int)i(count,)393 2130 y(MPI)p 468 2130 V 17 w(Datatype)f(datatype,)f (void)i(op,)f(MPI)p 1225 2130 V 17 w(Comm)g(comm)h(\))75 2248 y(MPI)p 150 2248 V 17 w(SCAN\(SENDBUF,)e(RECVBUF,)h(COUNT,)g (DATATYPE,)f(OP,)i(COMM\))170 2304 y()f(SENDBUF\(*\),)g (RECVBUF\(*\))170 2361 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 2478 y Fx(MPI)p 251 2478 14 2 v 16 w(SCAN)f FE(is)g(used)h(to)e(p)q(erform)g(a)h(parallel)h(pre\014x)f(with)g(resp) q(ect)g(to)f(an)h(asso)q(ciativ)o(e)g(and)75 2534 y(comm)o(utativ)o(e)e (reduction)i(op)q(eration)g(on)f(data)f(distributed)j(across)d(the)h (group.)40 b(The)23 b(op)q(eration)75 2591 y(returns)17 b(in)h(the)f(receiv)o(e)i(bu\013er)e(of)g(the)g(pro)q(cess)g(with)h (rank)f Fp(i)g FE(the)g(reduction)h(of)f(the)g(v)m(alues)i(in)f(the)75 2647 y(send)k(bu\013ers)f(of)f(pro)q(cesses)i(with)g(ranks)e Fp(0,...,i)p FE(.)37 b(The)21 b(t)o(yp)q(e)h(of)e(op)q(erations)i(supp) q(orted,)g(their)75 2704 y(seman)o(tics,)15 b(and)g(the)h(constrain)o (ts)e(on)h(send)h(and)g(receiv)o(e)g(bu\013ers)f(are)g(as)g(for)f Fx(MPI)p 1531 2704 V 16 w(REDUCE)p FE(.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 101 106 101 105 bop 75 -100 a Ft(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 860 b FE(101)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(USER)p 288 45 V 17 w(SCAN\()16 b(sendbuf,)h(recvbuf,)f(count,)g(datat)o(yp)q (e,)g(function,)g(comm\))117 123 y Fr(IN)171 b Fx(sendbuf)428 b Fr(address)15 b(of)f(input)f(bu\013er)117 201 y(OUT)124 b Fx(recvbuf)434 b Fr(address)15 b(of)f(output)g(bu\013er)117 278 y(IN)171 b Fx(count)466 b Fr(n)o(um)o(b)q(er)14 b(of)h(elemen)o(ts) g(in)f(input)h(and)g(output)g(bu\013er)h(\(in)o(te-)905 334 y(ger\))117 411 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(bu\013er)117 489 y(IN)171 b Fx(function)418 b Fr(user)15 b(pro)o(vided)f(function)117 566 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 691 y Fp(int)23 b(MPI)p 245 691 15 2 v 17 w(USER)p 358 691 V 17 w(SCAN\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)h(count,)393 748 y(MPI)p 468 748 V 17 w(Datatype)f(datatype,)f(void)i(function,)e (MPI)p 1368 748 V 17 w(Comm)h(comm\))75 835 y(MPI)p 150 835 V 17 w(USER)p 263 835 V 16 w(SCAN\(SENDBUF,)f(RECVBUF,)h(COUNT,)g (DATATYPE,)g(FUNCTION,)g(COMM\))170 892 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 948 y(EXTERNAL)g(FUNCTION)170 1005 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)166 1092 y FE(Same)12 b(as)g(the)h Fx(MPI)p 498 1092 14 2 v 15 w(SCAN)g FE(op)q(eration)g(function)g(except)g(that)f(a)g(user)g (supplied)j(function)e(is)g(used.)75 1148 y Fx(function)k FE(is)f(an)g(asso)q(ciativ)o(e)f(and)h(comm)o(utativ)o(e)f(function)h (with)g(an)g(input)g(v)o(ector,)f(an)g(inout)h(v)o(ector,)75 1205 y(and)g(a)g(length)g(argumen)o(t.)21 b(The)16 b(t)o(yp)q(es)g(of)g (the)g(t)o(w)o(o)e(v)o(ectors)h(and)h(of)g(the)g(returned)g(v)m(alues)h (all)g(agree.)75 1261 y(See)f Fx(MPI)p 241 1261 V 16 w(USER)p 369 1261 V 17 w(REDUCE)g FE(for)e(more)h(details.)75 1366 y Fx(MPI)p 160 1366 V 16 w(USER)p 288 1366 V 17 w(SCANA\()h(sendbuf,)h(recvbuf,)f(count,)g(datat)o(yp)q(e,)g(function,) h(comm\))117 1444 y Fr(IN)171 b Fx(sendbuf)428 b Fr(address)15 b(of)f(input)f(bu\013er)i(\(c)o(hoice\))117 1522 y(OUT)124 b Fx(recvbuf)434 b Fr(address)15 b(of)f(output)g(bu\013er)h(\(c)o (hoice\))117 1599 y(IN)171 b Fx(count)466 b Fr(n)o(um)o(b)q(er)14 b(of)h(elemen)o(ts)g(in)f(input)h(and)g(output)g(bu\013er)h(\(in)o(te-) 905 1655 y(ger\))117 1732 y(IN)171 b Fx(datat)o(yp)q(e)408 b Fr(data)14 b(t)o(yp)q(e)g(of)f(bu\013er)117 1810 y(IN)171 b Fx(function)418 b Fr(user)15 b(de\014ned)g(function)117 1887 y(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))75 2012 y Fp(int)23 b(MPI)p 245 2012 15 2 v 17 w(USER)p 358 2012 V 17 w(SCANA\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g (count,)393 2069 y(MPI)p 468 2069 V 17 w(Datatype)g(datatype,)f(void)i (function,)e(MPI)p 1368 2069 V 17 w(Comm)h(comm\))75 2156 y(MPI)p 150 2156 V 17 w(USER)p 263 2156 V 16 w(SCANA\(SENDBUF,)f (RECVBUF,)h(COUNT,)g(DATATYPE,)g(FUNCTION,)f(COMM\))170 2213 y()h(SENDBUF\(*\),)g(RECVBUF\(*\))170 2269 y(EXTERNAL)g(FUNCTION)170 2325 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g (IERROR)166 2413 y FE(Same)13 b(as)g Fx(MPI)p 424 2413 14 2 v 16 w(USER)p 552 2413 V 17 w(SCAN)p FE(,)h(except)f(that)g(the)g (user-de\014ned)i(op)q(eration)e(need)h(not)f(b)q(e)h(comm)o(u-)75 2469 y(tativ)o(e.)166 2603 y Fo(Implemen)o(tati)o(on)e(note:)166 2654 y Fm(MPI)p 243 2654 13 2 v 14 w(USER)p 361 2654 V 14 w(SCAN)j Fr(could)g(b)q(e)g(implemen)o(ted)e(as)i Fm(MPI)p 1040 2654 V 14 w(USER)p 1158 2654 V 15 w(SCANA)p Fr(,)e(though)i(for)g(some)f(arc)o(hitectures)75 2704 y(it)g(ma)o(y)d(b)q(e)k(p)q(ossible)f(to)g(arriv)o(e)g(at)f(a)h(faster) g Fm(MPI)p 864 2704 V 15 w(USER)p 983 2704 V 14 w(SCAN)f Fr(b)o(y)h(taking)f(adv)n(an)o(tage)g(of)g(comm)o(utativit)o(y)l(.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 102 107 102 106 bop 75 -100 a FE(102)680 b Ft(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fv(4.6)59 b(Co)n(rrectness)75 148 y Fq(4.6.1)49 b(Synchronization)18 b(Side-E\013ects)75 234 y FE(A)13 b(correct)f(program)g(should)i(in)o (v)o(ok)o(e)f(collectiv)o(e)h(comm)o(unications)g(so)e(that)g(deadlo)q (c)o(k)i(will)g(not)f(o)q(ccur,)75 290 y(whether)f(collectiv)o(e)h (comm)o(unication)g(is)f(sync)o(hronizing)h(or)f(not.)18 b(The)12 b(follo)o(wing)g(examples)h(illustrate)75 347 y(dangerous)i(use)h(of)e(collectiv)o(e)j(routines.)166 403 y(The)e(follo)o(wing)h(example)g(illustrates)h(an)e(erroneous)g (program.)75 499 y Fp(/*)24 b(Example)e(A)i(*/)75 555 y(switch\(rank\))147 611 y({)147 668 y(case)f(0:)g({)h (MPI_Bcast\(var1,)e(count,)h(type,)g(0,)h(comm\);)385 724 y(MPI_Send\(var2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)385 781 y(break;)337 837 y(})147 894 y(case)g(1:)g({)h(MPI_Recv\(var2,)e (count,)h(type,)g(0,)h(tag,)f(comm\);)385 950 y(MPI_Bcast\(var1,)f (count,)h(type,)g(0,)h(comm\);)385 1007 y(break;)337 1063 y(})147 1120 y(})166 1215 y FE(Pro)q(cess)16 b(zero)f(executes)h (a)g(broadcast,)f(follo)o(w)o(ed)h(b)o(y)f(a)h(blo)q(c)o(king)h(send)f (op)q(eration;)g(pro)q(cess)g(one)75 1271 y(\014rst)h(executes)h(a)g (matc)o(hing)f(blo)q(c)o(king)i(receiv)o(e,)g(follo)o(w)o(ed)f(b)o(y)g (the)f(matc)o(hing)h(broadcast)f(call.)28 b(This)75 1328 y(program)18 b(ma)o(y)h(deadlo)q(c)o(k.)32 b(The)20 b(broadcast)e(call) j(on)e(pro)q(cess)g(zero)g Fs(may)h FE(blo)q(c)o(k)g(un)o(til)g(pro)q (cess)g(one)75 1384 y(executes)f(the)g(matc)o(hing)f(broadcast)g(call,) i(so)e(that)f(the)i(send)g(is)g(not)f(executed.)30 b(Pro)q(cess)19 b(one)f(will)75 1441 y(de\014nitely)f(blo)q(c)o(k)f(on)f(the)h(receiv)o (e)g(and)f(so,)g(in)h(this)f(case,)g(nev)o(er)g(executes)h(the)f (broadcast.)166 1497 y(The)g(follo)o(wing)h(example)g(is)g(correct,)e (but)i(non-deterministic:)75 1592 y Fp(/*)24 b(Example)e(B)i(*/)75 1649 y(switch\(rank\))147 1705 y({)170 1762 y(case)g(0:)f({)h (MPI_Bcast\(var1,)e(count,)h(type,)g(0,)h(comm\);)409 1818 y(MPI_Send\(var2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)409 1874 y(break;)361 1931 y(})170 1987 y(case)h(1:)f({)h(MPI_Recv\(var2,)e (count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h(comm\);)409 2044 y(MPI_Bcast\(var1,)f(count,)h(type,)g(0,)h(comm\);)409 2100 y(MPI_Recv\(var2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h (comm\);)409 2157 y(break;)361 2213 y(})170 2270 y(case)h(2:)f({)h (MPI_Send\(var2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)409 2326 y(MPI_Bcast\(var1,)f(count,)h(type,)g(0,)h(comm\);)409 2383 y(break;)361 2439 y(})170 2496 y(})166 2591 y FE(All)19 b(three)f(pro)q(cesses)g(participate)h(in)g(a)e(broadcast.)28 b(Pro)q(cess)17 b(0)h(sends)g(a)g(message)f(to)h(pro)q(cess)75 2647 y(1)h(after)g(the)h(broadcast,)f(and)h(pro)q(cess)f(2)h(sends)g(a) f(message)g(to)g(pro)q(cess)h(1)f(b)q(efore)g(the)h(broadcast.)75 2704 y(Pro)q(cess)15 b(1)g(receiv)o(es)h(b)q(efore)g(and)f(after)f(the) i(broadcast,)e(with)h(a)g(wildcard)i(source)e(argumen)o(t.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 103 108 103 107 bop 75 -100 a Ft(4.6.)34 b(CORRECTNESS)1280 b FE(103)166 45 y(Tw)o(o)14 b(p)q(ossible)i(executions,)g(with)f (di\013eren)o(t)g(matc)o(hings)g(of)f(sends)h(and)g(receiv)o(es)h(are)f (illustrated)75 102 y(b)q(elo)o(w.)337 207 y Fp(First)24 b(Execution)170 320 y(0)311 b(1)357 b(2)648 376 y(/-----)47 b(send)457 432 y(recv)23 b(<-/)75 489 y(broadcast)118 b(broadcast)166 b(broadcast)123 545 y(send)23 b(---\\)337 602 y(\\-->)h(recv)337 707 y(Second)g(Execution)147 820 y(0)334 b(1)357 b(2)75 876 y(broadcast)123 933 y(send)23 b(---\\)337 989 y(\\-->)48 b(recv)433 1046 y(broadcast)166 b(broadcast)719 1102 y(/---)47 b(send)481 1159 y(recv)23 b(<---/)166 1264 y FE(Note)16 b(that)g(the)h(second)g(execution)h(has)f (the)f(p)q(eculiar)j(e\013ect)e(that)f(a)g(send)h(executed)h(after)e (the)75 1320 y(broadcast)i(is)h(receiv)o(ed)g(at)f(another)g(no)q(de)h (b)q(efore)g(the)f(broadcast.)29 b(This)19 b(example)g(illustrates)h (the)75 1377 y(fact)i(that)f(one)i(should)g(not)f(rely)h(on)f (collectiv)o(e)i(comm)o(unication)g(functions)f(to)e(ha)o(v)o(e)h (particular)75 1433 y(sync)o(hronization)d(e\013ects.)29 b(T)l(o)17 b(assume)h(that)g(collectiv)o(e)i(comm)o(unication)f (functions)g(do)f(or)f(do)h(not)75 1489 y(ha)o(v)o(e)d(certain)g(sync)o (hronizing)i(side-e\013ects)f(is)g(non-p)q(ortable.)75 1611 y Fq(4.6.2)49 b(Multiple)17 b(Calls)h(to)e(Collective)75 1697 y FE(It)d(is)h(the)g(user's)f(resp)q(onsibilit)o(y)j(to)d(mak)o(e) f(sure)i(that)f(there)g(are)g(no)h(t)o(w)o(o)e(concurren)o(tly)i (executing)g(col-)75 1753 y(lectiv)o(e)h(calls)g(that)f(use)g(the)g (same)g(comm)o(unicator)g(on)g(the)g(same)g(pro)q(cess.)20 b(Since)15 b(all)g(collectiv)o(e)h(com-)75 1810 y(m)o(unication)j (calls)g(are)f(blo)q(c)o(king)h(this)g(restriction)f(only)h(a\013ects)e (m)o(ultithreaded)i(implemen)o(tations.)75 1866 y(On)c(the)g(other)f (hand,)h(it)g(is)g(legitimate)h(for)e(one)g(pro)q(cess)h(to)f(start)g (a)g(new)h(collectiv)o(e)h(comm)o(unication)75 1923 y(call)i(ev)o(en)e (though)h(a)f(previous)h(call)h(that)e(uses)g(the)h(same)f(comm)o (unicator)g(has)h(not)f(y)o(et)g(terminated)75 1979 y(on)f(another)g (pro)q(cess.)20 b(As)15 b(illustrated)i(in)f(the)f(follo)o(wing)h (example:)75 2084 y Fp(/*)24 b(Example)e(C)i(*/)99 2141 y(MPI_Bcast\(var1,)e(count,)h(type,)g(0,)g(comm\);)99 2197 y(MPI_Bcast\(var2,)f(count,)h(type,)g(1,)g(comm\);)166 2302 y FE(In)17 b(a)e(nonsync)o(hronizing)j(implemen)o(tation)g(of)d (broadcast,)g(pro)q(cess)i(zero)f(ma)o(y)f(start)g(executing)75 2359 y(the)h(second)g(broadcast)f(b)q(efore)h(pro)q(cess)g(one)g (terminated)g(the)g(\014rst)f(broadcast.)21 b(Both)15 b(pro)q(cess)h(zero)75 2415 y(and)h(one)f(ma)o(y)g(terminate)g(their)h (t)o(w)o(o)e(broadcast)g(calls)j(b)q(efore)e(other)g(pro)q(cesses)h(ha) o(v)o(e)f(started)f(their)75 2471 y(calls.)21 b(It)15 b(is)h(the)f(implemen)o(tor's)h(resp)q(onsibilit)o(y)h(to)e(ensure)h (this)f(will)i(not)e(cause)h(an)o(y)e(error.)166 2604 y Fo(Implemen)o(tati)o(on)e(note:)166 2654 y Fr(Assume)e(that)g (broadcast)h(is)f(implemen)o(ted)e(using)i(p)q(oin)o(t-to-p)q(oin)o(t)e (MPI)j(comm)o(unicati)o(on.)j(The)d(follo)o(wing)75 2704 y(t)o(w)o(o)i(rules)i(are)f(satis\014ed:)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 104 109 104 108 bop 75 -100 a FE(104)680 b Ft(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)134 45 y Fr(1.)22 b(All)13 b(receiv)o(es)i(sp)q(ecify)g(their)f(source)h(explicitly)e (\(no)h(wildcards\).)134 120 y(2.)22 b(Eac)o(h)12 b(pro)q(cess)i(sends) g(all)d(messages)h(that)h(p)q(ertain)f(to)h(one)f(collectiv)o(e)g(call) g(b)q(efore)h(sending)f(an)o(y)g(message)189 170 y(that)i(p)q(ertain)g (to)g(a)f(subsequen)o(t)j(collectiv)o(e)e(call.)166 244 y(Then)i(messages)g(b)q(elonging)f(to)h(successiv)o(e)i(broadcasts)f (cannot)f(b)q(e)h(confused,)g(as)f(the)h(order)f(of)g(p)q(oin)o(t-)75 294 y(to-p)q(oin)o(t)d(messages)h(is)g(preserv)o(ed.)20 b(This)14 b(is)f(true,)i(in)e(general,)h(for)f(an)o(y)h(collectiv)o(e)g (library)m(.)166 433 y FE(A)k(collectiv)o(e)h(comm)o(unication)f(ma)o (y)f(execute)i(in)f(a)g(con)o(text)f(while)i(p)q(oin)o(t-to-p)q(oin)o (t)f(comm)o(uni-)75 490 y(cations)g(that)g(use)g(the)h(same)f(con)o (text)f(are)h(p)q(ending,)j(or)c(o)q(ccur)i(concurren)o(tly)l(.)30 b(This)19 b(is)g(illustrated)75 546 y(in)d(example)g(B)g(ab)q(o)o(v)o (e,)f(the)g(\014rst)g(pro)q(cess)h(ma)o(y)f(receiv)o(e)h(a)f(message)g (sen)o(t)g(with)h(the)f(con)o(text)g(of)g(com-)75 603 y(m)o(unicator)32 b Fx(comm)14 b FE(while)j(it)f(is)g(executing)h(a)f (broadcast)f(with)h(the)g(same)g(comm)o(unicator.)22 b(It)15 b(is)i(the)75 659 y(implemen)o(tor's)f(resp)q(onsibilit)o(y)h (to)e(ensure)h(this)f(will)i(not)e(cause)g(an)o(y)g(confusion.)166 792 y Fo(Implemen)o(tati)o(on)21 b(note:)65 b Fr(Assume)22 b(that)f(collectiv)o(e)g(comm)o(unications)d(are)k(implemen)o(ted)d (using)75 841 y(p)q(oin)o(t-to-p)q(oin)o(t)e(MPI)i(comm)o(unication.)29 b(Then,)20 b(in)e(order)i(to)e(a)o(v)o(oid)g(confusion,)h(whenev)o(er)h (a)e(comm)o(unica-)75 891 y(tor)g(is)f(created,)i(a)e(\\hidden)h(comm)o (unicator")c(need)19 b(b)q(e)f(created)h(for)e(collectiv)o(e)g(comm)o (unication.)26 b(A)17 b(direct)75 941 y(implemen)o(tatio)o(n)11 b(of)i(MPI)h(collectiv)o(e)g(comm)o(unicatio)o(n)d(can)j(ac)o(hiev)o(e) g(a)g(similar)d(e\013ect)k(more)e(c)o(heaply)m(,)g(e.g.,)f(b)o(y)75 991 y(using)j(a)g(hidden)h(tag)f(or)g(con)o(text)h(bit)g(to)f(indicate) g(whether)i(the)f(comm)o(unicator)d(is)i(used)h(for)f(p)q(oin)o(t-to-p) q(oin)o(t)75 1041 y(or)f(collectiv)o(e)g(comm)o(unicatio)o(n.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 105 110 105 109 bop 75 357 a Fw(Chapter)34 b(5)75 565 y FA(Groups,)40 b(Con)m(texts,)75 690 y(Comm)m(unicators,)g(and)g(Cac)m(heing)75 931 y Fv(5.1)59 b(Intro)r(duction)75 1034 y FE(W)l(e)18 b(b)q(egin)h(this)g(c)o(hapter)f(with)g(a)g(p)q(ersp)q(ectiv)o(e)i(of)d (wh)o(y)h(MPI)g(has)g(included)j(features)c(that)h(are)f(not)75 1091 y(found)11 b(in)h(existing)f(v)o(endor)g(systems,)g(and)f(wh)o(y)h (these)g(are)f(imp)q(ortan)o(t.)18 b(One)12 b(of)e(the)h(main)g(motiv)m (ations)75 1147 y(for)f(the)h(features)g(describ)q(ed)i(here)f(is)f(to) f(allo)o(w)i(for)e(the)h(systematic)g(dev)o(elopmen)o(t)h(of)e (parallel)j(libraries.)75 1273 y Fq(5.1.1)49 b(Why)16 b(w)o(e)g(need)g(lib)o(ra)o(ries)75 1360 y FE(P)o(arallel)k(libraries)g (are)e(needed)i(to)e(encapsulate)i(the)f(distracting)g(complications)h (asso)q(ciated)f(with)75 1417 y(distributed)f(realizations)f(of)e(k)o (ey)h(algorithms.)23 b(They)16 b(ensure)h(consisten)o(t)f(correctness)g (of)g(suc)o(h)g(pro-)75 1473 y(cedures,)h(and)g(pro)o(vide)g(a)f (\\higher)h(lev)o(el")h(of)e(p)q(ortabilit)o(y)h(than)g(MPI)f(itself)i (can)e(pro)o(vide.)25 b(As)16 b(suc)o(h,)75 1529 y(libraries)24 b(prev)o(en)o(t)e(eac)o(h)h(programmer)e(from)h(rep)q(eating)h(the)g(w) o(ork)f(of)g(de\014ning)i(consisten)o(t)f(data)75 1586 y(structures,)13 b(data)f(la)o(y)o(outs,)h(and)g(metho)q(ds)g(that)g (implemen)o(t)h(k)o(ey)f(algorithms)g(\(suc)o(h)g(as)g(matrix)g(op)q (er-)75 1642 y(ations\).)21 b(Since)c(the)f(b)q(est)g(libraries)i(come) d(with)i(sev)o(eral)e(v)m(ariations)i(on)e(parallel)j(systems)d (\(di\013eren)o(t)75 1699 y(data)d(la)o(y)o(outs,)g(di\013eren)o(t)h (strategies)f(dep)q(enden)o(t)i(on)f(size)g(of)g(system,)f(or)g(t)o(yp) q(e)h(of)f(\015oating)h(p)q(oin)o(t\),)g(this)75 1755 y(to)q(o)i(needs)h(to)e(b)q(e)i(hidden)h(from)d(the)i(user.)75 1881 y Fq(5.1.2)49 b(Why)16 b(existing)h(systems)e(do)i(not)f(supp)q(o) o(rt)f(lib)o(ra)o(ries)75 1968 y FE(It)f(is)g(di\016cult)h(to)e(write)h (libraries)h(with)f(message)g(passing)g(systems)f(that)g(v)o(endors)g (pro)q(duce,)i(b)q(ecause)75 2025 y(they)h(ha)o(v)o(e)g(no)g(w)o(a)o(y) f(for)g(the)h(library)h(to)f(insulate)h(itself)g(e\013ectiv)o(ely)g (from)e(the)i(other)e(p)q(oin)o(t-to-p)q(oin)o(t)75 2081 y(message)j(passing)h(that)e(go)q(es)i(on)f(in)h(the)g(system.)28 b(F)l(or)18 b(instance,)i(message)e(tags)f(are)h(a)g(means)h(for)75 2138 y(a)i(library)i(to)e(hide)i(its)e(messages)g(from)g(the)h(user.)39 b(Ho)o(w)o(ev)o(er,)22 b(man)o(y)f(libraries)i(ma)o(y)e(c)o(ho)q(ose)h (the)75 2194 y(same)16 b(tags,)f(or)h(the)g(v)o(endor's)g(library)h(ma) o(y)e(itself)i(indiscriminately)j(use)c(tags)f(when)i(implemen)o(ting) 75 2251 y(collectiv)o(e)f(op)q(erations)e(\()p Fs(e.g.)p FE(,)g(global)h(sum\);)e(in)j(that)d(case,)h(collectiv)o(e)i(and)f(p)q (oin)o(t-to-p)q(oin)o(t)f(messages)75 2307 y(ma)o(y)g(get)h(mixed)h (up.)166 2364 y(Collectiv)o(e)c(op)q(erations,)g(suc)o(h)g(as)f (reduce,)h(broadcast,)f(and)g(so)g(on,)h(are)f(not)f(de\014ned)j(in)f (all)g(v)o(endor)75 2421 y(systems,)j(and)i(when)f(de\014ned)i(are)d (most)h(often)f(only)i(a)o(v)m(ailable)h(o)o(v)o(er)d(the)h(full)i(set) d(of)h(pro)q(cessors)g(\(or)75 2477 y(pro)q(cesses\))d(allo)q(cated)h (to)e(the)i(user.)19 b(Libraries)14 b(need)g(to)e(w)o(ork)g(on)h (di\013eren)o(t)h(collections)g(of)f(pro)q(cesses,)75 2534 y(and)j(to)g(ha)o(v)o(e)g(a)o(v)m(ailable)h(collectiv)o(e)h(op)q (erations)e(that)g(only)h(impact)f(the)g(pro)q(cesses)h(that)e(care)h (ab)q(out)75 2590 y(the)f(collectiv)o(e)i(op)q(eration.)166 2647 y(A)k(third)g(issue)h(is)f(that)f(libraries)j(don't)d(w)o(an)o(t)g (to)g(describ)q(e)j(p)q(oin)o(t-to-p)q(oin)o(t)e(comm)o(unication)75 2704 y(in)e(terms)f(of)g(lo)o(w-lev)o(el,)j(hardw)o(are)c(dep)q(enden)o (t)j(names.)30 b(In)19 b(fact,)f(they)h(migh)o(t)f(lik)o(e)i(to)e(use)g (virtual)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 106 111 106 110 bop 75 -100 a FE(106)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y FE(names)e(for)f(pro)q(cesses)i(that)e(someho)o(w)g(re\015ect)i (the)f(relationship)i(of)d(pro)q(cesses)i(to)e(one)h(another)g(in)h (the)75 102 y(distributed)20 b(data)e(structure)g(natural)g(to)g(an)g (algorithm.)29 b(Libraries)20 b(\(or)d(program)h(mo)q(dules\))h(that)75 158 y(execute)h(concurren)o(tly)f(on)g(di\013eren)o(t)g(pro)q(cesses)g (ma)o(y)f(w)o(an)o(t)g(to)g(use)h(the)g(same)g(pro)q(cess)g(names)g (for)75 214 y(their)d(in)o(ternal)g(comm)o(unication.)75 344 y Fq(5.1.3)49 b(What)16 b(features)f(a)o(re)g(needed)h(to)g(supp)q (o)o(rt)g(lib)o(ra)o(ries)75 433 y FE(So,)f(w)o(e)g(ha)o(v)o(e)f (established)j(three)f(k)o(ey)f(areas)f(that)h(will)i(supp)q(ort)e(the) g(creation)h(of)e(libraries:)143 532 y FC(\017)23 b FE(Safe)11 b(comm)o(unication)i(space,)f(that)f(guaran)o(tees)g(that)g(a)g (library)i(can)f(comm)o(unicate)g(as)f(it)h(needs)189 589 y(to,)143 688 y FC(\017)23 b FE(Group)14 b(scop)q(e)h(for)g (collectiv)o(e)h(op)q(erations,)f(that)f(allo)o(w)h(libraries)h(to)e(a) o(v)o(oid)h(unnecessarily)i(syn-)189 745 y(c)o(hronizing)f(unin)o(v)o (olv)o(ed)h(pro)q(cessors,)143 844 y FC(\017)23 b FE(Abstract)15 b(pro)q(cess)h(naming)h(to)e(allo)o(w)h(libraries)i(to)d(describ)q(e)j (their)e(comm)o(unication)h(in)g(terms)189 901 y(suitable)f(to)f(their) g(o)o(wn)g(data)g(structures.)75 1030 y Fq(5.1.4)49 b(What)16 b(these)f(features)g(a)o(re)g(called)i(in)g(MPI)75 1119 y FE(The)e(features)g(that)g(implemen)o(t)h(these)g(imp)q(ortan)o(t)e (concepts)i(are)f(as)g(follo)o(ws,)g(in)h(MPI:)143 1218 y FC(\017)23 b Fz(Con)o(texts)18 b FE(pro)o(vide)j(the)e(abilit)o(y)i (to)e(ha)o(v)o(e)g(separate)g(safe)h(\\univ)o(erses")g(of)f (message-passing)189 1275 y(in)g(MPI.)g(A)g(con)o(text)f(is)h (something)g(lik)o(e)h(an)f(additional)i(tag)c(that)i(di\013eren)o (tiates)g(messages,)189 1331 y(but)g(the)f(system)h(manages)f(it,)h (and)g(the)g(user)g(nev)o(er)g(manipulates)h(con)o(texts.)30 b(W)l(e)19 b(sa)o(y)f(that)189 1388 y(comm)o(unicators)11 b(\(b)q(elo)o(w\))g(are)g(a)g(sp)q(eci\014c)j(con)o(text)c(of)h(comm)o (unication,)i(when)f(w)o(e)f(are)g(sp)q(eaking)189 1444 y(lo)q(osely)l(.)21 b(The)15 b(use)g(of)g(separate)g(comm)o(unication)g (con)o(texts)g(b)o(y)g(distinct)h(libraries)h(\(or)d(distinct)189 1501 y(library)e(in)o(v)o(o)q(cations\))g(will)h(insulate)g(comm)o (unication)g(in)o(ternal)f(to)g(the)f(library)i(execution)g(from)189 1557 y(external)k(comm)o(unication.)25 b(This)17 b(allo)o(ws)g(to)e(in) o(v)o(ok)o(e)i(the)g(library)g(ev)o(en)g(if)g(there)g(are)f(p)q(ending) 189 1613 y(comm)o(unications,)f(and)g(a)o(v)o(oids)g(the)h(need)g(to)e (sync)o(hronize)j(en)o(try)d(or)h(exit)h(in)o(to)f(library)h(co)q(de.) 143 1713 y FC(\017)23 b Fz(Groups)10 b FE(de\014ne)i(an)f(ordered)g (collection)i(of)e(pro)q(cesses,)h(eac)o(h)f(with)g(a)g(rank,)g(and)g (it)g(is)h(this)f(group)189 1769 y(that)k(de\014nes)i(the)f(lo)o(w-lev) o(el)h(names)f(for)g(in)o(ter-pro)q(cess)g(comm)o(unication)h(\(ranks)e (are)h(used)h(for)189 1826 y(sending)j(and)f(receiving\).)34 b(Th)o(us,)20 b(groups)e(de\014ne)j(a)e(scop)q(e)h(for)e(pro)q(cess)i (names)f(in)h(p)q(oin)o(t)g(to)189 1882 y(p)q(oin)o(t)15 b(comm)o(unication.)21 b(In)16 b(addition,)g(groups)f(de\014ne)h(the)f (scop)q(e)h(of)f(collectiv)o(e)i(op)q(erations.)143 1982 y FC(\017)23 b Fz(Comm)o(unicators)15 b FE(\(see)h([13)o(]\))f (encapsulate)i(all)g(of)f(these)g(ideas)h(in)g(an)f(ob)s(ject)f(that)g (pro)o(vides)189 2038 y(the)g(righ)o(t)g(scop)q(e)h(for)e(all)i(comm)o (unication)g(op)q(erations)g(in)g(MPI.)166 2138 y(The)f(curren)o(t)g (practice)h(in)f(man)o(y)g(comm)o(unication)g(libraries)i(is)e(that)g (there)g(is)g(a)g(a)f(unique,)j(pre-)75 2194 y(de\014ned)22 b(comm)o(unication)e(univ)o(erse)h(that)f(includes)i(all)f(pro)q (cesses)g(a)o(v)m(ailable)h(when)e(the)h(library)f(is)75 2251 y(initiated;)25 b(the)c(pro)q(cesses)g(are)f(assigned)i (consecutiv)o(e)f(ranks.)37 b(P)o(articipan)o(ts)20 b(in)i(a)e(p)q(oin) o(t-to-p)q(oin)o(t)75 2307 y(comm)o(unication)h(are)g(iden)o(ti\014ed)h (b)o(y)f(their)g(rank;)i(a)d(collectiv)o(e)j(comm)o(unication)e(\(suc)o (h)g(as)f(broad-)75 2363 y(cast\))c(in)o(v)o(olv)o(es)h(all)g(pro)q (cesses.)25 b(This)17 b(practice)g(can)g(b)q(e)g(follo)o(w)o(ed)g(in)g (MPI)g(b)o(y)f(using)i(the)e(prede\014ned)75 2420 y(comm)o(unicator)d Fx(MPI)p 455 2420 14 2 v 16 w(COMM)p 613 2420 V 17 w(W)o(ORLD)p FE(.)g(Users)h(that)f(are)g(satis\014ed)i(with)f(suc)o(h)g(practice)h (can)e(plug)i(in)75 2476 y Fx(MPI)p 160 2476 V 16 w(COMM)p 318 2476 V 16 w(W)o(ORLD)j FE(wherev)o(er)f(a)g(comm)o(unicator)g (argumen)o(t)g(is)h(required,)g(and)g(ignore)g(the)f(rest)75 2533 y(of)e(this)g(c)o(hapter.)166 2591 y(The)g(discussion)h(has)e (dealt)h(so)f(far)f(with)i Fz(in)o(tra-comm)o(unication)p FE(:)22 b(comm)o(unication)15 b(within)g(a)75 2647 y(group.)27 b(MPI)18 b(also)g(supp)q(orts)g Fz(in)o(ter-comm)o(unication)p FE(:)27 b(comm)o(unication)18 b(across)f(groups.)28 b(When)75 2704 y(an)15 b(application)i(is)f(built)h(b)o(y)e(comp)q(osing)h(sev)o (eral)f(parallel)i(mo)q(dules,)f(it)g(is)g(con)o(v)o(enien)o(t)g(to)e (allo)o(w)i(one)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 107 112 107 111 bop 75 -100 a Ft(5.2.)34 b(BASIC)16 b(CONCEPTS)1215 b FE(107)75 45 y(mo)q(dule)16 b(to)f(comm)o(unicate)g(with)g(the)h (other)e(using)i(lo)q(cal)g(ranks)f(within)h(the)g(second)f(mo)q(dule)h (for)f(ad-)75 102 y(dressing.)21 b(This)15 b(is)h(esp)q(ecially)h(con)o (v)o(enien)o(t)f(in)g(a)f(clien)o(t-serv)o(er)h(computing)f(paradigm,)g (where)g(either)75 158 y(clien)o(t)f(or)e(serv)o(er)g(are)g(parallel.) 21 b(The)12 b(supp)q(ort)h(of)f(in)o(ter-comm)o(unication)i(also)e(pro) o(vides)h(a)f(mec)o(hanism)75 214 y(for)g(the)h(extension)g(of)f(MPI)h (to)f(a)g(dynamic)h(mo)q(del)h(where)f(not)f(all)h(pro)q(cesses)g(are)g (preallo)q(cated)g(at)f(ini-)75 271 y(tialization)j(time.)k(In)14 b(suc)o(h)f(a)g(situation,)h(it)f(b)q(ecomes)h(necessary)f(to)g(supp)q (ort)g(comm)o(unication)h(across)75 327 y(\\univ)o(erses.")28 b(In)o(ter-comm)o(unication)19 b(is)f(supp)q(orted)h(b)o(y)e(ob)s (jects)h(called)h Fz(in)o(ter-comm)o(unicators)p FE(.)75 384 y(These)13 b(bind)h(t)o(w)o(o)e(groups)h(together)f(with)h(a)f(con) o(text)h(shared)g(b)o(y)g(b)q(oth)g(groups.)18 b(MPI)13 b(pro)o(vides)h(mec)o(h-)75 440 y(anisms)f(for)e(creating)i(and)f (manipulating)i(in)o(ter-comm)o(unicators.)19 b(They)12 b(are)g(used)h(in)g(p)q(oin)o(t-to-p)q(oin)o(t)75 497 y(comm)o(unication)k(in)f(the)g(same)g(manner)g(as)f(regular)h Fz(in)o(tra-comm)o(unicators)p FE(.)23 b(Users)15 b(that)h(do)f(not)75 553 y(need)h(in)o(ter-comm)o(unication)g(in)g(their)g(applications)h (can)e(safely)h(ignore)f(this)h(extension.)166 613 y(Comm)o(unicators)d (also)h(pro)o(vide)g(a)g(\\cac)o(heing")g(mec)o(hanism)h(that)e(allo)o (ws)h(one)g(to)g(asso)q(ciate)g(new)75 669 y(attributes)i(with)h(comm)o (unicators,)f(on)g(a)g(par)g(with)h(MPI)f(built-in)j(features.)k(This) 17 b(can)f(b)q(e)h(used)g(b)o(y)75 726 y(adv)m(anced)h(users)e(to)g (further)g(adorn)h(comm)o(unicators,)f(and)g(b)o(y)h(MPI)f(to)g (implemen)o(t)i(some)e(comm)o(u-)75 782 y(nicator)h(functions.)25 b(F)l(or)16 b(example,)h(the)g(virtual)h(top)q(ology)e(functions)h (describ)q(ed)i(in)f(Chapter)e(6)g(are)75 839 y(lik)o(ely)h(to)e(b)q(e) g(supp)q(orted)h(this)g(w)o(a)o(y)l(.)75 1000 y Fv(5.2)59 b(Basic)19 b(Concepts)75 1109 y Fq(5.2.1)49 b(Groups)75 1201 y FE(A)17 b Fz(group)g FE(is)g(an)g(ordered)g(set)f(of)h(pro)q (cess)g(iden)o(ti\014ers)h(\(henceforth)f(pro)q(cesses\);)g(pro)q(cess) g(iden)o(ti\014ers)75 1258 y(are)11 b(implemen)o(tation-dep)q(enden)o (t)i(ob)s(jects.)18 b(Eac)o(h)11 b(pro)q(cess)g(in)h(a)f(group)f(is)i (asso)q(ciated)f(with)g(an)g(in)o(teger)75 1314 y Fz(rank)p FE(,)18 b(starting)g(from)f(zero.)29 b(Groups)17 b(are)h(represen)o (ted)h(b)o(y)f(opaque)g Fz(group)j(ob)s(jects)p FE(,)e(and)f(hence)75 1371 y(cannot)k(b)q(e)h(directly)g(transferred)f(from)f(one)h(pro)q (cess)h(to)e(another.)41 b(A)22 b(group)g(is)g(used)h(within)g(a)75 1427 y(comm)o(unicator)13 b(to)f(describ)q(e)j(the)e(participan)o(ts)h (in)g(a)f(comm)o(unication)g(\\univ)o(erse")h(and)f(to)g(rank)g(suc)o (h)75 1484 y(participan)o(ts)j(\(th)o(us)e(giving)i(them)g(unique)g (names)f(within)i(that)d(\\univ)o(erse")i(of)f(comm)o(unication\).)166 1626 y Fo(Implemen)o(tati)o(on)h(note:)48 b Fr(A)17 b(group)g(ma)o(y)e (b)q(e)i(represen)o(ted)j(b)o(y)d(a)f(virtual-to-real)g(pro)q (cess-address-)75 1682 y(translation)d(table.)18 b(Eac)o(h)c(comm)o (unicator)d(ob)r(ject)k(w)o(ould)e(ha)o(v)o(e)h(a)f(p)q(oin)o(ter)i(to) e(suc)o(h)i(a)f(table.)75 1905 y Fq(5.2.2)49 b(Contexts)75 1997 y FE(A)15 b Fz(con)o(text)h FE(is)g(the)f(MPI)g(mec)o(hanism)h (for)f(partitioning)h(comm)o(unication)g(space.)21 b(A)15 b(de\014ning)i(prop-)75 2053 y(ert)o(y)d(of)g(a)g(con)o(text)g(is)h (that)e(a)h(message)g(sen)o(t)h(in)g(a)f(con)o(text)g(cannot)g(b)q(e)h (receiv)o(ed)h(in)f(another)f(con)o(text.)75 2110 y(Con)o(texts)20 b(are)h(not)g(explicit)j(MPI)d(ob)s(jects;)j(they)d(app)q(ear)h(only)g (along)f(with)h(groups)f(in)h(comm)o(u-)75 2166 y(nicators.)31 b(Distinct)20 b(comm)o(unicators)e(in)i(the)f(same)f(pro)q(cess)i(ha)o (v)o(e)e(distinct)i(con)o(texts.)31 b(A)19 b(con)o(text)75 2223 y(is)g(essen)o(tially)h(a)e(tag)g(\(or)g(tags\))f(needed)j(to)e (mak)o(e)g(a)g(comm)o(unicator)g(safe)h(for)e(p)q(oin)o(t-to-p)q(oin)o (t)j(and)75 2279 y(MPI-de\014ned)d(collectiv)o(e)g(comm)o(unication.) 166 2421 y Fo(Implemen)o(tati)o(on)11 b(note:)35 b Fr(A)14 b(p)q(ossible)f(implemen)o(tatio)o(n)d(is)j(for)g(a)g(con)o(text)h(to)e (b)q(e)i(a)f(lab)q(el)f(attac)o(hed)i(to)75 2478 y(messages)f(and)f (matc)o(hed)g(on)g(receiv)o(e.)19 b(Eac)o(h)13 b(in)o(tra-comm)o (unicator)d(stores)j(the)h(v)n(alue)e(of)g(that)g(lab)q(el.)17 b(Comm)o(u-)75 2534 y(nicator)f(generating)g(functions)h(use)f(a)g (collectiv)o(e)g(comm)o(unication)d(to)j(agree)h(on)f(a)g(new)g (group-wide)g(unique)75 2591 y(con)o(text.)25 b(In)15 b(in)o(ter-comm)o(unication,)e(it)i(is)h(more)f(con)o(v)o(enien)o(t)h (to)g(use)g(t)o(w)o(o)g(lab)q(els,)f(one)h(used)h(b)o(y)f(group)f(A)h (to)75 2647 y(send)e(and)e(group)h(B)g(to)g(receiv)o(e,)h(and)e(the)i (second)f(used)h(b)o(y)f(group)f(B)h(to)g(send)h(and)e(group)h(A)g(to)f (receiv)o(e.)19 b(Since)75 2704 y(con)o(texts)c(are)f(not)g(explicit)g (ob)r(jects,)g(other)h(implem)o(en)o(tations)c(are)k(p)q(ossible.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 108 113 108 112 bop 75 -100 a FE(108)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y Fq(5.2.3)49 b(Communicato)o(rs)75 131 y FE(Comm)o(unicators)18 b(bring)h(together)f(the)h(concepts)h(of)e(group)g(and)h(con)o(text.)30 b(F)l(urthermore,)19 b(to)f(sup-)75 188 y(p)q(ort)c(implemen)o (tation-sp)q(eci\014c)j(optimizations,)d(and)g(virtual)h(top)q (ologies,)f(they)g(\\cac)o(he")g(additional)75 244 y(information.)37 b(The)21 b(source)g(and)g(destination)h(of)e(a)g(message)h(is)g(iden)o (ti\014ed)i(b)o(y)e(the)g(rank)f(of)h(that)75 301 y(pro)q(cess)14 b(within)g(the)g(group.)19 b(F)l(or)13 b(collectiv)o(e)i(comm)o (unication,)f(the)f(comm)o(unicator)g(sp)q(eci\014es)j(the)d(set)75 357 y(of)k(pro)q(cesses)h(that)f(participate)i(in)f(the)g(collectiv)o (e)i(op)q(eration)d(and)h(their)g(order,)g(when)g(signi\014can)o(t.)75 414 y(Th)o(us,)c(the)g(comm)o(unicator)g(restricts)f(the)h(\\spatial")g (scop)q(e)h(of)f(comm)o(unication,)g(and)g(pro)o(vides)h(lo)q(cal)75 470 y(pro)q(cess)g(addressing.)166 527 y(Comm)o(unicators)10 b(are)h(represen)o(ted)h(b)o(y)g(opaque)f Fz(comm)o(unicator)i(ob)s (jects)p FE(,)f(and)g(hence)g(cannot)75 583 y(b)q(e)k(directly)g (transferred)f(from)g(one)g(pro)q(cess)g(to)g(another.)75 706 y Fq(5.2.4)49 b(Prede\014ned)15 b(Communicato)o(rs)75 793 y FE(An)g(initial)h(comm)o(unicator)27 b Fm(MPI)p 665 793 13 2 v 14 w(COMM)p 809 793 V 15 w(W)o(ORLD)14 b FE(of)g(all)h(pro)q(cesses)g(the)g(lo)q(cal)g(pro)q(cess)g(can)f (comm)o(u-)75 849 y(nicate)i(with)f(after)g(initialization)j(\(itself)e (included\))h(is)f(de\014ned)h(once)e Fx(MPI)p 1403 849 14 2 v 16 w(INIT)f FE(has)h(b)q(een)i(called.)166 906 y(In)f(a)f(static-pro)q(cess-mo)q(del)h(implemen)o(tation)g(of)f(MPI,)g (all)h(pro)q(cesses)g(that)e(participate)i(in)g(the)75 962 y(computation)i(are)g(a)o(v)m(ailable)i(after)d(MPI)i(is)f (initialize)q(d.)32 b(F)l(or)17 b(this)i(case,)g Fm(MPI)p 1481 962 13 2 v 14 w(COMM)p 1625 962 V 14 w(W)o(ORLD)g FE(is)f(a)75 1019 y(comm)o(unicator)h(of)g(all)i(pro)q(cesses)f(a)o(v)m (ailable)h(for)e(the)g(computation;)i(this)f(comm)o(unicator)g(has)f (the)75 1075 y(same)14 b(v)m(alue)h(in)g(all)g(pro)q(cesses.)20 b(In)15 b(an)f(implemen)o(tation)h(of)f(MPI)g(where)g(pro)q(cesses)h (can)f(dynamically)75 1132 y(join)23 b(an)f(MPI)g(execution,)j(it)d(ma) o(y)g(b)q(e)h(the)f(case)h(that)e(a)h(pro)q(cess)h(starts)e(an)h(MPI)g (computation)75 1188 y(without)d(ha)o(ving)g(access)g(to)f(all)i(other) f(pro)q(cesses.)31 b(In)20 b(suc)o(h)f(situations,)h Fm(MPI)p 1479 1188 V 14 w(COMM)p 1623 1188 V 15 w(W)o(ORLD)f FE(is)g(a)75 1245 y(comm)o(unicator)d(incorp)q(orating)h(all)h(pro)q (cesses)f(with)g(whic)o(h)g(the)g(joining)h(pro)q(cess)f(can)f (immediately)75 1301 y(comm)o(unicate.)33 b(Therefore,)20 b Fm(MPI)p 683 1301 V 14 w(COMM)p 827 1301 V 15 w(W)o(ORLD)f FE(ma)o(y)g(sim)o(ultaneouly)i(ha)o(v)o(e)e(di\013eren)o(t)h(v)m(alues) g(in)75 1358 y(di\013eren)o(t)15 b(pro)q(cesses.)166 1414 y(All)i(MPI)e(implemen)o(tations)i(are)e(required)i(to)d(pro)o (vide)i(the)g Fm(MPI)p 1316 1414 V 14 w(COMM)p 1460 1414 V 15 w(W)o(ORLD)f FE(comm)o(unica-)75 1471 y(tor.)34 b(It)21 b(cannot)f(b)q(e)h(deallo)q(cated)h(during)f(the)f(life)i(of)e (a)g(pro)q(cess.)36 b(The)20 b(group)g(corresp)q(onding)i(to)75 1527 y(this)d(comm)o(unicator)f(do)g(not)g(app)q(ear)h(as)f (pre-de\014ned)i(constan)o(ts,)e(but)h(it)f(ma)o(y)g(b)q(e)h(accessed)g (using)75 1584 y Fx(MPI)p 160 1584 14 2 v 16 w(COMM)p 318 1584 V 16 w(GROUP)12 b FE(\(see)f(b)q(elo)o(w\).)18 b(MPI)11 b(do)q(es)f(not)h(sp)q(ecify)g(the)g(corresp)q(ondence)h(b)q (et)o(w)o(een)e(the)h(pro-)75 1640 y(cess)g(rank)g(in)g Fm(MPI)p 388 1640 13 2 v 14 w(COMM)p 532 1640 V 15 w(W)o(ORLD)g FE(and)g(its)g(\(mac)o(hine-dep)q(enden)o(t\))h(absolute)g(address.)18 b(Neither)12 b(do)q(es)75 1697 y(MPI)k(sp)q(ecify)i(the)e(function)h (of)e(the)i(host)e(pro)q(cess,)h(if)h(an)o(y)l(.)23 b(An)16 b(inquiry)i(function)f Fx(MPI)p 1636 1697 14 2 v 15 w(GET)p 1739 1697 V 17 w(HOST)75 1753 y FE(is)i(pro)o(vided)g(in)g(Chapter)f(7) g(so)g(that)f(MPI)i(programs)e(can)h(determine)h(if)g(there)g(is)f(a)g (host)g(pro)q(cess,)75 1809 y(and)13 b(\014nd)g(its)g(rank)f(in)h Fm(MPI)p 539 1809 13 2 v 15 w(COMM)p 684 1809 V 14 w(W)o(ORLD)g FE(if)g(it)g(should)g(exist.)20 b(Other)12 b(implemen)o(tation-dep)q (end)q(en)o(t,)75 1866 y(prede\014ned)17 b(comm)o(unicators)e(ma)o(y)f (also)h(b)q(e)h(pro)o(vided.)75 2011 y Fv(5.3)59 b(Group)20 b(Management)75 2113 y FE(This)g(section)h(describ)q(es)g(the)f (manipulation)h(of)e(pro)q(cess)h(groups)g(in)g(MPI.)f(These)h(op)q (erations)g(are)75 2169 y(lo)q(cal)c(and)g(their)f(execution)i(do)e (not)g(require)h(in)o(terpro)q(cess)f(comm)o(unication.)75 2292 y Fq(5.3.1)49 b(Group)16 b(Accesso)o(rs)75 2426 y Fx(MPI)p 160 2426 14 2 v 16 w(GROUP)p 328 2426 V 18 w(SIZE\(group,)e(size\))117 2503 y Fr(IN)155 b Fx(group)479 b Fr(group)14 b(\(handle\))117 2579 y(OUT)108 b Fx(size)519 b Fr(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g(group)g(\(in)o (teger\))75 2704 y Fp(int)23 b(MPI)p 245 2704 15 2 v 17 w(Group)p 382 2704 V 17 w(size\(MPI)p 591 2704 V 16 w(Group)g(group,)g(int)g(*size\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 109 114 109 113 bop 75 -100 a Ft(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b FE(109)75 45 y Fp(MPI)p 150 45 15 2 v 17 w(GROUP)p 287 45 V 16 w(SIZE\(GROUP,)23 b(SIZE,)g(IERROR\))170 102 y(INTEGER)g(GROUP,)g(SIZE,)h(IERROR)75 236 y Fx(MPI)p 160 236 14 2 v 16 w(GROUP)p 328 236 V 18 w(RANK\(group,)14 b(rank\))117 313 y Fr(IN)155 b Fx(group)479 b Fr(group)14 b(\(handle\))117 389 y(OUT)108 b Fx(rank)504 b Fr(rank)50 b(of)g(the)h(calling)e(pro)q(cess)j(in)e(group,)59 b(or)905 446 y Fm(MPI)p 982 446 13 2 v 15 w(UNDEFINED)13 b Fr(if)h(the)h(pro)q (cess)i(is)d(not)h(a)f(mem)o(b)q(er)f(\(in-)905 502 y(teger\))75 627 y Fp(int)23 b(MPI)p 245 627 15 2 v 17 w(Group)p 382 627 V 17 w(rank\(MPI)p 591 627 V 16 w(Group)g(group,)g(int)g(*rank\))75 714 y(MPI)p 150 714 V 17 w(GROUP)p 287 714 V 16 w(RANK\(GROUP,)g(RANK,) g(IERROR\))170 770 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)75 904 y Fx(MPI)p 160 904 14 2 v 16 w(GROUP)p 328 904 V 18 w(TRANSLA)l(TE)p 602 904 V 17 w(RANKS)16 b(\(group1,)e(n,)h(ranks1,) g(group2,)g(ranks2\))117 982 y Fr(IN)155 b Fx(group1)456 b Fr(group1)14 b(\(handle\))117 1058 y(IN)155 b Fx(n)564 b Fr(n)o(um)o(b)q(er)11 b(of)h(ranks)g(in)23 b Fm(ranks1)13 b Fr(and)f Fm(ranks2)g Fr(arra)o(ys)g(\(in)o(teger\))117 1134 y(IN)155 b Fx(ranks1)464 b Fr(arra)o(y)14 b(of)f(zero)i(or)f(more) f(v)n(alid)f(ranks)i(in)g(group1)117 1209 y(IN)155 b Fx(group2)456 b Fr(group2)14 b(\(handle\))117 1285 y(OUT)108 b Fx(ranks2)464 b Fr(arra)o(y)64 b(of)f(corresp)q(onding)i(ranks)f(in)g (group2,)905 1342 y Fm(MPI)p 982 1342 13 2 v 15 w(UNDEFINED)12 b Fr(when)j(no)e(corresp)q(ondence)k(exists.)75 1467 y Fp(int)23 b(MPI)p 245 1467 15 2 v 17 w(Group)p 382 1467 V 17 w(translate)p 615 1467 V 16 w(ranks)g(\(MPI)p 870 1467 V 16 w(Group)h(group1,)f(int)g(n,)h(int)f(*ranks1,)393 1523 y(MPI)p 468 1523 V 17 w(group)g(group2,)g(int)g(*ranks2\))75 1610 y(MPI)p 150 1610 V 17 w(GROUP)p 287 1610 V 16 w(TRANSLATE)p 519 1610 V 16 w(RANKS\(GROUP1,)f(N,)i(RANKS1,)f(GROUP2,)g(RANKS2,)g (IERROR\))170 1666 y(INTEGER)g(GROUP1,)g(N,)h(RANKS1,)f(GROUP2,)g (RANKS2,)g(IERROR)75 1820 y Fq(5.3.2)49 b(Group)16 b(Constructo)o(rs)75 1907 y FE(These)k(functions)g(allo)o(w)g(construction)g(of)g(new)g (groups)f(from)g(existing)h(groups.)33 b(These)20 b(are)g(lo)q(cal)75 1963 y(op)q(erations,)d(and)h(distinct)g(groups)f(ma)o(y)f(b)q(e)i (de\014ned)h(on)e(di\013eren)o(t)g(pro)q(cesses;)h(a)f(pro)q(cess)h(ma) o(y)e(also)75 2020 y(de\014ne)f(a)f(group)g(that)f(do)q(es)i(not)f (include)i(itself.)21 b(Consisten)o(t)14 b(de\014nitions)i(are)d (required)j(when)e(groups)75 2076 y(are)h(used)g(as)g(argumen)o(ts)f (in)i(comm)o(unicator-building)i(functions.)i(MPI)15 b(do)q(es)h(not)e(pro)o(vide)i(a)f(mec)o(h-)75 2132 y(anism)h(to)g (build)i(a)e(group)g(from)f(scratc)o(h,)h(but)g(only)g(from)g(other,)f (previously)j(de\014ned)f(groups.)23 b(The)75 2189 y(base)13 b(group,)g(up)q(on)h(whic)o(h)g(all)g(other)f(groups)g(are)g (de\014ned,)i(is)e(the)h(group)f(asso)q(ciated)g(with)h(the)f(initial) 75 2245 y(comm)o(unicator)e Fx(MPI)p 453 2245 14 2 v 16 w(COMM)p 611 2245 V 17 w(W)o(ORLD)g FE(\(accessible)i(through)f(the) g(function)g Fx(MPI)p 1520 2245 V 16 w(COMM)p 1678 2245 V 16 w(GROUP)p FE(\).)75 2349 y Fx(MPI)p 160 2349 V 16 w(GROUP)p 328 2349 V 18 w(UNION\(group1,)i(group2,)h(newgroup\))117 2427 y Fr(IN)155 b Fx(group1)456 b Fr(\014rst)15 b(group)f(\(handle\)) 117 2503 y(IN)155 b Fx(group2)456 b Fr(second)15 b(group)f(\(handle\)) 117 2579 y(OUT)108 b Fx(newgroup)405 b Fr(union)14 b(group)f (\(handle\))75 2704 y Fp(int)23 b(MPI)p 245 2704 15 2 v 17 w(Group)p 382 2704 V 17 w(union\(MPI)p 615 2704 V 16 w(group)g(group1,)g(MPI)p 1037 2704 V 17 w(group)g(group2,)g(MPI)p 1460 2704 V 16 w(Group)h(*newgroup\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 110 115 110 114 bop 75 -100 a FE(110)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y Fp(MPI)p 150 45 15 2 v 17 w(GROUP)p 287 45 V 16 w(UNION\(GROUP1,)22 b(GROUP2,)h(NEWGROUP,)g(IERROR\))170 102 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 250 y Fx(MPI)p 160 250 14 2 v 16 w(GROUP)p 328 250 V 18 w(INTERSECTION\(group1,)15 b(group2,)f(newgroup\))117 342 y Fr(IN)155 b Fx(group1)456 b Fr(\014rst)15 b(group)f(\(handle\)) 117 447 y(IN)155 b Fx(group2)456 b Fr(second)15 b(group)f(\(handle\)) 117 551 y(OUT)108 b Fx(newgroup)405 b Fr(in)o(tersection)15 b(group)f(\(handle\))75 691 y Fp(int)23 b(MPI)p 245 691 15 2 v 17 w(Group)p 382 691 V 17 w(intersection\(MPI)p 783 691 V 15 w(group)g(group1,)g(MPI)p 1204 691 V 17 w(group)g(group2,)393 747 y(MPI)p 468 747 V 17 w(Group)g(*newgroup\))75 848 y(MPI)p 150 848 V 17 w(GROUP)p 287 848 V 16 w (INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\))170 905 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1053 y Fx(MPI)p 160 1053 14 2 v 16 w(GROUP)p 328 1053 V 18 w(DIFFERENCE\(group1,)14 b(group2,)g(newgroup\))117 1145 y Fr(IN)155 b Fx(group1)456 b Fr(\014rst)15 b(group)f(\(handle\)) 117 1250 y(IN)155 b Fx(group2)456 b Fr(second)15 b(group)f(\(handle\)) 117 1355 y(OUT)108 b Fx(newgroup)405 b Fr(di\013erence)16 b(group)e(\(handle\))75 1494 y Fp(int)23 b(MPI)p 245 1494 15 2 v 17 w(Group)p 382 1494 V 17 w(difference\(MPI)p 735 1494 V 15 w(group)g(group1,)g(MPI)p 1156 1494 V 17 w(group)g(group2,)393 1550 y(MPI)p 468 1550 V 17 w(Group)g(*newgroup\)) 75 1652 y(MPI)p 150 1652 V 17 w(GROUP)p 287 1652 V 16 w(DIFFERENCE\(GROUP1,)f(GROUP2,)h(NEWGROUP,)f(IERROR\))170 1708 y(INTEGER)h(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1809 y FE(The)15 b(set-lik)o(e)i(op)q(erations)e(are)g(de\014ned)i(as)d (follo)o(ws:)75 1962 y Fz(union)24 b FE(All)19 b(elemen)o(ts)g(of)f (the)h(\014rst)f(group)g(\()p Fx(group1)p FE(\),)g(follo)o(w)o(ed)h(b)o (y)f(all)h(elemen)o(ts)g(of)f(second)h(group)189 2019 y(\()p Fx(group2)p FE(\))14 b(not)h(in)h(\014rst.)75 2172 y Fz(in)o(tersect)23 b FE(all)18 b(elemen)o(ts)h(of)e(the)g (\014rst)h(group)f(that)g(are)g(also)g(in)i(the)f(second)g(group,)f (ordered)h(as)f(in)189 2228 y(\014rst)d(group.)75 2381 y Fz(di\013erence)22 b FE(all)c(elemen)o(ts)f(of)g(the)g(\014rst)f (group)g(that)g(are)h(not)f(in)h(the)g(second)g(group,)g(ordered)g(as)f (in)189 2438 y(the)f(\014rst)g(group.)75 2591 y(Note)k(that)g(for)g (these)h(op)q(erations)g(the)f(order)h(of)f(pro)q(cesses)h(in)g(the)g (output)f(group)h(is)g(determined)75 2647 y(primarily)g(b)o(y)f(order)g (in)h(the)f(\014rst)g(group)f(\(if)h(p)q(ossible\))i(and)e(then,)h(if)f (necessary)l(,)h(b)o(y)f(order)g(in)h(the)75 2704 y(second)c(group.)j (Neither)d(union)h(nor)e(in)o(tersection)h(are)e(comm)o(utativ)o(e,)h (but)g(b)q(oth)g(are)g(asso)q(ciativ)o(e.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 111 116 111 115 bop 75 -100 a Ft(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b FE(111)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(INCL\(group,)13 b(n,)j(ranks,)f(newgroup\))117 144 y Fr(IN)155 b Fx(group)479 b Fr(group)14 b(\(handle\))117 262 y(IN)155 b Fx(n)564 b Fr(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g (in)f(arra)o(y)h(ranks)g(\(and)g(size)h(of)e Fm(new-)905 318 y(group)p Fr(\))117 437 y(IN)155 b Fx(ranks)487 b Fr(ranks)13 b(of)f(pro)q(cesses)i(in)e Fm(group)h Fr(to)f(app)q(ear)h (in)f Fm(newgroup)i Fr(\(ar-)905 493 y(ra)o(y)g(of)f(in)o(tegers\))117 611 y(OUT)108 b Fx(newgroup)405 b Fr(new)12 b(group)f(deriv)o(ed)h (from)e(ab)q(o)o(v)o(e,)h(in)g(the)h(order)h(de\014ned)f(b)o(y)905 668 y Fm(ranks)75 814 y Fp(int)23 b(MPI)p 245 814 15 2 v 17 w(Group)p 382 814 V 17 w(incl\(MPI)p 591 814 V 16 w(group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 814 V 17 w(Group)h(*newgroup\))75 922 y(MPI)p 150 922 V 17 w(GROUP)p 287 922 V 16 w(INCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 978 y(INTEGER)g(GROUP,)g(N,)h(RANKS,)f (NEWGROUP,)g(IERROR)166 1086 y FE(The)17 b(function)h Fx(MPI)p 526 1086 14 2 v 16 w(GROUP)p 694 1086 V 18 w(INCL)f FE(creates)g(a)g(group)f Fx(newgroup)i FE(that)f(consists)g(of)g(the)g Fx(n)h FE(pro-)75 1142 y(cesses)e(in)h Fx(group)f FE(with)h(ranks)e Fx(rank[0],)p Fy(:)8 b(:)g(:)n Fx(,)16 b(rank[n-1])p FE(;)g(the)h(pro)q(cess)f(with)g(rank)g Fx(i)g FE(in)h Fx(newgroup)f FE(is)h(the)75 1199 y(pro)q(cess)e(with)f(rank)h Fx(ranks[i])g FE(in)g Fx(group)p FE(.)k(Eac)o(h)c(of)f(the)g Fx(n)h FE(elemen)o(ts)g(of)f Fx(ranks)h FE(m)o(ust)f(b)q(e)h(a)f(v)m (alid)i(rank)e(in)75 1255 y Fx(group)j FE(and)f(all)i(elemen)o(ts)f(m)o (ust)g(b)q(e)g(distinct,)h(or)e(else)h(the)g(program)f(is)h(erroneous.) 24 b(If)17 b Fx(n)g FE(=)32 b(0,)17 b(then)75 1312 y Fx(newgroup)i FE(is)f(empt)o(y)l(.)29 b(This)18 b(function)h(can,)g (for)e(instance,)i(b)q(e)g(used)g(to)e(reorder)h(the)g(elemen)o(ts)h (of)e(a)75 1368 y(group.)75 1493 y Fx(MPI)p 160 1493 V 16 w(GROUP)p 328 1493 V 18 w(EX)o(CL\(group,)d(n,)h(ranks,)g (newgroup\))117 1592 y Fr(IN)155 b Fx(group)479 b Fr(group)14 b(\(handle\))117 1710 y(IN)155 b Fx(n)564 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(arra)o(y)f(ranks)117 1828 y(IN)155 b Fx(ranks)487 b Fr(arra)o(y)15 b(of)f(in)o(teger)i(ranks)f(in)g Fm(group)g Fr(not)g(to)g(app)q(ear)g(in)g Fm(new-)905 1885 y(group)117 2003 y Fr(OUT)108 b Fx(newgroup)405 b Fr(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving) h(the)f(order)905 2059 y(de\014ned)d(b)o(y)28 b Fm(group)14 b Fr(\(handle\))75 2205 y Fp(int)23 b(MPI)p 245 2205 15 2 v 17 w(Group)p 382 2205 V 17 w(excl\(MPI)p 591 2205 V 16 w(group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 2205 V 17 w(Group)h(*newgroup\))75 2313 y(MPI)p 150 2313 V 17 w(GROUP)p 287 2313 V 16 w(EXCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 2370 y(INTEGER)g(GROUP,)g(N,)h(RANKS,)f (NEWGROUP,)g(IERROR)166 2478 y FE(The)12 b(function)h Fx(MPI)p 516 2478 14 2 v 16 w(GROUP)p 684 2478 V 17 w(EX)o(CL)f FE(creates)g(a)f(group)h(of)g(pro)q(cesses)g Fx(newgroup)h FE(that)e(is)h(obtained)75 2534 y(b)o(y)k(deleting)i(from)e Fx(group)g FE(those)g(pro)q(cesses)h(with)f(ranks)g Fx(ranks[0])h(,)p Fy(:)8 b(:)g(:)14 b Fx(ranks[n-1])p FE(.)24 b(The)17 b(ordering)f(of)75 2591 y(pro)q(cesses)f(in)h Fx(newgroup)g FE(is)f(iden)o(tical)i(to)e(the)g(ordering)g(in)h Fx(group)p FE(.)j(Eac)o(h)c(of)g(the)g Fx(n)g FE(elemen)o(ts)h(of)e Fx(ranks)75 2647 y FE(m)o(ust)h(b)q(e)h(a)f(v)m(alid)i(rank)e(in)h Fx(group)f FE(and)h(all)g(elemen)o(ts)g(m)o(ust)f(b)q(e)h(distinct;)g (otherwise,)f(the)g(program)f(is)75 2704 y(erroneous.)20 b(If)15 b Fx(n)h FE(=)28 b(0,)14 b(then)i Fx(newgroup)g FE(is)g(iden)o(tical)h(to)d Fx(group)p FE(.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 112 117 112 116 bop 75 -100 a FE(112)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(RANGE)p 494 45 V 17 w(INCL\(group,)f(n,)h(ranges,)g(newgroup\))117 122 y Fr(IN)155 b Fx(group)479 b Fr(group)14 b(\(handle\))117 197 y(IN)155 b Fx(n)564 b Fr(n)o(um)o(b)q(er)13 b(of)h(triplets)g(in)f (arra)o(y)28 b Fm(ranges)14 b Fr(\(in)o(teger\))117 272 y(IN)155 b Fx(ranges)466 b Fr(a)11 b(one-dimensional)e(arra)o(y)i(of)g (in)o(teger)g(triplets,)h(of)f(the)h(form)905 328 y(\(\014rst)i(rank,)e (last)g(rank,)h(stride\))g(indicating)e(ranks)i(in)f Fm(group)905 385 y Fr(or)i(pro)q(cesses)j(to)c(b)q(e)i(included)f(in)f Fm(newgroup)117 460 y Fr(OUT)108 b Fx(newgroup)405 b Fr(new)12 b(group)f(deriv)o(ed)h(from)e(ab)q(o)o(v)o(e,)h(in)g(the)h (order)h(de\014ned)f(b)o(y)905 516 y Fm(ranges)j Fr(\(handle\))75 641 y Fp(int)23 b(MPI)p 245 641 15 2 v 17 w(Group)p 382 641 V 17 w(range)p 519 641 V 16 w(incl\(MPI)p 727 641 V 16 w(group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 697 y(MPI)p 468 697 V 17 w(Group)g(*newgroup\))75 784 y(MPI)p 150 784 V 17 w(GROUP)p 287 784 V 16 w(RANGE)p 423 784 V 17 w(INCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 840 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 927 y FE(If)31 b Fx(ranges)15 b FE(consist)g(of)g(the)h(triplets)497 1027 y(\()p Fy(f)5 b(ir)q(st)617 1034 y Fj(1)637 1027 y Fy(;)j(l)q(ast)734 1034 y Fj(1)753 1027 y Fy(;)g(str)q(ide)894 1034 y Fj(1)913 1027 y FE(\))p Fy(;)g(:::;)f FE(\()p Fy(f)e(irst)1130 1034 y Fi(n)1152 1027 y Fy(;)j(l)q(ast)1249 1034 y Fi(n)1272 1027 y Fy(;)g(str)q(ide)1413 1034 y Fi(n)1436 1027 y FE(\))75 1128 y(then)16 b Fx(newgroup)f FE(consists)h(of)f(the)g(sequence)h(of)f(pro)q(cesses)h(in)g Fx(group)f FE(with)g(ranks)349 1247 y Fy(f)5 b(ir)q(st)451 1254 y Fj(1)471 1247 y Fy(;)j(f)d(ir)q(st)594 1254 y Fj(1)623 1247 y FE(+)11 b Fy(str)q(ide)789 1254 y Fj(1)808 1247 y Fy(;)d(:::;)g(f)d(ir)q(st)991 1254 y Fj(1)1019 1247 y FE(+)1064 1187 y Fh(\026)1093 1216 y Fy(l)q(ast)1169 1223 y Fj(1)1199 1216 y FC(\000)11 b Fy(f)5 b(ir)q(st)1347 1223 y Fj(1)p 1093 1236 274 2 v 1160 1278 a Fy(str)q(ide)1280 1285 y Fj(1)1372 1187 y Fh(\027)1404 1247 y Fy(str)q(ide)1524 1254 y Fj(1)1543 1247 y Fy(;)j(:::)358 1396 y(f)d(ir)q(st)460 1403 y Fi(n)484 1396 y Fy(;)j(f)d(ir)q(st)607 1403 y Fi(n)640 1396 y FE(+)11 b Fy(str)q(ide)806 1403 y Fi(n)829 1396 y Fy(;)d(:::;)g(f)d(ir)q(st)1012 1403 y Fi(n)1043 1396 y FE(+)1089 1336 y Fh(\026)1118 1365 y Fy(l)q(ast)1194 1372 y Fi(n)1228 1365 y FC(\000)10 b Fy(f)5 b(ir)q(st)1375 1372 y Fi(n)p 1118 1385 282 2 v 1187 1427 a Fy(str)q(ide)1307 1434 y Fi(n)1404 1336 y Fh(\027)1436 1396 y Fy(str)q(ide)1556 1403 y Fi(n)1579 1396 y Fy(:)75 1496 y FE(Eac)o(h)12 b(computed)g(rank)g(m)o(ust)f(b)q(e)i(a)f(v)m(alid)h(rank)f(in)h Fx(group)f FE(and)g(all)h(computed)f(ranks)g(m)o(ust)f(b)q(e)i (distinct,)75 1553 y(else)18 b(the)e(program)g(is)h(erroneous.)24 b(Note)16 b(that)g(w)o(e)h(ma)o(y)f(ha)o(v)o(e)g Fy(f)5 b(ir)q(st)1288 1560 y Fi(i)1317 1553 y Fy(>)16 b(l)q(ast)1444 1560 y Fi(i)1458 1553 y FE(,)h(and)g Fy(str)q(ide)1698 1560 y Fi(i)1728 1553 y FE(ma)o(y)f(b)q(e)75 1609 y(negativ)o(e,)f(but) g(cannot)g(b)q(e)h(zero.)166 1666 y(The)i(functionalit)o(y)h(of)e(this) i(routine)f(is)g(sp)q(eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e (expanding)i(the)f(arra)o(y)75 1722 y(of)g(ranges)g(to)g(an)h(arra)o(y) e(of)h(the)h(included)i(ranks)d(and)h(passing)g(the)g(resulting)g(arra) o(y)e(of)i(ranks)f(and)75 1778 y(other)e(argumen)o(ts)g(to)g Fx(MPI)p 560 1778 14 2 v 16 w(GROUP)p 728 1778 V 18 w(INCL)p FE(.)g(A)g(call)i(to)e Fx(MPI)p 1148 1778 V 16 w(GROUP)p 1316 1778 V 18 w(INCL)g FE(is)h(equiv)m(alen)o(t)h(to)e(a)h(call)75 1835 y(to)e Fx(MPI)p 216 1835 V 16 w(GROUP)p 384 1835 V 18 w(RANGE)p 550 1835 V 18 w(INCL)g FE(with)i(eac)o(h)f(rank)g Fx(i)g FE(in)h Fx(ranks)f FE(replaced)i(b)o(y)e(the)g(triplet)h Fp(\(i,i,1\))e FE(in)75 1891 y(the)g(argumen)o(t)g Fx(ranges)p FE(.)75 1995 y Fx(MPI)p 160 1995 V 16 w(GROUP)p 328 1995 V 18 w(RANGE)p 494 1995 V 17 w(EX)o(CL\(group,)g(n,)g(ranges,)g (newgroup\))117 2072 y Fr(IN)155 b Fx(group)479 b Fr(group)14 b(\(handle\))117 2147 y(IN)155 b Fx(n)564 b Fr(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 2222 y(IN)155 b Fx(ranges)466 b Fr(a)13 b(one-dimensional)d(arra)o(y)j (of)f(in)o(teger)h(triplets)g(of)f(the)i(form)905 2278 y(\(\014rst)j(rank,)f(last)g(rank,)g(stride\),)h(indicating)e(the)i (ranks)f(in)905 2335 y Fm(group)22 b Fr(of)f(pro)q(cesses)j(to)d(b)q(e) h(excluded)g(from)e(the)i(output)905 2391 y(group)14 b Fm(newgroup)p Fr(.)117 2466 y(OUT)108 b Fx(newgroup)405 b Fr(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving) h(the)f(order)905 2523 y(in)c Fm(group)g Fr(\(handle\))75 2647 y Fp(int)23 b(MPI)p 245 2647 15 2 v 17 w(Group)p 382 2647 V 17 w(range)p 519 2647 V 16 w(excl\(MPI)p 727 2647 V 16 w(group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 2704 y(MPI)p 468 2704 V 17 w(Group)g(*newgroup\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 113 118 113 117 bop 75 -100 a Ft(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b FE(113)75 45 y Fp(MPI)p 150 45 15 2 v 17 w(GROUP)p 287 45 V 16 w(RANGE)p 423 45 V 17 w(EXCL\(GROUP,)22 b(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 102 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 191 y FE(Eac)o(h)12 b(computed)g(rank)g(m)o(ust)f(b)q(e)i(a)f(v)m(alid) h(rank)f(in)h Fx(group)f FE(and)g(all)h(computed)f(ranks)g(m)o(ust)f(b) q(e)i(distinct,)75 248 y(or)i(else)h(the)f(program)f(is)i(erroneous.) 166 308 y(The)i(functionalit)o(y)h(of)e(this)i(routine)f(is)g(sp)q (eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i(the)f (arra)o(y)75 364 y(of)g(ranges)f(to)h(an)g(arra)o(y)f(of)h(the)g (excluded)i(ranks)e(and)g(passing)h(the)f(resulting)h(arra)o(y)e(of)h (ranks)g(and)75 421 y(other)c(argumen)o(ts)g(to)g Fx(MPI)p 554 421 14 2 v 16 w(GROUP)p 722 421 V 18 w(EX)o(CL)p FE(.)g(A)h(call)h(to)e Fx(MPI)p 1146 421 V 16 w(GROUP)p 1314 421 V 18 w(EX)o(CL)g FE(is)i(equiv)m(alen)o(t)g(to)e(a)h(call)75 477 y(to)g Fx(MPI)p 216 477 V 15 w(GROUP)p 383 477 V 18 w(RANGE)p 549 477 V 18 w(EX)o(CL)g FE(with)h(eac)o(h)f(rank)g Fx(i)g FE(in)h Fx(ranks)g FE(replaced)g(b)o(y)f(the)h(triplet)g Fp(\(i,i,1\))e FE(in)75 534 y(the)h(argumen)o(t)g Fx(ranges)p FE(.)166 593 y(Note:)20 b(the)c(range)g(op)q(erations)f(do)h(not)f (explicitly)k(en)o(umerate)d(ranks,)f(and)h(therefore)f(are)g(more)75 650 y(scalable)k(if)f(implemen)o(ted)h(e\016cien)o(tly)l(.)28 b(Hence,)18 b(w)o(e)f(recommend)h(MPI)f(programmers)f(to)h(use)h(them) 75 706 y(whenenev)o(er)e(p)q(ossible,)h(as)d(high-qualit)o(y)j (implemen)o(tations)g(will)g(tak)o(e)d(adv)m(an)o(tage)h(of)f(this)i (fact.)75 848 y Fq(5.3.3)49 b(Group)16 b(Destructo)o(rs)75 987 y Fx(MPI)p 160 987 V 16 w(GROUP)p 328 987 V 18 w(FREE\(group\))117 1068 y Fr(INOUT)62 b Fx(group)479 b Fr(group)14 b(\(handle\))75 1196 y Fp(int)23 b(MPI)p 245 1196 15 2 v 17 w(Group)p 382 1196 V 17 w(free\(MPI)p 591 1196 V 16 w(group)g(*group\))75 1286 y(MPI)p 150 1286 V 17 w(GROUP)p 287 1286 V 16 w(FREE\(GROUP,)g (IERROR\))170 1342 y(INTEGER)g(GROUP,)g(IERROR)166 1432 y FE(This)12 b(op)q(eration)g(marks)f(a)g(group)g(ob)s(ject)g(for)g (deallo)q(cation.)20 b(The)12 b(handle)h Fx(group)f FE(is)g(set)f(to)g (n)o(ull)i(b)o(y)75 1489 y(the)g(call.)20 b(The)13 b(group)f(ob)s(ject) g(is)h(actually)h(deallo)q(cated)g(only)f(if)g(there)g(are)f(no)g (other)h(activ)o(e)f(references)75 1545 y(to)j(it.)23 b(If)17 b(a)e(group)h(has)g(b)q(een)h(b)q(ound)g(to)f(one)g(or)g(more)f (comm)o(unicators,)h(then)g(the)g(user)h(has)e(to)h(free)75 1602 y(all)g(\\copies")g(of)e(the)i(group.)166 1737 y Fo(Implemen)o(tati)o(on)c(note:)166 1791 y Fr(One)18 b(can)f(k)o(eep)h(a)f(reference)i(coun)o(t)f(that)f(is)g(incremen)o (ted)h(for)e(eac)o(h)i(call)e(to)h Fm(MPI)p 1516 1791 13 2 v 15 w(COMM)p 1661 1791 V 14 w(MAKE)g Fr(and)75 1841 y Fm(MPI)p 152 1841 V 14 w(COMM)p 296 1841 V 15 w(DUP)p Fr(,)d(and)h(and)g(decremen)o(ted)i(for)e(eac)o(h)h(call)e(to)h Fm(MPI)p 1194 1841 V 15 w(GROUP)p 1351 1841 V 13 w(FREE)f Fr(or)h Fm(MPI)p 1606 1841 V 15 w(COMM)p 1751 1841 V 14 w(FREE)p Fr(;)75 1890 y(the)f(group)g(ob)r(ject)h(is)f(ultimately)d (deallo)q(cated)j(when)g(the)h(reference)h(coun)o(t)e(drops)h(to)f (zero.)75 2136 y Fv(5.4)59 b(Communicato)n(r)19 b(Management)75 2244 y FE(This)d(section)h(describ)q(es)g(the)f(manipulation)h(of)f (comm)o(unicators)f(in)i(MPI.)e(Op)q(erations)i(that)e(access)75 2300 y(comm)o(unicators)g(are)g(lo)q(cal)h(and)f(their)h(execution)g (do)q(es)g(not)f(require)h(in)o(terpro)q(cess)g(comm)o(unication.)75 2357 y(Op)q(erations)22 b(that)e(create)h(comm)o(unicators)g(are)g (collectiv)o(e)i(and)e(ma)o(y)f(require)i(in)o(terpro)q(cess)g(com-)75 2413 y(m)o(unication.)28 b(\(High)18 b(qualit)o(y)g(implemen)o(tations) g(will)h(b)q(e)g(able)f(to)f(amortize)g(these)h(o)o(v)o(erheads)f(o)o (v)o(er)75 2470 y(sev)o(eral)e(calls.\))75 2611 y Fq(5.4.1)49 b(Communicato)o(r)16 b(Accesso)o(rs)75 2704 y FE(The)f(follo)o(wing)h (are)f(all)h(lo)q(cal)h(op)q(erations.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 114 119 114 118 bop 75 -100 a FE(114)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(COMM)p 318 45 V 16 w(SIZE\(comm,)e (size\))117 127 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 211 y(OUT)108 b Fx(size)519 b Fr(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g(group)g(of)27 b Fm(comm)12 b Fr(\(in)o(teger\))75 340 y Fp(int)23 b(MPI)p 245 340 15 2 v 17 w(Comm)p 358 340 V 17 w(size\(MPI)p 567 340 V 16 w(Comm)g(comm,)g(int)h(*size\))75 431 y(MPI)p 150 431 V 17 w(COMM)p 263 431 V 16 w(SIZE\(COMM,)f(SIZE,)g(IERROR\))170 488 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 626 y Fx(MPI)p 160 626 14 2 v 16 w(COMM)p 318 626 V 16 w(RANK\(comm,)13 b(rank\))117 708 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 792 y(OUT)108 b Fx(rank)504 b Fr(rank)12 b(of)f(the)h(calling)e(pro)q(cess)j(in)f(group)f(of)23 b Fm(comm)9 b Fr(\(in)o(teger\))75 921 y Fp(int)23 b(MPI)p 245 921 15 2 v 17 w(Comm)p 358 921 V 17 w(rank\(MPI)p 567 921 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 1012 y(MPI)p 150 1012 V 17 w(COMM)p 263 1012 V 16 w(RANK\(COMM,)f(RANK,)g(IERROR\)) 170 1069 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 1207 y Fx(MPI)p 160 1207 14 2 v 16 w(COMM)p 318 1207 V 16 w(GROUP\(comm,)14 b(group\))117 1289 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 1373 y(OUT)108 b Fx(group)479 b Fr(group)14 b(corresp)q(onding)h(to)f Fm(comm)e Fr(\(handle\))75 1502 y Fp(int)23 b(MPI)p 245 1502 15 2 v 17 w(Comm)p 358 1502 V 17 w(group\(MPI)p 591 1502 V 16 w(Comm)g(comm,)g(MPI)p 941 1502 V 17 w(group)g(*group\))75 1593 y(MPI)p 150 1593 V 17 w(COMM)p 263 1593 V 16 w(GROUP\(COMM,)g(GROUP,)g(IERROR\))170 1650 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)75 1828 y Fq(5.4.2)49 b(Communicato)o(r)16 b(Constructo)o(rs)75 1922 y FE(The)f(follo)o(wing) g(are)g(collectiv)o(e)h(functions)f(that)f(are)h(in)o(v)o(ok)o(ed)g(b)o (y)f(all)i(pro)q(cesses)f(in)g(the)g(group)f(asso)q(ci-)75 1979 y(ated)h(with)h Fx(comm)p FE(.)166 2040 y(Note)f(that)g(there)g (is)h(a)g(c)o(hic)o(k)o(en-and-egg)g(asp)q(ect)g(to)f(MPI)g(in)i(that)d (a)h(comm)o(unicator)h(is)g(needed)75 2096 y(to)21 b(create)h(a)f(new)h (comm)o(unicator.)39 b(The)22 b(base)f(comm)o(unicator)h(for)f(all)i (MPI)e(comm)o(unicators)g(is)75 2153 y(prede\014ned)c(outside)f(of)e (MPI,)h(and)h(is)f Fx(MPI)p 839 2153 14 2 v 16 w(COMM)p 997 2153 V 17 w(W)o(ORLD)g FE(\(de\014ned)h(ab)q(o)o(v)o(e\).)75 2261 y Fx(MPI)p 160 2261 V 16 w(COMM)p 318 2261 V 16 w(DUP\(comm,)d(new)o(comm\))117 2343 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 2427 y(OUT)108 b Fx(new)o(comm)397 b Fr(cop)o(y)14 b(of)f Fm(comm)f Fr(\(handle\))75 2556 y Fp(int)23 b(MPI)p 245 2556 15 2 v 17 w(Comm)p 358 2556 V 17 w(dup\(MPI)p 543 2556 V 16 w(Comm)g(comm,)h(MPI)p 894 2556 V 16 w(Comm)g(*newcomm\))75 2647 y(MPI)p 150 2647 V 17 w(COMM)p 263 2647 V 16 w(DUP\(COMM,)f (NEWCOMM,)g(IERROR\))170 2704 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR) 1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 115 120 115 119 bop 75 -100 a Ft(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b FE(115)166 45 y Fx(MPI)p 251 45 14 2 v 16 w(COMM)p 409 45 V 16 w(DUP)17 b FE(Duplicates)h(the)f (existing)h(comm)o(unicator)e Fx(comm)f FE(with)i(all)h(its)f(cac)o (hed)g(in-)75 102 y(formation.)26 b(Returns)18 b(in)g Fx(new)o(comm)e FE(a)i(new)f(comm)o(unicator)g(with)h(the)g(same)f (group,)g(same)g(cac)o(hed)75 158 y(information,)e(but)g(a)g(new)h(con) o(text.)166 298 y Fo(Implemen)o(tati)o(on)g(note:)49 b Fr(One)17 b(need)h(not)f(actually)g(cop)o(y)g(the)g(group)g (information,)e(but)i(only)f(add)75 354 y(a)j(new)h(reference)i(and)d (incremen)o(t)h(the)g(reference)i(coun)o(t.)35 b(Cop)o(y)19 b(on)g(write)h(can)f(b)q(e)h(used)h(for)e(the)h(cac)o(hed)75 410 y(information.)75 597 y Fx(MPI)p 160 597 V 16 w(COMM)p 318 597 V 16 w(MAKE\(comm,)13 b(group,)h(new)o(comm\))117 675 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 751 y(IN)155 b Fx(group)479 b Fr(Group,)14 b(whic)o(h)h(is)f(a)g(subset)i(of)e(the)h(group)g(of)e Fm(comm)g Fr(\(han-)905 807 y(dle\))117 883 y(OUT)108 b Fx(new)o(comm)397 b Fr(new)15 b(comm)o(uni)o(cator)d(\(handle\))75 1008 y Fp(int)23 b(MPI)p 245 1008 15 2 v 17 w(Comm)p 358 1008 V 17 w(make\(MPI)p 567 1008 V 16 w(Comm)g(comm,)g(MPI)p 917 1008 V 17 w(group)g(group,)g(MPI)p 1316 1008 V 17 w(Comm)h(*newcomm\))75 1095 y(MPI)p 150 1095 V 17 w(COMM)p 263 1095 V 16 w(MAKE\(COMM,)f(GROUP,)g(NEWCOMM,)g(IERROR\))170 1152 y(INTEGER)g(COMM,)h(GROUP,)f(NEWCOMM,)f(IERROR)75 1238 y FE(This)14 b(function)g(creates)f(a)g(new)h(comm)o(unicator)f Fx(new)o(comm)f FE(with)i(comm)o(unication)g(group)f(de\014ned)i(b)o(y) 75 1295 y Fx(group)f FE(and)h(a)e(new)i(con)o(text.)k(No)14 b(cac)o(hed)h(information)f(propagates)f(from)h Fx(comm)e FE(to)i Fx(new)o(comm)p FE(.)k(The)75 1351 y(function)e(returns)f(a)g (n)o(ull)i(handle)f(to)f(pro)q(cesses)g(that)g(are)g(not)f(in)i Fx(group)p FE(.)166 1408 y(The)f(call)h(is)f(erroneous)g(if)g(not)f (all)i Fx(group)f FE(argumen)o(ts)f(ha)o(v)o(e)g(the)h(same)f(v)m (alue,)i(or)e(if)h Fx(group)g FE(is)g(not)75 1465 y(a)j(subset)h(of)f (the)h(group)f(asso)q(ciated)h(with)g Fx(comm)p FE(.)28 b(Note)18 b(that)g(the)g(call)i(is)f(to)f(b)q(e)h(executed)h(b)o(y)e (all)75 1521 y(pro)q(cesses)e(in)g Fx(comm)p FE(,)d(ev)o(en)i(if)h (they)f(do)g(not)g(b)q(elong)h(to)f(the)g(new)g(group.)166 1661 y Fo(Implemen)o(tati)o(on)e(note:)40 b Fr(Since)15 b(all)e(pro)q(cesses)18 b(calling)13 b Fm(MPI)p 1215 1661 13 2 v 14 w(COMM)p 1359 1661 V 15 w(DUP)h Fr(or)g Fm(MPI)p 1600 1661 V 15 w(COMM)p 1745 1661 V 14 w(MAKE)75 1717 y Fr(pro)o(vide)d(the)g(same)f Fm(group)h Fr(argumen)o(t,)f(it)g (is)h(theoretically)g(p)q(ossible)g(to)f(agree)i(on)e(a)h(group-wide)f (unique)h(con)o(text)75 1774 y(with)h(no)h(comm)o(unication.)h(Ho)o(w)o (ev)o(er,)f(lo)q(cal)f(execution)i(of)e(these)i(functions)f(requires)h (use)g(of)e(a)g(larger)h(con)o(text)75 1830 y(name)h(space)i(and)e (reduces)j(error-c)o(hec)o(king.)22 b(Implemen)o(tations)12 b(ma)o(y)h(strik)o(e)i(v)n(arious)g(compromises)e(b)q(et)o(w)o(een)75 1887 y(these)k(con\015icting)f(goals.)23 b(If)16 b(new)g(comm)o (unicators)e(are)i(created)i(without)d(sync)o(hronizing)h(the)h(pro)q (cesses)i(in-)75 1943 y(v)o(olv)o(ed)d(then)i(the)g(comm)o(ucatio)o(n)d (system)h(should)h(b)q(e)h(able)f(to)f(cop)q(e)i(with)f(messages)g (arriving)f(in)h(a)f(con)o(text)75 1999 y(that)e(has)g(not)g(y)o(et)g (b)q(een)h(allo)q(cated)e(at)h(the)h(receiving)f(pro)q(cess.)75 2186 y Fx(MPI)p 160 2186 14 2 v 16 w(COMM)p 318 2186 V 16 w(SPLIT\(comm,)e(colo)o(r,)i(k)o(ey)l(,)h(new)o(comm\))117 2264 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 2340 y(IN)155 b Fx(colo)o(r)495 b Fr(con)o(trol)14 b(of)f(subset)i(assignmen)o(t)e(\(in)o(teger\))117 2416 y(IN)155 b Fx(k)o(ey)525 b Fr(con)o(trol)14 b(of)f(rank)h(assigmen)o(t) f(\(in)o(teger\))117 2492 y(OUT)108 b Fx(new)o(comm)397 b Fr(new)15 b(comm)o(uni)o(cator)d(\(handle\))75 2617 y Fp(int)23 b(MPI)p 245 2617 15 2 v 17 w(Comm)p 358 2617 V 17 w(split\(MPI)p 591 2617 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g (key,)h(MPI)p 1419 2617 V 16 w(Comm)g(*newcomm\))75 2704 y(MPI)p 150 2704 V 17 w(COMM)p 263 2704 V 16 w(SPLIT\(COMM,)f(COLOR,)g (KEY,)g(NEWCOMM,)g(IERROR\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 116 121 116 120 bop 75 -100 a FE(116)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)170 45 y Fp(INTEGER)23 b(COMM,)h(COLOR,)f(KEY,)g(NEWCOMM,)g(IERROR)75 135 y FE(This)c(function)f(partitions)g(the)h(group)e(asso)q(ciated)h (with)h Fx(comm)d FE(in)o(to)i(disjoin)o(t)g(subgroups,)h(one)f(for)75 191 y(eac)o(h)g(v)m(alue)g(of)f Fx(colo)o(r)p FE(.)26 b(Eac)o(h)17 b(subgroup)h(con)o(tains)f(all)i(pro)q(cesses)f(of)f(the)g (same)g(color.)27 b(Within)19 b(eac)o(h)75 248 y(subgroup,)k(the)f(pro) q(cesses)g(are)f(rank)o(ed)h(in)g(the)g(order)f(de\014ned)i(b)o(y)f (the)g(v)m(alue)h(of)e(the)h(parameter)75 304 y Fx(k)o(ey)p FE(,)e(with)g(ties)g(brok)o(en)g(according)g(to)f(their)h(rank)f(in)i (the)f(old)g(group.)33 b(A)19 b(new)h(comm)o(unicator)f(is)75 361 y(created)14 b(for)f(eac)o(h)h(subgroup)g(and)g(returned)g(in)g Fx(new)o(comm)p FE(.)k(A)c(pro)q(cess)g(ma)o(y)f(supply)i(the)f(color)g (v)m(alue)75 417 y Fx(MPI)p 160 417 14 2 v 16 w(UNDEFINED)p FE(,)i(in)h(whic)o(h)h(case)f Fx(new)o(comm)e FE(returns)h(a)g(n)o(ull) i(comm)o(unicator)f(handle.)25 b(This)17 b(is)g(a)75 474 y(collectiv)o(e)g(call,)f(but)f(eac)o(h)h(pro)q(cess)f(can)g(pro)o (vide)h(di\013eren)o(t)f(v)m(alues)i(for)d Fx(colo)o(r)g FE(and)i Fx(k)o(ey)p FE(.)166 534 y(A)27 b(call)i(to)d Fx(MPI)p 475 534 V 16 w(COMM)p 633 534 V 17 w(MAKE\(comm,)h(group,)j (new)o(comm\))c FE(is)h(equiv)m(alen)o(t)i(to)e(a)g(call)i(to)75 590 y Fx(MPI)p 160 590 V 16 w(COMM)p 318 590 V 16 w(SPLIT\(comm,)8 b(colo)o(r,)i(k)o(ey)l(,)h(new)o(comm\))p FE(,)e(where)h(all)h(mem)o(b) q(ers)g(of)e Fx(group)h FE(pro)o(vide)h Fx(colo)o(r)e FE(=)23 b(0)75 647 y(and)15 b Fx(k)o(ey)f FE(=)29 b(rank)15 b(in)g Fx(group)p FE(,)f(and)h(all)g(pro)q(cesses)g(that)f(are)g(not)g (mem)o(b)q(ers)h(of)f Fx(group)g FE(pro)o(vide)h Fx(colo)o(r)e FE(=)75 703 y Fx(MPI)p 160 703 V 16 w(UNDEFINED)p FE(.)k(The)i (function)f Fx(MPI)p 819 703 V 16 w(COMM)p 977 703 V 17 w(SPLIT)f FE(allo)o(ws)h(more)f(general)i(partitioning)g(of)e(a)75 760 y(group)e(in)o(to)g(one)g(or)g(more)g(subgroups)g(with)h(optional)f (reordering.)75 901 y Fq(5.4.3)49 b(Communicato)o(r)16 b(Destructo)o(rs)75 1041 y Fx(MPI)p 160 1041 V 16 w(COMM)p 318 1041 V 16 w(FREE\(comm\))117 1122 y Fr(INOUT)62 b Fx(comm)470 b Fr(comm)o(unicator)11 b(to)j(b)q(e)h(destro)o(y)o(ed)g (\(handle\))75 1250 y Fp(int)23 b(MPI)p 245 1250 15 2 v 17 w(Comm)p 358 1250 V 17 w(free\(MPI)p 567 1250 V 16 w(Comm)g(*comm\))75 1340 y(MPI)p 150 1340 V 17 w(COMM)p 263 1340 V 16 w(FREE\(COMM,)g(IERROR\))170 1396 y(INTEGER)g(COMM,)h (IERROR)166 1486 y FE(This)13 b(collectiv)o(e)g(op)q(eration)g(marks)e (the)h(comm)o(unication)h(ob)s(ject)f(for)f(deallo)q(cation.)20 b(The)13 b(handle)75 1542 y(is)i(set)g(to)f(n)o(ull.)21 b(The)15 b(ob)s(ject)f(is)h(actually)g(deallo)q(cated)h(only)g(if)f (there)f(are)h(no)f(other)h(activ)o(e)f(references)75 1599 y(to)h(it.)21 b(The)15 b(asso)q(ciated)h(group)f(ma)o(y)g(also)g (b)q(e)h(deallo)q(cated,)h(if)f(there)f(are)g(no)h(activ)o(e)f (references)h(to)f(it.)75 1655 y(It)g(is)h(the)f(user)h(resp)q (onsibilit)o(y)h(to)e(free)g(eac)o(h)g(cop)o(y)g(of)g(a)g(comm)o (unicator)g(ob)s(ject.)166 1791 y Fo(Implemen)o(tati)o(on)g(note:)47 b Fr(A)17 b(reference)i(coun)o(t)e(mec)o(hanism)d(can)j(b)q(e)g(used:) 24 b(the)17 b(reference)i(coun)o(t)e(is)75 1841 y(incremen)o(ted)d(b)o (y)g(eac)o(h)g(call)f(to)g Fm(MPI)p 663 1841 13 2 v 15 w(COMM)p 808 1841 V 14 w(DUP)p Fr(,)g(and)g(decremen)o(ted)i(b)o(y)f (eac)o(h)g(call)f(to)g Fm(MPI)p 1606 1841 V 15 w(COMM)p 1751 1841 V 14 w(FREE)p Fr(.)75 1891 y(The)h(ob)r(ject)h(is)f (ultimately)d(deallo)q(cated)j(when)h(the)f(coun)o(t)g(reac)o(hes)i (zero.)166 1951 y(Though)k(collectiv)o(e,)i(it)e(is)h(an)o(ticipated)f (that)h(this)f(op)q(eration)h(will)e(normally)f(b)q(e)j(lo)q(cal,)g (though)f(the)75 2007 y(debugging)13 b(v)o(ersion)h(of)g(an)f(MPI)h (library)g(migh)o(t)d(c)o(ho)q(ose)k(to)f(sync)o(hronize.)75 2253 y Fv(5.5)59 b(Inter-Communication)75 2361 y FE(This)18 b(section)h(in)o(tro)q(duces)f(the)g(concept)g(of)f(in)o(ter-comm)o (unication)i(and)f(describ)q(es)i(the)d(p)q(ortions)h(of)75 2418 y(MPI)j(that)g(supp)q(ort)g(it.)38 b(It)22 b(describ)q(es)h(supp)q (ort)e(for)g(writing)g(programs)f(that)h(con)o(tain)g(user-lev)o(el)75 2474 y(serv)o(ers.)h(It)16 b(also)f(describ)q(es)j(a)e(name)g(service)g (that)g(simpli\014es)i(writing)e(programs)f(con)o(taining)i(in)o(ter-) 75 2531 y(comm)o(unication.)166 2591 y(All)h(p)q(oin)o(t-to-p)q(oin)o (t)g(comm)o(unication)g(describ)q(ed)h(th)o(us)d(far)h(has)g(in)o(v)o (olv)o(ed)g(comm)o(unication)h(b)q(e-)75 2647 y(t)o(w)o(een)c(pro)q (cesses)g(that)f(are)h(mem)o(b)q(ers)g(of)f(the)h(same)g(group.)19 b(This)c(t)o(yp)q(e)f(of)f(comm)o(unication)i(is)f(called)75 2704 y(\\in)o(tra-comm)o(unication")h(and)h(the)f(comm)o(unicator)g (used)h(is)f(called)i(an)e(\\in)o(tra-comm)o(unicator.")1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 117 122 117 121 bop 75 -100 a Ft(5.5.)34 b(INTER-COMMUNICA)l(TION)1047 b FE(117)166 45 y(In)11 b(mo)q(dular)g(and)f(m)o(ulti-discipl)q(inary)j (applications,)g(di\013eren)o(t)d(pro)q(cess)h(groups)f(execute)h (distinct)75 102 y(mo)q(dules)i(and)f(pro)q(cesses)h(within)g (di\013eren)o(t)f(mo)q(dules)h(comm)o(unicate)f(with)h(one)f(another)f (in)i(a)f(pip)q(eline)75 158 y(or)j(a)f(more)h(general)h(mo)q(dule)g (graph.)j(In)d(these)f(applications,)i(the)e(most)f(natural)h(w)o(a)o (y)f(for)h(a)g(pro)q(cess)75 214 y(to)g(sp)q(ecify)i(a)e(target)f(pro)q (cess)i(is)g(b)o(y)g(the)g(rank)f(of)g(the)h(target)e(pro)q(cess)i (within)h(the)e(target)g(group.)20 b(In)75 271 y(applications)13 b(that)e(con)o(tain)h(in)o(ternal)h(user-lev)o(el)g(serv)o(ers,)f(eac)o (h)f(serv)o(er)h(ma)o(y)f(b)q(e)h(a)f(pro)q(cess)h(group)g(that)75 327 y(pro)o(vides)17 b(services)f(to)g(one)g(or)g(more)f(clien)o(ts,)i (and)g(eac)o(h)f(clien)o(t)h(ma)o(y)f(b)q(e)g(a)g(pro)q(cess)h(group)e (that)h(uses)75 384 y(the)h(services)h(of)f(one)g(or)g(more)g(serv)o (ers.)25 b(It)17 b(is)h(again)f(most)f(natural)i(to)e(sp)q(ecify)j(the) e(target)f(pro)q(cess)75 440 y(b)o(y)d(rank)g(within)i(the)e(target)f (group)i(in)g(these)f(applications.)21 b(This)14 b(t)o(yp)q(e)g(of)e (comm)o(unication)j(is)e(called)75 497 y(\\in)o(ter-comm)o(unication")j (and)f(the)g(comm)o(unicator)g(used)h(is)g(called)h(an)e(\\in)o (ter-comm)o(unicator.")166 555 y(An)23 b(in)o(ter-comm)o(unication)g (op)q(eration)g(is)f(a)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o(unication) g(b)q(et)o(w)o(een)g(pro-)75 612 y(cesses)16 b(in)h(di\013eren)o(t)f (groups.)21 b(The)16 b(group)f(con)o(taining)h(a)g(pro)q(cess)g(that)f (initiates)i(an)e(in)o(ter-comm)o(un-)75 668 y(ication)i(op)q(eration)f (is)g(called)h(the)f(\\lo)q(cal)h(group,")e(that)g(is,)h(the)g(sender)h (in)f(a)g(send)g(and)g(the)g(receiv)o(er)75 725 y(in)g(a)e(receiv)o(e.) 21 b(The)15 b(group)g(con)o(taining)g(the)g(target)f(pro)q(cess)h(is)g (called)i(the)e(\\remote)f(group,")g(that)g(is,)75 781 y(the)j(receiv)o(er)i(in)f(a)f(send)h(and)f(the)h(sender)g(in)g(a)f (receiv)o(e.)27 b(As)17 b(in)i(in)o(tra-comm)o(unication,)f(the)f (target)75 838 y(pro)q(cess)12 b(is)g(sp)q(eci\014ed)i(using)e(a)f Fx(\(communicato)o(r,)f(rank\))i FE(pair.)19 b(Unlik)o(e)13 b(in)o(tra-comm)o(unication,)g(the)e(rank)75 894 y(is)16 b(relativ)o(e)f(to)g(a)g(second,)g(remote)g(group.)166 953 y(Lo)q(cal)g(and)g(remote)e(groups)h(of)g(an)g(in)o(ter-comm)o (unicator)h(are)f(not)f(required)j(to)d(b)q(e)i(disjoin)o(t.)21 b(An)75 1009 y(in)o(tra-comm)o(unicator)14 b(can)h(b)q(e)h(seen)f(as)f (a)g(particular)i(case)e(of)h(an)f(in)o(ter-comm)o(unicator,)h(where)g (lo)q(cal)75 1066 y(and)h(remote)g(groups)f(happ)q(en)j(to)d(b)q(e)i (iden)o(tical.)24 b(The)16 b(seman)o(tics)h(of)e(accessor)h(function)h (are)f(de\014ned)75 1122 y(so)k(as)g(to)g(b)q(e)h(consisten)o(t)g(with) g(this)g(in)o(terpretation.)36 b(Here)20 b(is)h(a)g(summary)f(of)g(the) g(prop)q(erties)h(of)75 1178 y(in)o(ter-comm)o(unication)16 b(and)g(in)o(ter-comm)o(unicators:)143 1281 y FC(\017)23 b FE(The)c(syn)o(tax)g(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)g(is)g(the)f(same)g(for)g(b)q(oth)h(in)o(ter-)f(and)h(in)o (tra-)189 1337 y(comm)o(unication.)26 b(The)17 b(same)g(comm)o (unicator)g(can)g(b)q(e)g(used)h(b)q(oth)f(for)g(send)g(and)h(for)e (receiv)o(e)189 1394 y(op)q(erations.)143 1496 y FC(\017)23 b FE(A)14 b(target)g(pro)q(cess)h(is)g(addressed)h(b)o(y)f(its)g(rank)f (in)i(the)f(remote)f(group,)g(b)q(oth)h(for)f(sends)i(and)f(for)189 1553 y(receiv)o(es.)143 1655 y FC(\017)23 b FE(Comm)o(unications)12 b(using)i(an)e(in)o(ter-comm)o(unicator)h(are)f(guaran)o(teed)g(not)g (to)g(con\015ict)h(with)g(an)o(y)189 1712 y(comm)o(unications)i(that)g (use)h(a)e(di\013eren)o(t)i(comm)o(unicator.)143 1814 y FC(\017)23 b FE(An)15 b(in)o(ter-comm)o(unicator)g(cannot)g(b)q(e)h (used)g(for)f(collectiv)o(e)i(comm)o(unication.)143 1916 y FC(\017)23 b FE(A)15 b(comm)o(unicator)g(will)i(pro)o(vide)e(either)h (in)o(tra-)f(or)g(in)o(ter-comm)o(unication,)h(nev)o(er)f(b)q(oth.)75 2019 y(The)i(routine)f Fx(MPI)p 412 2019 14 2 v 16 w(COMM)p 570 2019 V 17 w(TEST)p 701 2019 V 16 w(INTER)h FE(ma)o(y)e(b)q(e)i (used)g(to)f(determine)h(if)g(a)f(comm)o(unicator)g(is)h(an)75 2075 y(in)o(ter-)i(or)e(in)o(tra-comm)o(unicator.)29 b(In)o(ter-comm)o(unicators)18 b(can)g(b)q(e)h(used)g(as)f(argumen)o (ts)f(to)g(some)h(of)75 2132 y(the)f(other)f(comm)o(unicator)h(access)g (routines.)25 b(In)o(ter-comm)o(unicators)17 b(cannot)f(b)q(e)i(used)f (as)g(input)h(to)75 2188 y(an)o(y)d(of)g(the)g(constructor)f(routines)i (for)f(in)o(tra-comm)o(unicators.)166 2323 y Fo(Implemen)o(tati)o(on)e (note:)38 b Fr(Comm)o(unicators)12 b(can)i(b)q(e)h(represen)o(ted)j(in) c(eac)o(h)h(pro)q(cess)h(b)o(y)e(a)g(tuple)h(con-)75 2373 y(sisting)f(of:)75 2454 y Fo(group)75 2537 y(send)p 173 2537 15 2 v 16 w(con)o(text)75 2620 y(receiv)o(e)p 223 2620 V 16 w(con)o(text)75 2704 y(source)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 118 123 118 122 bop 75 -100 a FE(118)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y Fr(F)m(or)e(in)o(ter-comm)o(unicators,)e Fo(group)g Fr(describ)q(es)16 b(the)e(remote)f(group,)g(and)g Fo(source)f Fr(is)h(the)h(rank)f(of)g(the)h(pro)q(cess)75 95 y(in)j(the)i(lo)q(cal) e(group.)30 b(F)m(or)17 b(in)o(tra-comm)o(unicators,)f Fo(group)g Fr(is)i(the)g(comm)o(unicator)d(group)j(\(remote=lo)q (cal\),)75 145 y Fo(source)h Fr(is)h(the)g(rank)g(of)f(the)i(pro)q (cess)h(in)d(this)h(group,)h(and)f Fo(send)h(con)o(text)e Fr(and)h Fo(receiv)o(e)h(con)o(text)e Fr(are)75 195 y(iden)o(tical.)e (A)d(group)g(is)g(represen)o(ted)i(b)o(y)e(a)g (rank-to-absolute-address)g(translation)g(table.)166 244 y(The)e(in)o(ter-comm)o(unicator)c(cannot)j(b)q(e)h(discussed)h (sensibly)e(without)g(considering)h(pro)q(cesses)h(in)e(b)q(oth)g(the) 75 294 y(lo)q(cal)k(and)g(remote)h(groups.)23 b(Imagine)14 b(a)i(pro)q(cess)h Fo(P)e Fr(in)h(group)f Fd(P)s Fr(,)h(whic)o(h)g(has) f(an)h(in)o(ter-comm)o(unicator)d Fo(C)1835 300 y Fc(P)1863 294 y Fr(,)75 344 y(and)j(a)h(pro)q(cess)h Fo(Q)e Fr(in)g(group)h Fd(Q)p Fr(,)f(whic)o(h)g(has)h(an)f(in)o(ter-comm)o(unicator)e Fo(C)1293 350 y Fc(Q)1323 344 y Fr(.)26 b(\(Note)17 b(that)f Fd(P)k Fr(and)c Fd(Q)h Fr(do)f(not)75 394 y(ha)o(v)o(e)e(to)f(b)q(e)i (distinct.\))j(Then)145 469 y Fd(\017)23 b Fo(C)223 475 y Fc(P)251 469 y Fo(.group)13 b Fr(describ)q(es)j(the)e(group)g Fd(Q)g Fr(and)f Fo(C)927 475 y Fc(Q)957 469 y Fo(.group)g Fr(describ)q(es)j(the)e(group)g Fd(P)s Fr(.)145 543 y Fd(\017)23 b Fo(C)223 549 y Fc(P)251 543 y Fo(.send)p 362 543 15 2 v 16 w(con)o(text)15 b(=)h Fr(C)629 549 y Fc(Q)658 543 y Fo(.receiv)o(e)p 819 543 V 16 w(con)o(text)c Fr(and)i(the)h(con)o(text)f(is)g(unique)g(in)f Fd(Q)p Fr(;)189 593 y Fo(C)223 599 y Fc(P)251 593 y Fo(.receiv)o(e)p 412 593 V 16 w(con)o(text)i(=)32 b(C)699 599 y Fc(Q)728 593 y Fo(.send)p 839 593 V 17 w(con)o(text)12 b Fr(and)i(this)g(con)o (text)g(is)g(unique)g(in)f Fd(P)s Fr(.)145 668 y Fd(\017)23 b Fo(C)223 674 y Fc(P)251 668 y Fo(.source)13 b Fr(is)h(rank)g(of)f Fo(P)h Fr(in)f Fd(P)k Fr(and)d Fo(C)851 674 y Fc(Q)881 668 y Fo(.source)f Fr(is)g(rank)h(of)f Fo(Q)h Fr(in)g Fd(Q)p Fr(.)166 743 y(Assume)g(that)h Fo(P)f Fr(sends)i(a)f(message)f (to)g Fo(Q)h Fr(using)f(the)i(in)o(ter-comm)o(unicator.)h(Then)e Fo(P)f Fr(uses)i(the)g Fo(group)75 793 y Fr(table)e(to)g(\014nd)g(the)g (absolute)g(address)h(of)e Fo(Q)p Fr(;)h Fo(source)e Fr(and)i Fo(send)p 1129 793 V 16 w(con)o(text)e Fr(are)j(app)q(ended)g (to)e(the)i(message.)166 843 y(Assume)c(that)g Fo(Q)g Fr(p)q(osts)i(a)d(receiv)o(e)j(with)e(an)g(explicit)g(source)h (parameter)f(using)g(the)h(in)o(ter-comm)o(unicator.)75 892 y(Then)18 b Fo(Q)g Fr(matc)o(hes)f Fo(receiv)o(e)p 553 892 V 16 w(con)o(text)f Fr(to)h(the)h(message)g(con)o(text)g(and)f (source)i(parameter)f(to)f(the)h(message)75 942 y(source.)166 999 y(The)c(same)f(algorithm)e(is)j(appropriate)g(for)g(in)o(tra-comm)n (unicators)e(as)i(w)o(ell.)75 1203 y Fq(5.5.1)49 b(Communicato)o(r)16 b(Accesso)o(rs)75 1336 y Fx(MPI)p 160 1336 14 2 v 16 w(COMM)p 318 1336 V 16 w(TEST)p 448 1336 V 17 w(INTER\(comm,)d (\015ag\))117 1414 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))117 1489 y(OUT)108 b Fx(\015ag)518 b Fr(\(logical\))75 1613 y Fp(int)23 b(MPI)p 245 1613 15 2 v 17 w(Comm)p 358 1613 V 17 w(test)p 471 1613 V 16 w(inter\(MPI)p 703 1613 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 1700 y(MPI)p 150 1700 V 17 w(COMM)p 263 1700 V 16 w(TEST)p 375 1700 V 17 w(INTER\(COMM,)f(FLAG,)i(IERROR\))170 1756 y(INTEGER)f(COMM,)h (FLAG,)f(IERROR)75 1843 y FE(This)d(lo)q(cal)g(routine)g(allo)o(ws)f (the)g(calling)i(pro)q(cess)f(to)e(determine)i(if)g(a)f(comm)o (unicator)g(is)g(an)h(in)o(ter-)75 1899 y(comm)o(unicator)15 b(or)g(an)g(in)o(tra-comm)o(unicator.)k(It)c(returns)g Fm(true)i FE(if)e(it)h(is)g(an)f(in)o(tercomm)o(unicator.)166 1956 y(When)j(an)f(in)o(ter-comm)o(unicator)h(is)g(used)g(as)f(an)g (input)i(argumen)o(t)d(to)h(the)h(comm)o(unicator)f(ac-)75 2012 y(cessors)e(describ)q(ed)i(under)f(in)o(tra-comm)o(unication,)f (the)h(follo)o(wing)g(table)f(describ)q(es)i(b)q(eha)o(vior.)p 302 2107 1346 2 v 301 2163 2 57 v 652 2146 a Fx(MPI)p 737 2146 14 2 v 16 w(COMM)p 895 2146 V 17 w(*)d FE(F)l(unction)j(Beha)o (vior)p 1647 2163 2 57 v 301 2220 V 655 2203 a(\(in)f(In)o(ter-Comm)o (unication)f(Mo)q(de\))p 1647 2220 V 302 2221 1346 2 v 302 2231 V 301 2288 2 57 v 327 2271 a Fx(MPI)p 412 2271 14 2 v 16 w(Comm)p 554 2271 V 14 w(size)p 697 2288 2 57 v 90 w FE(returns)g(the)g(size)h(of)f(the)g(remote)g(group.)p 1647 2288 V 301 2344 V 327 2327 a Fx(MPI)p 412 2327 14 2 v 16 w(Comm)p 554 2327 V 14 w(group)p 697 2344 2 57 v 50 w FE(returns)g(the)g(remote)g(group.)p 1647 2344 V 301 2401 V 327 2384 a Fx(MPI)p 412 2384 14 2 v 16 w(Comm)p 554 2384 V 14 w(rank)p 697 2401 2 57 v 75 w FE(returns)g(the)g(rank)g (in)h(the)g(lo)q(cal)g(group)p 1647 2401 V 302 2402 1346 2 v 75 2505 a Fq(5.5.2)49 b(Intercommunicato)o(r)14 b(Constructo)o(rs)h (and)h(Destructo)o(rs)75 2591 y FE(In)o(tercomm)o(unicators)21 b(can)h(b)q(e)g(created)g(using)h(the)e(function)i Fx(MPI)p 1296 2591 14 2 v 16 w(COMM)p 1454 2591 V 16 w(DUP)p FE(.)e(The)h (functions)75 2647 y Fx(MPI)p 160 2647 V 16 w(COMM)p 318 2647 V 16 w(MAKE)16 b FE(and)g Fx(MPI)p 650 2647 V 16 w(COMM)p 808 2647 V 17 w(SPLIT)f FE(do)h(not)g(apply)h(to)f(in)o (tercomm)o(unicators.)22 b(In)17 b(addi-)75 2704 y(tion,)f(the)g (functions)h Fx(MPI)p 543 2704 V 16 w(INTERCOMM)p 833 2704 V 16 w(ST)l(ART)g FE(and)f Fx(MPI)p 1179 2704 V 16 w(INTERCOMM)p 1469 2704 V 17 w(FINISH)g FE(can)g(b)q(e)g(used)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 119 124 119 123 bop 75 -100 a Ft(5.5.)34 b(INTER-COMMUNICA)l(TION)1047 b FE(119)75 45 y(to)20 b(bind)i(t)o(w)o(o)e(groups)g(in)o(to)h(an)f(in) o(tercomm)o(unicator;)j(the)e(function)h Fx(MPI)p 1418 45 14 2 v 16 w(INTERCOMM)p 1708 45 V 16 w(MERGE)75 102 y FE(creates)15 b(an)g(in)o(tracomm)o(unicator)g(b)o(y)g(merging)g(the) g(t)o(w)o(o)f(groups)h(of)g(an)g(in)o(tercomm)o(unicator.)166 158 y(Construction)d(of)g(an)h(in)o(ter-comm)o(unicator)f(from)g(t)o(w) o(o)f(in)o(tracomm)o(unicators)h(requires)h(separate)75 215 y(collectiv)o(e)k(op)q(erations)f(in)g(the)f(lo)q(cal)i(group)e (and)h(in)g(the)f(remote)g(group,)g(b)q(oth)h(for)e(the)i(starting)f (and)75 271 y(the)k(\014nishing)i(op)q(erations,)e(as)g(w)o(ell)g(as)g (a)f(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)f(b)q(et)o(w)o(een)h (a)e(pro)q(cess)h(in)75 327 y(the)c(lo)q(cal)i(group)e(and)g(a)g(pro)q (cess)h(in)g(the)f(remote)g(group.)20 b(The)c(explicit)h(sync)o (hronization)f(can)g(cause)75 384 y(deadlo)q(c)o(k)g(in)g(mo)q(dular)g (programs)e(with)h(cyclic)i(comm)o(unication)f(graphs,)f(ev)o(en)g(if:) 143 478 y FC(\017)23 b FE(within)16 b(eac)o(h)f(lo)q(cal)i(group)e (calls)h(are)f(executed)h(in)g(the)f(same)g(order,)143 572 y FC(\017)23 b FE(the)f(lo)q(cal)h(and)f(remote)g(op)q(erations)g (are)g(decoupled)i(and)e(the)h(construction)f(is)h(p)q(erformed)189 629 y(\\lo)q(osely)13 b(sync)o(hronously")g(\(b)o(y)f(calling)i(the)f (t)o(w)o(o)e(routines)i Fx(MPI)p 1308 629 V 15 w(INTERCOMM)p 1597 629 V 17 w(ST)l(ART\(\))g FE(and)189 685 y Fx(MPI)p 274 685 V 15 w(INTERCOMM)p 563 685 V 17 w(FINISH\(\))p FE(\).)166 779 y Fx(MPI)p 251 779 V 16 w(INTERCOMM)p 541 779 V 16 w(ST)l(ART)21 b FE(and)e Fx(MPI)p 894 779 V 16 w(INTERCOMM)p 1184 779 V 17 w(FINISH)g FE(are)f(b)q(oth)i (collectiv)o(e)h(op)q(era-)75 836 y(tions)14 b(in)g(their)g(lo)q(cal)h (group,)e(but)h(the)g(call)g(within)h(eac)o(h)f(group)f(need)i(not)e(b) q(e)h(sync)o(hronized)h(with)f(the)75 892 y(call)i(in)g(the)g(other)e (group.)166 949 y(The)g(in)o(ter-comm)o(unicator)h(ob)s(jects)f(are)g (destro)o(y)o(ed)f(in)j(the)e(same)g(w)o(a)o(y)f(as)h(in)o(tra-comm)o (unicator)75 1005 y(ob)s(jects,)g(that)h(is,)g(b)o(y)g(calling)i Fx(MPI)p 692 1005 V 16 w(COMM)p 850 1005 V 16 w(FREE)p FE(.)166 1062 y(The)f(functions)h Fx(MPI)p 542 1062 V 16 w(INTERCOMM)p 832 1062 V 17 w(ST)l(ART)g FE(and)f Fx(MPI)p 1179 1062 V 16 w(INTERCOMM)p 1469 1062 V 16 w(FINISH)g FE(can)h(b)q(e)f(used)75 1118 y(to)21 b(create)g(an)h(in)o (tercomm)o(unicator)g(from)f(t)o(w)o(o)f(existing)j(in)o(tracomm)o (unicators,)f(in)g(the)g(follo)o(wing)75 1175 y(situation:)33 b(A)o(t)21 b(least)h(one)f(selected)i(mem)o(b)q(er)f(from)f(eac)o(h)g (group)g(\(the)h(\\group)f(leader"\))g(has)h(the)75 1231 y(abilit)o(y)f(to)d(comm)o(unicate)i(with)g(the)f(selected)i(mem)o(b)q (er)f(from)e(the)i(other)f(group;)i(i.e.)33 b(a)19 b(\\paren)o(t")75 1287 y(comm)o(unicator)h(exists)h(where)g(b)q(oth)f(leaders)h(b)q (elong,)i(and)d(eac)o(h)h(leader)g(kno)o(ws)f(the)h(rank)f(of)g(the)75 1344 y(other)h(leader)i(in)f(this)g(paren)o(t)f(comm)o(unicator)h (\(the)f(t)o(w)o(o)f(leaders)i(could)h(b)q(e)f(the)g(same)f(pro)q (cess\).)75 1400 y(F)l(urthermore,)14 b(mem)o(b)q(ers)i(of)e(eac)o(h)i (group)f(kno)o(w)f(the)h(rank)g(of)g(their)h(leader.)166 1457 y(In)d(static)g(MPI)g(implemen)o(tation)h(the)f Fx(MPI)p 927 1457 V 16 w(COMM)p 1085 1457 V 16 w(W)o(ORLD)g FE(comm)o(unicator)f(can)h(b)q(e)h(this)f(par-)75 1513 y(en)o(t)k(comm)o(unicator.)24 b(Since)18 b(this)g(paren)o(t)e(comm)o (unicator)g(ma)o(y)h(b)q(e)g(shared)g(for)f(m)o(ultiple)j(in)o(tercom-) 75 1570 y(m)o(unicator)e(creating)h(functions,)g(and)g(the)f(same)g (pro)q(cess)h(ma)o(y)f(b)q(e)h(leaders)g(for)f(sev)o(eral)h(suc)o(h)f (calls,)75 1626 y(an)e(additional)i(user-pro)o(vided)f(tag)e(is)i(used) g(to)e(di\013eren)o(tiate)i(calls.)166 1683 y(In)23 b(dynamic)g(MPI)g (implemen)o(tations,)i(where,)f(e.g.,)f(a)g(pro)q(cess)f(ma)o(y)g(spa)o (wn)g(new)h(c)o(hildren)75 1739 y(pro)q(cesses)18 b(during)h(an)f(MPI)f (execution,)j(the)e(paren)o(t)f(pro)q(cess)h(ma)o(y)f(b)q(e)i(the)f (\\bridge")g(b)q(et)o(w)o(een)g(the)75 1796 y(old)i(comm)o(unication)h (univ)o(erse)f(and)g(the)g(new)g(comm)o(unication)g(w)o(orld)g(that)f (includes)j(the)d(paren)o(t)75 1852 y(and)c(its)h(c)o(hildren.)75 1956 y Fx(MPI)p 160 1956 V 16 w(INTERCOMM)p 450 1956 V 16 w(ST)l(ART\(lo)q(cal)p 714 1956 V 17 w(comm,)281 b(lo)q(cal)p 1230 1956 V 16 w(leader,)h(p)q(eer)p 1733 1956 V 17 w(comm,)75 2012 y(remote)p 210 2012 V 16 w(leader,)14 b(tag,)h(inter)p 540 2012 V 17 w(request\))117 2090 y Fr(IN)155 b Fx(lo)q(cal)p 409 2090 V 17 w(comm)365 b Fr(lo)q(cal)13 b(in)o(tra-comm)o(unicator)e(\(handle\))117 2165 y(IN)155 b Fx(lo)q(cal)p 409 2165 V 17 w(leader)370 b Fr(rank)14 b(of)f(lo)q(cal)g(group)h(leader)g(in)g Fm(lo)q(cal)p 1517 2165 13 2 v 15 w(comm)e Fr(\(in)o(teger\))117 2240 y(IN)155 b Fx(p)q(eer)p 401 2240 14 2 v 18 w(comm)372 b Fr(\\paren)o(t")17 b(in)o(tra-comm)o(unicator;)e(signi\014can)o(t)i (only)f(at)h(the)905 2297 y Fb(mpi)p 990 2297 13 2 v 31 w Fr(arglo)q(cal)p 1148 2297 V 13 w(leader)e(\(handle\))117 2372 y(IN)155 b Fx(remote)p 453 2372 14 2 v 16 w(leader)327 b Fr(rank)12 b(of)f(remote)g(group)h(leader)g(in)23 b Fm(p)q(eer)p 1548 2372 13 2 v 16 w(comm)p Fr(;)10 b(signi\014can)o(t) 905 2428 y(only)j(at)h(the)h Fm(lo)q(cal)p 1201 2428 V 15 w(leader)g Fr(\(in)o(teger\))117 2504 y(IN)155 b Fx(tag)526 b Fr(\\safe")14 b(tag)f(\(in)o(teger\))117 2579 y(OUT)108 b Fx(inter)p 407 2579 14 2 v 17 w(request)349 b Fr(handle)14 b(for)27 b Fm(MPI)p 1193 2579 13 2 v 15 w(INTERCOMM)p 1459 2579 V 14 w(\014nish\(\))75 2704 y Fp(int)c(MPI)p 245 2704 15 2 v 17 w(Intercomm)p 478 2704 V 16 w(start\(MPI)p 710 2704 V 16 w(Comm)g(local)p 965 2704 V 17 w(comm,)g(int)h(local)p 1341 2704 V 16 w(leader,)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 120 125 120 124 bop 75 -100 a FE(120)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)393 45 y Fp(MPI)p 468 45 15 2 v 17 w(Comm)23 b(peer)p 700 45 V 17 w(comm,)g(int)h(remote)p 1100 45 V 16 w(leader,)f(int)g(tag,) 393 102 y(MPI)p 468 102 V 17 w(Intercomm)p 701 102 V 16 w(request)g(*inter)p 1052 102 V 16 w(request\))75 188 y(MPI)p 150 188 V 17 w(INTERCOMM)p 383 188 V 16 w(START\(LOCAL)p 663 188 V 15 w(COMM,)h(LOCAL)p 942 188 V 16 w(LEADER,)f(PEER)p 1245 188 V 17 w(COMM,)g(REMOTE)p 1549 188 V 16 w(LEADER,)g(TAG,)393 245 y(NEW)p 468 245 V 17 w(COMM,)g(INTER)p 748 245 V 17 w(REQUEST,)f(IERROR\))170 301 y(INTEGER)h(LOCAL)p 484 301 V 17 w(COMM,)g(LOCAL)p 764 301 V 17 w(LEADER,)f(PEER)p 1067 301 V 17 w(COMM,)h(REMOTE)p 1371 301 V 17 w(LEADER,)g(TAG,)170 358 y(INTER)p 293 358 V 17 w(REQUEST,)g(IERROR)75 444 y FE(This)d(call)g(starts)e(o\013)g(an)h(in)o(ter-comm)o(unicator)g (creation)g(op)q(eration,)h(returning)f(a)g(handle)h(for)f(the)75 501 y(completion)d(of)e(the)h(op)q(eration)g(in)g Fx(handle)p FE(;)h(it)f(is)g(collectiv)o(e)h(in)g Fx(lo)q(cal)p 1272 501 14 2 v 16 w(comm)p FE(.)i(The)c(call)i(do)q(es)f(not)f(w)o(ait)75 557 y(for)20 b(the)h(remote)f(group)h(to)f(execute)h Fx(MPI)p 840 557 V 16 w(INTERCOMM)p 1130 557 V 17 w(sta)o(rt\(\))p FE(.)36 b Fx(inter)p 1404 557 V 17 w(request)22 b FE(is)f(conceptually) 75 613 y(similar)c(to)f(the)g(comm)o(unication)h(requesthandle)h(used)f (b)o(y)f(non-blo)q(c)o(king)i(p)q(oin)o(t-to-p)q(oin)o(t)f(routines.)75 670 y(The)d(handle)h(is)f(constructed)f(b)o(y)h(a)p 700 670 V 29 w Fx(sta)o(rt)g FE(routine)g(and)g(destro)o(y)o(ed)f(b)o(y)g (the)h(matc)o(hing)p 1594 670 V 30 w Fx(\014nish)h FE(routine;)75 726 y(these)i(handles)h(are)e(not)h(v)m(alid)h(for)e(an)o(y)g(other)h (use.)24 b(It)17 b(is)g(erroneous)g(to)f(call)i(this)f(routine)g(a)f (second)75 783 y(time)g(with)f(the)g(same)g Fx(b)o(ridge)p 592 783 V 17 w(comm)p FE(,)d Fx(remote)p 883 783 V 16 w(leader)j FE(and)h Fx(tag)f FE(argumen)o(ts,)f(without)h(an)g(in)o (terv)o(ening)75 839 y(call)h(to)f Fx(MPI)p 299 839 V 16 w(INTERCOMM)p 589 839 V 16 w(\014nish\(\))h FE(to)f(complete)h(the)f (p)q(ending)i(call.)166 896 y(Pro)q(cesses)h(should)h(pro)o(vide)g (iden)o(tical)h Fx(comm)p FE(,)c Fx(lo)q(cal)p 1106 896 V 16 w(leader)i FE(and)g Fx(tag)g FE(argumen)o(ts)g(within)h(eac)o(h)75 952 y(group.)75 1056 y Fx(MPI)p 160 1056 V 16 w(INTERCOMM)p 450 1056 V 16 w(FINISH\(inter)p 711 1056 V 17 w(request,)d(new)o (comm\))117 1134 y Fr(IN)155 b Fx(inter)p 407 1134 V 17 w(request)349 b Fr(handle)14 b(returned)i(b)o(y)d Fm(MPI)p 1342 1134 13 2 v 15 w(INTERCOMM)p 1608 1134 V 14 w(sta)o(rt\(\))117 1209 y Fr(OUT)108 b Fx(new)o(comm)397 b Fr(new)15 b(in)o(ter-comm)o(unicator)c(\(handle\))75 1334 y Fp(int)23 b(MPI)p 245 1334 15 2 v 17 w(Intercomm)p 478 1334 V 16 w(finish\(MPI)p 734 1334 V 16 w(Intercomm)p 966 1334 V 16 w(request)g(*inter)p 1317 1334 V 16 w(request,)393 1390 y(MPI)p 468 1390 V 17 w(Comm)g(*newcomm\))75 1477 y(MPI)p 150 1477 V 17 w(INTERCOMM)p 383 1477 V 16 w(FINISH\(INTER)p 687 1477 V 15 w(REQUEST,)g(IERROR\))170 1533 y(INTEGER)g(INTER)p 484 1533 V 17 w(REQUEST,)g(NEWCOMM,)f(IERROR)75 1620 y Fx(MPI)p 160 1620 14 2 v 16 w(INTERCOMM)p 450 1620 V 16 w(FINISH)d FE(completes)g(an)g(async)o(hronous)f(in)o(ter-comm)o (unicator)h(creation)g(op)q(er-)75 1676 y(ation,)g(This)g(routine)f(is) h(collectiv)o(e)h(in)f(the)g(goup)f(asso)q(ciated)g(with)h(comm)o (unicator)f Fx(lo)q(cal)p 1692 1676 V 16 w(comm)e FE(of)75 1733 y(the)g(corresp)q(onding)i(call)f(to)e Fx(MPI)p 673 1733 V 16 w(INTERCOMM)p 963 1733 V 17 w(ST)l(ART)p FE(.)i(It)f(w)o(aits)g(for)f(the)i(remote)e(group)h(to)g(call)75 1789 y Fx(MPI)p 160 1789 V 16 w(INTERCOMM)p 450 1789 V 16 w(ST)l(ART)h FE(but)e(do)q(es)g(not)g(w)o(ait)g(for)g(the)g (remote)g(group)f(to)h(call)75 1845 y Fx(MPI)p 160 1845 V 16 w(INTERCOMM)p 450 1845 V 16 w(FINISH)p FE(.)166 1978 y Fo(Implemen)o(tati)o(on)8 b(note:)33 b Fm(MPI)p 723 1978 13 2 v 14 w(INTERCOMM)p 988 1978 V 15 w(ST)m(ART)10 b Fr(initiates)g(async)o(hronous)h(sends)h(and)e(receiv)o(es)75 2028 y(for)i(an)h(exc)o(hange)g(of)g(information)d(b)q(et)o(w)o(een)k (leaders.)k Fm(MPI)p 1031 2028 V 15 w(INTERCOMM)p 1297 2028 V 14 w(FINISH)12 b Fr(completes)h(the)g(exc)o(hange)75 2078 y(of)g(information,)e(next)j(execute)i(a)e(broadcast)g(in)f(eac)o (h)i(lo)q(cal)e(group,)g(with)h(the)g(lo)q(cal)f(leader)h(at)g(the)h (ro)q(ot.)75 2264 y Fx(MPI)p 160 2264 14 2 v 16 w(INTERCOMM)p 450 2264 V 16 w(MERGE\(intercomm,)f(k)o(ey)l(,)h(newintracomm)117 2342 y Fr(IN)155 b Fx(intercomm)384 b Fr(In)o(tercomm)o(unicator)12 b(\(handle\))117 2417 y(IN)155 b Fx(k)o(ey)525 b Fr(In)o(teger)15 b(k)o(ey)117 2492 y(OUT)108 b Fx(newintracomm)308 b Fr(new)15 b(in)o(tra-comm)n(unicator)c(\(handle\))75 2617 y Fp(int)23 b(MPI)p 245 2617 15 2 v 17 w(Intercomm)p 478 2617 V 16 w(merge\(MPI)p 710 2617 V 16 w(Comm)g(intercomm,)g(int)g(key,)h(MPI)p 1395 2617 V 16 w(Comm)g(*newintracomm\))75 2704 y(MPI)p 150 2704 V 17 w(INTERCOMM)p 383 2704 V 16 w(MERGE\(INTERCOMM,)d(KEY,)j (NEWINTERCOMM,)e(IERROR\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 121 126 121 125 bop 75 -100 a Ft(5.6.)34 b(CA)o(CHEING)1376 b FE(121)170 45 y Fp(INTEGER)23 b(INTERCOMM,)g(KEY,)g(NEWINTRACOMM,)f (IERROR)75 133 y FE(This)h(function)g(returns)g(an)f(in)o(tra-comm)o (unicator)g(for)g(the)h(union)g(of)g(the)f(t)o(w)o(o)f(groups)h(that)g (are)75 189 y(asso)q(ciated)16 b(with)f Fx(intercomm)p FE(.)k(All)d(pro)q(cesses)g(should)g(pro)o(vide)g(the)g(same)f Fx(k)o(ey)g FE(v)m(alue)i(within)f(eac)o(h)f(of)75 246 y(these)i(t)o(w)o(o)f(groups)h(\(there)f(should)i(b)q(e)g(exactly)g(t)o (w)o(o)d(k)o(eys)i(v)m(alues)h(used,)g(one)f(unique)i(v)m(alue)f(in)g (eac)o(h)75 302 y(group\).)h(The)14 b(pro)q(cess)h(in)g(the)f(group)g (that)f(pro)o(vided)i(a)f(lo)o(w)o(er)g(v)m(alue)h(are)f(ordered)g(b)q (efore)h(those)f(that)75 358 y(pro)o(vided)j(the)f(higher)g(v)m(alue.) 23 b(Pro)q(cesses)16 b(in)h(the)f(in)o(tersection)g(ma)o(y)f(c)o(ho)q (ose)h(either)h(k)o(ey)e(v)m(alue.)23 b(Ties)75 415 y(are)15 b(brok)o(en)g(arbitrarily)l(.)21 b(This)15 b(call)i(is)f(blo)q(c)o (king)g(and)f(collectiv)o(e)i(within)g(the)e(union)h(of)f(the)g (groups.)75 543 y Fq(5.5.3)49 b(Name)16 b(Service)75 630 y FE(MPI)e(pro)o(vides)h(a)g(name)f(service)h(to)f(simplify)j (construction)d(of)g(in)o(ter-comm)o(unicators.)20 b(This)15 b(service)75 687 y(allo)o(ws)f(a)g(lo)q(cal)h(pro)q(cess)f(group)g(to)f (create)h(an)g(in)o(ter-comm)o(unicator)g(when)g(the)g(only)h(a)o(v)m (ailable)h(infor-)75 743 y(mation)10 b(ab)q(out)h(the)f(remote)g(group) h(is)g(a)f(user-de\014ned)i(c)o(haracter)e(string.)18 b(A)11 b(lo)q(osely)h(sync)o(hronous)e(v)o(er-)75 800 y(sion)328 b(is)h(pro)o(vided)g(b)o(y)f(routines)75 856 y Fx(MPI)p 160 856 14 2 v 16 w(INTERCOMM)p 450 856 V 16 w(NAME)p 595 856 V 17 w(ST)l(ART)16 b FE(and)g Fx(MPI)p 941 856 V 16 w(INTERCOMM)p 1231 856 V 16 w(FINISH)p FE(.)75 961 y Fx(MPI)p 160 961 V 16 w(INTERCOMM)p 450 961 V 16 w(NAME)p 595 961 V 17 w(ST)l(ART\(comm,)e(name,)g(inter)p 1131 961 V 17 w(request\))117 1039 y Fr(IN)155 b Fx(comm)470 b Fr(lo)q(cal)13 b(in)o(tra-comm)o(unicator)e(\(handle\))117 1117 y(IN)155 b Fx(name)485 b Fr(c)o(haracter)15 b(string)117 1194 y(OUT)108 b Fx(inter)p 407 1194 V 17 w(request)349 b Fr(handle)14 b(for)g(MPI)p 1186 1194 13 2 v 15 w(INTER)o(COMM)p 1473 1194 V 15 w(FINISH\(\))75 1319 y Fp(int)23 b(MPI)p 245 1319 15 2 v 17 w(Intercomm)p 478 1319 V 16 w(name)p 590 1319 V 17 w(start\(MPI)p 823 1319 V 16 w(Comm)g(comm,)g(char)h (*name,)393 1376 y(MPI)p 468 1376 V 17 w(Intercomm)p 701 1376 V 16 w(request)f(*inter)p 1052 1376 V 16 w(request\))75 1463 y(MPI)p 150 1463 V 17 w(INTERCOMM)p 383 1463 V 16 w(NAME)p 495 1463 V 16 w(START\(COMM,)g(NAME,)g(NEW)p 1013 1463 V 17 w(COMM,)g(INTER)p 1293 1463 V 16 w(REQUEST,)g(IERROR\)) 170 1520 y(INTEGER)g(COMM,)h(NAME,)f(NEW)p 723 1520 V 16 w(COMM,)h(INTER)p 1003 1520 V 16 w(REQUEST,)f(IERROR)75 1607 y FE(This)d(is)h(the)f(name-serv)o(ed)g(equiv)m(alen)o(t)h(of)f Fx(MPI)p 944 1607 14 2 v 16 w(INTERCOMM)p 1234 1607 V 16 w(ST)l(ART)p FE(.)h(The)f(op)q(eration)g(is)g(com-)75 1664 y(pleted)c(b)o(y)f(a)g(call)i(to)d Fx(MPI)p 536 1664 V 16 w(INTERCOMM)p 826 1664 V 17 w(FINISH)p FE(.)75 1813 y Fv(5.6)59 b(Cacheing)75 1916 y FE(MPI)20 b(pro)o(vides)h(a)f (\\cac)o(heing")h(facilit)o(y)g(that)f(allo)o(ws)h(an)f(application)i (to)e(attac)o(h)f(arbitrary)h(pieces)75 1973 y(of)f(information,)g (called)i Fs(attributes)p FE(,)f(to)f(comm)o(unicators.)31 b(More)18 b(precisely)l(,)k(the)d(cac)o(heing)h(facilit)o(y)75 2029 y(allo)o(ws)15 b(a)g(p)q(ortable)h(library)g(to)e(do)i(the)f (follo)o(wing:)143 2127 y FC(\017)23 b FE(pass)15 b(information)g(b)q (et)o(w)o(een)g(calls)i(b)o(y)e(asso)q(ciating)g(it)h(with)f(an)h(MPI)f (comm)o(unicator,)143 2225 y FC(\017)23 b FE(quic)o(kly)16 b(retriev)o(e)g(that)e(information,)h(and)143 2323 y FC(\017)23 b FE(b)q(e)18 b(guaran)o(teed)e(that)h(out-of-date)g (information)g(is)h(nev)o(er)g(retriev)o(ed,)f(ev)o(en)h(if)g(the)f (comm)o(uni-)189 2380 y(cator)d(is)i(freed)f(and)h(its)f(handle)h (subsequen)o(tly)h(reused)f(b)o(y)f(MPI.)75 2478 y(These)j (capabilities,)i(in)f(some)e(form,)g(are)h(required)g(b)o(y)g(built-in) i(MPI)e(routines)g(suc)o(h)g(as)f(collectiv)o(e)75 2534 y(comm)o(unication)g(and)g(virtual)g(top)q(ology)l(.)23 b(De\014ning)17 b(an)g(in)o(terface)f(to)g(these)h(capabilities)i(as)d (part)f(of)75 2591 y(the)f(MPI)f(standard)g(is)h(v)m(aluable)h(b)q (ecause)g(it)f(p)q(ermits)g(routines)g(lik)o(e)g(collectiv)o(e)i(comm)o (unication)e(and)75 2647 y(virtual)h(top)q(ology)g(to)f(b)q(e)h (implemen)o(ted)h(as)f(p)q(ortable)g(co)q(de,)g(and)g(also)f(b)q (ecause)i(it)f(mak)o(es)f(MPI)g(more)75 2704 y(extensible)j(b)o(y)e (allo)o(wing)h(user-written)g(routines)f(to)g(use)h(standard)e(MPI)h (calling)i(sequences.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 122 127 122 126 bop 75 -100 a FE(122)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y Fq(5.6.1)49 b(F)o(unctionalit)o(y)75 135 y FE(A)o(ttributes)21 b(are)f(lo)q(cal)i(to)e(the)h(pro)q(cess)g(and)g(sp)q(eci\014c)i(to)d (the)h(comm)o(unicator)g(to)f(whic)o(h)h(they)g(are)75 191 y(attac)o(hed.)d(They)13 b(are)e(not)h(propagated)f(b)o(y)h(MPI)g (from)g(one)g(comm)o(unicator)g(to)f(another)h(except)g(when)75 248 y(the)f(comm)o(unicator)f(is)h(duplicated)i(using)e Fx(MPI)p 899 248 14 2 v 16 w(COMM)p 1057 248 V 16 w(DUP)g FE(\(and)g(ev)o(en)g(then)g(the)f(application)j(m)o(ust)75 304 y(giv)o(e)j(sp)q(eci\014c)i(p)q(ermission)f(for)e(the)h(attribute)g (to)f(b)q(e)i(copied\).)22 b(A)o(ttributes)16 b(are)g(scalar)f(v)m (alues,)i(equal)75 361 y(in)f(size)g(or)f(larger)g(than)g(a)g (C-language)g(p)q(oin)o(ter.)166 419 y(The)10 b(cac)o(heing)h(in)o (terface)g(de\014ned)g(here)g(represen)o(ts)f(that)f(attributes)h(are)g (stored)g(b)o(y)g(MPI)g(opaquely)75 475 y(within)16 b(a)f(comm)o (unicator.)20 b(Accessor)15 b(functions)h(include)i(the)d(follo)o (wing:)143 577 y FC(\017)23 b FE(obtain)16 b(a)f(k)o(ey)h(v)m(alue)h (\(used)f(to)g(iden)o(tify)h(an)f(attribute\);)f(the)h(user)g(sp)q (eci\014es)i(\\callbac)o(k")e(func-)189 634 y(tions)g(b)o(y)h(whic)o(h) g(MPI)f(informs)h(the)g(application)h(when)f(the)f(group)g(or)g(comm)o (unicator)g(is)h(de-)189 690 y(stro)o(y)o(ed)d(or)h(copied.)143 792 y FC(\017)23 b FE(store)14 b(and)i(retriev)o(e)f(the)g(v)m(alue)i (of)d(an)i(attribute;)75 894 y(MPI)f(pro)o(vides)h(the)f(follo)o(wing)h (services)g(related)g(to)e(cac)o(heing.)21 b(They)15 b(are)g(all)h(lo)q(cal.)75 999 y Fx(MPI)p 160 999 V 16 w(A)l(TTR)p 293 999 V 17 w(GET)p 398 999 V 17 w(KEY\(cop)o(y)p 608 999 V 16 w(fn,)f(delete)p 799 999 V 18 w(fn,)g(k)o(eyval,)f(extra)p 1120 999 V 17 w(state\))117 1078 y Fr(IN)155 b Fx(cop)o(y)p 407 1078 V 17 w(fn)447 b Fr(Cop)o(y)14 b(callbac)o(k)f(function)g(for)h Fm(k)o(eyval)117 1158 y Fr(IN)155 b Fx(delete)p 431 1158 V 18 w(fn)422 b Fr(Delete)15 b(callbac)o(k)e(function)g(for)h Fm(k)o(eyval)117 1237 y Fr(OUT)108 b Fx(k)o(eyval)471 b Fr(k)o(ey)14 b(v)n(alue)f(for)h(future)g(access)i(\(in)o(teger\))117 1316 y(IN)155 b Fx(extra)p 416 1316 V 17 w(state)384 b Fr(Extra)14 b(state)h(for)f(callbac)o(k)f(functions)75 1442 y Fp(int)23 b(MPI)p 245 1442 15 2 v 17 w(Attr)p 358 1442 V 17 w(get)p 447 1442 V 17 w(key\(copy)p 656 1442 V 16 w(fn,)g(delete)p 911 1442 V 17 w(fn,)g(int)g(keyval,)g(void*) h(extra)p 1573 1442 V 16 w(state\))75 1531 y(MPI)p 150 1531 V 17 w(ATTR)p 263 1531 V 16 w(GET)p 351 1531 V 17 w(KEY\(COPY)p 560 1531 V 16 w(FN,)g(DELETE)p 816 1531 V 16 w(FN,)g(KEYVAL,)e(EXTRA)p 1238 1531 V 17 w(STATE,)h(IERROR\))170 1587 y(INTEGER)g(COPY)p 460 1587 V 17 w(FN,)g(DELETE)p 716 1587 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1139 1587 V 17 w(STATE,)g(IERROR)166 1676 y FE(Generates)13 b(a)g(new)h(attribute)g(k) o(ey)l(.)19 b(Keys)14 b(are)g(lo)q(cally)h(unique)g(in)g(pro)q(cess,)f (and)f(opaque)h(to)f(user,)75 1732 y(though)i(they)h(are)f(explicitly)k (stored)c(in)h(in)o(tegers.)21 b(Once)c(allo)q(cated,)f(the)f(k)o(ey)h (v)m(alue)h(can)e(b)q(e)h(used)h(to)75 1788 y(asso)q(ciate)e (attributes)g(and)h(access)f(them)g(on)g(an)o(y)g(lo)q(cally)i (de\014ned)g(comm)o(unicator.)166 1847 y(The)h Fx(cop)o(y)p 351 1847 14 2 v 17 w(fn)g FE(function)h(is)g(in)o(v)o(ok)o(ed)f(when)h (a)f(comm)o(unicator)g(is)h(duplicated.)30 b(A)19 b(C)f(protot)o(yp)q (e)75 1903 y(for)d(suc)o(h)g(a)g(function)h(is)166 1962 y Fx(cop)o(y)p 255 1962 V 16 w(fn\(oldcomm,)d(new)o(comm,)h(k)o(eyval,) g(extra)p 981 1962 V 17 w(state\))166 2020 y FE(This)22 b(function)h(do)q(es)g(whatev)o(er)e(it)h(needs)h(to)e(do)h(with)g(the) h(new)f(comm)o(unicator)f Fx(new)o(comm)75 2077 y FE(based)c(on)g(what) f(attribute)h(information)g(w)o(as)f(in)h(the)g(old)h(comm)o(unicator)e Fx(olcomm)p FE(;)f(it)i(returns)g(1)f(if)75 2133 y(successful,)f(0)e (if)h(not.)19 b(A)14 b(v)m(alid)h(cop)o(y)f(function)g(is)g(one)g(that) f(completely)i(duplicates)g(the)f(information)75 2190 y(b)o(y)20 b(making)g(a)f(full)i(duplicate)g(cop)o(y)f(of)f(the)h(data) f(structures)h(implied)i(b)o(y)d(an)h(attribute;)h(another)75 2246 y(migh)o(t)f(just)g(mak)o(e)g(another)g(reference)h(to)e(that)h (data)g(structure,)g(while)i(using)f(a)f(reference-coun)o(t)75 2302 y(mec)o(hanism.)37 b(Other)21 b(t)o(yp)q(es)g(of)f(attributes)h (migh)o(t)g(not)f(cop)o(y)h(at)f(all)h(\(they)g(migh)o(t)g(b)q(e)g(sp)q (eci\014c)i(to)75 2359 y Fx(oldcomm)15 b FE(only\).)24 b Fx(MPI)p 488 2359 V 16 w(COMM)p 646 2359 V 16 w(DUP)17 b FE(is)g(the)g(only)g(function)g(that)f(calls)i(the)e Fx(cop)o(y)p 1547 2359 V 17 w(fn)p FE(.)24 b Fx(cop)o(y)p 1724 2359 V 17 w(fn)17 b FE(ma)o(y)75 2415 y(b)q(e)f(sp)q(eci\014ed)h (as)e(n)o(ull,)h(in)g(whic)o(h)g(case)g(no)f(cop)o(y)g(callbac)o(k)h(o) q(ccurs)g(for)e Fx(k)o(eyval)p FE(.)166 2474 y(Analogous)h(to)g Fx(cop)o(y)p 532 2474 V 16 w(fn)h FE(is)f(a)g(callbac)o(k)i(deletion)f (function,)g(de\014ned)g(as)f(follo)o(ws:)166 2532 y Fx(delete)p 279 2532 V 18 w(fn\(comm,)e(k)o(eyval,)h(extra)p 733 2532 V 16 w(state\))166 2591 y FE(This)g(function)h(do)q(es)f (whatev)o(er)g(is)g(needed)h(to)e(remo)o(v)o(e)h(the)f(attribute)h (from)f(the)h(comm)o(unicator)75 2647 y(sp)q(eci\014ed)k(b)o(y)e Fx(comm)p FE(.)k(It)c(ma)o(y)f(b)q(e)h(sp)q(eci\014ed)i(as)e(n)o(ull,)h (in)f(whic)o(h)h(case)f(no)g(delete)h(callbac)o(k)f(o)q(ccurs)h(for)75 2704 y Fx(k)o(eyval)p FE(.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 123 128 123 127 bop 75 -100 a Ft(5.6.)34 b(CA)o(CHEING)1376 b FE(123)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(A)l(TTR)p 293 45 V 17 w(FREE)p 419 45 V 17 w(KEY\(k)o(eyval\))117 134 y Fr(IN)155 b Fx(k)o(eyval)471 b Fr(F)m(rees)15 b(the)g(in)o(teger) f(k)o(ey)g(v)n(alue)f(\(in)o(teger\))75 270 y Fp(int)23 b(MPI)p 245 270 15 2 v 17 w(Attr)p 358 270 V 17 w(free)p 471 270 V 16 w(key\(int)g(keyval\))75 368 y(MPI)p 150 368 V 17 w(ATTR)p 263 368 V 16 w(FREE)p 375 368 V 17 w(KEY\(KEYVAL,)f(IERROR\))170 424 y(INTEGER)h(KEYVAL,)g(IERROR)166 522 y FE(F)l(rees)13 b(an)g(extan)o(t)f(attribute)h(k)o(ey)l(.)19 b(It)14 b(is)f(not)g(erroneous)g(to)f(free)h(an)g(attribute)g(k)o(ey)g (that)g(is)g(in)h(use,)75 578 y(b)q(ecause)h(the)e(actual)h(free)f(do)q (es)h(not)f(transpire)h(un)o(til)h(after)e(all)h(references)g(\(in)g (other)f(comm)o(unicators)75 635 y(on)i(a)g(pro)q(cess\))g(to)g(the)g (k)o(ey)g(ha)o(v)o(e)g(b)q(een)h(freed.)75 750 y Fx(MPI)p 160 750 14 2 v 16 w(A)l(TTR)p 293 750 V 17 w(PUT)p 401 750 V 16 w(V)l(ALUE\(comm,)e(k)o(eyval,)g(attribute)p 1024 750 V 19 w(val\))117 839 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)7 b(to)i(whic)o(h)g(attribute)h(will)e(b)q(e)h (attac)o(hed)h(\(han-)905 895 y(dle\))117 993 y(IN)155 b Fx(k)o(eyval)471 b Fr(k)o(ey)14 b(v)n(alue,)f(as)h(returned)h(b)o(y) 905 1049 y Fm(MPI)p 982 1049 13 2 v 15 w(A)m(TTR)p 1105 1049 V 14 w(get)p 1173 1049 V 15 w(k)o(ey)g Fr(\(in)o(teger\))117 1147 y(IN)155 b Fx(attribute)p 484 1147 14 2 v 18 w(val)352 b Fr(attribute)15 b(v)n(alue)75 1283 y Fp(int)23 b(MPI)p 245 1283 15 2 v 17 w(Attr)p 358 1283 V 17 w(put)p 447 1283 V 17 w(value\(MPI)p 680 1283 V 16 w(Comm)g(comm,)g(int)h(keyval,)e (void*)i(attribute)p 1604 1283 V 16 w(val\))75 1381 y(MPI)p 150 1381 V 17 w(ATTR)p 263 1381 V 16 w(PUT)p 351 1381 V 17 w(VALUE\(COMM,)f(KEYVAL,)f(ATTRIBUTE)p 1061 1381 V 16 w(VAL,)i(IERROR\))170 1438 y(INTEGER)f(COMM,)h(KEYVAL,)e (ATTRIBUTE)p 914 1438 V 16 w(VAL,)i(IERROR)166 1535 y FE(This)h(function)g(stores)f(the)g(stipulated)i(attribute)e(v)m(alue)i Fx(attribute)p 1408 1535 14 2 v 18 w(val)e FE(for)g(subsequen)o(t)h (re-)75 1592 y(triev)m(al)e(b)o(y)e Fx(MPI)p 376 1592 V 16 w(A)l(TTR)p 509 1592 V 17 w(GET)p 614 1592 V 17 w(V)l(ALUE)p FE(.)h(If)g(the)g(v)m(alue)h(is)f(already)g(presen)o(t,)h (a)e(subsequen)o(t)i(call)g(to)75 1648 y Fx(MPI)p 160 1648 V 16 w(A)l(TTR)p 293 1648 V 17 w(PUT)p 401 1648 V 16 w(V)l(ALUE)59 b FE(replaces)f(the)g(old)g(v)m(alue,)69 b(exactly)58 b(as)f(if)h(a)f(call)i(to)75 1705 y Fx(MPI)p 160 1705 V 16 w(A)l(TTR)p 293 1705 V 17 w(DELETE)p 480 1705 V 16 w(V)l(ALUE\(comm,)14 b(k)o(eyval\))g FE(had)i(\014rst)e(b)q (een)j(called)f(\(see)g(b)q(elo)o(w\).)75 1820 y Fx(MPI)p 160 1820 V 16 w(A)l(TTR)p 293 1820 V 17 w(GET)p 398 1820 V 17 w(V)l(ALUE\(comm,)d(k)o(eyval,)i(attribute)p 1022 1820 V 18 w(val,)f(found\))117 1909 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)10 b(to)i(whic)o(h)g(attribute)h(is)f(attac)o(hed) h(\(handle\))117 2007 y(IN)155 b Fx(k)o(eyval)471 b Fr(k)o(ey)14 b(v)n(alue)f(\(in)o(teger\))117 2105 y(OUT)108 b Fx(attribute)p 484 2105 V 18 w(val)352 b Fr(attribute)15 b(v)n(alue,)d(unless)j Fm(found)g Fr(=)f(false)117 2203 y(OUT)108 b Fx(found)481 b Fr(true)11 b(or)f(false,)g(dep)q(ending)h(on)e(whether)j(the)e (attribute)h(v)n(alue)905 2259 y(has)j(b)q(een)h(put)f(in)o(to)g(this)f (attribute)i(set)75 2395 y Fp(int)23 b(MPI)p 245 2395 15 2 v 17 w(Attr)p 358 2395 V 17 w(get)p 447 2395 V 17 w(value\(MPI)p 680 2395 V 16 w(Comm)g(comm,)g(int)h(keyval,)e(void*)i (attribute)p 1604 2395 V 16 w(val,)393 2451 y(int)g(*found\))75 2549 y(MPI)p 150 2549 V 17 w(ATTR)p 263 2549 V 16 w(GET)p 351 2549 V 17 w(VALUE\(COMM,)f(KEYVAL,)f(ATTRIBUTE)p 1061 2549 V 16 w(VAL,)i(FOUND,)f(IERROR\))170 2606 y(INTEGER)g(COMM,)h (KEYVAL,)e(ATTRIBUTE)p 914 2606 V 16 w(VAL,)i(FOUND,)f(IERROR)166 2704 y FE(Retriev)o(es)16 b(attribute)f(v)m(alue)h(b)o(y)f(k)o(ey)l(.) -32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 124 129 124 128 bop 75 -100 a FE(124)92 b Ft(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(A)l(TTR)p 293 45 V 17 w(DELETE)p 480 45 V 16 w(V)l(ALUE\(comm,)f(k)o(eyval\))117 124 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)10 b(to)i(whic)o(h)g(attribute)h(is)f(attac)o(hed)h(\(handle\))117 202 y(IN)155 b Fx(k)o(eyval)471 b Fr(The)15 b(in)o(teger)f(k)o(ey)g(v)n (alue)f(to)h(b)q(e)g(erased)h(from)e Fm(comm)p Fr(.)75 328 y Fp(int)23 b(MPI)p 245 328 15 2 v 17 w(Attr)p 358 328 V 17 w(delete)p 519 328 V 16 w(value\(MPI)p 751 328 V 16 w(Comm)g(comm,)h(int)f(keyval\))75 416 y(MPI)p 150 416 V 17 w(ATTR)p 263 416 V 16 w(DELETE)p 423 416 V 17 w(VALUE\(COMM,)f(KEYVAL,)h(IERROR\))170 473 y(INTEGER)g(COMM,)h (KEYVAL,)e(IERROR)166 561 y FE(Delete)14 b(attribute)g(from)f(cac)o(he) h(b)o(y)f(k)o(ey)l(.)20 b(This)14 b(function)g(in)o(v)o(ok)o(es)g(the)g (attribute)f(delete)i(function)75 617 y Fx(delete)p 188 617 14 2 v 18 w(fn)g FE(sp)q(eci\014ed)j(when)d(the)h Fx(k)o(eyval)e FE(w)o(as)h(created.)166 676 y(Whenev)o(er)i(a)g(comm)o (unicator)g(is)h(replicated)g(using)g(the)f(function)h Fx(MPI)p 1445 676 V 16 w(COMM)p 1603 676 V 16 w(DUP)g FE(all)g(call-)75 732 y(bac)o(k)h(cop)o(y)h(functions)g(for)f (attributes)h(that)f(are)g(curren)o(tly)h(set)f(are)h(in)o(v)o(ok)o (ed.)33 b(Whenev)o(er)20 b(a)f(com-)75 788 y(m)o(unicator)d(is)g (deleted)h(using)g(the)e(function)i Fx(MPI)p 957 788 V 16 w(COMM)p 1115 788 V 16 w(FREE)g FE(all)f(callbac)o(k)h(delete)g (functions)g(for)75 845 y(attributes)e(that)f(are)h(curren)o(tly)h(set) f(are)g(in)o(v)o(ok)o(ed.)75 974 y Fx(Example)75 1063 y FE(This)h(example)g(sho)o(ws)f(ho)o(w)g(to)g(write)g(a)h(collectiv)o (e)h(comm)o(unication)f(op)q(eration)g(that)e(uses)i(cac)o(heing)75 1120 y(to)e(b)q(e)h(more)f(e\016cien)o(t)h(after)f(the)h(\014rst)f (call.)21 b(The)14 b(co)q(ding)i(st)o(yle)f(assumes)f(that)g(MPI)g (function)i(results)75 1176 y(return)f(only)h(error)e(statuses.)147 1292 y Fp(static)23 b(int)g(gop_key_assigned)f(=)h(0;)48 b(/*)24 b(0)f(only)h(on)f(first)g(entry)h(*/)147 1349 y(static)f(int)g(gop_key;)357 b(/*)24 b(key)f(for)g(this)h(module's)f (stuff)g(*/)147 1462 y(typedef)f(struct)147 1518 y({)218 1574 y(int)i(ref_count;)237 b(/*)23 b(reference)g(count)g(*/)218 1631 y(/*)h(other)f(stuff,)g(whatever)g(else)g(we)h(want)f(*/)147 1687 y(})g(gop_stuff_type;)147 1800 y(Efficient_Collective_Op)d (\(comm,)j(...\))147 1857 y(MPI_Comm)f(comm;)147 1913 y({)194 1970 y(gop_stuff_type)g(*gop_stuff;)194 2026 y(MPI_Group)166 b(group;)194 2083 y(int)310 b(foundflag;)194 2195 y(MPI_Comm_group\(comm,)21 b(&group\);)194 2308 y(if)j(\(!gop_key_assigned\))117 b(/*)23 b(get)h(a)f(key)h(on)f(first)h (call)f(ever)g(*/)194 2365 y({)242 2421 y(gop_key_assigned)f(=)h(1;)242 2478 y(if)h(\()f(!)h(MPI_Attr_get_key\()e(gop_stuff_copier,)839 2534 y(gop_stuff_destructor,)839 2591 y(&gop_key,)g(\(void)h(*\)0\)\);) 242 2647 y(/*)h(get)f(the)h(key)f(while)g(assigning)g(its)g(copy)h(and) f(delete)g(callback)314 2704 y(behavior.)f(*/)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 125 130 125 129 bop 75 -100 a Ft(5.6.)29 b(CA)o(CHEING)1381 b FE(125)242 102 y Fp(MPI_Abort)23 b(\("Insufficient)f(keys)h (available"\);)194 158 y(})194 271 y(MPI_Attr_get_value)f(\(comm,)h (gop_key,)g(&gop_stuff,)f(&foundflag\);)194 327 y(if)i(\(foundflag)e (==)i(MPI_FOUND\))194 384 y({)g(/*)g(This)f(module)g(has)g(executed)g (in)h(this)f(group)g(before.)314 440 y(We)g(will)h(use)f(the)g(cached)g (information)g(*/)194 497 y(})194 553 y(else)194 610 y({)h(/*)g(This)f(is)h(a)f(group)g(that)h(we)f(have)h(not)f(yet)h (cached)f(anything)f(in.)314 666 y(We)h(will)h(now)f(do)h(so.)242 723 y(*/)242 835 y(/*)g(First,)f(allocate)f(storage)h(for)h(the)f (stuff)g(we)h(want,)314 892 y(and)f(initialize)g(the)g(reference)g (count)g(*/)242 1005 y(gop_stuff)g(=)g(\(gop_stuff_type)f(*\))i(malloc) f(\(sizeof\(gop_stuff_type\)\);)242 1061 y(if)h(\(gop_stuff)e(==)i (NULL\))f({)h(/*)f(abort)g(on)h(out-of-memory)e(error)h(*/)h(})242 1174 y(gop_stuff)f(->)g(ref_count)g(=)h(1;)242 1287 y(/*)g(Second,)f (fill)g(in)g(*gop_stuff)g(with)g(whatever)g(we)h(want.)314 1344 y(This)f(part)g(isn't)g(shown)h(here)f(*/)242 1456 y(/*)h(Third,)f(store)g(gop_stuff)f(as)i(the)g(attribute)e(value)h(*/) 242 1513 y(MPI_Attr_put_value)e(\()j(comm,)f(gop_key,)g(gop_stuff\);) 194 1569 y(})194 1626 y(/*)h(Then,)f(in)h(any)f(case,)g(use)h(contents) f(of)g(*gop_stuff)266 1682 y(to)g(do)h(the)g(global)f(op)g(...)h(*/)147 1739 y(})147 1852 y(/*)f(The)h(following)e(routine)h(is)h(called)f(by)g (MPI)h(when)f(a)h(group)f(is)h(freed)f(*/)147 1965 y (gop_stuff_destructor)e(\(comm,)i(keyval,)g(gop_stuff,)f(extra\))147 2021 y(MPI_Comm)g(comm;)147 2077 y(int)h(keyval;)147 2134 y(gop_stuff_type)f(*gop_stuff;)147 2190 y(void)h(*extra;)147 2247 y({)194 2303 y(if)h(\(keyval)f(!=)g(gop_key\))g({)h(/*)f(abort)h (--)f(programming)g(error)g(*/)g(})194 2416 y(/*)h(The)f(group's)g (being)g(freed)h(removes)f(one)g(reference)g(to)g(gop_stuff)g(*/)194 2473 y(gop_stuff)g(->)h(ref_count)e(-=)i(1;)194 2586 y(/*)g(If)f(no)h(references)f(remain,)f(then)i(free)f(the)h(storage)e (*/)194 2642 y(if)i(\(gop_stuff)e(->)i(ref_count)f(==)g(0\))h({)242 2698 y(free\(\(void)f(*\)gop_stuff\);)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 126 131 126 130 bop 75 -100 a FE(126)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)194 45 y Fp(})147 102 y(})147 214 y(/*)23 b(The)h(following)e(routine)h(is) h(called)f(by)g(MPI)h(when)f(a)h(group)f(is)h(copied)f(*/)147 271 y(gop_stuff_copier)e(\(comm,)i(keyval,)g(gop_stuff,)g(extra\))147 327 y(MPI_Comm)f(comm;)147 384 y(int)h(keyval;)147 440 y(gop_stuff_type)f(*gop_stuff;)147 497 y(void)h(*extra;)147 553 y({)194 610 y(if)h(\(keyval)f(!=)g(gop_key\))g({)h(/*)f(abort)h(--) f(programming)g(error)g(*/)g(})194 723 y(/*)h(The)f(new)h(group)f(adds) g(one)h(reference)e(to)i(this)f(gop_stuff)g(*/)194 779 y(gop_stuff)g(->)h(ref_count)e(+=)i(1;)147 835 y(})75 977 y Fv(5.7)59 b(F)n(o)n(rmalizing)21 b(the)e(Lo)r(osely)g (Synchronous)g(Mo)r(del)g(\(Usage,)g(Safet)n(y\))75 1080 y Fq(5.7.1)49 b(Basic)17 b(Statements)75 1165 y FE(When)e(a)g(caller)h (passes)e(a)h(comm)o(unicator)f(\(that)g(con)o(tains)h(a)g(con)o(text)f (and)h(group\))f(to)g(a)h(callee,)h(that)75 1222 y(comm)o(unicator)f(m) o(ust)h(b)q(e)g(free)g(of)f(side)i(e\013ects)e(throughout)g(execution)i (of)e(the)h(subprogram)f(\(quies-)75 1278 y(cen)o(t\).)26 b(This)18 b(pro)o(vides)g(one)f(mo)q(del)h(in)g(whic)o(h)h(libraries)g (can)e(b)q(e)h(written,)f(and)h(w)o(ork)e(\\safely)l(.")27 b(F)l(or)75 1335 y(libraries)13 b(so)f(designated,)h(the)f(callee)h (has)f(p)q(ermission)h(to)e(do)h(whatev)o(er)f(comm)o(unication)i(it)f (lik)o(es)h(with)75 1391 y(the)19 b(comm)o(unicator,)g(and)g(under)h (the)f(ab)q(o)o(v)o(e)g(guaran)o(tee)f(kno)o(ws)g(that)h(no)g(other)f (comm)o(unications)75 1448 y(will)d(in)o(terfere.)k(Since)c(w)o(e)e(p)q (ermit)g(go)q(o)q(d)g(implemen)o(tations)i(to)d(create)h(new)g(comm)o (unicators)g(without)75 1504 y(sync)o(hronization)h(\(suc)o(h)g(as)f(b) o(y)g(preallo)q(cated)i(con)o(texts)e(on)g(comm)o(unicators\),)g(this)g (do)q(es)h(not)f(imp)q(ose)75 1561 y(a)i(signi\014can)o(t)h(o)o(v)o (erhead.)166 1617 y(This)k(form)f(of)g(safet)o(y)f(is)i(analogous)f(to) g(other)g(common)g(computer-science)j(usages,)e(suc)o(h)f(as)75 1674 y(passing)c(a)f(descriptor)h(of)f(an)h(arra)o(y)e(to)h(a)g (library)i(routine.)k(The)15 b(library)g(routine)g(has)g(ev)o(ery)f (righ)o(t)h(to)75 1730 y(exp)q(ect)h(suc)o(h)f(a)g(descriptor)h(to)f(b) q(e)g(v)m(alid)i(and)f(mo)q(di\014able.)75 1850 y Fq(5.7.2)49 b(Mo)q(dels)17 b(of)g(Execution)75 1935 y FE(W)l(e)i(sa)o(y)f(that)g(a) g(parallel)j(pro)q(cedure)e(is)g Fs(active)g FE(in)h(a)e(pro)q(cess)h (if)g(the)g(pro)q(cess)g(b)q(elongs)h(to)e(a)g(group)75 1992 y(that)e(ma)o(y)h(collectiv)o(ely)i(execute)f(the)f(pro)q(cedure,) h(and)f(some)f(mem)o(b)q(er)h(of)g(that)f(group)h(is)g(curren)o(tly)75 2048 y(executing)c(the)f(pro)q(cedure)h(co)q(de.)19 b(If)13 b(a)e(parallel)j(pro)q(cedure)f(is)f(activ)o(e)g(in)h(a)f(pro)q(cess,)g (then)g(this)h(pro)q(cess)75 2105 y(ma)o(y)e(b)q(e)i(receiving)g (messages)e(p)q(ertaining)i(to)f(this)g(pro)q(cedure,)h(ev)o(en)f(if)g (it)g(do)q(es)h(not)e(curren)o(tly)h(execute)75 2161 y(the)j(co)q(de)h(of)f(this)g(pro)q(cedure.)75 2279 y Fx(Non-reentrant)i(pa)o(rallel)d(p)o(ro)q(cedures)75 2365 y FE(This)22 b(co)o(v)o(ers)f(the)h(case)g(where,)h(at)e(an)o(y)g (p)q(oin)o(t)h(in)h(time,)g(at)e(most)g(one)h(in)o(v)o(o)q(cation)g(of) f(a)g(parallel)75 2421 y(pro)q(cedure)14 b(can)f(b)q(e)g(activ)o(e)g (at)f(an)o(y)h(pro)q(cess.)19 b(That)12 b(is,)i(concurren)o(t)e(in)o(v) o(o)q(cations)i(of)e(the)h(same)f(parallel)75 2478 y(pro)q(cedure)k(ma) o(y)e(o)q(ccur)h(only)g(within)h(disjoin)o(t)f(groups)f(of)h(pro)q (cesses.)20 b(F)l(or)14 b(example,)h(all)h(in)o(v)o(o)q(cations)75 2534 y(of)f(parallel)i(pro)q(cedures)f(in)o(v)o(olv)o(e)g(all)h(pro)q (cesses,)e(pro)q(cesses)h(are)f(single-threaded,)i(and)e(there)h(are)f (no)75 2591 y(recursiv)o(e)h(in)o(v)o(o)q(cations.)166 2647 y(In)21 b(suc)o(h)f(a)g(case,)h(a)f(con)o(text)f(can)i(b)q(e)f (statically)h(allo)q(cated)g(to)f(eac)o(h)g(pro)q(cedure.)36 b(The)20 b(static)75 2704 y(allo)q(cation)h(can)e(b)q(e)h(done)g(in)h (a)e(pream)o(ble,)i(as)e(part)g(of)g(initialization)j(co)q(de.)34 b(Or,)20 b(it)g(can)f(b)q(e)i(done)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 127 132 127 131 bop 75 -100 a Ft(5.7.)29 b(F)o(ORMALIZING)11 b(THE)g(LOOSEL)l(Y)i(SYNCHR)o(ONOUS)g(MODEL)e(\(USA)o(GE,)f(SAFETY\))p FE(127)75 45 y(a)18 b(compile/link)j(time,)e(if)g(the)g(implemen)o (tation)g(has)g(additional)h(mec)o(hanisms)f(to)e(reserv)o(e)i(con)o (text)75 102 y(v)m(alues.)k(Comm)o(unicators)15 b(to)g(b)q(e)i(used)f (b)o(y)g(the)g(di\013eren)o(t)g(pro)q(cedures)h(can)f(b)q(e)g(built)h (in)g(a)f(pream)o(ble,)75 158 y(if)h(the)g(executing)h(groups)e(are)h (statically)g(de\014ned;)i(if)e(the)g(executing)h(groups)e(c)o(hange)h (dynamically)l(,)75 214 y(then)f(a)g(new)h(comm)o(unicator)e(has)h(to)g (b)q(e)h(built)g(whenev)o(er)g(the)f(executing)h(group)f(c)o(hanges,)g (but)h(this)75 271 y(new)d(comm)o(unicator)g(can)g(b)q(e)h(built)g (using)g(the)f(same)g(t)o(yp)q(e)g(of)g(preallo)q(cated)h(con)o(text.)k (If)14 b(the)g(parallel)75 327 y(pro)q(cedures)k(can)f(b)q(e)g (organized)g(in)o(to)g(libraries,)h(so)f(that)f(only)h(one)g(pro)q (cedure)h(of)e(eac)o(h)h(library)h(can)75 384 y(b)q(e)i(concurren)o (tly)g(activ)o(e)f(in)i(eac)o(h)e(pro)q(cessor,)h(then)f(it)h(is)g (su\016cien)o(t)g(to)e(allo)q(cate)i(one)g(con)o(text)e(p)q(er)75 440 y(library)l(.)75 610 y Fx(P)o(a)o(rallel)c(p)o(ro)q(cedures)i(that) g(a)o(re)f(non-reentrant)i(within)f(each)g(executing)h(group)75 712 y FE(This)k(co)o(v)o(ers)g(the)g(case)f(where,)j(at)d(an)o(y)g(p)q (oin)o(t)i(in)f(time,)h(for)f(eac)o(h)g(pro)q(cess)g(group,)g(there)g (can)g(b)q(e)75 769 y(at)d(most)g(one)h(activ)o(e)g(in)o(v)o(o)q (cation)g(of)g(a)f(parallel)i(pro)q(cedure)g(b)o(y)f(a)f(pro)q(cess)h (mem)o(b)q(er.)31 b(Ho)o(w)o(ev)o(er,)19 b(it)75 825 y(migh)o(t)i(b)q(e)h(p)q(ossible)i(that)c(the)i(same)f(pro)q(cedure)h (is)g(concurren)o(tly)g(in)o(v)o(ok)o(ed)g(in)g(t)o(w)o(o)e(partially)j (\(or)75 882 y(completely\))17 b(o)o(v)o(erlapping)f(groups.)k(F)l(or) 15 b(example,)i(the)e(same)g(collectiv)o(e)j(comm)o(unication)e (function)75 938 y(ma)o(y)e(b)q(e)i(concurren)o(tly)g(in)o(v)o(ok)o(ed) g(on)f(t)o(w)o(o)f(partially)i(o)o(v)o(erlapping)g(groups.)166 1003 y(In)f(suc)o(h)f(a)g(case,)g(a)f(con)o(text)h(is)g(asso)q(ciated)h (with)f(eac)o(h)g(parallel)i(pro)q(cedure)f(and)f(eac)o(h)g(executing) 75 1060 y(group,)j(so)h(that)e(o)o(v)o(erlapping)j(execution)f(groups)f (ha)o(v)o(e)g(distinct)i(comm)o(unication)f(con)o(texts.)27 b(\(One)75 1116 y(do)q(es)19 b(not)f(need)i(a)f(di\013eren)o(t)f(con)o (text)h(from)f(eac)o(h)h(group;)g(one)g(merely)h(needs)f(a)g (\\coloring")g(of)f(the)75 1172 y(groups,)e(so)g(that)g(one)h(can)g (generate)f(the)h(comm)o(unicators)f(for)g(eac)o(h)g(parallel)i(pro)q (cedure)g(when)f(the)75 1229 y(execution)g(groups)e(are)g(de\014ned.)22 b(Here)16 b(again,)f(one)h(only)g(need)h(one)e(con)o(text)g(for)g(eac)o (h)h(library)l(,)g(if)g(no)75 1285 y(t)o(w)o(o)e(pro)q(cedures)i(from)e (the)i(same)f(library)h(can)f(b)q(e)h(concurren)o(tly)g(activ)o(e)f(in) h(the)f(same)g(group.)166 1350 y(Note)f(that,)g(for)h(collectiv)o(e)h (comm)o(unication)g(libraries,)g(w)o(e)f(do)g(allo)o(w)g(sev)o(eral)g (concurren)o(t)g(in)o(v)o(o-)75 1407 y(cations)h(within)h(the)g(same)e (group:)22 b(a)15 b(broadcast)h(in)h(a)e(group)h(ma)o(y)g(b)q(e)g (started)g(in)h(a)e(pro)q(cess)i(b)q(efore)75 1463 y(the)e(previous)h (broadcast)f(in)h(that)e(group)h(ended)h(at)f(another)g(pro)q(cess.)20 b(In)c(suc)o(h)f(a)g(case,)g(one)g(cannot)75 1520 y(rely)21 b(on)g(con)o(text)g(mec)o(hanisms)g(to)f(disam)o(biguate)i(successiv)o (e)g(in)o(v)o(o)q(cations)f(of)g(the)g(same)f(parallel)75 1576 y(pro)q(cedure)f(within)f(the)g(same)g(group:)24 b(the)18 b(pro)q(cedure)g(needs)h(to)e(b)q(e)h(implemen)o(ted)i(so)d (as)g(to)g(a)o(v)o(oid)75 1633 y(confusion)i(\(bac)o(k)e(masking\).)28 b(F)l(or)17 b(example,)i(for)e(broadcast,)h(one)g(ma)o(y)f(need)i(to)e (carry)h(additional)75 1689 y(information)g(in)g(messages,)f(suc)o(h)h (as)g(the)f(broadcast)g(ro)q(ot,)g(to)g(help)i(in)f(suc)o(h)g(disam)o (biguation;)i(one)75 1746 y(also)15 b(relies)i(on)e(the)g(pair-wise)h (preserv)m(ation)g(of)f(message)g(order)g(b)o(y)g(MPI.)g(With)g(suc)o (h)h(an)f(approac)o(h,)75 1802 y(w)o(e)f(ma)o(y)g(b)q(e)h(gaining)g(p)q (erformance,)f(but)g(w)o(e)h(lose)f(mo)q(dularit)o(y)l(.)21 b(It)14 b(is)h(insu\016cien)o(t)h(to)d(implemen)o(t)j(the)75 1859 y(parallel)h(pro)q(cedure)g(so)e(that)g(it)h(w)o(orks)f(correctly) h(in)g(isolation,)h(when)f(in)o(v)o(ok)o(ed)g(only)g(once;)g(it)g (needs)75 1915 y(to)f(b)q(e)h(implemen)o(ted)i(so)d(that)g(an)o(y)h(n)o (um)o(b)q(er)g(of)f(successiv)o(e)i(in)o(v)o(o)q(cations)f(will)h (execute)g(correctly)l(.)22 b(Of)75 1971 y(course,)15 b(the)g(same)g(approac)o(h)g(can)g(b)q(e)h(used)g(for)e(other)h (parallel)i(libraries.)75 2141 y Fx(W)o(ell-nested)g(pa)o(rallel)d(p)o (ro)q(cedures)75 2243 y FE(Calls)h(of)g(parallel)h(pro)q(cedures)g(are) e(w)o(ell-nested)i(if)g(a)e(new)h(parallel)h(pro)q(cedure)g(is)f(alw)o (a)o(ys)f(in)o(v)o(ok)o(ed)h(in)75 2300 y(a)j(subset)h(of)e(a)h(group)g (executing)i(the)e(same)g(parallel)i(pro)q(cedure.)30 b(Th)o(us,)19 b(pro)q(cesses)g(that)e(execute)75 2356 y(the)e(same)g(parallel)i(pro)q(cedure)f(ha)o(v)o(e)f(the)g(same)g (execution)h(stac)o(k.)166 2421 y(In)h(suc)o(h)f(a)g(case,)g(a)g(new)g (con)o(text)f(needs)i(to)f(b)q(e)h(dynamically)h(allo)q(cated)f(for)e (eac)o(h)h(new)h(in)o(v)o(o)q(ca-)75 2478 y(tion)h(of)f(a)g(parallel)i (pro)q(cedure.)28 b(Ho)o(w)o(ev)o(er,)16 b(a)i(stac)o(k)e(mec)o(hanism) i(can)g(b)q(e)g(used)g(for)f(allo)q(cating)i(new)75 2534 y(con)o(texts.)29 b(Th)o(us,)19 b(a)g(p)q(ossible)h(mec)o(hanism)f(is)g (\014rst)g(to)f(allo)q(cate)h(a)f(large)h(n)o(um)o(b)q(er)g(of)f(con)o (texts)g(\(up)75 2591 y(to)e(the)h(upp)q(er)h(b)q(ound)g(on)f(the)g (depth)g(of)g(nested)g(parallel)h(pro)q(cedure)g(calls\),)g(and)f(then) g(use)g(a)g(lo)q(cal)75 2647 y(stac)o(k)d(managemen)o(t)g(of)g(these)h (con)o(text's)f(on)g(eac)o(h)h(pro)q(cess)g(to)f(create)h(a)f(new)h (comm)o(unicator)g(\(using)75 2704 y Fx(MPI)p 160 2704 14 2 v 16 w(COMM)p 318 2704 V 16 w(MAKE)p FE(\))g(for)f(eac)o(h)i(new)f (in)o(v)o(o)q(cation.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 128 133 128 132 bop 75 -100 a FE(128)87 b Ft(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y Fx(The)h(General)f(case)75 134 y FE(In)22 b(the)g(general)g(case,) h(there)e(ma)o(y)g(b)q(e)h(m)o(ultiple)i(concurren)o(tly)e(activ)o(e)g (in)o(v)o(o)q(cations)g(of)f(the)g(same)75 190 y(parallel)i(pro)q (cedure)g(within)g(the)f(same)f(group;)j(in)o(v)o(o)q(cations)e(ma)o(y) f(not)h(b)q(e)g(w)o(ell-nested.)41 b(A)22 b(new)75 247 y(con)o(text)17 b(need)i(to)e(b)q(e)h(created)g(for)f(eac)o(h)g(in)o(v) o(o)q(cation.)28 b(It)18 b(is)g(the)g(user)g(resp)q(onsibilit)o(y)i(to) d(mak)o(e)g(sure)75 303 y(that,)k(if)h(t)o(w)o(o)d(distinct)j(parallel) h(pro)q(cedures)e(are)g(in)o(v)o(ok)o(ed)g(concurren)o(tly)h(on)e(o)o (v)o(erlapping)i(sets)e(of)75 360 y(pro)q(cesses,)15 b(then)h(con)o(text)e(allo)q(cation)j(or)d(comm)o(unicator)h(creation)h (is)f(prop)q(erly)h(co)q(ordinated.)75 511 y Fv(5.8)59 b(Motivating)19 b(Examples)75 617 y Fq(5.8.1)49 b(Current)15 b(Practice)g(#1)75 706 y FE(Example)h(#1a:)147 807 y Fp(main\(int)22 b(argc,)i(char)f(**argv\))147 864 y({)194 920 y(int)h(me,)f(size;)194 977 y(...)194 1033 y(MPI_Init)g(\()h (&argc,)f(argv)g(\);)194 1090 y(MPI_Comm_rank)f(\(MPI_COMM_WORLD,)g (&me\);)194 1146 y(MPI_Comm_size)g(\(MPI_COMM_WORLD,)g(&size\);)194 1259 y(\(void\)printf)h(\("Process)f(\045d)i(size)f(\045d\\n",)g(me,)h (size\);)194 1315 y(...)194 1372 y(MPI_Finalize\(\);)147 1428 y(})75 1528 y FE(Example)19 b(#1a)f(is)h(a)f(do-nothing)h(program) e(that)h(initializes)k(itself)d(legally)l(,)i(and)d(refers)g(to)g(the)h (the)75 1585 y(\\all")f(comm)o(unicator,)g(and)g(prin)o(ts)g(a)f (message.)27 b(It)18 b(terminates)g(itself)h(legally)g(to)q(o.)27 b(This)18 b(example)75 1641 y(do)q(es)e(not)e(imply)j(that)d(MPI)i (supp)q(orts)f Fp(printf)p FE(-lik)o(e)g(comm)o(unication)h(itself.)75 1699 y(Example)g(#1b:)170 1800 y Fp(main\(int)23 b(argc,)g(char)h (**argv\))170 1857 y({)242 1913 y(int)f(me,)h(size;)242 1970 y(int)f(SOME_TAG)g(=)h(0;)242 2026 y(...)242 2083 y(MPI_Init\(&argc,)e(argv\);)242 2195 y(MPI_Comm_rank\(MPI_COMM_WOR)o (LD,)f(&me\);)71 b(/*)23 b(local)h(*/)242 2252 y (MPI_Comm_size\(MPI_COMM_WOR)o(LD,)d(&size\);)i(/*)g(local)h(*/)242 2365 y(if\(\(me)f(\045)h(2\))f(==)h(0\))314 2421 y(MPI_Send\(...,)e (\(\(me)h(+)h(1\))f(\045)h(size\),)f(SOME_TAG,)g(MPI_COMM_WORLD\);)242 2478 y(else)314 2534 y(MPI_Recv\(...,)f(\(\(me)h(-)h(1)f(+)h(size\))f (\045)h(size\),)f(SOME_TAG,)g(MPI_COMM_WORLD\);)242 2591 y(...)242 2647 y(MPI_Finalize\(\);)170 2704 y(})1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 129 134 129 133 bop 75 -100 a Ft(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FE(129)75 45 y(Example)16 b(#1b)f(sc)o(hematically)i (illustrates)g(message)e(exc)o(hanges)h(b)q(et)o(w)o(een)g(\\ev)o(en")f (and)h(\\o)q(dd")f(pro-)75 102 y(cesses)h(in)g(the)f(\\all")g(comm)o (unicator.)75 222 y Fq(5.8.2)49 b(Current)15 b(Practice)g(#2)147 308 y Fp(main\(int)22 b(argc,)i(char)f(**argv\))147 365 y({)194 421 y(int)h(me,)f(count;)194 478 y(void)h(*data;)194 534 y(...)194 647 y(MPI_Init\(&argc,)e(argv\);)194 703 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&me\);)194 816 y(if\(me)i(==)h(0\))194 873 y({)290 929 y(/*)f(get)h(input,)f(create)g (buffer)g(``data'')g(*/)290 986 y(...)194 1042 y(})194 1155 y(MPI_Bcast\(data,)f(count,)h(MPI_BYTE,)g(0,)g(MPI_COMM_WORLD\);) 194 1268 y(...)194 1381 y(MPI_Finalize\(\);)147 1437 y(})75 1537 y FE(This)16 b(example)g(illustrates)g(the)f(use)h(of)f(a)g (collectiv)o(e)h(comm)o(unication.)75 1658 y Fq(5.8.3)49 b(\(App)o(ro)o(ximate\))16 b(Current)f(Practice)g(#3)123 1744 y Fp(main\(int)22 b(argc,)i(char)f(**argv\))123 1800 y({)170 1857 y(int)h(me,)f(count,)g(count2;)170 1913 y(void)h(*send_buf,)e(*recv_buf,)h(*send_buf2,)f(*recv_buf2;)170 1970 y(MPI_Group)h(MPI_GROUP_WORLD,)f(grp0,)h(grprem;)170 2026 y(MPI_Comm)g(commslave;)170 2083 y(static)g(int)h(ranks[])f(=)h ({0};)170 2139 y(...)170 2195 y(MPI_Init\(&argc,)e(argv\);)170 2252 y(MPI_Comm_group\(MPI_COMM_WORLD)o(,)f(&MPI_GROUP_WORLD\);)170 2308 y(MPI_Comm_rank\(MPI_COMM_WORLD,)f(&me\);)47 b(/*)24 b(local)f(*/)170 2421 y(MPI_Group_incl\(MPI_GROUP_WORL)o(D,)e(1,)j (ranks,)f(&grp0\);)94 b(/*)24 b(local)f(*/)170 2478 y (MPI_Group_difference\(MPI_GROU)o(P_WORLD,)d(grp0,)j(&grprem\);/*)g (local)g(*/)170 2534 y(MPI_Comm_make\(MPI_COMM_WORLD,)d(grprem,)j (&commslave\);)170 2647 y(if\(me)h(!=)f(0\))170 2704 y({)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 130 135 130 134 bop 75 -100 a FE(130)92 b Ft(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)218 45 y Fp(/*)24 b(compute)f(on)g(slave)g(*/)218 102 y(...)218 158 y(MPI_Reduce\(send_buf,recv_bu)o(ff,count)o(,)e(MPI_INT,)i (MPI_SUM,)g(1,)g(commslave\);)218 214 y(...)170 271 y(})170 327 y(/*)h(zero)f(falls)h(through)e(immediately)h(to)g(this)h(reduce,)f (others)g(do)g(later...)g(*/)170 384 y(MPI_Reduce\(send_buf2,)e (recv_buff2,)i(count2,)433 440 y(MPI_INT,)g(MPI_SUM,)f(0,)i (MPI_COMM_WORLD\);)170 497 y(MPI_Finalize\(\);)123 553 y(})75 656 y FE(This)18 b(example)h(illustrates)g(ho)o(w)e(a)h(group)f (consisting)i(of)e(all)i(but)f(the)g(zeroth)f(pro)q(cess)h(of)g(the)g (\\all")75 712 y(group)13 b(is)h(created,)f(and)h(then)g(ho)o(w)f(a)g (comm)o(unicator)g(is)h(formed)f(\()g Fx(commslave)p FE(\))e(for)i(that)g(new)h(group.)75 768 y(The)f(new)h(comm)o(unicator) e(is)i(used)g(in)g(a)e(collectiv)o(e)j(call,)g(and)e(all)h(pro)q (cesses)f(execute)h(a)f(collectiv)o(e)i(call)75 825 y(in)j(the)33 b Fm(MPI)p 303 825 13 2 v 15 w(COMM)p 448 825 V 14 w(W)o(ORLD)18 b FE(con)o(text.)25 b(This)18 b(example)h(illustrates)f(ho)o(w)f(the)g (t)o(w)o(o)f(comm)o(unicators)75 881 y(\(that)c(inheren)o(tly)k(p)q (ossess)d(distinct)i(con)o(texts\))d(protect)h(comm)o(unication.)20 b(That)13 b(is,)h(comm)o(unication)75 938 y(in)30 b Fm(MPI)p 219 938 V 14 w(COMM)p 363 938 V 15 w(W)o(ORLD)15 b FE(is)h(insulated)g (from)f(comm)o(unication)h(in)31 b Fx(commslave)p FE(,)13 b(and)i(vice)i(v)o(ersa.)166 994 y(In)h(summary)l(,)g(\\group)f(safet)o (y")f(is)i(ac)o(hiev)o(ed)h(via)f(comm)o(unicators)f(b)q(ecause)i (distinct)g(con)o(texts)75 1051 y(within)d(comm)o(unicators)f(are)g (enforced)h(to)e(b)q(e)i(unique)h(on)e(an)o(y)g(pro)q(cess.)75 1172 y Fq(5.8.4)49 b(Example)17 b(#4)75 1258 y FE(The)e(follo)o(wing)g (example)g(is)g(mean)o(t)f(to)f(illustrate)j(\\safet)o(y")d(b)q(et)o(w) o(een)h(p)q(oin)o(t-to-p)q(oin)o(t)i(and)e(collectiv)o(e)75 1314 y(comm)o(unication.)20 b(MPI)12 b(guaran)o(tees)g(that)g(a)g (single)i(comm)o(unicator)e(can)h(do)f(safe)g(p)q(oin)o(t-to-p)q(oin)o (t)i(and)75 1371 y(collectiv)o(e)j(comm)o(unication.)147 1462 y Fp(#define)22 b(TAG_ARBITRARY)h(12345)147 1518 y(#define)f(SOME_COUNT)166 b(50)147 1631 y(main\(int)22 b(argc,)i(char)f(**argv\))147 1687 y({)194 1744 y(int)h(me;)194 1800 y(MPI_Group)f(MPI_GROUP_WORLD,)f(subgroup;)194 1857 y(int)i(ranks[])f(=)g({2,)h(4,)f(6,)h(8};)194 1913 y(...)194 1970 y(MPI_Init\(&argc,)e(argv\);)194 2026 y (MPI_Comm_group\(MPI_COMM_WORL)o(D,)f(&MPI_GROUP_WORLD\);)194 2139 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)g(4,)j(ranks,)f (&subgroup\);)f(/*)i(local)f(*/)194 2195 y(MPI_Group_rank\(subgroup,)e (&me\);)119 b(/*)23 b(local)g(*/)194 2308 y (MPI_Comm_make\(MPI_COMM_WORLD)o(,)e(subgroup,)i(&the_comm\);)194 2421 y(if\(me)g(!=)h(MPI_UNDEFINED_RANK\))194 2478 y({)266 2534 y(/*)f(asynchronous)g(receive:)f(*/)266 2591 y(MPI_Irecv\(buffer,) f(count,)i(MPI_DOUBLE,)g(MPI_ANY_SOURCE,)505 2647 y(TAG_ARBITRARY,)e (the_comm,)i(request\);)194 2704 y(})1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 131 136 131 135 bop 75 -100 a Ft(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FE(131)194 102 y Fp(for\(i)23 b(=)h(0;)g(i)g(<)f (SOME_COUNT,)g(i++\))242 158 y(MPI_Reduce\(...,)f(the_comm\);)194 214 y(MPI_Finalize\(\);)147 271 y(})75 399 y Fq(5.8.5)49 b(Lib)o(ra)o(ry)16 b(Example)h(#1)75 487 y FE(The)e(main)h(program:)147 586 y Fp(main\(int)22 b(argc,)i(char)f(**argv\))147 643 y({)194 699 y(int)h(done)f(=)h(0;)194 756 y(user_lib_t)f(*libh_a,)g (*libh_b;)194 812 y(void)h(*dataset1,)e(*dataset2;)194 869 y(...)194 925 y(MPI_Init\(&argc,)g(argv\);)194 982 y(...)194 1038 y(init_user_lib\(MPI_COMM_WORLD)o(,)f(&libh_a\);)194 1095 y(init_user_lib\(MPI_COMM_WORLD)o(,)g(&libh_b\);)194 1151 y(...)194 1207 y(user_start_op\(libh_a,)g(dataset1\);)194 1264 y(user_start_op\(libh_a,)g(dataset2\);)194 1320 y(...)194 1377 y(while\(!done\))194 1433 y({)266 1490 y(/*)i(work)h(*/)266 1546 y(...)266 1603 y(MPI_Reduce\(...,)e (MPI_COMM_WORLD\);)266 1659 y(...)266 1716 y(/*)h(see)h(if)g(done)f(*/) 266 1772 y(...)194 1828 y(})194 1885 y(user_end_op\(libh_a\);)194 1941 y(user_end_op\(libh_b\);)194 1998 y(MPI_Finalize\(\);)147 2054 y(})75 2153 y FE(The)15 b(user)h(library)g(initialization)i(co)q (de:)147 2252 y Fp(void)23 b(init_user_lib\(void)e(*comm,)i(user_lib_t) g(**handle\))147 2308 y({)194 2365 y(user_lib_t)g(*save;)194 2478 y(user_lib_initsave\(&save\);)e(/*)i(local)h(*/)194 2534 y(MPI_Comm_dup\(comm,)e(&\(save)h(->)g(comm\)\);)194 2647 y(/*)h(other)f(inits)g(*/)194 2704 y(...)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 132 137 132 136 bop 75 -100 a FE(132)92 b Ft(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)194 102 y Fp(*handle)23 b(=)h(save;)147 158 y(})75 253 y FE(User)15 b(start-up)g(co)q(de:)147 347 y Fp(void)23 b(user_start_op\(user_lib_t)d(*handle,)j(void)h(*data\))147 404 y({)194 460 y(MPI_Irecv\()f(...,)g(handle->comm,)f(&\(handle)h(->)h (irecv_handle\))e(\);)194 517 y(MPI_Isend\()h(...,)g(handle->comm,)f (&\(handle)h(->)h(isend_handle\))e(\);)147 573 y(})75 668 y FE(User)15 b(clean-up)i(co)q(de:)147 763 y Fp(void)23 b(user_end_op\(user_lib_t)e(*handle\))147 819 y({)194 875 y(MPI_Status)i(*status;)194 932 y(MPI_Wait\(handle)f(->)i (isend_handle,)e(status\);)194 988 y(MPI_Wait\(handle)g(->)i (irecv_handle,)e(status\);)147 1045 y(})75 1168 y Fq(5.8.6)49 b(Lib)o(ra)o(ry)16 b(Example)h(#2)75 1254 y FE(The)e(main)h(program:) 147 1349 y Fp(main\(int)22 b(argc,)i(char)f(**argv\))147 1405 y({)194 1462 y(int)h(ma,)f(mb;)194 1518 y(...)194 1631 y(static)g(int)h(list_a[])f(=)g({0,)h(1};)75 1687 y(#if)47 b(defined\(EXAMPLE_2B\))22 b(|)h(defined\(EXAMPLE_2C\))194 1744 y(static)g(int)h(list_b[])f(=)g({0,)h(2)f(,3};)75 1800 y(#else/*)g(EXAMPLE_2A)f(*/)194 1857 y(static)h(int)h(list_b[])f (=)g({0,)h(2};)75 1913 y(#endif)194 1970 y(int)g(size_list_a)e(=)i (sizeof\(list_a\)/sizeof\(int)o(\);)194 2026 y(int)g(size_list_b)e(=)i (sizeof\(list_b\)/sizeof\(int)o(\);)194 2139 y(...)194 2195 y(MPI_Init\(&argc,)e(argv\);)194 2252 y(...)194 2365 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)f(size_list_a,)h(list_a,)h (&group_a\);)194 2421 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)e (size_list_b,)h(list_b,)h(&group_b\);)194 2534 y (MPI_Comm_make\(MPI_COMM_WORLD)o(,)e(group_a,)i(&comm_a\);)194 2591 y(MPI_Comm_make\(MPI_COMM_WORLD)o(,)e(group_b,)i(&comm_b\);)194 2704 y(MPI_Comm_rank\(comm_a,)e(&ma\);)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 133 138 133 137 bop 75 -100 a Ft(5.8.)34 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1050 b FE(133)194 45 y Fp(MPI_Comm_rank\(comm_b,)21 b(&mb\);)194 158 y(if\(ma)i(!=)h(MPI_UNDEFINED_RANK\))266 214 y(lib_call\(comm_a\);)194 327 y(if\(mb)f(!=)h(MPI_UNDEFINED_RANK\)) 194 384 y({)242 440 y(lib_call\(comm_b\);)242 497 y (lib_call\(comm_b\);)194 553 y(})194 666 y(MPI_Finalize\(\);)147 723 y(})75 818 y FE(The)15 b(library:)147 914 y Fp(void)23 b(lib_call\(MPI_Comm)e(comm\))147 970 y({)194 1027 y(int)j(me,)f(done)h (=)f(0;)194 1083 y(MPI_Comm_rank\(comm,)f(&me\);)194 1140 y(if\(me)h(==)h(0\))266 1196 y(while\(!done\))266 1253 y({)337 1309 y(MPI_Recv\(...,)f(MPI_ANY_SOURCE,)e(MPI_ANY_TAG,)i (comm\);)337 1366 y(...)266 1422 y(})194 1479 y(else)194 1535 y({)242 1591 y(/*)h(work)f(*/)242 1648 y(MPI_Send\(...,)f(0,)i (ARBITRARY_TAG,)e(comm\);)242 1704 y(....)194 1761 y(})75 1817 y(#ifdef)h(EXAMPLE_2C)194 1874 y(/*)h(include)f(\(resp,)g (exclude\))g(for)g(safety)g(\(resp,)g(no)h(safety\):)f(*/)194 1930 y(MPI_Sync\(comm\);)75 1987 y(#endif)147 2043 y(})75 2139 y FE(The)e(ab)q(o)o(v)o(e)f(example)h(is)g(really)g(three)g (examples,)h(dep)q(ending)g(on)f(whether)f(or)g(not)g(one)h(includes)75 2195 y(rank)f(3)h(in)g Fx(list)p 345 2195 14 2 v 17 w(b)p FE(,)h(and)f(whether)g(or)f(not)g(a)g(sync)o(hronize)i(is)f(included)i (in)f Fx(lib)p 1477 2195 V 16 w(call)p FE(.)36 b(This)21 b(example)75 2252 y(illustrates)14 b(that,)e(despite)j(con)o(texts,)d (subsequen)o(t)i(calls)g(to)e Fx(lib)p 1166 2252 V 17 w(call)h FE(with)g(the)g(same)g(con)o(text)f(need)i(not)75 2308 y(b)q(e)h(safe)e(from)g(one)i(another)e(\(collo)q(quially)l(,)k (\\bac)o(k-masking"\).)i(Safet)o(y)13 b(is)h(realized)i(if)e(the)28 b Fx(MPI)p 1772 2308 V 16 w(Sync)75 2364 y FE(is)17 b(added.)25 b(What)16 b(this)h(demonstrates)f(is)h(that)f(libraries)i(ha)o(v)o(e)e (to)g(b)q(e)h(written)g(carefully)l(,)h(ev)o(en)f(with)75 2421 y(con)o(texts.)23 b(When)17 b(rank)f(3)g(is)h(excluded,)i(then)d (the)h(sync)o(hronize)g(is)g(not)g(needed)g(to)f(get)g(safet)o(y)g (from)75 2477 y(bac)o(k)f(masking.)166 2534 y(Algorithms)d(lik)o(e)i (\\com)o(bine")e(ha)o(v)o(e)g(strong)f(enough)i(source)f(selectivit)o (y)h(so)f(that)g(they)g(are)g(inher-)75 2591 y(en)o(tly)k(ok)m(a)o(y)l (.)22 b(So)16 b(are)f(m)o(ultiple)j(calls)f(to)e(a)h(t)o(ypical)g (tree-broadcast)g(algorithm)f(with)i(the)f(same)f(ro)q(ot.)75 2647 y(Ho)o(w)o(ev)o(er,)21 b(m)o(ultiple)i(calls)f(to)f(a)g(t)o (ypical)h(tree-broadcast)e(algorithm)h(|)h(with)f(di\013eren)o(t)g(ro)q (ots)g(|)75 2704 y(could)16 b(bac)o(k-mask,)f(pro)q(ducing)i (undesirable)g(results.)k(Therefore,)15 b(suc)o(h)h(algorithms)f(w)o (ould)h(ha)o(v)o(e)f(to)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 134 139 134 138 bop 75 -100 a FE(134)92 b Ft(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y FE(utilize)f(the)e(monotonically)h(increasing)g(tags)e(\(with)g (comm)o(unicator)h(scop)q(e\))g(to)f(k)o(eep)h(things)g(straigh)o(t.)75 102 y(All)19 b(of)e(the)h(foregoing)f(is)h(a)f(supp)q(osition)i(of)f (\\collectiv)o(e)h(calls")f(implemen)o(ted)h(with)f(p)q(oin)o(t-to-p)q (oin)o(t)75 158 y(op)q(erations.)25 b(MPI)17 b(implemen)o(tations)h(ma) o(y)e(or)g(ma)o(y)h(not)f(implemen)o(t)i(collectiv)o(e)h(calls)f(using) f(p)q(oin)o(t-)75 214 y(to-p)q(oin)o(t)h(op)q(erations.)28 b(These)19 b(algorithms)f(are)f(used)i(to)e(illustrate)i(the)g(issues)f (of)g(correctness)g(and)75 271 y(safet)o(y)l(,)c(indep)q(enden)o(t)k (of)c(ho)o(w)h(MPI)g(implemen)o(ts)i(its)e(collectiv)o(e)i(calls.)75 395 y Fq(5.8.7)49 b(Inter-Communication)16 b(Examples)75 481 y Fx(Example)f(1:)k(Three-Group)e(\\Pip)q(eline")409 624 y Fp(+---------+)213 b(+---------+)h(+---------+)409 681 y(|)h(|)f(|)h(|)g(|)f(|)409 737 y(|)24 b(Group)f(0)h(|)f(<----->)g (|)h(Group)f(1)h(|)g(<----->)f(|)g(Group)h(2)f(|)409 793 y(|)215 b(|)f(|)h(|)g(|)f(|)409 850 y(+---------+)f(+---------+)h (+---------+)75 1014 y FE(Groups)16 b(0)g(and)h(1)f(comm)o(unicate.)24 b(Groups)16 b(1)h(and)f(2)g(comm)o(unicate.)24 b(Therefore,)17 b(group)f(0)g(requires)75 1071 y(one)f(in)o(ter-comm)o(unicator,)f (group)g(1)g(requires)i(t)o(w)o(o)d(in)o(ter-comm)o(unicators,)h(and)h (group)f(2)g(requires)h(1)75 1127 y(in)o(ter-comm)o(unicator.)147 1236 y Fp(main\(int)22 b(argc,)i(char)f(**argv\))147 1292 y({)194 1349 y(MPI_Comm)g(myComm;)166 b(/*)24 b (intra-communicator)d(of)j(local)f(sub-group)g(*/)194 1405 y(MPI_Comm)g(myFirstComm;)46 b(/*)24 b(inter-communicator)d(*/)194 1462 y(MPI_Comm)i(mySecondComm;)f(/*)i(second)f(inter-communicator)e (\(group)i(B)h(only\))f(*/)194 1518 y(MPI_Intercomm_request)e (inter_request1,)h(inter_request2;)194 1574 y(int)i(membershipKey;)194 1631 y(int)g(rank;)194 1744 y(MPI_Init\(&argc,)e(argv\);)194 1800 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&rank\);)194 1913 y(/*)j(User)f(code)h(must)f(generate)g(membershipKey)f(in)h(the)h (range)f([0,)g(1,)h(2])g(*/)194 1970 y(membershipKey)e(=)i(rank)f(\045) h(3;)194 2083 y(/*)g(Build)f(intra-communicator)e(for)j(local)f (sub-group)g(*/)194 2139 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e (membershipKey,)h(rank,)h(&myComm\);)194 2252 y(/*)h(Build)f (inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 2308 y(if)i(\(membershipKey)e(==)h(0\))194 2365 y({)501 b(/*)24 b(Group)f(0)h(communicates)e(with)h(group)h(1.)f(*/)242 2421 y(MPI_Intercomm_start\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d(1,)743 2478 y(1,)j(&inter_request1)e(\);)242 2534 y(MPI_Intercomm_finish\()f (inter_request1,)h(myFirstComm)g(\);)194 2591 y(})194 2647 y(else)i(if)f(\(membershipKey)f(==)i(1\))194 2704 y({)334 b(/*)24 b(Group)f(1)h(communicates)e(with)h(groups)g(0)h(and)g (2.)f(*/)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 135 140 135 139 bop 75 -100 a Ft(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FE(135)242 45 y Fp(MPI_Intercomm_start\()21 b(myComm,)i(0,)h(MPI_COMM_WORLD,)d(0,)743 102 y(1,)j(&inter_request1)e (\);)242 158 y(MPI_Intercomm_start\()f(myComm,)i(0,)h(MPI_COMM_WORLD,)d (2,)743 214 y(12,)j(&inter_request2)d(\);)242 271 y (MPI_Intercomm_finish\()g(inter_request1,)h(myFirstComm)g(\);)242 327 y(MPI_Intercomm_finish\()f(inter_request2,)h(mySecondComm)g(\);)194 384 y(})194 440 y(else)i(if)f(\(membershipKey)f(==)i(2\))194 497 y({)501 b(/*)24 b(Group)f(2)h(communicates)e(with)h(group)h(1.)f (*/)242 553 y(MPI_Intercomm_start\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d (1,)743 610 y(12,)j(&inter_request1)d(\);)242 666 y (MPI_Intercomm_finish\()g(inter_request1,)h(myFirstComm)g(\);)194 723 y(})194 835 y(/*)i(Do)f(work)h(...)f(*/)194 948 y (MPI_Finalize\(\);)147 1005 y(})75 1124 y Fx(Example)15 b(2:)k(Three-Group)e(\\Ring")290 1210 y Fp(+-----------------------)o (--------)o(-------)o(-------)o(--------)o(------+)290 1266 y(|)1408 b(|)290 1322 y(|)95 b(+---------+)213 b(+---------+)h (+---------+)94 b(|)290 1379 y(|)h(|)215 b(|)f(|)h(|)g(|)f(|)96 b(|)290 1435 y(+-->)23 b(|)h(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h (|)g(<----->)f(|)g(Group)h(2)f(|)h(<--+)409 1492 y(|)215 b(|)f(|)h(|)g(|)f(|)409 1548 y(+---------+)f(+---------+)h(+---------+) 166 1646 y FE(Groups)14 b(0)g(and)h(1)f(comm)o(unicate.)20 b(Groups)14 b(1)g(and)h(2)f(comm)o(unicate.)20 b(Groups)14 b(0)g(and)h(2)f(comm)o(u-)75 1703 y(nicate.)21 b(Therefore,)14 b(eac)o(h)h(requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators.)147 1800 y Fp(main\(int)22 b(argc,)i(char)f(**argv\))147 1857 y({)194 1913 y(MPI_Comm)47 b(myComm;)118 b(/*)24 b(intra-communicator)d(of)j(local)f(sub-group)g(*/)194 1970 y(MPI_Comm)g(myFirstComm;)f(/*)i(inter-communicators)d(*/)194 2026 y(MPI_Comm)i(mySecondComm;)194 2083 y(MPI_Intercomm_request)e (inter_request1,)h(inter_request2;)194 2139 y(int)i(membershipKey;)194 2195 y(int)g(rank;)194 2308 y(MPI_Init\(&argc,)e(argv\);)194 2365 y(MPI_Comm_rank\(MPI_COMMM_WORL)o(D,)f(&rank\);)194 2421 y(...)194 2534 y(/*)j(User)f(code)h(must)f(generate)g (membershipKey)f(in)h(the)h(range)f([0,)g(1,)h(2])g(*/)194 2591 y(membershipKey)e(=)i(rank)f(\045)h(3;)194 2704 y(/*)g(Build)f(intra-communicator)e(for)j(local)f(sub-group)g(*/)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 136 141 136 140 bop 75 -100 a FE(136)92 b Ft(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)194 45 y Fp(MPI_Comm_split\(MPI_COMM_WORL)o(D,)21 b(membershipKey,)h(rank,) h(&myComm\);)194 158 y(/*)h(Build)f(inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 214 y(if)i(\(membershipKey)e(==) h(0\))194 271 y({)310 b(/*)24 b(Group)f(0)h(communicates)e(with)h (groups)g(1)h(and)g(2.)f(*/)242 327 y(MPI_Intercomm_start\()e(myComm,)i (0,)h(MPI_COMM_WORLD,)d(1,)743 384 y(1,)j(&inter_request1)e(\);)242 440 y(MPI_Intercomm_start\()f(myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 497 y(2,)j(&inter_request2)e(\);)194 553 y(})194 610 y(else)i(if)f(\(membershipKey)f(==)i(1\))194 666 y({)215 b(/*)24 b(Group)f(1)g(communicates)g(with)g(groups)g(0)h(and)f(2.)h(*/) 242 723 y(MPI_Intercomm_start\()d(myComm,)i(0,)h(MPI_COMM_WORLD,)d(0,) 743 779 y(1,)j(&inter_request1)e(\);)242 835 y(MPI_Intercomm_start\()f (myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 892 y(12,)j(&inter_request2)d (\);)194 948 y(})194 1005 y(else)j(if)f(\(membershipKey)f(==)i(2\))194 1061 y({)191 b(/*)24 b(Group)f(2)h(communicates)e(with)h(groups)g(0)h (and)f(1.)h(*/)242 1118 y(MPI_Intercomm_start\()d(myComm,)i(0,)h (MPI_COMM_WORLD,)d(0,)743 1174 y(2,)j(&inter_request1)e(\);)242 1231 y(MPI_Intercomm_start\()f(myComm,)i(0,)h(MPI_COMM_WORLD,)d(1,)743 1287 y(12,)j(&inter_request2)d(\);)194 1344 y(})194 1456 y(/*)j(Everyone)f(has)g(the)h(same)f("FINISH")g(code...)g(*/)194 1513 y(MPI_Intercomm_finish\(inter_r)o(equest1,)d(&myFirstComm\);)194 1569 y(MPI_Intercomm_finish\(inter_r)o(equest2,)g(&mySecondComm\);)194 1682 y(/*)k(Do)f(some)h(work)f(...)h(*/)194 1795 y(MPI_Finalize\(\);) 147 1852 y(})75 1971 y Fx(Example)15 b(3:)k(Three-Group)e(\\Pip)q (eline")e(Using)h(Name)f(Service)409 2056 y Fp(+---------+)213 b(+---------+)h(+---------+)409 2113 y(|)h(|)f(|)h(|)g(|)f(|)409 2169 y(|)24 b(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h(|)g(<----->)f (|)g(Group)h(2)f(|)409 2226 y(|)215 b(|)f(|)h(|)g(|)f(|)409 2282 y(+---------+)f(+---------+)h(+---------+)166 2380 y FE(Groups)21 b(0)g(and)h(1)f(comm)o(unicate.)39 b(Groups)21 b(1)g(and)h(2)f(comm)o(unicate.)39 b(Therefore,)23 b(group)e(0)75 2436 y(requires)e(one)f(in)o(ter-comm)o(unicator,)h(group)e(1)h (requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators,)h(and)g(group)g(2)75 2493 y(requires)e(1)f(in)o(ter-comm)o(unicator.)147 2591 y Fp(main\(int)22 b(argc,)i(char)f(**argv\))147 2647 y({)194 2704 y(MPI_Comm)g(myComm;)142 b(/*)24 b(intra-communicator)d (of)j(local)f(sub-group)g(*/)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 137 142 137 141 bop 75 -100 a Ft(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FE(137)194 45 y Fp(MPI_Comm)23 b(myFirstComm;)f(/*)i (inter-communicator)d(*/)194 102 y(MPI_Comm)i(mySecondComm;/*)f(second) h(inter-communicator)e(\(group)i(B)h(only\))f(*/)194 158 y(MPI_Intercomm_request)e(inter_request1,)h(inter_request2;)194 214 y(int)i(color,)f(rank,)g(size;)194 327 y(MPI_Init\(&argc,)f (argv\);)194 384 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&rank\);)194 440 y(MPI_Comm_size\(MPI_COMM_WORLD)o(,)g(&size\);)194 497 y(...)194 610 y(/*)j(User)f(code)h(must)f(generate)g(color)g(in)g (the)h(range)f([0,)g(1,)h(2])g(*/)194 666 y(if)g(\()g(rank)f(<)h (\(\(size+1\)/3\))e(\))242 723 y(color)h(=)h(0;)194 779 y(else)g(if)f(\()h(rank)f(<)h(\(2*\(size+1\)/3\))e(\))242 835 y(color)h(=)h(1;)194 892 y(else)242 948 y(color)f(=)h(2;)194 1061 y(/*)g(Build)f(intra-communicator)e(for)j(local)f(sub-group)g(*/) 194 1118 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e(color,)i(rank,)g (&myComm\);)194 1231 y(/*)h(Build)f(inter-communicators.)45 b(Group)23 b(membership)g(conditions)f(must)i(be)f(*/)194 1287 y(/*)h(provided)f(by)g(the)h(user.)f(*/)194 1344 y(if)h(\(color)f(==)h(0\))194 1400 y({)167 b(/*)24 b(Group)f(0)h (communicates)e(with)h(group)g(1.)h(*/)242 1456 y (MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(01",)i (&inter_request1\);)242 1513 y(MPI_Intercomm_finish)d (\(inter_request1,)h(&myFirstComm\);)194 1569 y(})194 1626 y(else)i(if)f(\(color)g(==)h(1\))194 1682 y({)167 b(/*)24 b(Group)f(1)h(communicates)e(with)h(groups)g(0)h(and)f(2.)h(*/) 242 1739 y(MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(10",)i (&inter_request1\);)242 1795 y(MPI_Intercomm_name_start\(m)o(yComm,)d ("Connect)h(12",)i(&inter_request2\);)242 1852 y(MPI_Intercomm_finish)d (\(inter_request1,)h(&myFirstComm\);)242 1908 y(MPI_Intercomm_finish)f (\(inter_request2,)h(&mySecondComm\);)194 1965 y(})194 2021 y(else)i(if)f(\(color)g(==)h(2\))194 2077 y({)167 b(/*)24 b(Group)f(2)h(communicates)e(with)h(group)g(1.)h(*/)242 2134 y(MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(21",)i (&inter_request1\);)242 2190 y(MPI_Intercomm_finish)d (\(inter_request1,)h(&myFirstComm\);)194 2247 y(})194 2360 y(/*)i(Do)f(some)h(work)f(...)h(*/)194 2473 y(MPI_Finalize\(\);) 147 2529 y(})-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 138 143 138 142 bop 75 -100 a FE(138)92 b Ft(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHEING)75 45 y Fx(Example)g(4:)k(Three-Group)e(\\Ring")e(Using)g(Name)g(Service) 290 131 y Fp(+-----------------------)o(--------)o(-------)o(-------)o (--------)o(------+)290 187 y(|)1408 b(|)290 244 y(|)95 b(+---------+)213 b(+---------+)h(+---------+)94 b(|)290 300 y(|)h(|)215 b(|)f(|)h(|)g(|)f(|)96 b(|)290 357 y(+-->)23 b(|)h(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h(|)g(<----->)f(|)g (Group)h(2)f(|)h(<--+)409 413 y(|)215 b(|)f(|)h(|)g(|)f(|)409 470 y(+---------+)f(+---------+)h(+---------+)75 570 y FE(Groups)12 b(0)g(and)h(1)f(comm)o(unicate.)19 b(Groups)13 b(1)f(and)g(2)h(comm)o(unicate.)19 b(Groups)12 b(0)g(and)h(2)f(comm)o (unicate.)75 627 y(Therefore,)j(eac)o(h)g(requires)h(t)o(w)o(o)e(in)o (ter-comm)o(unicators.)147 728 y Fp(main\(int)22 b(argc,)i(char)f (**argv\))147 784 y({)194 841 y(MPI_Comm)47 b(myComm;)71 b(/*)23 b(intra-communicator)f(of)h(local)g(sub-group)g(*/)194 897 y(MPI_Comm)47 b(myFirstComm;)380 b(/*)24 b(inter-communicators)d (*/)194 953 y(MPI_Comm)47 b(mySecondComm;)194 1010 y (MPI_Intercomm_request)21 b(inter_request1,)h(inter_request2;)194 1066 y(int)i(color,)f(rank,)g(size;)194 1179 y(MPI_Init\(&argc,)f (argv\);)194 1236 y(MPI_Comm_size\(MPI_COMM_WORLD)o(,)f(&size\);)194 1292 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)g(&rank\);)194 1349 y(...)194 1462 y(/*)j(User)f(code)h(must)f(generate)g(color)g(in)g (the)h(range)f([0,)g(1,)h(2])g(*/)194 1518 y(if)g(\()g(rank)f(==)h(0)f (\))242 1574 y(color)g(=)h(0;)194 1631 y(else)g(if)f(\()h(rank)f(==)h (1)g(\))242 1687 y(color)f(=)h(1;)194 1744 y(else)242 1800 y(color)f(=)h(2;)194 1913 y(/*)g(Build)f(intra-communicator)e(for) j(local)f(sub-group)g(*/)194 1970 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,) e(color,)i(rank,)g(&myComm\);)194 2083 y(/*)h(Build)f (inter-communicators.)45 b(Group)23 b(membership)g(conditions)f(must)i (be)f(*/)194 2139 y(/*)h(provided)f(by)g(the)h(user.)f(*/)194 2195 y(if)h(\(color)f(==)h(0\))194 2252 y({)119 b(/*)24 b(Group)f(0)h(communicates)e(with)i(groups)f(1)g(and)h(2.)f(*/)242 2308 y(MPI_Intercomm_name_start\(m)o(yComm,)e("Connect)h(01",)i (&inter_request1\);)242 2365 y(MPI_Intercomm_name_start\(m)o(yComm,)d ("Connect)h(02",)i(&inter_request2\);)194 2421 y(})194 2478 y(else)g(if)f(\(color)g(==)h(1\))194 2534 y({)119 b(/*)24 b(Group)f(1)h(communicates)e(with)i(groups)f(0)g(and)h(2.)f(*/) 242 2591 y(MPI_Intercomm_name_start\(m)o(yComm,)e("Connect)h(10",)i (&inter_request1\);)242 2647 y(MPI_Intercomm_name_start\(m)o(yComm,)d ("Connect)h(12",)i(&inter_request2\);)194 2704 y(})1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 139 144 139 143 bop 75 -100 a Ft(5.8.)34 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1050 b FE(139)194 45 y Fp(else)24 b(if)f(\(\))194 102 y({)96 b(/*)23 b(Group)g(2)h(communicates)e(with) i(groups)f(0)g(and)h(1.)f(*/)242 158 y(MPI_Intercomm_name_start\(m)o (yComm,)e("Connect)h(20",)i(&inter_request1\);)242 214 y(MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(21",)i (&inter_request2\);)194 271 y(})194 384 y(/*)g(Everyone)f(has)g(the)h (same)f("FINISH")g(code...)g(*/)194 440 y(MPI_Intercomm_finish)e (\(inter_request1,)h(&myFirstComm\);)194 497 y(MPI_Intercomm_finish)f (\(inter_request2,)h(&mySecondComm\);)194 610 y(/*)i(Do)f(some)h(work)f (...)h(*/)194 723 y(MPI_Finalize\(\);)147 779 y(})-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 140 145 140 144 bop 75 357 a Fw(Chapter)34 b(6)75 564 y FA(Pro)s(cess)40 b(T)-10 b(op)s(ologi)q(es)75 805 y Fv(6.1)59 b(Intro)r(duction)75 907 y FE(A)13 b(pro)q(cess)h(group)f(in)h(MPI)g(is)g(a)f(collection)i (of)e Fy(n)g FE(pro)q(cesses.)20 b(Eac)o(h)13 b(pro)q(cess)h(in)g(the)f (group)g(is)h(assigned)75 963 y(a)j(rank)g(b)q(et)o(w)o(een)g(0)g(and)g Fy(n)12 b FC(\000)f FE(1.)25 b(In)18 b(man)o(y)f(parallel)h (applications)h(a)e(linear)h(pro)q(cess)f(ordering)h(do)q(es)75 1020 y(not)c(re\015ect)h(the)g(comm)o(unication)g(pattern)f(of)g(the)h (pro)q(cesses,)f(whic)o(h)i(is)f(usually)h(determined)f(b)o(y)g(the)75 1076 y(underlying)20 b(problem)e(geometry)f(and)h(the)f(n)o(umerical)i (algorithm)f(used.)28 b(Often)18 b(the)f(pro)q(cesses)h(are)75 1133 y(arranged)13 b(in)g(top)q(ological)h(patterns)f(as)g(simple)h(as) f(a)g(t)o(w)o(o{)e(or)i(three{dimensional)i(grid.)k(In)14 b(the)f(more)75 1189 y(general)19 b(case)f(the)h(logical)g(pro)q(cess)g (arrangemen)o(t)e(is)i(describ)q(ed)h(b)o(y)e(a)g(graph.)29 b(In)19 b(this)g(c)o(hapter)f(w)o(e)75 1246 y(will)f(refer)e(to)f(this) i(logical)h(pro)q(cess)e(arrangemen)o(t)f(as)h(the)g(\\virtual)h(top)q (ology)l(.")166 1302 y(A)f(clear)h(distinction)h(m)o(ust)d(b)q(e)i (made)f(b)q(et)o(w)o(een)h(the)f(virtual)h(pro)q(cess)f(top)q(ology)g (and)g(the)h(top)q(ol-)75 1359 y(ogy)21 b(of)g(the)g(ph)o(ysical)i (hardw)o(are)e(pro)q(cessors.)38 b(The)21 b(application-sp)q(eci)q (\014c,)26 b(virtual)c(top)q(ology)f(can)75 1415 y(b)q(e)f(exploited)h (b)o(y)f(the)f(system)g(in)i(the)e(assignmen)o(t)h(of)f(pro)q(cesses)h (to)f(ph)o(ysical)i(pro)q(cessors,)f(if)g(this)75 1472 y(helps)14 b(to)f(impro)o(v)o(e)g(the)g(comm)o(unication)h(p)q (erformance)f(on)g(a)g(giv)o(en)h(mac)o(hine.)20 b(Ho)o(w)12 b(this)i(mapping)f(is)75 1528 y(done)i(in)h(detail,)f(ho)o(w)o(ev)o (er,)f(is)h(outside)g(the)g(scop)q(e)g(of)g(MPI.)f(The)h(description)h (of)f(the)f(virtual)i(pro)q(cess)75 1584 y(top)q(ology)l(,)g(on)h(the)f (other)g(hand,)h(only)g(dep)q(ends)h(on)e(the)h(application,)h(and)e (is,)h(therefore,)f(mac)o(hine{)75 1641 y(indep)q(enden)o(t.)22 b(The)13 b(functions)h(whic)o(h)g(are)f(prop)q(osed)h(in)g(this)g(pap)q (er)g(only)f(deal)h(with)g(this)g(side)g(of)f(the)75 1697 y(mapping)j(problem.)166 1754 y(There)e(are)g(w)o(ell{kno)o(wn)h (tec)o(hniques)g(for)f(mapping)h(grid/torus)e(structures)h(to)g(hardw)o (are)f(top)q(o-)75 1810 y(logies)j(suc)o(h)g(as)g(h)o(yp)q(ercub)q(es)h (or)e(grids.)22 b(F)l(or)15 b(more)g(complicated)i(graph)f(structures)f (go)q(o)q(d)g(heuristics)75 1867 y(often)j(yield)j(near{optimal)e (results)g([16)o(].)30 b(On)19 b(the)g(other)g(hand,)g(if)g(there)g(is) g(no)g(w)o(a)o(y)f(for)g(the)h(user)75 1923 y(to)f(sp)q(ecify)i(the)f (logical)h(pro)q(cess)f(arrangemen)o(t)f(as)g(a)g(\\virtual)h(top)q (ology)l(,")g(a)g(random)f(mapping)h(is)75 1980 y(most)f(lik)o(ely)i (the)e(consequence.)31 b(On)19 b(some)g(mac)o(hines)g(this)g(will)h (lead)f(to)f(unnecessary)h(con)o(ten)o(tion)75 2036 y(in)c(the)g(in)o (terconnect)g(net)o(w)o(ork.)k(Therefore,)14 b(though)g(w)o(e)h(don't)f (discuss)h(the)g(ph)o(ysical)h(mapping,)f(the)75 2093 y(mere)i(existence)i(of)d(the)i(virtual)f(top)q(ology)g(information)g (ma)o(y)g(b)q(e)h(used)f(as)g(advice)h(b)o(y)f(the)h(run)o(time)75 2149 y(system.)41 b(Some)23 b(details)g(ab)q(out)f(predicted)i(and)f (measured)f(p)q(erformance)h(impro)o(v)o(emen)o(ts)f(whic)o(h)75 2206 y(result)14 b(from)e(go)q(o)q(d)h(pro)q(cess{to{pro)q(cessor)f (mapping)i(on)f(mo)q(dern)h(w)o(ormhole{routing)f(arc)o(hitectures)75 2262 y(can)i(b)q(e)h(found)g(in)g([8)o(,)f(7].)75 2406 y Fv(6.2)59 b(Virtual)20 b(T)-5 b(op)r(ologies)75 2508 y FE(In)16 b(the)f(most)g(general)g(case)h(the)f(comm)o(unication)h (pattern)f(of)f(a)h(set)g(of)g(pro)q(cesses)h(is)g(b)q(e)f(represen)o (ted)75 2564 y(b)o(y)e(a)g(graph.)19 b(The)14 b(no)q(des)f(stand)h(for) e(the)i(pro)q(cesses,)f(and)h(the)f(edges)h(connect)f(pro)q(cesses)h (whic)o(h)g(com-)75 2621 y(m)o(unicate)h(with)g(eac)o(h)g(other.)20 b(MPI)14 b(pro)o(vides)i(message{passing)e(b)q(et)o(w)o(een)h(an)o(y)g (pair)g(of)f(pro)q(cesses)h(in)p 75 2661 720 2 v 127 2688 a FB(1)144 2704 y Fu(V)m(ersion)f(as)f(of)g(Oct.)j(16,)d(1993)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 141 146 141 145 bop 75 -100 a Ft(6.3.)34 b(EMBEDDING)15 b(IN)g(MPI)1165 b FE(141)75 45 y(a)17 b(group.)27 b(There)18 b(is)g(no)g(requiremen)o (t)g(for)f(op)q(ening)i(a)e(c)o(hannel)i(explicitly)l(.)30 b(Therefore,)18 b(a)f(\\missing)75 102 y(link")g(in)f(the)f (user{de\014ned)i(pro)q(cess)f(graph)f(do)q(es)h(not)f(prev)o(en)o(t)g (the)g(corresp)q(onding)i(pro)q(cesses)f(from)75 158 y(exc)o(hanging)h(messages.)k(It)16 b(only)g(means)g(that)f(this)i (connection)g(is)f(neglected)h(in)g(the)f(virtual)h(top)q(ol-)75 214 y(ogy)l(.)32 b(This)20 b(implies)i(that)d(the)g(top)q(ology)g(giv)o (es)h(no)f(con)o(v)o(enien)o(t)h(w)o(a)o(y)f(of)g(naming)g(this)h(path) o(w)o(a)o(y)e(of)75 271 y(comm)o(unication.)25 b(Another)16 b(p)q(ossible)j(consequence)f(is)f(that,)f(an)g(automatic)g(mapping)i (to)q(ol)e(\(if)h(one)75 327 y(exists)e(for)e(the)i(run)o(time)f(en)o (vironmen)o(t\))g(will)i(not)e(try)g(to)f(assign)i(the)f(pro)q(cesses)h (to)e(neigh)o(b)q(oring)j(pro-)75 384 y(cessors.)i(Edges)12 b(in)h(the)e(comm)o(unication)i(graph)e(are)h(not)f(w)o(eigh)o(ted.)19 b(Pro)q(cesses)11 b(are)h(either)g(neigh)o(b)q(ors)75 440 y(or)i(not.)20 b(Exp)q(erience)d(with)e(similar)h(tec)o(hniques)g (in)g(P)l(ARMA)o(CS)g([4)o(,)e(6])h(sho)o(w)f(that)g(this)i (information)75 497 y(is)i(usually)h(su\016cien)o(t)g(for)e(a)g(go)q(o) q(d)h(mapping.)28 b(Additionally)l(,)20 b(a)e(more)f(precise)i(sp)q (eci\014cation)h(w)o(ould)75 553 y(b)q(e)g(more)f(di\016cult)i(for)d (the)i(user)f(to)g(set)g(up,)h(and)f(it)h(w)o(ould)g(mak)o(e)e(the)i (in)o(terface)f(functions)h(more)75 610 y(complicated.)166 670 y(In)c(principle,)h(a)e(function)h(for)f(the)g(de\014nition)i(of)e (suc)o(h)h(a)f(graph)g(structure)g(w)o(ould)g(b)q(e)h(su\016cien)o(t)75 726 y(for)d(all)i(applications.)21 b(Ho)o(w)o(ev)o(er,)13 b(in)h(man)o(y)g(applications)h(the)f(graph)g(structure)f(is)h(v)o(ery) g(regular,)g(and)75 783 y(the)e(detailed)i(set{up)f(of)e(the)i(graph)f (w)o(ould)h(b)q(e)g(incon)o(v)o(enien)o(t)g(for)f(the)g(user)h(and)f (resource{consuming)75 839 y(during)g(run)g(time.)19 b(A)12 b(large)f(fraction)g(of)g(all)i(parallel)g(applications)g(use)f (pro)q(cess)g(top)q(ologies)f(lik)o(e)i(rings,)75 896 y(t)o(w)o(o{)i(or)i(higher{dimensional)i(grids,)e(or)g(tori.)24 b(These)18 b(structures)e(are)h(completely)h(de\014ned)g(b)o(y)f(the)75 952 y(n)o(um)o(b)q(er)d(of)g(dimensions)i(and)e(the)g(n)o(um)o(b)q(ers) h(of)f(pro)q(cesses)g(in)h(eac)o(h)f(co)q(ordinate)h(direction.)21 b(Also,)14 b(the)75 1008 y(mapping)h(of)e(grids)h(and)h(tori)e(is)i (generally)g(an)f(easier)g(problem)h(then)f(that)f(of)h(general)g (graphs.)19 b(Th)o(us,)75 1065 y(it)c(is)h(desirable)h(to)d(separate)h (out)g(this)h(case)f(explicitly)l(.)166 1125 y(Pro)q(cess)i(co)q (ordinates)g(in)h(a)e(Cartesian)h(structure)g(b)q(egin)h(their)f(n)o (um)o(b)q(ering)h(at)e(0.)25 b(Ro)o(w)16 b(ma)s(jor)75 1181 y(n)o(um)o(b)q(ering)f(is)g(used)f(for)g(the)g(pro)q(cesses)g(in)h (a)f(cartesian)g(structure.)19 b(This)c(means)f(that,)f(for)g(example,) 75 1238 y(the)k(relation)h(b)q(et)o(w)o(een)g(group)e(rank)h(and)h(co)q (ordinates)f(for)g(four)g(pro)q(cesses)g(in)h(a)f(\(2)11 b FC(\002)h FE(2\))k(grid)i(is)f(as)75 1294 y(follo)o(ws.)206 1377 y(co)q(ord)e(\(0,0\):)48 b(rank)15 b(0)206 1434 y(co)q(ord)g(\(0,1\):)48 b(rank)15 b(1)206 1490 y(co)q(ord)g(\(1,0\):) 48 b(rank)15 b(2)206 1546 y(co)q(ord)g(\(1,1\):)48 b(rank)15 b(3)75 1698 y Fv(6.3)59 b(Emb)r(edding)18 b(in)i(MPI)75 1807 y FE(The)c(supp)q(ort)g(for)f(virtual)i(top)q(ologies)f(as)g (de\014ned)h(in)g(this)f(c)o(hapter)g(is)g(consisten)o(t)g(with)g (other)g(parts)75 1863 y(of)d(MPI,)f(and,)h(whenev)o(er)h(p)q(ossible,) g(mak)o(es)f(use)g(of)f(functions)i(whic)o(h)g(are)f(de\014ned)h (elsewhere.)20 b(T)l(op)q(ol-)75 1920 y(ogy)f(information)h(is)g(asso)q (ciated)g(with)g(comm)o(unicators.)32 b(It)20 b(can)g(main)o(tained)g (using)h(the)e(cac)o(hing)75 1976 y(mec)o(hanism)f(describ)q(ed)g(in)g (Chapter)f(5.)25 b(The)17 b(top)q(ology)f(functions)i(complemen)o(t)g (and)f(in)o(terop)q(erate)75 2032 y(with)f(the)f(collectiv)o(e)i(comm)o (unication)f(functions)g(describ)q(ed)h(in)f(Chapter)f(4.)166 2093 y(The)d(curren)o(t)g(design)g(of)g(the)f(pro)q(cess)h(top)q(ology) g(functions)g(assumes)g(that)f(MPI)h(is)g(free)g(to)f(c)o(ho)q(ose)75 2149 y(an)o(y)g(pro)q(cess)h(in)h(a)e(group)g(for)g(an)o(y)h(lo)q (cation)g(in)h(the)e(top)q(ology)l(.)19 b(There)12 b(are)f(no)h(sp)q (ecial)h(pro)q(cesses)f(whic)o(h)75 2205 y(ha)o(v)o(e)g(to)f(b)q(e)i (aligned)g(with)g(top)q(ology)e(substructures.)19 b(This)13 b(assumption)f(\014ts)g(b)q(est)g(to)g(a)g(\\lo)q(cal)g(SPMD)75 2262 y(mo)q(del")k(in)g(whic)o(h)g(all)g(pro)q(cesses)g(in)g(the)f (same)g(group)g(execute)h(the)f(same)g(program.)75 2426 y Fv(6.4)59 b(Overview)19 b(of)h(the)f(p)n(rop)r(osed)g(MPI)h (functions)75 2534 y FE(The)13 b(functions)h Fx(MPI)p 445 2534 14 2 v 16 w(MAKE)p 590 2534 V 16 w(GRAPH)g FE(and)f Fx(MPI)p 938 2534 V 16 w(MAKE)p 1083 2534 V 16 w(CART)g FE(are)g(used)g(to)f(create)h(general)g(virtual)75 2591 y(top)q(ologies)21 b(and)f(Cartesian)g(top)q(ologies,)h(resp)q(ectiv)o (ely)l(.)37 b(These)20 b(top)q(ology)g(creation)g(functions)h(are)75 2647 y(collectiv)o(e.)f(As)11 b(for)g(other)g(collectiv)o(e)i(calls,)f (the)f(co)q(de)h(should)g(b)q(e)g(written)f(to)g(w)o(ork)f(correctly)l (,)i(whether)75 2704 y(the)j(call)i(sync)o(hronizes)f(or)f(not.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 142 147 142 146 bop 75 -100 a FE(142)882 b Ft(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)166 45 y FE(The)g(top)q(ology)g (de\014nition)i(function)g Fx(MPI)p 912 45 14 2 v 15 w(MAKE)p 1056 45 V 17 w(CART)e FE(can)h(b)q(e)g(used)g(to)e(describ)q (e)j(Cartesian)75 102 y(structures)k(of)g(arbitrary)f(dimension.)40 b(F)l(or)21 b(eac)o(h)g(co)q(ordinate)h(direction)g(it)g(is)f(sp)q (eci\014ed)j(whether)75 158 y(the)16 b(pro)q(cess)g(structure)g(is)g(p) q(erio)q(dic)i(or)d(not.)22 b(Similar)17 b(functions)g(are)e(con)o (tained)i(in)g(EXPRESS)f([17)o(])75 214 y(and)g(P)l(ARMA)o(CS.)h(Note)f (that)f(an)h Fy(n)p FE({dimensional)j(h)o(yp)q(ercub)q(e)f(is)e(an)h Fy(n)p FE({dimensional)h(torus)e(with)g(2)75 271 y(pro)q(cesses)i(p)q (er)h(co)q(ordinate)f(direction.)29 b(Th)o(us,)18 b(sp)q(ecial)i(supp)q (ort)e(for)f(h)o(yp)q(ercub)q(e)i(structures)f(is)g(not)75 327 y(necessary)l(.)h(The)12 b(lo)q(cal)h(auxiliary)g(function)g Fx(MPI)p 929 327 V 16 w(MAKE)p 1074 327 V 16 w(DIMS)e FE(can)h(b)q(e)h(used)f(to)f(compute)h(an)g(\\ev)o(en")75 384 y(distribution)17 b(of)e(pro)q(cesses)g(along)g(a)g(giv)o(en)h(n)o (um)o(b)q(er)g(of)e(dimensions.)166 440 y(The)k(function)g Fx(MPI)p 527 440 V 16 w(TOPO)p 669 440 V 18 w(ST)l(A)l(TUS)h FE(can)e(b)q(e)i(used)f(to)f(inquire)j(on)d(the)h(top)q(ology)g(asso)q (ciated)75 497 y(with)e(a)f(comm)o(unicator.)20 b(The)c(top)q(ological) g(information)f(can)h(b)q(e)g(extracted)f(from)g(the)g(comm)o(unica-)75 553 y(tor)h(using)h(the)g(functions)h Fx(MPI)p 633 553 V 15 w(GET)p 736 553 V 17 w(GRAPHDIMS)f FE(and)g Fx(MPI)p 1203 553 V 16 w(GET)p 1307 553 V 17 w(GRAPH)p FE(,)g(for)f(general)i (graphs,)75 610 y(and)f Fx(MPI)p 250 610 V 15 w(GET)p 353 610 V 17 w(CARTDIM)f FE(and)h Fx(MPI)p 763 610 V 16 w(GET)p 867 610 V 17 w(CART)p FE(,)f(for)g(Cartesian)g(toplogies.)25 b(Sev)o(eral)17 b(additional)75 666 y(functions)d(are)e(pro)o(vided)i (to)e(manipulate)i(Cartesian)f(top)q(ologies:)19 b(the)13 b(functions)h Fx(MPI)p 1603 666 V 16 w(CART)p 1738 666 V 17 w(RANK)75 723 y FE(and)h Fx(MPI)p 248 723 V 16 w(CART)p 383 723 V 17 w(COORD)h FE(translate)e(Cartesian)h(co)q(ordinates)g(in)o (to)g(a)g(group)f(rank,)h(and)g(vice-v)o(ersa;)75 779 y(the)e(function)h Fx(MPI)p 412 779 V 16 w(CART)p 547 779 V 17 w(SUB)g FE(can)f(b)q(e)h(used)f(to)g(extract)f(a)h(Cartesian)g (subspace;)h(and)f(the)g(function)75 835 y Fx(MPI)p 160 835 V 16 w(CART)p 295 835 V 17 w(SHIFT)20 b FE(pro)o(vides)h(the)f (information)g(needed)i(to)e(execute)g(a)g(shift)h(in)g(a)f(Cartesian)g (di-)75 892 y(mension.)j(The)16 b(t)o(w)o(o)f(functions)h Fx(MPI)p 734 892 V 16 w(GRAPH)p 900 892 V 18 w(NNEIGBORS)i FE(and)e Fx(MPI)p 1358 892 V 16 w(GRAPH)p 1524 892 V 17 w(NEIGHBORS)i FE(can)75 948 y(b)q(e)i(used)f(to)g(extract)f(the)h (neigh)o(b)q(ors)h(of)f(a)f(no)q(de)i(in)g(a)f(graph.)31 b(The)19 b(function)h Fx(MPI)p 1590 948 V 16 w(CART)p 1725 948 V 17 w(SUB)f FE(is)75 1005 y(collectiv)o(e;)e(all)f(other)f (functions)h(are)e(lo)q(cal.)166 1061 y(Tw)o(o)20 b(additional)j (functions,)g Fx(MPI)p 796 1061 V 16 w(MAP)p 911 1061 V 16 w(GRAPH)g FE(and)e Fx(MPI)p 1276 1061 V 16 w(MAP)p 1391 1061 V 17 w(CART)g FE(are)g(presen)o(ted)h(in)75 1118 y(the)c(last)f(section.)29 b(These)18 b(functions)g(are)g(not)f (part)g(of)g(the)h(MPI)g(de\014nition.)29 b(Ho)o(w)o(ev)o(er,)17 b(these)h(t)o(w)o(o)75 1174 y(functions,)d(together)e(with)i(the)f (comm)o(unicator)g(manipulation)i(functions)f(presen)o(ted)g(in)g (Chapter)f(5,)75 1231 y(are)21 b(su\016cien)o(t)h(to)e(implemen)o(t)j (all)f(other)f(top)q(ology)f(functions.)39 b(The)21 b(last)g(section)h (outlines)h(suc)o(h)75 1287 y(implemen)o(tation.)75 1428 y Fv(6.5)59 b(T)-5 b(op)r(ology)19 b(Constructo)n(rs)75 1576 y Fx(MPI)p 160 1576 V 16 w(MAKE)p 305 1576 V 16 w(CART\(comm)p 575 1576 V 15 w(old,)c(ndims,)g(dims,)f(p)q(erio)q(ds,)i (reo)o(rder,)e(comm)p 1370 1576 V 15 w(ca)o(rt\))117 1653 y Fr(IN)171 b Fx(comm)p 454 1653 V 14 w(old)383 b Fr(comm)o(unicator)11 b(for)j(paren)o(t)g(group)g(\(handle\))117 1726 y(IN)171 b Fx(ndims)460 b Fr(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of)i(Cartesian)g(grid)f(\(in)o(teger\))117 1798 y(IN)171 b Fx(dims)483 b Fr(in)o(teger)19 b(arra)o(y)f(of)f(size)i Fm(ndims)f Fr(sp)q(ecifying)g(the)g(n)o(um)o(b)q(er)g(of)905 1854 y(pro)q(cesses)f(in)c(eac)o(h)i(dimension)117 1927 y(IN)171 b Fx(p)q(erio)q(ds)436 b Fr(logical)12 b(arra)o(y)h(of)g(size) h Fm(ndims)f Fr(sp)q(ecifying)g(whether)h(the)g(grid)905 1983 y(is)d(p)q(erio)q(dic)g(\()p Fm(true)p Fr(\))h(or)f(ap)q(erio)q (dic)g(\()p Fm(false)p Fr(\))g(in)f(eac)o(h)i(dimension)117 2055 y(IN)171 b Fx(reo)o(rder)438 b Fr(ranking)10 b(ma)o(y)g(b)q(e)h (reordered)i(\()p Fm(true)p Fr(\))f(or)f(not)g(\()p Fm(false)p Fr(\))g(\(logical\))117 2127 y(OUT)124 b Fx(comm)p 454 2127 V 14 w(ca)o(rt)367 b Fr(comm)o(unicator)11 b(for)j(new)g (Cartesian)g(group)g(\(handle\))166 2252 y FE(The)f(function)h(returns) f(a)g(handle)h(to)e(a)h(new)g(comm)o(unicator)g(to)f(whic)o(h)i(the)g (Cartesian)e(top)q(ology)75 2308 y(information)i(is)g(attac)o(hed.)19 b(If)14 b Fx(reo)o(rder)f(=)h(false)h FE(then)f(the)g(rank)f(of)h(eac)o (h)g(pro)q(cess)g(in)h(the)f(new)g(group)f(is)75 2365 y(iden)o(tical)19 b(to)d(its)h(rank)g(in)g(the)g(old)h(group.)25 b(Otherwise,)17 b(the)g(function)h(ma)o(y)e(reorder)h(the)g(pro)q (cesses,)75 2421 y(so)i(as)f(to)g(c)o(ho)q(ose)h(a)g(go)q(o)q(d)g(em)o (b)q(edding)i(of)d(the)h(virtual)h(top)q(ology)e(on)o(to)g(the)h(ph)o (ysical)i(mac)o(hine.)32 b(If)75 2478 y(the)16 b(total)e(size)j(of)e (the)g(Cartesian)h(grid)f(is)h(smaller)h(than)e(the)h(size)g(of)f(the)h (group)f(of)g Fx(comm)p FE(,)e(then)j(the)75 2534 y(pro)q(cesses)f (that)e(are)h(not)g(mem)o(b)q(er)g(of)g(the)h(grid)f(are)g(returned)h (a)f(n)o(ull)i(handle.)k(The)15 b(call)g(is)g(erroneous)75 2591 y(if)h(it)f(sp)q(eci\014es)i(a)e(grid)h(that)e(is)i(larger)f(than) g(the)g(the)g(group)g(size.)166 2647 y(In)22 b(a)g(p)q(ortable)g (program)f(the)h(n)o(um)o(b)q(er)g(of)f(a)o(v)m(ailable)j(pro)q (cessors)d(is)h(usually)i(not)d(kno)o(wn)h(at)75 2704 y(compile)g(time.)36 b(In)21 b(the)f(case)h(of)f(a)g(Cartesian)g(top)q (ology)g(the)h(function)g Fx(MPI)p 1485 2704 V 16 w(MAKE)p 1630 2704 V 16 w(DIMS)f FE(helps)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 143 148 143 147 bop 75 -100 a Ft(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FE(143)75 45 y(in)18 b(setting)e(the)h(n)o(um)o(b)q(ers)g(of)f(pro)q (cesses)i(p)q(er)f(co)q(ordinate)g(direction,)h(dep)q(ending)h(on)e (the)f(n)o(um)o(b)q(er)h(of)75 102 y(pro)q(cesses)f(in)g(the)f(group)g (and)g(optional)h(constrain)o(ts)f(whic)o(h)h(can)f(b)q(e)h(sp)q (eci\014ed)h(b)o(y)e(the)h(user:)75 208 y Fx(MPI)p 160 208 14 2 v 16 w(MAKE)p 305 208 V 16 w(DIMS\(nno)q(des,)g(ndims,)f (dims\))117 287 y Fr(IN)171 b Fx(nno)q(des)441 b Fr(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(a)f(grid)h(\(in)o(teger\))117 367 y(IN)171 b Fx(ndims)460 b Fr(n)o(um)o(b)q(er)13 b(of)h(Cartesian)g (dimensions)e(\(in)o(teger\))117 447 y(INOUT)78 b Fx(dims)483 b Fr(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fl(ndims)e Fr(sp)q(ecifying)h(the)h(n)o(um)o(b)q(er)f(of)905 503 y(no)q(des)e(in)e(eac)o(h)i(dimension)166 630 y FE(The)d(en)o(tries)h (in)g(the)f(arra)o(y)f Fx(dims)h FE(are)g(set)g(to)f(describ)q(e)j(a)e (Cartesian)g(grid)g(with)h Fx(ndims)f FE(dimensions)75 686 y(and)h(a)g(total)g(of)f Fx(nno)q(des)k FE(no)q(des.)j(The)14 b(dimensions)h(are)d(set)h(to)g(b)q(e)h(as)e(close)i(to)f(eac)o(h)g (other)g(as)g(p)q(ossible.)75 743 y(The)20 b(caller)h(ma)o(y)e(further) h(constrain)g(the)g(op)q(eration)g(of)g(this)g(routine)g(b)o(y)g(sp)q (ecifying)i(elemen)o(ts)e(of)75 799 y(arra)o(y)13 b Fp(dims)p FE(.)19 b(If)14 b Fp(dims[i])g FE(is)g(set)g(to)g(a)g(p)q(ositiv)o(e)h (n)o(um)o(b)q(er,)g(the)f(routine)h(will)h(not)d(mo)q(dify)i(the)g(n)o (um)o(b)q(er)75 856 y(of)20 b(no)q(des)h(in)g(the)f(corresp)q(onding)h (dimension;)k(only)20 b(those)g(en)o(tries)h(where)g Fp(dims[i])i(=)g(0)d FE(will)i(b)q(e)75 912 y(mo)q(di\014ed)17 b(b)o(y)e(the)g(call.)166 971 y(Note)g(that)f(an)h(error)g(will)i(o)q (ccur)e(if)h Fp(nnodes)e FE(is)i(not)f(a)g(m)o(ultiple)i(of)1413 931 y Fh(Y)1352 1025 y Fi(i;dims)p Fj([)p Fi(i)p Fj(])p Fk(6)p Fj(=0)1535 971 y Fy(dims)p FE([)p Fy(i)p FE(].)166 1075 y(Negativ)o(e)f(input)i(v)m(alues)f(of)f Fp(dims[i])g FE(will)i(also)e(cause)h(an)f(error.)23 b(All)18 b(elemen)o(ts)f(of)f Fp(dims)g FE(m)o(ust)75 1132 y(b)q(e)g(set)f(to)f(0)h(for)g(completely) h(automatic)f(pro)q(cess)g(distribution.)166 1190 y(F)l(or)j Fp(dims[i])g FE(set)g(b)o(y)g(the)h(call,)h Fp(dims[i])e FE(will)i(b)q(e)g(ordered)e(in)i(nonincreasing)g(order.)30 b(Arra)o(y)75 1247 y Fp(dims)15 b FE(is)g(suitable)i(for)d(use)i(as)f (input)h(to)e(routine)i Fx(MPI)p 1023 1247 V 16 w(MAKE)p 1168 1247 V 16 w(CART)p FE(.)166 1306 y(Examples:)166 1364 y(Assume)f(that)g Fp(dims)f FE(=)i(\(0,0\).)i(Then)75 1421 y Fx(MPI)p 160 1421 V 16 w(MAKE)p 305 1421 V 16 w(DIM\(6,)13 b(2,)i(dims\))g FE(returns)g(\(3,2\);)75 1477 y Fx(MPI)p 160 1477 V 16 w(MAKE)p 305 1477 V 16 w(DIM\(7,)e(2,)i(dims\))g FE(returns)g(\(7,1\).)166 1536 y(Assume)g(that)g Fp(dims)f FE(=)i(\(0,3,0\).)i(Then)75 1593 y Fx(MPI)p 160 1593 V 16 w(MAKE)p 305 1593 V 16 w(DIM\(6,)13 b(3,)i(dims\))g FE(returns)g(\(2,3,1\);)75 1649 y Fx(MPI)p 160 1649 V 16 w(MAKE)p 305 1649 V 16 w(DIM\(7,)e(3,)i(dims\))g FE(is)g(erroneous.)166 1708 y(The)g(function)h Fx(MPI)p 522 1708 V 16 w(MAKE)p 667 1708 V 16 w(DIMS)f FE(is)h(lo)q(cal.)166 1767 y(F)l(or)e(general)i (graph)f(structure,)g(the)g(constructor)f(is)75 1873 y Fx(MPI)p 160 1873 V 16 w(MAKE)p 305 1873 V 16 w(GRAPH\(comm)p 606 1873 V 15 w(old,)h(nno)q(des,)i(index,)f(edges,)g(reo)o(rder,)d (comm)p 1401 1873 V 15 w(graph\))117 1952 y Fr(IN)171 b Fx(comm)p 454 1952 V 14 w(old)383 b Fr(comm)o(unicator)11 b(for)j(paren)o(t)g(group)g(\(handle\))117 2032 y(IN)171 b Fx(nno)q(des)441 b Fr(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g (graph)g(\(in)o(teger\))117 2112 y(IN)171 b Fx(index)473 b Fr(arra)o(y)14 b(of)f(in)o(tegers)i(describing)f(no)q(de)h(degrees)g (\(see)g(b)q(elo)o(w\))117 2191 y(IN)171 b Fx(edges)468 b Fr(arra)o(y)14 b(of)f(in)o(tegers)i(describing)f(graph)g(edges)h (\(see)g(b)q(elo)o(w\))117 2271 y(IN)171 b Fx(reo)o(rder)438 b Fr(ranking)10 b(ma)o(y)g(b)q(e)h(reordered)i(\()p Fm(true)p Fr(\))f(or)f(not)g(\()p Fm(false)p Fr(\))g(\(logical\))117 2351 y(OUT)124 b Fx(comm)p 454 2351 V 14 w(graph)333 b Fr(comm)o(unicator)11 b(for)j(new)g(graph)g(group)g(\(handle\))166 2478 y FE(The)19 b(function)h(returns)f(a)f(handle)i(to)e(a)h(new)g (comm)o(unicator)g(to)f(whic)o(h)i(the)f(graph)f(top)q(ology)75 2534 y(information)c(is)g(attac)o(hed.)19 b(If)14 b Fx(reo)o(rder)f(=)h (false)h FE(then)f(the)g(rank)f(of)h(eac)o(h)g(pro)q(cess)g(in)h(the)f (new)g(group)f(is)75 2591 y(iden)o(tical)i(to)e(its)g(rank)g(in)h(the)f (old)h(group.)19 b(Otherwise,)14 b(the)f(function)h(ma)o(y)f(reorder)f (the)i(pro)q(cesses,)f(so)75 2647 y(as)f(to)g(c)o(ho)q(ose)g(a)g(go)q (o)q(d)g(em)o(b)q(edding)i(of)e(the)g(virtual)h(top)q(ology)f(on)o(to)f (the)i(ph)o(ysical)g(mac)o(hine.)20 b(If)13 b(the)f(size)75 2704 y Fx(nno)q(des)18 b FE(of)d(the)g(graph)g(is)h(smaller)h(than)e (the)h(size)g(of)f(the)h(group)f(of)g Fx(comm)p FE(,)e(then)j(the)g (pro)q(cesses)g(that)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 144 149 144 148 bop 75 -100 a FE(144)882 b Ft(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y FE(are)h(not)g(mem)o(b)q(er)g(of)g (the)g(grid)g(are)g(returned)h(a)e(n)o(ull)j(handle.)24 b(The)16 b(call)i(is)e(erroneous)g(if)h(it)f(sp)q(eci\014es)75 102 y(a)f(graph)g(that)f(is)i(larger)f(than)g(then)h(the)f(group)g (size.)166 161 y(The)g(three)h(parameters)e Fx(nno)q(des,)j(index)f FE(and)f Fx(edges)i FE(de\014ne)f(the)f(graph)g(structure.)166 220 y Fx(nno)q(des)g FE(is)e(the)g(n)o(um)o(b)q(er)g(of)f(no)q(des)i (of)e(the)h(graph.)19 b(The)13 b(no)q(des)g(are)f(n)o(um)o(b)q(ered)i (from)e Fp(0)h FE(to)f Fp(nnodes)75 276 y(-1)p FE(.)166 335 y(The)j Fp(i)p FE(-th)g(en)o(try)g(of)g(arra)o(y)f Fx(index)i FE(stores)e(the)i(total)e(n)o(um)o(b)q(er)i(of)f(neigh)o(b)q (ors)g(of)g(the)g(\014rst)g Fp(i)g FE(graph)75 392 y(no)q(des.)40 b(The)21 b(lists)i(of)e(neigh)o(b)q(ors)h(of)f(no)q(des)h Fp(0,)i(1,)g Fy(:)8 b(:)g(:)d Fp(,)24 b(nnodes-1)c FE(are)i(stored)f (in)h(consecutiv)o(e)75 448 y(lo)q(cations)16 b(in)g(arra)o(y)e Fx(edges)p FE(.)166 507 y(The)19 b(de\014nitions)h(of)e(the)h(argumen)o (ts)f Fp(nnodes)p FE(,)g Fp(index)p FE(,)h(and)g Fp(edges)f FE(are)g(illustrated)i(with)f(the)75 564 y(follo)o(wing)d(simple)h (example:)166 623 y(Assume)e(there)h(are)f(four)f(pro)q(cesses)i(0,)e (1,)h(2,)f(3)h(with)h(the)f(follo)o(wing)h(adjacency)g(matrix:)p 181 676 433 2 v 180 733 2 57 v 206 716 a(pro)q(cess)p 373 733 V 50 w(neigh)o(b)q(ors)p 613 733 V 181 735 433 2 v 180 791 2 57 v 266 774 a(0)p 373 791 V 110 w(1,)f(3)p 613 791 V 180 847 V 266 831 a(1)p 373 847 V 110 w(0)p 613 847 V 180 904 V 266 887 a(2)p 373 904 V 110 w(3)p 613 904 V 180 960 V 266 943 a(3)p 373 960 V 110 w(0,)g(2)p 613 960 V 181 962 433 2 v 166 1040 a(Then,)g(the)h(input)g(argumen)o (ts)e(are:)206 1131 y(nno)q(des)i(=)50 b(4)206 1188 y(index)16 b(=)81 b(2,)15 b(3,)f(4,)h(6)206 1244 y(edges)g(=)82 b(1,)15 b(3,)f(0,)h(3,)f(0,)h(2)166 1338 y(Th)o(us,)h(in)h(C,)e Fp(index[0])g FE(is)i(the)f(degree)g(of)g(no)q(de)h(zero,)f(and)g Fp(index[i])23 b(-)h(index[i-1])14 b FE(is)j(the)75 1395 y(degree)22 b(of)g(no)q(de)g Fp(i,)i(i=1,)f Fy(:)8 b(:)g(:)e Fp(,)23 b(nnodes-1)p FE(;)h(the)e(list)h(of)e(neigh)o(b)q(ors)i(of)e (no)q(de)i(zero)e(is)i(stored)e(in)75 1451 y Fp(edges[j])p FE(,)d Fp(0)g FC(\024)g Fp(j)g FC(\024)g Fp(index)p FE([)p Fp(0)p FE(])11 b FC(\000)h Fp(1)18 b FE(and)h(the)g(list)g(of)f(neigh)o (b)q(ors)h(of)f(no)q(de)h Fp(i)p FE(,)f Fp(i)g Fy(>)h Fp(0)p FE(,)f(is)h(stored)f(in)75 1508 y Fp(edges[j])p FE(,)c Fp(index)p FE([)p Fp(i)9 b FC(\000)h Fp(1)p FE(])i FC(\024)h Fp(j)g FC(\024)g Fp(index)p FE([)p Fp(i)p FE(])8 b FC(\000)j Fp(1)p FE(.)166 1567 y(In)17 b(F)l(ortran,)f Fp(index[1])f FE(is)j(the)e(degree)i(of)e(no)q(de)h(zero,)g(and)g Fp(index[i+1])22 b(-)i(index[i])16 b FE(is)h(the)75 1623 y(degree)22 b(of)g(no)q(de)g Fp(i,)i(i=1,)f Fy(:)8 b(:)g(:)e Fp(,)23 b(nnodes-1)p FE(;)h(the)e(list)h(of)e(neigh)o(b)q(ors)i(of)e (no)q(de)i(zero)e(is)i(stored)e(in)75 1680 y Fp(edges[j])p FE(,)h Fp(1)i FC(\024)g Fp(j)g FC(\024)g Fp(index)p FE([)p Fp(0)p FE(])c(and)i(the)g(list)h(of)f(neigh)o(b)q(ors)g(of)g(no)q(de)g Fp(i)p FE(,)h Fp(i)h Fy(>)g Fp(0)p FE(,)f(is)g(stored)e(in)75 1736 y Fp(edges[j])p FE(,)14 b Fp(index)p FE([)p Fp(i)9 b FC(\000)h Fp(1)p FE(])g(+)g Fp(1)j FC(\024)g Fp(j)f FC(\024)h Fp(index)p FE([)p Fp(i)p FE(].)166 1795 y(The)h(total)f(n)o (um)o(b)q(er)h(of)g(en)o(tries)g(in)h Fx(index)f FE(is)h Fp(nnodes)e FE(and)h(the)f(total)h(n)o(um)o(b)q(er)g(of)f(en)o(tries)h (in)h Fx(edges)75 1852 y FE(is)h(equal)g(to)e(the)h(n)o(um)o(b)q(er)h (of)f(graph)g(edges.)166 1987 y Fo(Implemen)o(tati)o(on)i(note:)54 b Fr(The)19 b(follo)o(wing)c(top)q(ology)i(information)f(is)i(lik)o (ely)f(to)h(b)q(e)h(stored)g(with)f(a)75 2037 y(comm)o(unicator:)134 2119 y(1.)k(T)o(yp)q(e)14 b(of)f(top)q(ology)g(\(Cartesian/graph\))134 2204 y(2.)22 b(F)m(or)13 b(a)h(Cartesian)g(top)q(ology:)245 2296 y Fd(\017)23 b Fr(ndims)12 b(\(n)o(um)o(b)q(er)h(of)h (dimensions\),)245 2367 y Fd(\017)23 b Fr(dims)12 b(\(n)o(um)o(b)q(ers) i(of)f(pro)q(cesses)k(p)q(er)d(co)q(ordinate)h(direction\),)245 2439 y Fd(\017)23 b Fr(p)q(erio)q(ds)14 b(\(p)q(erio)q(dicit)o(y)g (information\),)245 2511 y Fd(\017)23 b Fr(o)o(wn)p 364 2511 13 2 v 14 w(p)q(osition)13 b(\(o)o(wn)h(p)q(osition)f(in)h(grid,)f (could)g(also)h(b)q(e)g(computed)g(from)e(rank)h(and)h(dims\))134 2602 y(3.)22 b(F)m(or)13 b(a)h(graph)f(top)q(ology:)245 2693 y Fd(\017)23 b Fr(index,)13 b(edges)i(\(v)o(ectors)g(de\014ning)f (graph)g(structure\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 145 150 145 149 bop 75 -100 a Ft(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FE(145)166 45 y Fr(F)m(or)15 b(a)h(graph)f(structure)j(the)e(n)o(um)o (b)q(er)f(of)g(no)q(des)i(is)e(equal)h(to)f(the)i(n)o(um)o(b)q(er)e(of) g(pro)q(cesses)j(in)d(the)i(group.)75 102 y(Therefore,)g(it)e(do)q(es)h (not)f(ha)o(v)o(e)g(to)g(b)q(e)h(stored)h(explicitely)m(.)k(An)16 b(additional)d(zero)j(en)o(try)g(at)g(the)f(start)h(of)f(arra)o(y)75 158 y Fm(index)g Fr(will)e(simplify)e(access)k(to)f(the)h(top)q(ology)d (information.)75 367 y Fq(6.5.1)49 b(T)l(op)q(ology)19 b(inquiry)e(functions)75 455 y FE(If)h(a)f(top)q(ology)g(has)h(b)q(een) h(de\014ned)g(with)e(one)h(of)f(the)h(ab)q(o)o(v)o(e)f(functions,)i (the)f(top)q(ology)f(information)75 511 y(can)g(b)q(e)g(lo)q(ok)o(ed)g (up)g(using)g(inquiry)h(functions.)24 b(They)17 b(all)h(are)e(lo)q(cal) h(calls,)h(and)e(do)h(not)f(require)h(an)o(y)75 567 y(comm)o (unication.)166 625 y(The)f(function)g Fx(MPI)p 523 625 14 2 v 16 w(TOPO)p 665 625 V 17 w(ST)l(A)l(TUS)h FE(returns)f(the)f(t)o (yp)q(e)h(of)f(top)q(ology)h(whic)o(h)g(is)g(assigned)g(to)f(a)75 681 y(comm)o(unicator.)75 786 y Fx(MPI)p 160 786 V 16 w(TOPO)p 302 786 V 17 w(ST)l(A)l(TUS\(comm,)f(status\))117 864 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(\(handle\))117 941 y(OUT)108 b Fx(status)476 b Fr(top)q(ology)13 b(t)o(yp)q(e)h(of)g(comm)o(uni)o(cator)e Fl(comm)h Fr(\(c)o(hoice\))166 1066 y FE(The)i(output)g(v)m(alue)i Fp(status)d FE(is)i(one)f(of)g(the) g(follo)o(wing:)117 1169 y Fm(MPI)p 194 1169 13 2 v 14 w(GRAPH)602 b FE(graph)15 b(top)q(ology)117 1226 y Fm(MPI)p 194 1226 V 14 w(CART)631 b FE(Cartesian)15 b(top)q(ology)117 1282 y Fm(MPI)p 194 1282 V 14 w(UNDEFINED)508 b FE(no)15 b(top)q(ology)166 1365 y(F)l(unctions)k Fx(MPI)p 462 1365 14 2 v 16 w(GET)p 566 1365 V 16 w(GRAPHDIMS)g FE(and)g Fx(MPI)p 1036 1365 V 16 w(GET)p 1140 1365 V 16 w(GRAPH)h FE(retriev)o(e)e(the)g(graph)g(top)q(ology)75 1422 y(information)d (that)g(w)o(as)f(asso)q(ciated)i(with)f(a)g(comm)o(unicator)g(b)o(y)g Fx(MPI)p 1310 1422 V 16 w(MAKE)p 1455 1422 V 16 w(GRAPH)p FE(.)75 1526 y Fx(MPI)p 160 1526 V 16 w(GET)p 264 1526 V 17 w(GRAPHDIMS\(comm,)e(nno)q(des,)k(nedges\))117 1604 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)7 b(for)i(group)h(with)f (graph)h(structure)i(\(handle\))117 1681 y(OUT)124 b Fx(nno)q(des)441 b Fr(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(graph)g (\(in)o(teger\))117 1758 y(OUT)124 b Fx(nedges)445 b Fr(n)o(um)o(b)q(er)13 b(of)h(edges)h(in)e(graph)h(\(in)o(teger\))166 1883 y FE(The)21 b(information)g(pro)o(vided)h(b)o(y)f Fx(MPI)p 864 1883 V 16 w(GET)p 968 1883 V 16 w(GRAPHDIMS)h FE(can)f(b)q(e)g(used)h(to)e(dimension)j(the)75 1940 y(v)o(ectors)14 b Fp(index)h FE(and)g Fp(edges)g FE(long)g(enough)h(in) g(the)f(follo)o(wing)h(call)g(to)f Fx(MPI)p 1389 1940 V 16 w(GET)p 1493 1940 V 16 w(GRAPH)p FE(.)75 2044 y Fx(MPI)p 160 2044 V 16 w(GET)p 264 2044 V 17 w(GRAPH\(comm,)e (maxindex,)i(maxedges,)g(index,)h(edges\))117 2122 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)7 b(for)i(group)h(with)f (graph)h(structure)i(\(handle\))117 2199 y(IN)171 b Fx(maxindex)393 b Fr(length)14 b(of)f(v)o(ector)i Fm(index)g Fr(in)f(the)g(calling)f (program)905 2255 y(\(in)o(teger\))117 2332 y(IN)171 b Fx(maxedges)388 b Fr(length)14 b(of)f(v)o(ector)i Fm(edges)g Fr(in)e(the)i(calling)d(program)905 2389 y(\(in)o(teger\))117 2465 y(OUT)124 b Fx(index,)15 b(edges)342 b Fr(v)o(ectors)15 b(con)o(taining)d(the)h(graph)g(structure)j(\(for)d(details)g(see)905 2522 y(the)i(de\014nition)e(of)g(MPI)p 1291 2522 13 2 v 16 w(MAKE)p 1436 2522 V 15 w(GRAPH\))166 2647 y FE(The)h(functions)h Fx(MPI)p 538 2647 14 2 v 16 w(GET)p 642 2647 V 16 w(CARTDIM)f FE(and)g Fx(MPI)p 1046 2647 V 16 w(GET)p 1150 2647 V 17 w(CART)g FE(return)g(the)g(cartesian)g(top)q(ology)75 2704 y(information)h(that)g(w)o(as)f(asso)q(ciated)i(with)f(a)g(comm)o (unicator)g(b)o(y)g(the)g(function)h Fx(MPI)p 1566 2704 V 16 w(MAKE)p 1711 2704 V 16 w(CART)p FE(.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 146 151 146 150 bop 75 -100 a FE(146)882 b Ft(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(GET)p 264 45 V 17 w(CARTDIM\(comm,)d(ndims\))117 126 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(for)j(group)g(with)f(Cartesian)h(structure)905 182 y(\(handle\))117 265 y(OUT)124 b Fx(ndims)460 b Fr(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of)i(the)g(Cartesian)g(structure)905 322 y(\(in)o(teger\))75 497 y Fx(MPI)p 160 497 V 16 w(GET)p 264 497 V 17 w(CART\(comm,)f(maxdims,)g(dims,)i(p)q(erio)q(d,)h(co)q(o) o(rd\))117 578 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(for)j(group)g(with)f(Cartesian)h(structure)905 634 y(\(handle\))117 717 y(IN)171 b Fx(maxdims)403 b Fr(length)10 b(of)f(v)o(ectors)i Fl(size,)21 b(period)p Fr(,)9 b(and)g Fl(coord)g Fr(in)g(the)i(call-)905 773 y(ing)i(program)g(\(in)o (teger\))117 856 y(OUT)124 b Fx(dims)483 b Fr(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(for)e(eac)o(h)g(Cartesian)g(dimension)905 912 y(\(arra)o(y)g(of)f(in)o(teger\))117 995 y(OUT)124 b Fx(p)q(erio)q(ds)436 b Fr(p)q(erio)q(dicit)o(y)18 b(\()p Fm(true)p Fr(/)p Fm(false)p Fr(\))i(for)e(eac)o(h)h(Cartesian)g (dimension)905 1052 y(\(arra)o(y)14 b(of)f(logical\))117 1134 y(OUT)124 b Fx(co)q(o)o(rd)466 b Fr(co)q(ordinates)15 b(of)e(calling)g(pro)q(cess)i(in)f(Cartesian)g(structure)905 1191 y(\(arra)o(y)g(of)f(in)o(teger\))166 1319 y FE(F)l(or)e(a)g(pro)q (cess)h(group)f(with)h(Cartesian)g(structure,)f(the)h(function)h Fx(MPI)p 1399 1319 V 15 w(CART)p 1533 1319 V 17 w(RANK)f FE(translates)75 1375 y(the)21 b(logical)h(pro)q(cess)f(co)q(ordinates) g(to)f(pro)q(cess)h(ranks)f(as)g(they)h(are)f(used)i(b)o(y)e(the)h(p)q (oin)o(t{to{p)q(oin)o(t)75 1432 y(routines:)75 1539 y Fx(MPI)p 160 1539 V 16 w(CART)p 295 1539 V 17 w(RANK\(comm,)13 b(co)q(o)o(rd,)h(rank\))117 1620 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(for)j(group)g(with)f(Cartesian)h(structure) 905 1677 y(\(handle\))117 1759 y(IN)171 b Fx(co)q(o)o(rd)466 b Fr(in)o(teger)15 b(arra)o(y)g(\(of)f(size)h Fl(ndims)p Fr(\))f(sp)q(ecifying)g(the)i(Cartesian)905 1816 y(co)q(ordinates)f(of) e(a)h(pro)q(cess)117 1898 y(OUT)124 b Fx(rank)488 b Fr(rank)14 b(of)f(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o(teger\))166 2026 y FE(The)g(in)o(v)o(erse)h(rank-to-co)q(ordinates)f(translation)g (is)h(pro)o(vided)g(b)o(y)f(the)g(function:)75 2134 y Fx(MPI)p 160 2134 V 16 w(CART)p 295 2134 V 17 w(COORD\(comm,)f(rank,)g (maxdims,)g(co)q(o)o(rd\))117 2215 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(for)j(group)g(with)f(Cartesian)h(structure) 905 2271 y(\(handle\))117 2354 y(IN)171 b Fx(rank)488 b Fr(rank)14 b(of)f(a)h(pro)q(cess)i(within)d(group)g(of)h Fm(comm)e Fr(\(in)o(teger\))117 2436 y(IN)171 b Fx(maxdims)403 b Fr(length)11 b(of)f(v)o(ector)i Fl(coord)d Fr(in)h(the)i(calling)d (program)g(\(in)o(teger\))117 2519 y(OUT)124 b Fx(co)q(o)o(rd)466 b Fr(in)o(teger)14 b(arra)o(y)f(\(of)g(size)h Fl(ndims)p Fr(\))e(con)o(taining)h(the)h(Cartesian)905 2575 y(co)q(ordinates)h(of) e(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o(teger\))166 2704 y FE(The)f(follo)o(wing)h(functions)h(pro)o(vide)e(\\adjacency")h (information)f(for)g(a)g(general)g(graph)g(top)q(ology)l(.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 147 152 147 151 bop 75 -100 a Ft(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FE(147)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(GRAPH)p 326 45 V 17 w(NNEIGHBORS\(comm,)15 b(rank,)g(nneighb)q(o)o(rs\))117 123 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(for)j(a)f(group)h(with)g(graph)f(top)q(ology)117 199 y(IN)171 b Fx(rank)488 b Fr(rank)14 b(of)f(pro)q(cess)j(in)d(group)h (\(in)o(teger\))117 276 y(OUT)124 b Fx(nneighb)q(o)o(rs)369 b Fr(n)o(um)o(b)q(er)13 b(of)h(neigh)o(b)q(ors)g(of)f(sp)q(eci\014ed)j (pro)q(cess)f(\(in)o(teger\))75 448 y Fx(MPI)p 160 448 V 16 w(GRAPH)p 326 448 V 17 w(NEIGHBORS\(comm,)g(rank,)g(maxneighb)q(o) o(rs,)f(neighb)q(o)o(rs\))117 526 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(for)j(a)f(group)h(with)g(graph)f(top)q(ology) 117 603 y(IN)171 b Fx(rank)488 b Fr(rank)14 b(of)f(pro)q(cess)j(in)d (group)h(\(in)o(teger\))117 679 y(OUT)124 b Fx(maxneighb)q(o)o(rs)312 b Fr(size)15 b(of)e(arra)o(y)h Fm(neighb)q(o)o(rs)117 755 y Fr(OUT)124 b Fx(neighb)q(o)o(rs)392 b Fr(ranks)13 b(of)g(pro)q(cesses)i(that)e(are)h(neigh)o(b)q(ors)f(to)g(sp)q (eci\014ed)h(pro-)905 812 y(cess)i(\(arra)o(y)d(of)h(in)o(teger\))166 937 y FE(Example:)26 b(Supp)q(ose)19 b(that)f Fp(comm)f FE(is)i(a)e(comm)o(unicator)h(for)g(a)f(group)h(with)h(a)e(sh)o (u\017e-exc)o(hange)75 993 y(top)q(ology:)32 b(The)21 b(group)g(has)g(2)636 977 y Fi(n)680 993 y FE(mem)o(b)q(ers.)38 b(Eac)o(h)21 b(pro)q(cess)h Fy(a)1216 1000 y Fj(1)1236 993 y Fy(;)8 b(:)g(:)g(:)d(;)j(a)1362 1000 y Fi(n)1405 993 y FE(\()p Fy(a)1447 1000 y Fi(i)1484 993 y FC(2)23 b(f)p FE(0)p Fy(;)8 b FE(1)p FC(g)p FE(\))19 b(has)i(three)75 1050 y(neigh)o(b)q(ors:)34 b Fy(exchang)r(e)p FE(\()p Fy(a)542 1057 y Fj(1)561 1050 y Fy(;)8 b(:)g(:)g(:)d(;)j(a)687 1057 y Fi(n)709 1050 y FE(\))24 b(=)g Fy(a)834 1057 y Fj(1)853 1050 y Fy(;)8 b(:)g(:)g(:)d(;)j(a)979 1057 y Fi(n)p Fk(\000)p Fj(1)1047 1050 y Fy(;)g FE(\026)-23 b Fy(a)1092 1057 y Fi(n)1136 1050 y FE(\()q(\026)f Fy(a)24 b FE(=)f(1)15 b FC(\000)f Fy(a)p FE(\),)23 b Fy(shuf)5 b(f)g(l)q(e)p FE(\()p Fy(a)1630 1057 y Fj(1)1650 1050 y Fy(;)j(:)g(:)g(:)d(;)j(a)1776 1057 y Fi(n)1798 1050 y FE(\))24 b(=)75 1106 y Fy(a)99 1113 y Fj(2)119 1106 y Fy(;)8 b(:)g(:)g(:)d(;)j(a)245 1113 y Fi(n)267 1106 y Fy(;)g(a)312 1113 y Fj(1)331 1106 y FE(,)20 b(and)f Fy(unshuf)5 b(f)g(l)q(e)p FE(\()p Fy(a)714 1113 y Fj(1)734 1106 y Fy(;)j(:)g(:)g(:)d(;)j(a)860 1113 y Fi(n)882 1106 y FE(\))19 b(=)g Fy(a)997 1113 y Fi(n)1020 1106 y Fy(;)8 b(a)1065 1113 y Fj(1)1084 1106 y Fy(;)g(:)g(:)g(:)d(;)j(a)1210 1113 y Fi(n)p Fk(\000)p Fj(1)1278 1106 y FE(.)31 b(The)19 b(graph)f(adjacency)i(list)f(is)75 1163 y(illustrated)e(b)q(elo)o(w)e (for)g Fy(n)e FE(=)g(3)p 75 1169 930 2 v 74 1225 2 57 v 133 1208 a Fz(no)q(de)p 300 1225 V 84 w(exc)o(hange)50 b(sh)o(u\017e)e(unsh)o(u\017e)p 1004 1225 V 75 1227 930 2 v 74 1283 2 57 v 100 1266 a FE(0)h(\(000\))p 300 1283 V 141 w(1)203 b(0)d(0)p 1004 1283 V 74 1340 V 100 1323 a(1)49 b(\(001\))p 300 1340 V 141 w(0)203 b(2)d(4)p 1004 1340 V 74 1396 V 100 1379 a(2)49 b(\(010\))p 300 1396 V 141 w(3)203 b(4)d(1)p 1004 1396 V 74 1453 V 100 1436 a(3)49 b(\(011\))p 300 1453 V 141 w(2)203 b(6)d(5)p 1004 1453 V 74 1509 V 100 1492 a(4)49 b(\(100\))p 300 1509 V 141 w(5)203 b(1)d(2)p 1004 1509 V 74 1566 V 100 1549 a(5)49 b(\(101\))p 300 1566 V 141 w(4)203 b(3)d(6)p 1004 1566 V 74 1622 V 100 1605 a(6)49 b(\(110\))p 300 1622 V 141 w(7)203 b(5)d(3)p 1004 1622 V 74 1678 V 100 1661 a(7)49 b(\(111\))p 300 1678 V 141 w(6)203 b(7)d(7)p 1004 1678 V 75 1680 930 2 v 166 1716 a(Eac)o(h)15 b(pro)q(cess)g(stores)g(a) g(real)g(n)o(um)o(b)q(er)h Fp(A)p FE(.)75 1814 y Fp(C)48 b(extract)22 b(neighborhood)h(information)218 1870 y(CALL)g (MPI_COMM_RANK\(comm,)f(myrank,)h(ierr\))218 1926 y(CALL)g (MPI_GRAPH_NEIGHBORS\(comm,)e(myrank,)i(3,)g(neighbors,)g(ierr\))75 1983 y(C)h(perform)f(exchange)f(permutation)218 2039 y(CALL)h(MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e(neighbors\(1\),)h (0,)337 2096 y(neighbors\(1\),)g(0,)g(comm,)g(status,)g(ierr\))75 2152 y(C)h(perform)f(shuffle)g(permutation)218 2209 y(CALL)g (MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e(neighbors\(2\),)h(0,)337 2265 y(neighbors\(3\),)g(0,)g(comm,)g(status,)g(ierr\))75 2322 y(C)h(perform)f(unshuffle)f(permutation)218 2378 y(CALL)h(MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e(neighbors\(3\),)h (0,)337 2435 y(neighbors\(2\),)g(0,)g(comm,)g(status,)g(ierr\))75 2560 y Fq(6.5.2)49 b(Ca)o(rtesian)16 b(shift)g(co)q(o)o(rdinates)75 2647 y FE(If)i(the)f(pro)q(cess)h(top)q(ology)f(is)h(a)f(Cartesian)g (structure,)h(a)f(collectiv)o(e)i(shift)f(op)q(eration)g(is)g(most)e (lik)o(ely)75 2704 y(to)g(b)q(e)g(used)h(along)f(a)g(co)q(ordinate)h (direction.)24 b(In)17 b(MPI)f(shift)g(op)q(erations)h(are)e(handled)j (b)o(y)e(the)g(more)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 148 153 148 152 bop 75 -100 a FE(148)882 b Ft(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y FE(general)20 b(send{receiv)o(e)i (functions.)35 b(As)20 b(input)h(they)f(tak)o(e)f(the)h(rank)g(of)f(a)h (source)g(pro)q(cess)g(for)g(the)75 102 y(receiv)o(e,)c(and)f(the)g (rank)g(of)g(a)g(destination)h(pro)q(cess)g(for)e(the)h(send.)166 171 y(If)j(the)g(function)h Fx(MPI)p 561 171 14 2 v 16 w(CART)p 696 171 V 17 w(SHIFT)f FE(is)g(called)i(for)d(a)h(Cartesian)g (pro)q(cess)g(group,)g(it)g(pro)o(vides)75 228 y(the)c(calling)i(pro)q (cess)e(with)h(the)f(ab)q(o)o(v)o(e)g(iden)o(ti\014ers,)h(whic)o(h)g (then)g(can)f(b)q(e)h(passed)f(to)f(the)i(send{receiv)o(e)75 284 y(function.)21 b(The)15 b(user)g(sp)q(eci\014es)i(the)e(co)q (ordinate)h(direction)g(for)f(the)g(shift)h(and)f(the)g(step)g(length.) 21 b(The)75 341 y(function)16 b(is)g(lo)q(cal.)75 458 y Fx(MPI)p 160 458 V 16 w(CART)p 295 458 V 17 w(SHIFT\(comm,)c (direction,)k(disp,)g(rank)p 977 458 V 16 w(source,)g(rank)p 1224 458 V 16 w(dest\))117 548 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(for)j(group)g(with)f(Cartesian)h(structure) 905 604 y(\(handle\))117 706 y(IN)171 b Fx(direction)408 b Fr(co)q(ordinate)14 b(dimension)f(of)g(shift)h(\(in)o(teger\))117 808 y(IN)171 b Fx(disp)497 b Fr(shift)18 b(amoun)o(t)f(\()p Fa(>)i Fr(0:)26 b(up)o(w)o(ards)19 b(shift,)g Fa(<)g Fr(0:)26 b(do)o(wn)o(w)o(ards)905 864 y(shift\))14 b(\(in)o(teger\))117 966 y(OUT)124 b Fx(rank)p 420 966 V 16 w(source)353 b Fr(rank)14 b(of)f(source)j(pro)q(cess)f(\(in)o(teger\))117 1067 y(OUT)124 b Fx(rank)p 420 1067 V 16 w(dest)396 b Fr(rank)14 b(of)f(destination)h(pro)q(cess)i(\(in)o(teger\))166 1205 y FE(Dep)q(ending)h(on)e(the)g(p)q(erio)q(dicit)o(y)i(of)e(the)g (Cartesian)g(group)g(in)h(the)f(sp)q(eci\014ed)i(co)q(ordinate)f (direc-)75 1262 y(tion,)d Fx(MPI)p 264 1262 V 16 w(CART)p 399 1262 V 17 w(SHIFT)f FE(pro)o(vides)h(the)g(iden)o(ti\014ers)h(for)e (a)h(circular)h(or)e(an)g(end{o\013)h(shift.)19 b(In)13 b(the)g(case)75 1318 y(of)j(an)h(end{o\013)g(shift,)g(the)g(v)m(alue)h Fm(MPI)p 746 1318 13 2 v 14 w(PROCNULL)e FE(ma)o(y)g(b)q(e)i(returned)f (in)g Fx(rank)p 1478 1318 14 2 v 17 w(source)g FE(or)f Fx(rank)p 1771 1318 V 17 w(dest)p FE(,)75 1375 y(indicating)h(that)e (the)g(source)g(or)g(the)g(destination)h(for)f(the)g(shift)h(is)f(out)g (of)g(range.)166 1444 y(Example:)37 b(The)24 b(comm)o(unicator)f Fx(comm)f FE(is)i(asso)q(ciated)f(with)h(a)f(t)o(w)o(o-dimensional)i(p) q(erio)q(dic)75 1501 y(Cartesian)14 b(top)q(ology)l(.)19 b(A)14 b(t)o(w)o(o-dimensional)i(arra)o(y)c(of)i Fp(REAL)p FE(s)g(is)g(stored)g(one)g(elemen)o(t)h(p)q(er)g(pro)q(cess,)f(in)75 1557 y(v)m(ariable)j Fp(A)p FE(.)d(One)i(wishes)g(to)f(sk)o(ew)f(this)i (arra)o(y)l(,)e(b)o(y)h(shifting)h(column)g Fy(i)f FE(b)o(y)g Fy(i)p FE(.)75 1717 y Fp(....)75 1774 y(C)24 b(find)f(process)g(rank) 218 1830 y(CALL)g(MPI_COMM_RANK\(comm,)f(rank,)h(ierr,)g(ierr\)\))75 1887 y(C)h(find)f(Cartesian)g(coordinates)218 1943 y(CALL)g (MPI_CART_COORD\(comm,)f(rank,)h(coord,)g(ierr\))75 1999 y(C)h(compute)f(shift)g(source)g(and)g(destination)218 2056 y(CALL)g(MPI_CART_SHIFT\(comm,)f(2,)h(coord\(2\),)g(source,)g (dest,)g(ierr\))75 2112 y(C)h(skew)f(array)218 2169 y(CALL)g (MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e(dest,)i(0,)f(source,)g(0,) h(comm,)839 2225 y(status,)f(ierr\))75 2423 y Fq(6.5.3)49 b(P)o(a)o(rtitioning)17 b(of)g(Ca)o(rtesian)e(structures)75 2534 y FE(If)10 b(a)g(Cartesian)g(top)q(ology)g(has)g(b)q(een)h (created)f(with)h Fx(MPI)p 1046 2534 V 16 w(MAKE)p 1191 2534 V 16 w(CART)p FE(,)f(the)g(function)h Fx(MPI)p 1677 2534 V 16 w(CART)p 1812 2534 V 17 w(SUB)75 2591 y FE(can)f(b)q(e)h (used)g(to)e(partition)i(the)f(comm)o(unicator)g(group)g(in)o(to)g (subgroups)g(whic)o(h)h(form)e(lo)o(w)o(er{dimensional)75 2647 y(Cartesian)15 b(subsgrids,)h(and)f(built)i(for)e(eac)o(h)g (subgroup)h(a)f(comm)o(unicator)g(with)g(the)h(asso)q(ciated)f(sub-)75 2704 y(grid)h(Cartesian)e(top)q(ology)l(.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 149 154 149 153 bop 75 -100 a Ft(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FE(149)75 45 y Fx(MPI)p 160 45 14 2 v 16 w(CART)p 295 45 V 17 w(SUB\(comm,)13 b(remain)p 688 45 V 16 w(dims,)h(comm)p 936 45 V 15 w(new\))117 125 y Fr(IN)171 b Fx(comm)454 b Fr(comm)o(unicator)11 b(for)j(group)g(with)f(Cartesian)h(structure) 905 182 y(\(handle\))117 263 y(IN)171 b Fx(remain)p 466 263 V 15 w(dims)339 b Fr(the)15 b Fl(i)p Fr(-th)f(en)o(try)g(of)g Fm(remain)p 1341 263 13 2 v 14 w(dims)g Fr(sp)q(eci\014es)h(whether)h (the)e Fl(i)p Fr(-)905 319 y(th)c(dimnsion)f(is)h(k)o(ept)g(in)g(the)g (subgrid)h(\()p Fl(true)p Fr(\))e(or)h(is)g(dropp)q(ed)905 376 y(\()p Fl(false)p Fr(\))j(\(logical)g(v)o(ector\))117 457 y(OUT)124 b Fx(comm)p 454 457 14 2 v 14 w(new)366 b Fr(comm)o(unicator)14 b(con)o(taining)h(the)i(subgrid)g(whic)o(h)f (includes)905 513 y(the)f(calling)d(pro)q(cess)k(\(handle\))166 641 y FE(Example:)166 700 y(Assume)11 b(that)e Fx(MPI)p 508 700 V 16 w(MAKE)p 653 700 V 27 w(CART\(...,)h(comm\))e FE(has)j(de\014ned)h(a)e(2)q FC(\002)q FE(3)q FC(\002)q FE(4)f(grid.)18 b(Let)11 b Fp(remain)p 1766 700 15 2 v 17 w(dims)75 756 y(=)24 b(\(true,)f(false,)g(true\))p FE(.)k(Then)18 b(a)g(call)h(to)e Fx(MPI)p 1008 756 14 2 v 16 w(SUBP)l(ART\(comm,)h(remain)p 1520 756 V 15 w(dims,)g(comm)p 1771 756 V 15 w(new\))75 813 y FE(will)13 b(create)e(comm)o(unicators)f (for)h(three)g(new)h(groups)e(with)i(eigh)o(t)f(pro)q(cesses,)h(eac)o (h)f(with)h(a)f(2)r FC(\002)r FE(4)g(Carte-)75 869 y(sian)g(top)q (ology)l(.)18 b(If)10 b Fp(remain)p 550 869 15 2 v 16 w(dims)24 b(=)g(\(false,)e(false,)h(true\))10 b FE(then)g(the)h(call)g (to)e Fx(MPI)p 1603 869 14 2 v 16 w(SUBP)l(ART\(comm,)75 926 y(remain)p 207 926 V 15 w(dims,)15 b(comm)p 455 926 V 14 w(new\))g FE(will)i(create)d(comm)o(unicators)g(for)g(six)h(new)g (groups)g(with)g(four)f(pro)q(cesses,)75 982 y(eac)o(h)h(with)h(a)f (one-dimensional)i(Cartesian)e(top)q(ology)l(.)75 1121 y Fq(6.5.4)49 b(Lo)o(w{level)18 b(top)q(ology)g(functions)75 1212 y FE(The)d(t)o(w)o(o)f(new)h(functions)h(in)o(tro)q(duced)g(in)g (this)g(section)f(are)g(not)g(part)f(of)h(the)g(MPI)g(de\014nition.)21 b(Ho)o(w-)75 1269 y(ev)o(er,)15 b(they)g(can)g(b)q(e)h(used)g(to)f (implemen)o(t)h(all)g(other)f(top)q(ology)g(functions.)75 1376 y Fx(MPI)p 160 1376 V 16 w(MAP)p 275 1376 V 16 w(CART\(comm)p 545 1376 V 15 w(old,)g(ndims,)g(dims,)g(p)q(erio)q(d,)g(newrank\))117 1456 y Fr(IN)171 b Fx(comm)p 454 1456 V 14 w(old)383 b Fr(comm)o(unicator)11 b(for)j(paren)o(t)g(group)g(\(handle\))117 1537 y(IN)171 b Fx(ndims)460 b Fr(n)o(um)o(b)q(er)12 b(of)f(dimensions)h(of)f(Cartesian)i(structure)h(\(in)o(teger\))117 1618 y(IN)171 b Fx(dims)483 b Fr(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fl(ndims)e Fr(sp)q(ecifying)h(the)h(n)o(um)o(b)q(er)f(of)905 1674 y(pro)q(cesses)g(in)c(eac)o(h)i(co)q(ordinate)f(direction)117 1755 y(IN)155 b Fx(p)q(erio)q(ds)452 b Fr(logical)16 b(arra)o(y)h(of)g(size)h Fl(ndims)f Fr(sp)q(ecifying)g(the)h(p)q(erio)q (dicit)o(y)905 1812 y(sp)q(eci\014cation)d(in)e(eac)o(h)i(co)q (ordinate)f(direction)117 1893 y(OUT)124 b Fx(newrank)414 b Fr(reordered)11 b(rank)f(of)e(the)i(calling)e(pro)q(cess;)k Fm(MPI)p 1643 1893 13 2 v 15 w(UNDEFINED)905 1949 y Fr(if)h(calling)g (pro)q(cess)i(do)q(es)g(not)f(b)q(elong)f(to)h(grid)g(\(in)o(teger\)) 166 2077 y FE(This)k(function)g(computes)g(an)g(optimal)g(placemen)o(t) g(for)f(the)g(calling)j(pro)q(cess)e(on)f(the)h(ph)o(ysical)75 2133 y(mac)o(hine.)31 b(A)19 b(p)q(ossible)i(implemen)o(tation)f(of)e (this)h(function)h(is)f(to)f(alw)o(a)o(ys)g(return)h(the)g(rank)g(of)f (the)75 2190 y(calling)f(pro)q(cess)e({)g(i.e.,)g(not)g(to)g(p)q (erform)f(an)o(y)h(reordering.)166 2249 y(The)k(function)g Fx(MPI)p 529 2249 14 2 v 16 w(MAKE)p 674 2249 V 16 w(CART\(comm)p 944 2249 V 15 w(old,)g(ndims,)g(dims,)g(p)q(erio)q(ds,)i(reo)o(rder,)c (comm)p 1760 2249 V 15 w(ca)o(rt\))p FE(,)75 2305 y(with)j Fp(reorder)j(=)g(true)c FE(can)h(b)q(e)g(implemen)o(ted)h(b)o(y)f (calling)h Fx(MPI)p 1264 2305 V 16 w(MAP)p 1379 2305 V 16 w(CART\(comm)p 1649 2305 V 15 w(old,)f(ndims,)75 2362 y(dims,)15 b(p)q(erio)q(d,)h(newrank\))p FE(,)g(then)g(calling)i Fx(MPI)p 876 2362 V 15 w(COMM)p 1033 2362 V 17 w(SPLIT\(comm,)13 b(colo)o(r,)h(k)o(ey)l(,)h(comm)p 1656 2362 V 15 w(ca)o(rt\))p FE(,)g(with)75 2418 y Fp(color)23 b(=)h(0)17 b FE(if)h Fp(newrank)23 b FC(6)p FE(=)h Fp(MPI)p 676 2418 15 2 v 17 w(UNDEFINED)p FE(,)16 b Fp(color)23 b(=)h(MPI)p 1201 2418 V 17 w(UNDEFINED)16 b FE(otherwise,)i(and)g Fp(key)23 b(=)75 2475 y(newrank)p FE(.)166 2534 y(The)17 b(function)g Fx(MPI)p 525 2534 14 2 v 16 w(SUBCART\(comm,)f(remain)p 1039 2534 V 15 w(dims,)g(comm)p 1288 2534 V 15 w(new\))h FE(can)g(b)q(e)g(implemen)o(ted)h(b)o(y)75 2591 y(a)d(call)i(to)e Fx(MPI)p 338 2591 V 15 w(COMM)p 495 2591 V 17 w(SPLIT\(comm,)d(colo)o (r,)i(k)o(ey)l(,)i(comm)p 1118 2591 V 14 w(new\))p FE(,)g(using)g(a)f (single)i(n)o(um)o(b)q(er)e(enco)q(ding)75 2647 y(of)g(the)g(lost)g (dimensions)i(as)e Fp(color)g FE(and)g(a)g(single)i(n)o(um)o(b)q(er)f (enco)q(ding)g(of)f(the)h(preserv)o(ed)f(dimensions)75 2704 y(as)g Fp(key)p FE(.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 150 155 150 154 bop 75 -100 a FE(150)882 b Ft(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)166 45 y FE(All)i(other)e(cartesian)h(top) q(ology)f(functions)h(can)g(b)q(e)h(implemen)o(ted)g(lo)q(cally)l(,)g (using)g(the)e(top)q(ology)75 102 y(information)g(that)g(is)h(cac)o (hed)f(with)h(the)f(comm)o(unicator.)166 158 y(The)g(corresp)q(onding)i (new)e(function)h(for)f(general)g(graph)g(structures)g(is)75 262 y Fx(MPI)p 160 262 14 2 v 16 w(MAP)p 275 262 V 16 w(GRAPH\(comm)p 576 262 V 16 w(old,)g(nno)q(des,)h(index,)g(edges,)g (newrank\))117 339 y Fr(IN)171 b Fx(comm)p 454 339 V 14 w(old)383 b Fr(comm)o(unicator)11 b(for)j(paren)o(t)g(group)g (\(handle\))117 414 y(IN)171 b Fx(nno)q(des)441 b Fr(n)o(um)o(b)q(er)13 b(of)h(graph)f(no)q(des)i(\(in)o(teger\))117 489 y(IN)171 b Fx(index,)15 b(edges)342 b Fr(in)o(teger)10 b(arra)o(ys)f(sp)q (ecifying)h(the)f(graph)h(structure,)h(see)g Fm(MPI)p 1852 489 13 2 v 14 w(MAKE)p 1984 489 V 15 w(GRAPH)117 564 y Fr(OUT)124 b Fx(newrank)414 b Fr(reordered)11 b(rank)f(of)e(the)i (calling)e(pro)q(cess;)k Fm(MPI)p 1643 564 V 15 w(UNDEFINED)905 621 y Fr(if)j(the)i(calling)d(pro)q(cess)k(do)q(es)e(not)g(b)q(elong)g (to)f(graph)h(\(in)o(te-)905 677 y(ger\))166 802 y FE(The)f(function)g Fx(MPI)p 521 802 14 2 v 16 w(MAKE)p 666 802 V 16 w(GRAPH\(comm)p 967 802 V 16 w(old,)f(nno)q(des,)i(index,)g(edges,)f(reo)o(rder,)e (comm)p 1760 802 V 15 w(ca)o(rt\))p FE(,)75 858 y(with)g Fp(reorder)23 b(=)h(true)12 b FE(can)h(b)q(e)h(implemen)o(ted)h(b)o(y)e (calling)i Fx(MPI)p 1219 858 V 15 w(MAP)p 1333 858 V 17 w(GRAPH\(comm)p 1635 858 V 15 w(old,)e(nno)q(des,)75 915 y(index,)18 b(edges,)h(newrank\))e FE(then)h(calling)h Fx(MPI)p 868 915 V 16 w(COMM)p 1026 915 V 16 w(SPLIT\(comm,)c(colo)o (r,)h(k)o(ey)l(,)h(comm)p 1654 915 V 15 w(ca)o(rt\))p FE(,)g(with)75 971 y Fp(color)23 b(=)h(0)17 b FE(if)h Fp(newrank)23 b FC(6)p FE(=)h Fp(MPI)p 676 971 15 2 v 17 w(UNDEFINED)p FE(,)16 b Fp(color)23 b(=)h(MPI)p 1201 971 V 17 w(UNDEFINED)16 b FE(otherwise,)i(and)g Fp(key)23 b(=)75 1028 y(newrank)p FE(.)166 1084 y(All)f(other)f(graph)g(top)q (ology)g(functions)h(can)g(b)q(e)g(implemen)o(ted)h(lo)q(cally)l(,)h (using)e(the)f(top)q(ology)75 1140 y(information)15 b(that)g(is)h(cac)o (hed)f(with)h(the)f(comm)o(unicator.)75 1284 y Fv(6.6)59 b(A)20 b(simple)f(example)75 1385 y FE(The)12 b(example)h(in)g (\014gure)f(6.1)f(sho)o(ws)g(ho)o(w)h(the)g(grid)g(de\014nition)i(and)e (inquiry)i(functions)f(can)f(b)q(e)h(used)f(in)75 1442 y(an)k(application)h(program.)k(A)16 b(partial)g(di\013eren)o(tial)h (equation,)f(for)f(example)h(the)g(P)o(oisson)g(equation,)75 1498 y(is)g(to)f(b)q(e)h(solv)o(ed)f(on)h(a)f(rectangular)g(domain.)21 b(First)15 b(the)g(pro)q(cesses)h(organize)f(themselv)o(es)h(in)h(a)e (t)o(w)o(o{)75 1554 y(dimensional)h(structure.)k(Eac)o(h)14 b(pro)q(cess)h(then)f(inquires)j(for)c(the)i(ranks)f(of)g(its)g(neigh)o (b)q(ors)i(in)f(the)f(four)75 1611 y(directions.)29 b(The)18 b(n)o(umerical)h(problem)g(is)f(solv)o(ed)h(b)o(y)f(an)f(iterativ)o(e)h (metho)q(d,)h(the)f(details)h(of)e(whic)o(h)75 1667 y(are)e(hidden)i (in)f(the)f(subroutine)h Fp(relax)p FE(.)166 1724 y(In)e(eac)o(h)f (relaxation)g(step)g(eac)o(h)g(pro)q(cess)h(computes)f(new)g(v)m(alues) h(for)f(the)g(solution)h(grid)f(function)75 1780 y(at)i(all)i(p)q(oin)o (ts)f(o)o(wned)g(b)o(y)f(the)h(pro)q(cess.)22 b(Then)16 b(the)g(v)m(alues)h(at)e(in)o(terpro)q(cess)i(b)q(oundaries)g(ha)o(v)o (e)e(to)g(b)q(e)75 1837 y(exc)o(hanged)d(with)h(neigh)o(b)q(oring)g (pro)q(cesses.)19 b(F)l(or)11 b(example,)i(the)f(exc)o(hange)g (subroutine)h(migh)o(t)f(con)o(tain)75 1893 y(a)i(call)h(lik)o(e)g Fx(MPI)p 361 1893 14 2 v 15 w(SEND\(...,neigh)p 663 1893 V 16 w(rank\(1\),...\))i FE(to)c(send)i(up)q(dated)g(v)m(alues)g(to)e (the)h(left{hand)g(neigh)o(b)q(or)75 1950 y(\(i-1,j\).)19 b(Note)c(that)g(there)g(is)h(only)f(lo)q(cal)i(comm)o(unication)f(in)g (this)f(exc)o(hange)h(op)q(eration.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 151 156 151 155 bop 75 -100 a Ft(6.6.)34 b(A)15 b(SIMPLE)h(EXAMPLE)1157 b FE(151)252 163 y(in)o(teger)15 b(ndims,)h(n)o(um)p 639 163 14 2 v 16 w(neigh)252 219 y(logical)g(reorder)252 276 y(parameter)e(\(ndims=2,)i(n)o(um)p 782 276 V 16 w(neigh=4,)g(reorder=.true.\))252 332 y(in)o(teger)f(comm,)g(comm)p 671 332 V 16 w(cart,)f(dims\(ndims\),)i(neigh)p 1174 332 V 17 w(def\(2\),)e(neigh)p 1441 332 V 18 w(rank\(n)o(um)p 1654 332 V 16 w(neigh\))252 388 y(in)o(teger)h(o)o(wn)p 485 388 V 16 w(p)q(osition\(ndims\),)h(i,)g(j,)e(rank)252 445 y(logical)i(p)q(erio)q(d\(ndims\))252 501 y(real)p FC(\003)p FE(8)f(u\(0:101,0:101)o(\),)d(f\(0:101,0:10)o(1\))252 558 y(data)i(dims)i(/)f(ndims)h FC(\003)f FE(0)g(/)252 614 y(comm)g(=)g(MPI)p 528 614 V 16 w(COMM)p 696 614 V 16 w(W)o(ORLD)75 671 y(C)89 b(Set)15 b(pro)q(cess)h(grid)f(size)i (and)e(p)q(erio)q(dicit)o(y)252 727 y(call)h(MPI)p 427 727 V 16 w(MAKDIM\(comm,)f(ndims,)g(dims\))252 784 y(p)q(erio)q(d\(1\)) g(=)h(.TR)o(UE.)252 840 y(p)q(erio)q(d\(2\))f(=)h(.TR)o(UE.)75 897 y(C)89 b(Create)15 b(a)g(grid)g(structure)g(in)h(W)o(ORLD)g(group)f (and)g(inquire)i(ab)q(out)e(o)o(wn)g(p)q(osition)252 953 y(call)h(MPI)p 427 953 V 16 w(MAKE)p 585 953 V 17 w(CAR)l(T)f(\(comm,)g(ndims,)g(dims,)h(p)q(erio)q(d,)g(reorder,)f (rank,)f(comm)p 1739 953 V 16 w(cart\))252 1009 y(call)i(MPI)p 427 1009 V 16 w(GA)l(T)p 542 1009 V 17 w(CAR)l(T)f(\(comm)p 840 1009 V 16 w(cart,)f(ndims,)i(dims,)f(p)q(erio)q(d,)h(o)o(wn)p 1464 1009 V 16 w(p)q(osition\))75 1066 y(C)89 b(Lo)q(ok)15 b(up)h(the)f(ranks)g(for)g(the)g(neigh)o(b)q(ors.)21 b(Own)15 b(pro)q(cess)h(co)q(ordinates)f(are)g(\(i,j\).)75 1122 y(C)89 b(Neigh)o(b)q(ors)16 b(are)f(\(i{1,j\),)f(\(i+1,j\),)g (\(i,j{1\),)g(\(i,j+1\))252 1179 y(i)i(=)f(o)o(wn)p 413 1179 V 16 w(p)q(osition\(1\))252 1235 y(j)g(=)g(o)o(wn)p 414 1235 V 16 w(p)q(osition\(2\))252 1292 y(neigh)p 360 1292 V 17 w(def\(1\))g(=)g(i{1)252 1348 y(neigh)p 360 1348 V 17 w(def\(2\))g(=)g(j)252 1405 y(call)h(MPI)p 427 1405 V 16 w(CAR)l(T)p 572 1405 V 17 w(RANK)g(\(comm)p 878 1405 V 16 w(cart,)e(neigh)p 1105 1405 V 18 w(def,)h(neigh)p 1315 1405 V 17 w(rank\(1\)\))252 1461 y(neigh)p 360 1461 V 17 w(def\(1\))g(=)g(i+1)252 1518 y(neigh)p 360 1518 V 17 w(def\(2\))g(=)g(j)252 1574 y(call)h(MPI)p 427 1574 V 16 w(CAR)l(T)p 572 1574 V 17 w(RANK)g(\(comm)p 878 1574 V 16 w(cart,)e(neigh)p 1105 1574 V 18 w(def,)h(neigh)p 1315 1574 V 17 w(rank\(2\)\))252 1630 y(neigh)p 360 1630 V 17 w(def\(1\))g(=)g(i)252 1687 y(neigh)p 360 1687 V 17 w(def\(2\))g(=)g(j{1)252 1743 y(call)h(MPI)p 427 1743 V 16 w(CAR)l(T)p 572 1743 V 17 w(RANK)g(\(comm)p 878 1743 V 16 w(cart,)e(neigh)p 1105 1743 V 18 w(def,)h(neigh)p 1315 1743 V 17 w(rank\(3\)\))252 1800 y(neigh)p 360 1800 V 17 w(def\(1\))g(=)g(i)252 1856 y(neigh)p 360 1856 V 17 w(def\(2\))g(=)g(j+1)252 1913 y(call)h(MPI)p 427 1913 V 16 w(CAR)l(T)p 572 1913 V 17 w(RANK)g(\(comm)p 878 1913 V 16 w(cart,)e(neigh)p 1105 1913 V 18 w(def,)h(neigh)p 1315 1913 V 17 w(rank\(4\)\))75 1969 y(C)89 b(Initialize)18 b(the)e(grid)f(functions)h(and)g(start)d(the)j(iteration)252 2026 y(call)g(init)g(\(u,)f(f)t(\))252 2082 y(do)g(10)g(it=1,100)322 2139 y(call)i(relax)e(\(u,)g(f)t(\))75 2195 y(C)89 b(Exc)o(hange)15 b(data)g(with)g(neigh)o(b)q(or)h(pro)q(cesses)322 2251 y(call)h(exc)o(hange)e(\(u,)g(comm)p 793 2251 V 16 w(cart,)f(neigh)p 1020 2251 V 18 w(rank,)g(n)o(um)p 1242 2251 V 17 w(neigh\))75 2308 y(10)131 b(con)o(tin)o(ue)252 2364 y(call)16 b(output)f(\(u\))252 2421 y(end)147 2625 y(Figure)g(6.1:)k(Set{up)d(of)e(pro)q(cess)i (structure)f(for)f(t)o(w)o(o{dimensional)i(parallel)h(P)o(oisson)e (solv)o(er.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 152 157 152 156 bop 75 359 a Fw(Chapter)34 b(7)75 570 y FA(MPI)40 b(En)m(vironmen)m(tal)h(Managemen)m(t)75 796 y FE(This)15 b(c)o(hapter)e(discusses)j(routines)e(for)g(getting)f(and,)i(where)f (appropriate,)g(setting)g(v)m(arious)g(parame-)75 852 y(ters)h(that)g(relate)g(to)g(the)g(MPI)h(implemen)o(tation)g(and)g (the)f(execution)i(en)o(vironmen)o(t.)j(\(suc)o(h)c(as)f(error)75 908 y(handling)i(and)f(comm)o(unication)g(bu\013ering\).)21 b(The)16 b(pro)q(cedures)g(for)f(en)o(tering)h(and)g(lea)o(ving)g(the)g (MPI)75 965 y(execution)g(en)o(vironmen)o(t)g(are)f(also)g(dexcrib)q (ed)i(here.)75 1125 y Fv(7.1)59 b(Implementation)17 b(info)n(rmation)75 1234 y Fq(7.1.1)49 b(Environmental)17 b(Inquiries)75 1326 y FE(The)e(follo)o(wing)h(routines)g(allo)o(w)f(to)g(inquire)i(on) e(the)g(execution)i(en)o(vironmen)o(t)e(of)g(an)g(MPI)g(program.)75 1433 y Fx(MPI)p 160 1433 14 2 v 16 w(GET)p 264 1433 V 17 w(V)l(ALID)p 408 1433 V 16 w(T)l(A)o(G)p 510 1433 V 17 w(RANGE)h(\()f(lo)o(w,)f(high)i(\))117 1513 y Fr(OUT)108 b Fx(lo)o(w)523 b Fr(Smallest)12 b(v)n(alid)h(tag)g(v)n(alue)117 1594 y(OUT)108 b Fx(high)507 b Fr(Largest)15 b(v)n(alid)d(tag)i(v)n (alue)75 1722 y Fp(int)23 b(MPI)p 245 1722 15 2 v 17 w(GET)p 334 1722 V 17 w(VALID)p 471 1722 V 16 w(TAG)p 559 1722 V 17 w(RANGE\(int)g(*low,)g(int)h(*high\))75 1811 y(MPI)p 150 1811 V 17 w(GET)p 239 1811 V 17 w(VALID)p 376 1811 V 16 w(TAG)p 464 1811 V 17 w(RANGE\(LOW,)e(HIGH,)h(IERROR\)) 170 1868 y(INTEGER)g(LOW,)h(HIGH,)f(IERROR)75 1957 y FE(This)16 b(routine)g(gets)f(the)g(range)g(of)g(v)m(alid)i(tags;)e (that)f(is,)i(the)f(user)h(ma)o(y)f(use)h(an)o(y)f(tag)f(v)m(alue)j (from)d Fp(low)75 2014 y FE(to)h Fp(high)f FE(inclusiv)o(e.)75 2120 y Fx(MPI)p 160 2120 14 2 v 16 w(GET)p 264 2120 V 17 w(PROCESSOR)p 540 2120 V 18 w(NAME\()i(len,)f(name)g(\))117 2201 y Fr(IN)155 b Fx(len)533 b Fr(Length)14 b(\(in)g(c)o(haracters\))i (of)d(the)i(space)g(in)e Fl(name)117 2282 y Fr(OUT)108 b Fx(name)485 b Fr(A)17 b(unique)g(sp)q(eci\014er)i(for)e(the)g(actual) g(\(as)g(opp)q(osed)h(to)e(vir-)905 2338 y(tual\))e(no)q(de.)75 2466 y Fp(int)23 b(MPI)p 245 2466 15 2 v 17 w(GET)p 334 2466 V 17 w(PROCESSOR)p 567 2466 V 16 w(NAME\(int)g(len,)g(char)g (*name\))75 2555 y(MPI)p 150 2555 V 17 w(GET)p 239 2555 V 17 w(PROCESSOR)p 472 2555 V 15 w(NAME\(LEN,)g(NAME,)g(IERROR\))170 2612 y(INTEGER)g(LEN,)h(NAME,)f(IERROR)p 75 2661 720 2 v 127 2688 a FB(1)144 2704 y Fu(V)m(ersion)14 b(as)f(of)g(Octob)q(er) g(14,)g(1993)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 153 158 153 157 bop 75 -100 a Ft(7.1.)34 b(IMPLEMENT)l(A)l(TION)16 b(INF)o(ORMA)l(TION)841 b FE(153)166 45 y(The)19 b(name)g(is)g(a)f(c)o (haracter)g(string)h(for)f(maxim)o(um)h(\015exibilit)o(y)l(.)33 b(F)l(rom)18 b(this)h(v)m(alue)h(it)f(m)o(ust)f(b)q(e)75 102 y(p)q(ossible)23 b(to)d(iden)o(tify)i(a)e(sp)q(eci\014c)j(piece)f (of)e(hardw)o(are;)j(p)q(ossible)f(v)m(alues)g(include)h(\\pro)q (cessor)e(9)f(in)75 158 y(rac)o(k)f(4)h(of)f(mpp.cs.org")g(and)h (\\231")e(\(where)i(231)f(is)h(the)g(actual)g(pro)q(cessor)g(n)o(um)o (b)q(er)g(in)h(the)f(run-)75 214 y(ning)f(homogeneous)f(system\).)29 b(A)18 b(error)g(co)q(de)h(of)e Fm(MPI)p 1062 214 13 2 v 15 w(UNDEFINED)g FE(is)i(allo)o(w)o(ed)g(but)f(strongly)g(dis-)75 271 y(couraged.)31 b(The)19 b(v)m(alue)h(of)e Fm(MPI)p 646 271 V 14 w(MAX)p 752 271 V 15 w(PROCESSOR)p 1008 271 V 13 w(NAME)h FE(is)g(the)g(maxim)o(um)g(length)g(required)h(b)o(y) 75 327 y Fx(MPI)p 160 327 14 2 v 16 w(GET)p 264 327 V 17 w(PROCESSOR)p 540 327 V 18 w(NAME)c FE(\(including,)h(in)f(the)f(C)g (binding,)i(the)e(terminating)h(n)o(ull\).)75 459 y Fx(MPI)p 160 459 V 16 w(GET)p 264 459 V 17 w(HOST\()f(comm,)e(rank\))117 565 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(\(handle\))117 696 y(OUT)108 b Fx(rank)504 b Fr(rank)21 b(of)f Fl(HOST)g Fr(in)h(group)f(of)g Fl(comm)p Fr(;)j Fm(MPI)p 1604 696 13 2 v 15 w(PROCNULL)d Fr(if)905 752 y(there)15 b(is)f(no)g Fl(HOST)75 905 y Fp(int)23 b(MPI)p 245 905 15 2 v 17 w(GET)p 334 905 V 17 w(HOST\(MPI)p 543 905 V 16 w(Comm)g(comm,)h(int)f(*rank\))75 1020 y(MPI)p 150 1020 V 17 w(GET)p 239 1020 V 17 w(HOST\(COMM,)f(RANK,)h(IERROR\)) 170 1076 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)166 1191 y FE(This)17 b(routines)g(gets)f(the)g(rank)g(of)g(the)h Fp(HOST)e FE(pro)q(cess)i(in)g(the)g(group)f(asso)q(ciated)g(with)h (comm)o(u-)75 1247 y(nicator)h Fp(comm)p FE(,)g(if)g(there)g(is)h(suc)o (h.)28 b(MPI)18 b(do)q(es)h(not)e(sp)q(ecify)i(what)f(it)g(means)g(for) f(a)h(pro)q(cess)g(to)g(b)q(e)g(a)75 1304 y Fp(HOST)p FE(,)c(nor)h(do)q(es)h(it)f(requires)h(that)f(a)f Fp(HOST)h FE(exists.)75 1586 y Fq(7.1.2)49 b(Bu\013ering)75 1727 y FE(While)15 b(it)g(is)f(p)q(ossible)i(to)d(design)i(algorithms)f (that)f(do)h(not)f(require)i(that)e(an)o(y)h(bu\013ering)h(b)q(e)f(pro) o(vided)75 1783 y(for)e(sending)i(and)e(receiving)i(\\blo)q(c)o(king")g (messages,)e(man)o(y)g(practitioners)h(exp)q(ect)g(that)f(the)g (message-)75 1839 y(passing)k(system)e(will)j(pro)o(vide)f(some)f (bu\013ering.)21 b(That)15 b(is,)g(there)g(is)h(often)f(an)g(exp)q (ectation)h(that)f(the)75 1896 y(program)f(fragmen)o(t,)g(executed)i(b) o(y)f(all)h(tasks,)75 2130 y Fp(MPI_send\()23 b(buf,)g(size,)g (datatype,)g(nbr,)g(tag,)g(comm)h(\))75 2187 y(MPI_recv\()f(buf,)g (size,)g(datatype,)g(nbr,)g(tag,)g(comm)h(\))75 2393 y FE(will)17 b(execute)f(for)f(messages)g(whose)h Fm(size)g FE(is)g(not)f(to)q(o)g(large.)21 b(The)15 b(purp)q(ose)i(of)e(this)h (section)g(is)g(to)f(giv)o(e)75 2450 y(the)j(user)g(a)g(w)o(a)o(y)e(to) i(determine)h(what)e(\\to)q(o)g(large")g(is)i(and)f(to)f(help)i(the)f (implemen)o(tation)h(pro)o(vide)75 2506 y(enough)d(bu\013ering)f(for)g (the)g(user's)g(need.)166 2591 y(Note)f(that)g(nothing)h(in)g(this)g (section)g Fs(r)n(e)n(quir)n(es)i FE(an)e(implemen)o(tation)h(to)e(pro) o(vide)h(an)o(y)f(bu\013ering;)75 2647 y(that)f(is,)h(an)g(implemen)o (tation)h(ma)o(y)f(de\014ne)h(the)f(ab)q(o)o(v)o(e)f(program)g(fragmen) o(t)g(to)g(b)q(e)h(erroneous)g(for)f(an)o(y)75 2704 y(p)q(ositiv)o(e)j (v)m(alue)h(of)d Fm(size)p FE(.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 154 159 154 158 bop 75 -100 a FE(154)532 b Ft(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fx(General)g(bu\013ering)f(pa)o(rameters)75 178 y(MPI)p 160 178 14 2 v 16 w(GET)p 264 178 V 17 w(BUFFER)p 450 178 V 17 w(P)l(ARAMS\()d(numsend,)i(sizesend,)g(numrecv,)e(sizerecv,)g (totalnumsednd,)i(totalsize-)75 235 y(sen,)i(totalnumrecv,)f (totalsizerecv\))117 312 y Fr(OUT)108 b Fx(numsend)421 b Fr(Bound)10 b(on)g(the)g(n)o(um)o(b)q(er)f(of)g(p)q(ending)h (messages)g(with)g(source)905 368 y(at)k(calling)e(pro)q(cess)k(and)e (same)f(destination)117 442 y(OUT)108 b Fx(sizesend)436 b Fr(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 498 y(with)f(source)h(at)f(calling)e(pro)q(cess) k(and)e(same)f(destination)117 571 y(OUT)108 b Fx(numrecv)427 b Fr(Bound)16 b(on)e(the)i(n)o(um)o(b)q(er)e(of)g(p)q(ending)i (messages)f(with)f(des-)905 628 y(tination)f(at)h(calling)e(pro)q(cess) k(and)e(same)f(source)117 701 y(OUT)108 b Fx(sizerecv)442 b Fr(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 758 y(with)f(destination)f(at)h(calling)f(pro)q (cess)i(and)f(same)f(source)117 831 y(OUT)108 b Fx(totalnumsend)333 b Fr(Bound)10 b(on)g(the)g(n)o(um)o(b)q(er)f(of)g(p)q(ending)h (messages)g(with)g(source)905 888 y(at)k(the)g(calling)f(pro)q(cess)j (and)d(an)o(y)h(destination)117 961 y(OUT)108 b Fx(totalsizesend)348 b Fr(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 1017 y(with)d(source)i(at)f(the)g(calling)e(pro) q(cess)k(and)d(an)o(y)g(destination)117 1091 y(OUT)108 b Fx(totalnumrecv)339 b Fr(Bound)16 b(on)f(the)h(n)o(um)o(b)q(er)f(of)g (p)q(ending)g(messages)h(from)d(an)o(y)905 1147 y(source)i(with)f(the)g (calling)f(pro)q(cess)j(as)e(destination)117 1221 y(OUT)108 b Fx(totalsizerecv)354 b Fr(Bound)15 b(on)f(the)h(total)f(size)h(in)f (b)o(ytes)h(of)f(p)q(ending)h(messages)905 1277 y(from)d(an)o(y)i (source)h(with)e(destination)h(at)g(the)g(calling)f(no)q(de)75 1402 y Fp(int)23 b(MPI)p 245 1402 15 2 v 17 w(GET)p 334 1402 V 17 w(BUFFER)p 495 1402 V 16 w(PARAMS\(int)g(*numsend,)f(int)i (*sizesend,)e(int)i(*numrecv,)393 1458 y(int)g(*sizerecv,)e(int)i (*totalnumsednd,)d(int)j(*totalsizesen,)393 1514 y(int)g (*totalnumrecv,)e(int)h(*totalsizerecv\))75 1601 y(MPI)p 150 1601 V 17 w(GET)p 239 1601 V 17 w(BUFFER)p 400 1601 V 16 w(PARAMS\(NUMSEND,)f(SIZESEND,)g(NUMRECV,)h(SIZERECV,)g (TOTALNUMSEDND,)393 1657 y(TOTALSIZESEN,)f(TOTALNUMRECV,)g (TOTALSIZERECV,)g(IERROR\))170 1714 y(INTEGER)h(NUMSEND,)g(SIZESEND,)g (NUMRECV,)g(SIZERECV,)f(TOTALNUMSEDND,)170 1770 y(TOTALSIZESEN,)h (TOTALNUMRECV,)f(TOTALSIZERECV,)g(IERROR)166 1857 y FE(A)e(program)f (where)h(at)g(an)o(y)g(p)q(oin)o(t)g(in)h(time)g(no)f(pro)q(cess)g(has) g(more)g(then)g Fx(numsend)i FE(p)q(ending)75 1913 y(messages)13 b(sen)o(t)g(b)o(y)g(it)h(to)f(the)h(same)f(destination,)h(no)f(more)g (than)h Fx(sizesend)h FE(b)o(ytes)e(of)g(data)g(in)h(p)q(ending)75 1970 y(messages)h(sen)o(t)g(b)o(y)h(it)f(to)g(the)h(same)f (destination,)h(no)f(more)g(than)h Fx(totalnumsend)h FE(p)q(ending)g(messages)75 2026 y(sen)o(t)d(b)o(y)h(it)g(to)e(an)o(y)i (destination,)g(no)f(more)g(than)h Fx(totalsizesend)h FE(b)o(ytes)f(of)f(data)g(in)h(p)q(ending)h(messages)75 2083 y(sen)o(t)g(b)o(y)g(it)h(to)e(an)o(y)h(destination,)h(no)f(more)g (than)g Fx(numrecv)h FE(p)q(ending)h(messages)d(sen)o(t)h(to)g(it)g (from)g(the)75 2139 y(same)k(source,)h(no)e(more)h(than)g Fx(sizerecv)g FE(b)o(ytes)g(in)g(p)q(ending)i(messages)e(sen)o(t)f(to)h (it)g(from)f(the)h(same)75 2195 y(source,)g(no)f(more)g(than)g Fx(totalnumrecv)g FE(p)q(ending)i(messages)e(sen)o(t)g(to)g(it)g(from)g (an)o(y)g(source,)h(and)f(no)75 2252 y(more)c(than)g Fx(totalsizerecv)h FE(b)o(ytes)f(in)h(p)q(ending)h(messages)e(sen)o(t)g (to)f(it)i(from)e(an)o(y)h(source)h(is)f(guaran)o(teed)75 2308 y(no)g(to)g(deadlo)q(c)o(k)h(b)q(ecause)g(of)f(the)g(lac)o(k)g(of) g(bu\013er)g(space)h(for)e(p)q(ending)j(messages.)166 2365 y(Examples:)166 2421 y(1.)30 b(Supp)q(ose)20 b(that)e(p)q(ending)i (messages)f(are)f(alw)o(a)o(ys)g(bu\013ered)h(at)f(the)h(sender,)h(all) g(bu\013ers)e(are)75 2478 y(shared,)11 b(total)f(bu\013er)g(size)i(at)d (the)i(no)q(de)g(is)g Fp(B)p FE(,)e(eac)o(h)i(p)q(ending)h(message)e (tak)o(es)g Fp(h)g FE(b)o(ytes)g(of)g(storage,)g(in)h(ad-)75 2534 y(dition)j(to)e(the)h(storage)e(for)h(its)h(data,)f(and)h(no)g (storage)e(is)i(lost)g(due)g(to)f(fragmen)o(tation.)18 b(Then)13 b(p)q(ossible)75 2591 y(return)f(v)m(alues)h(for)f Fx(MPI)p 495 2591 14 2 v 16 w(GET)p 599 2591 V 16 w(BUFFER)p 784 2591 V 18 w(P)l(ARAMS)h FE(are)f Fp(numsend)23 b(=)g (totalnumsend-k,)f(sizesend)75 2647 y(=)i(totalsizesend)e(=)i(B-kh)p FE(,)10 b(and)g Fp(numrecv)23 b(=)h(totalnumrecv)e(=)i(sizerecv)e(=)i (totalsizerecv)75 2704 y(=)g(INT)p 198 2704 15 2 v 16 w(MAX)p FE(,)15 b(for)f(some)h Fp(0)e Fy(<)g Fp(k)f Fy(<)h Fp(B)p Fy(=)p Fp(h)p FE(.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 155 160 155 159 bop 75 -100 a Ft(7.1.)34 b(IMPLEMENT)l(A)l(TION)16 b(INF)o(ORMA)l(TION)841 b FE(155)166 45 y(2.)36 b(Supp)q(ose)22 b(that)e(messages)g(are)h(alw)o(a)o(ys)f(bu\013ered)h(at)f(the)h (receiv)o(er,)h(and)f(that)f(a)g(separate)75 102 y(bu\013er)15 b(with)h Fp(C)e FE(b)o(ytes)h(is)h(allo)q(cated)g(for)e(eac)o(h)i(pair) f(of)g(comm)o(unicating)h(pro)q(cesses;)f(en)o(tries)g(for)g(p)q(end-) 75 158 y(ing)j(messages)g(are)f(k)o(ept)h(in)g(a)g(seeparate)f(table,)h (with)h(ro)q(om)e(for)g(total)g(of)g Fp(m)h FE(p)q(ending)h(messages)f (at)75 214 y(eac)o(h)13 b(receiving)i(pro)q(cess;)e(the)g(total)g(n)o (um)o(b)q(er)g(of)g(pro)q(cesses)g(is)h Fp(p)p FE(,)e(and)i(no)f (storage)e(is)j(lost)f(due)g(to)g(frag-)75 271 y(men)o(tation.)23 b(Then)16 b(p)q(ossible)i(return)e(v)m(alues)i(for)d Fx(MPI)p 1035 271 14 2 v 16 w(GET)p 1139 271 V 17 w(BUFFER)p 1325 271 V 17 w(P)l(ARAMS)j FE(are)e Fp(sendcount)22 b(=)75 327 y(totalsendcount)g(=)i(sendsize)e(=)i(totalsendsize)e(=)i (INT)p 1200 327 15 2 v 17 w(MAX,)f(recvcount)g(=)g(totalrecvcount)75 384 y(=)h(m,)f(recsize)g(=)h(C)15 b FE(and)g Fp(totalrecvsize)22 b(=)i(\(p-1\))p FC(\002)p Fp(C)p FE(.)166 442 y(The)18 b(call)g(ma)o(y)f(return)g(the)h(v)m(alue)h Fm(MPI)p 867 442 13 2 v 14 w(UNDEFINED)e FE(in)h(an)o(y)f(parameter.)26 b(If)18 b(this)g(is)g(the)f(case,)75 498 y(then)f(the)f(MPI)g(implemen) o(tation)h(mak)o(es)f(no)g(guaran)o(tee)g(ab)q(out)g(bu\013ering.)166 556 y(Some)k(implemen)o(tations)h(ma)o(y)f(b)q(e)g(able)h(to)f(adjust)f (their)i(bu\013ering.)32 b(In)20 b(addition,)h(since)f(the)75 612 y(parameters)13 b(returned)i(b)o(y)f Fx(MPI)p 640 612 14 2 v 16 w(GET)p 744 612 V 17 w(BUFFER)p 930 612 V 17 w(P)l(ARAMS)h FE(represen)o(t)f(a)g(guaran)o(tee)g(ab)q(out)g(a)g (partic-)75 669 y(ular)g(c)o(hoice)h(of)e(n)o(um)o(b)q(er)h(of)g (bu\013ers,)f(it)i(ma)o(y)e(not)g(guaran)o(tee)g(bu\013ering)i(that)e (it)h(can)g(in)g(fact)g(p)q(erform.)75 725 y(Th)o(us,)k(in)h(example)g (1)f(ab)q(o)o(v)o(e,)g(the)g(implemen)o(tation)h(pic)o(ks)g(an)f (arbitrary)f(v)m(alue)i(for)f Fp(k)p FE(,)g(trading)g(o\013)75 782 y(n)o(um)o(b)q(er)h(of)f(messages)h(for)f(total)g(message)g(size,)j (rather)d(than)g(letting)i(the)f(user)g(mak)o(e)f(this)h(trade-)75 838 y(o\013.)k(In)18 b(order)e(to)g(allo)o(w)h(the)g(MPI)f(user)h(to)f (inquire)j(ab)q(out)d(a)g(sp)q(eci\014c)j(bu\013ering)e(need,)h(the)e (routine)75 895 y Fx(MPI)p 160 895 V 16 w(SUGGEST)p 375 895 V 18 w(BUFFER)p 562 895 V 17 w(P)l(ARAMS)g FE(is)g(pro)o(vided.)75 999 y Fx(MPI)p 160 999 V 16 w(SUGGEST)p 375 999 V 18 w(BUFFER)p 562 999 V 17 w(P)l(ARAMS\()h(numsend,)i(sizesend,)f (numrecv,)f(sizerecv,)g(totalnumsend,)i(to-)75 1056 y(talsizesen,)d (totalnumrecv,)f(totalsizerecv,)g(\015ag\))117 1134 y Fr(IN)155 b Fx(numsend)421 b Fr(Bound)13 b(on)f(n)o(um)o(b)q(er)g(of)f (p)q(ending)i(messages)f(with)g(source)i(at)905 1191 y(calling)f(pro)q(cess)i(and)f(same)f(destination)117 1269 y(IN)155 b Fx(sizesend)436 b Fr(Bound)12 b(on)g(total)f(size)i(in) e(b)o(ytes)i(of)e(p)q(ending)h(messages)g(with)905 1325 y(source)j(at)f(calling)f(no)q(de)h(and)g(same)f(destination)117 1403 y(IN)155 b Fx(numrecv)427 b Fr(Bound)16 b(on)e(the)i(n)o(um)o(b)q (er)e(of)g(p)q(ending)i(messages)f(with)f(des-)905 1459 y(tination)f(and)h(calling)e(pro)q(cess)k(and)e(same)f(source)117 1537 y(IN)155 b Fx(sizerecv)442 b Fr(Bound)12 b(on)g(total)f(size)i(in) e(b)o(ytes)i(of)e(p)q(ending)h(messages)g(with)905 1593 y(destination)i(at)g(calling)e(pro)q(cess)k(and)e(same)f(source)117 1671 y(IN)155 b Fx(totalnumsend)333 b Fr(Bound)10 b(on)g(the)g(n)o(um)o (b)q(er)f(of)g(p)q(ending)h(messages)g(with)g(source)905 1728 y(at)k(the)g(calling)f(pro)q(cess)j(and)d(an)o(y)h(destination)117 1805 y(IN)155 b Fx(totalsizesend)348 b Fr(Bound)12 b(on)g(total)f(size) i(in)e(b)o(ytes)i(of)e(p)q(ending)h(messages)g(with)905 1862 y(source)j(at)f(the)h(calling)d(pro)q(cess)k(and)e(an)o(y)f (destination)117 1939 y(IN)155 b Fx(totalnumrecv)339 b Fr(Bound)16 b(on)f(the)h(n)o(um)o(b)q(er)f(of)g(p)q(ending)g (messages)h(from)d(an)o(y)905 1996 y(source)i(with)f(the)g(calling)f (pro)q(cess)j(as)e(destination)117 2074 y(IN)155 b Fx(totalsizerecv)354 b Fr(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 2130 y(from)d(an)o(y)i(source)h(with)e (destination)h(at)g(the)g(calling)f(no)q(de)117 2208 y(OUT)108 b Fx(\015ag)518 b Fm(TRUE)17 b Fr(if)f(the)h(suggestion)h(w)o (as)f(accepted,)i Fm(F)m(ALSE)p Fr(,)c(other-)905 2264 y(wise)75 2390 y Fp(int)23 b(MPI)p 245 2390 15 2 v 17 w(SUGGEST)p 430 2390 V 16 w(BUFFER)p 590 2390 V 17 w(PARAMS\(int)f (numsend,)h(int)h(sizesend,)e(int)i(numrecv,)393 2446 y(int)g(sizerecv,)e(int)i(totalnumsend,)e(int)h(totalsizesen,)393 2503 y(int)h(totalnumrecv,)e(int)h(totalsizerecv,)f(int)i(*flag\))75 2591 y(MPI)p 150 2591 V 17 w(SUGGEST)p 335 2591 V 16 w(BUFFER)p 495 2591 V 16 w(PARAMS\(NUMSEND,)e(SIZESEND,)h(NUMRECV,)g (SIZERECV,)393 2647 y(TOTALNUMSEND,)f(TOTALSIZESEN,)g(TOTALNUMRECV,)g (TOTALSIZERECV,)g(FLAG,)393 2704 y(IERROR\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 156 161 156 160 bop 75 -100 a FE(156)532 b Ft(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)170 45 y Fp(INTEGER)23 b(NUMSEND,)g(SIZESEND,)g(NUMRECV,)g(SIZERECV,)f (TOTALNUMSEND,)170 102 y(TOTALSIZESEN,)h(TOTALNUMRECV,)f (TOTALSIZERECV,)g(FLAG,)h(IERROR)166 189 y FE(If)15 b(a)g(v)m(alue)h (of)e(true)h(is)g(returned)g(in)h Fx(const)g FE(then)f(it)h(is)f(as)f (if)i(the)f(v)m(alues)h(pro)o(vided)f(b)o(y)g(the)g(user)g(in)75 245 y Fx(MPI)p 160 245 14 2 v 16 w(SUGGEST)p 375 245 V 18 w(BUFFER)p 562 245 V 17 w(P)l(ARAMS)g FE(w)o(ere)f(returned)h(b)o (y)f(a)g(call)h(to)e Fx(MPI)p 1378 245 V 16 w(GET)p 1482 245 V 17 w(BUFFER)p 1668 245 V 17 w(P)l(ARAMS)p FE(:)75 301 y(the)i(user)h(is)g(guaran)o(teed)e(that)h(no)g(deadlo)q(c)o(k)i (will)g(o)q(ccur)e(due)h(to)f(lac)o(k)g(of)g(bu\013ers,)g(pro)o(vided)h (it)g(nev)o(er)75 358 y(exceeds)11 b(the)g(limitations)g(on)g(n)o(um)o (b)q(er)f(and)h(size)g(of)f(p)q(ending)i(messages)e(implied)i(b)o(y)f (these)f(parameters.)166 415 y(This)19 b(routines)g(allo)o(ws)f(the)h (user)f(to)g(tell)i(the)e(MPI)g(implemen)o(tation)i(that)e(a)g(certain) h(amoun)o(t)75 471 y(of)c(bu\013ering)i(w)o(ould)f(b)q(e)h(desireable,) g(The)f(MPI)g(implemen)o(tation)h(is)g(not)e(required)i(to)e(pro)o (vide)i(suc)o(h)75 528 y(bu\013ering.)166 585 y(It)e(is)h(exp)q(ected)g (that)f(a)g(common)f(use)i(of)f(these)g(routines)h(will)h(b)q(e)75 681 y Fp(MPI_Get_buffer_params\()k(&sendnum,)h(&sendsize,)h(...)g(\);) 75 738 y(if)h(\(sendsize)e(<)i(buffer_needed\))e({)170 794 y(fprintf\()h(stderr,)g("Program)g(requires)g(more)g(buffering)g (than)g(is)h(available\\n")e(\);)170 851 y(MPI_Abort\()h(1)h(\);)170 907 y(})75 964 y(else)170 1020 y(exploit)f(buffering)75 1116 y FE(This)17 b(allo)o(ws)g(a)f(program)f(to)h(fail)h(rather)f (than)g(deadlo)q(c)o(k.)25 b(It)16 b(is)h(also)g(p)q(ossible)h(for)e (an)g(algorithm)h(to)75 1173 y(switc)o(h)e(to)g(a)g(di\013eren)o(t)g (algorithm)g(that)g(requires)h(less)g(or)e(no)i(bu\013ering.)166 1230 y(Another)j(use)g(is)h(to)f(inform)g(the)g(MPI)g(implemen)o (tation)i(of)d(the)i(bu\013er)f(space)g(desires)h(of)f(the)75 1286 y(application:)75 1383 y Fp(MPI_suggest_buffer_params\()h(1,)k (buffer_needed,)e(1,)h(buffer_needed,)f(0,)i(0,)g(0,)f(0)48 b(&flag)23 b(\);)75 1439 y(if)h(\(flag\))f({)170 1495 y(exploit)g(buffering)170 1552 y(})75 1608 y(else)g({)170 1665 y(fprintf\()g(stderr,)g("Program)g(requires)g(more)g(buffering)g (than)g(is)h(available\\n")e(\);)170 1721 y(MPI_abort\()h(1)h(\);)170 1778 y(})75 1901 y Fx(Application)17 b(p)o(rovided)e(bu\013ering)75 1988 y FE(An)k(application)h(program)e(can)h(optionally)h(pro)o(vide)f (space)g(for)f(MPI's)g(use)i(in)f(bu\013ering)g(message)75 2044 y(data)c(via)g(the)g(follo)o(wing)h(call:)75 2148 y Fx(MPI)p 160 2148 V 16 w(USER)p 288 2148 V 17 w(SPECIFIES)p 519 2148 V 17 w(BUFFER\()f(comm,)e(bu\013er,)i(size\))117 2226 y Fr(IN)155 b Fx(comm)470 b Fr(is)16 b(a)f(comm)o(unicator)e(with) j(whic)o(h)f(the)i(bu\013er)f(is)g(to)g(b)q(e)g(as-)905 2283 y(so)q(ciated)f(\(handle\))117 2359 y(IN)155 b Fx(bu\013er)478 b Fr(initial)12 b(bu\013er)j(address)g(\(c)o(hoice\))117 2435 y(IN)155 b Fx(size)519 b Fr(bu\013er)15 b(size,)f(in)g(b)o(ytes)g (\(in)o(teger\))75 2560 y Fp(int)23 b(MPI)p 245 2560 15 2 v 17 w(USER)p 358 2560 V 17 w(SPECIFIES)p 591 2560 V 16 w(BUFFER\(MPI)p 847 2560 V 16 w(Comm)g(comm,)g(void*)g(buffer,)g (int)h(size\))75 2647 y(MPI)p 150 2647 V 17 w(USER)p 263 2647 V 16 w(SPECIFIES)p 495 2647 V 16 w(BUFFER\(COMM,)f(BUFFER,)f (SIZE,)i(IERROR\))170 2704 y()f(SENDBUF\(*\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 157 162 157 161 bop 75 -100 a Ft(7.2.)34 b(ERR)o(OR)17 b(HANDLING)1194 b FE(157)170 45 y Fp(INTEGER)23 b(COMM,)h(BUFFER,)e(SIZE,)i(IERROR)166 132 y FE(If)d Fx(MPI)p 302 132 14 2 v 15 w(USER)p 429 132 V 18 w(SPECIFIES)p 661 132 V 16 w(BUFFER)g FE(is)g(called,)i(then)e (for)f(subsequen)o(t)h(regular)f(sends)h(using)75 188 y(the)h(sp)q(eci\014ed)j(comm)o(unicator,)e(MPI)f(m)o(ust)g(pro)o(vide) h(as)f(m)o(uc)o(h)g(safet)o(y)f Fs(as)i(if)31 b FE(outgoing)22 b(message)75 245 y(data)c(w)o(ere)g(bu\013ered)i(b)o(y)e(the)h(sending) h(pro)q(cess,)f(in)h(the)e(pro)o(vided)i(bu\013er)f(space,)g(using)g(a) g(circular)75 301 y(con)o(tiguous-space)h(allo)q(cation)g(p)q(olicy)l (.)33 b(F)l(ollo)o(wing)20 b(the)f(call)i(to)d Fx(MPI)p 1320 301 V 16 w(USER)p 1448 301 V 17 w(SPECIFIES)p 1679 301 V 17 w(BUFFER)p FE(,)75 358 y(the)d(application)h(is)f(not)f(p)q (ermitted)h(to)f(access)h(the)f(bu\013er)h(un)o(til)g(the)g(corresp)q (onding)g(comm)o(unicator)75 414 y(has)g(b)q(een)h(freed.)166 471 y(Access)j(to)f(the)h(pro)o(vided)h(bu\013er)f(space)g(is)g(not)f (inherited)j(b)o(y)e(comm)o(unicators)f(deriv)o(ed)i(from)75 527 y(the)15 b(initially)j(sp)q(eci\014ed)g(one.)166 660 y Fo(Implemen)o(tati)o(on)12 b(note:)37 b Fr(One)15 b(approac)o(h)f(is)g(for)g(MPI)g(to)g(simply)e(transform)h(all)g(blo)q (c)o(king)g(sends)i(on)75 710 y(the)f(sp)q(eci\014ed)i(comm)o(unicator) 11 b(in)o(to)145 785 y Fd(\017)23 b Fr(cop)o(y)13 b(data)h(to)g (\(application-pro)o(vided\))e(bu\013er)j(space)145 860 y Fd(\017)23 b Fr(issue)14 b(non-blo)q(c)o(king)f(send)i(from)d(the)i (bu\013er)h(cop)o(y)145 936 y Fd(\017)23 b Fr(return)15 b(to)e(application)145 1011 y Fd(\017)23 b Fr(c)o(hec)o(k)14 b(completion)f(on)g(subsequen)o(t)j(MPI)e(call\(s\))166 1093 y(No)i(doubt)g(man)o(y)e(optimizations)g(within)h(MPI)h(are)h(p)q (ossible|the)f(prop)q(osal)g(just)g(requires)i(that)e(MPI)75 1149 y(pro)o(vide)e(at)f(least)i(this)e(m)o(uc)o(h)g(safet)o(y)m(.)166 1289 y FE(Note)e(that)g(collectiv)o(e)j(comm)o(unications)e(and)g (other)f(nonlo)q(cal)i(op)q(erations)f(will)i(share)d(suggested)75 1345 y(or)k(user-supplied)j(comm)o(unication)d(bu\013ers)h(with)f(p)q (oin)o(t)h(to)e(p)q(oin)o(t)i(comm)o(unication.)75 1489 y Fv(7.2)59 b(Erro)n(r)21 b(handling)75 1591 y FE(This)15 b(section)h(con)o(tains)f(the)g(suggested)g(POSIX)g(binding)i(for)e (error)f(handling)i(in)g(MPI.)e(MPI)h(imple-)75 1648 y(men)o(tations)d(that)h(do)f(not)h(run)g(in)g(a)g(POSIX-complian)o(t)i (en)o(vironmen)o(t)e(ma)o(y)f(need)h(to)g(use)g(alternativ)o(e)75 1704 y(mec)o(hanisms.)166 1761 y(An)j(MPI)g(implemen)o(tation)i(can)e (not)f(or)h(ma)o(y)f(c)o(ho)q(ose)h(not)g(to)f(handle)i(some)f(errors)f (that)h(o)q(ccur)75 1817 y(during)23 b(MPI)f(calls.)41 b(These)22 b(can)g(include)i(errors)d(that)h(generate)f(exceptions)i (or)f(traps,)g(suc)o(h)g(as)75 1874 y(\015oating)15 b(p)q(oin)o(t)h (errors)f(or)g(access)g(violations.)22 b(The)15 b(set)g(of)g(errors)g (that)g(are)g(not)g(handled)i(b)o(y)e(MPI)g(is)75 1930 y(implemen)o(tation-dep)q(endent.)166 1987 y(There)d(are)g(three)h(p)q (ossible)h(error)e(handling)i(mo)q(des)e(for)g(those)g(errors)f(that)h (MPI)g(handles,)i(whic)o(h)75 2043 y(are)h(iden)o(ti\014ed)i(b)o(y)e (three)h(constan)o(t)e(v)m(alues:)75 2138 y Fm(MPI)p 152 2138 13 2 v 14 w(ERRORSF)m(A)m(T)m(AL)20 b FE(The)c(error)e(is)i Fs(fatal)f FE(and)h(causes)f(the)g(program)f(to)h(ab)q(ort.)75 2232 y Fm(MPI)p 152 2232 V 14 w(ERRORSRETURN)21 b FE(The)13 b(error)e(causes)i(the)f(MPI)g(routine)h(where)f(it)h(o)q(ccurred)g(to) e(return)i(an)f(error)189 2289 y(co)q(de.)75 2383 y Fm(MPI)p 152 2383 V 14 w(ERRORSCALLUSERROUTINE)20 b FE(The)11 b(error)f(causes)h(MPI)f(to)g(call)i(a)e(user-sp)q(eci\014ed)j(error)d (handling)189 2440 y(routine.)20 b(Suc)o(h)c(handler)g(can)f(b)q(e)h (sp)q(eci\014ed)i(b)o(y)d(a)g(call)h(to)f Fx(MPI)p 1291 2440 14 2 v 15 w(SET)p 1389 2440 V 17 w(HANDLER)p FE(.)166 2534 y(A)e(user)h(can)f(asso)q(ciate)g(an)g(error)g(handling)i(mo)q(de) e(with)h(a)f(comm)o(unicator.)19 b(The)13 b(sp)q(eci\014ed)j(error)75 2591 y(handling)j(mo)q(de)f(will)i(b)q(e)e(used)g(for)g(an)o(y)f(error) g(that)g(o)q(ccurs)h(during)h(a)e(call)i(to)e(MPI)h(for)f(a)g(comm)o (u-)75 2647 y(nication)g(with)f(this)g(comm)o(unicator.)21 b(MPI)16 b(calls)h(that)e(are)g(not)g(related)h(to)g(an)o(y)f(comm)o (unicator)g(are)75 2704 y(considered)i(to)d(b)q(e)i(attac)o(hed)f(to)f (the)h(comm)o(unicator)g Fm(MPI)p 1108 2704 13 2 v 15 w(W)o(ORLD)p FE(.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 158 163 158 162 bop 75 -100 a FE(158)532 b Ft(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)166 45 y FE(A)f(user)h(ma)o(y)e(also)h(attac)o(h)f(an)i(error)e(handling)j (routine)f(with)f(a)g(comm)o(unicator;)g(This)h(attac)o(he-)75 102 y(men)o(t)j(is)g(purely)h(lo)q(cal:)29 b(di\013eren)o(t)19 b(pro)q(cesses)h(ma)o(y)e(attac)o(h)g(di\013eren)o(t)h(error)g (handlers)h(to)e(the)h(same)75 158 y(comm)o(unicator.)166 214 y(A)13 b(newly)h(created)g(comm)o(unicator)f(inherits)i(the)e (error)g(mo)q(de)g(and)h(the)f(error)g(handling)i(routine,)75 271 y(if)22 b(an)o(y)l(,)g(that)f(is)h(asso)q(ciated)g(with)f(the)h (\\paren)o(t")e(comm)o(unicator.)38 b(In)23 b(particular,)g(the)e(user) h(can)75 327 y(sp)q(ecify)14 b(a)e(\\global")h(error-handling)h(mo)q (de)f(for)f(all)i(comm)o(unicators)e(b)o(y)g(asso)q(ciating)i(this)f (mo)q(de)g(with)75 384 y(the)i(comm)o(unicator)g Fm(MPI)p 527 384 13 2 v 14 w(COMM)p 671 384 V 15 w(W)o(ORLD)g FE(immediatly)i(after)d(initialization.)166 440 y(The)h(default)h (error)f(handling)h(mo)q(de)g(for)f Fm(MPI)p 980 440 V 14 w(COMM)p 1124 440 V 15 w(W)o(ORLD)g FE(is)g Fm(MPI)p 1422 440 V 15 w(ERRORSF)m(A)m(T)m(A)o(L)p FE(.)75 544 y Fx(MPI)p 160 544 14 2 v 16 w(GET)p 264 544 V 17 w(ERRORMODE)i(\()d (comm,)f(mo)q(de)i(\))117 621 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(to)j(get)g(error)h(mo)q(de)e(from)f (\(handle\))117 696 y(OUT)108 b Fx(mo)q(de)483 b Fr(curren)o(t)16 b(error)e(handling)f(mo)q(de)g(\(state\))75 821 y Fp(int)23 b(MPI)p 245 821 15 2 v 17 w(GET)p 334 821 V 17 w(ERRORMODE\(MPI)p 663 821 V 15 w(Comm)h(comm,)f(MPI)p 1013 821 V 17 w(Errmode)g(mode\))75 907 y(MPI)p 150 907 V 17 w(GET)p 239 907 V 17 w(ERRORMODE\(COMM,)e (MODE,)j(IERROR\))170 963 y(INTEGER)f(COMM,)h(MODE,)f(IERROR)166 1050 y FE(Gets)15 b(the)g(error)f(handling)j(mo)q(de.)75 1154 y Fx(MPI)p 160 1154 14 2 v 16 w(SET)p 259 1154 V 17 w(ERRORMODE)g(\()d(comm,)f(mo)q(de)i(\))117 1231 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(to)j(set)h(error)g (mo)q(de)e(for)g(\(handle\))117 1306 y(IN)155 b Fx(mo)q(de)483 b Fr(new)15 b(error)f(mo)q(de)f(\(state\))75 1430 y Fp(int)23 b(MPI)p 245 1430 15 2 v 17 w(SET)p 334 1430 V 17 w(ERRORMODE\(MPI)p 663 1430 V 15 w(Comm)h(comm,)f(MPI)p 1013 1430 V 17 w(Errmode)g(mode\)) 75 1517 y(MPI)p 150 1517 V 17 w(SET)p 239 1517 V 17 w(ERRORMODE\(COMM,) e(MODE,)j(IERROR\))170 1573 y(INTEGER)f(COMM,)h(MODE,)f(IERROR)166 1659 y FE(Sets)15 b(the)g(error)g(handling)i(mo)q(de.)75 1763 y Fx(MPI)p 160 1763 14 2 v 16 w(SET)p 259 1763 V 17 w(ERRHANDLER\()f(comm,)d(handler)j(\))117 1840 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(to)j(set)h(the)f(error)h (handler)f(for)g(\(handle\))117 1915 y(IN)155 b Fx(handler)449 b Fr(user-de\014ned)16 b(error)f(handler)f(to)g(call)75 2040 y Fp(int)23 b(MPI)p 245 2040 15 2 v 17 w(SET)p 334 2040 V 17 w(ERRHANDLER\(MPI)p 687 2040 V 15 w(Comm)h(comm,)f(MPI)p 1037 2040 V 17 w(Errhandler)f(handler\))75 2126 y(MPI)p 150 2126 V 17 w(SET)p 239 2126 V 17 w(ERRHANDLER\(COMM,)f(HANDLER,)i (IERROR\))170 2183 y(INTEGER)g(COMM,)h(HANDLER,)e(IERROR)166 2269 y FE(This)16 b(routine)f(returns)g(the)h(previous)g(error)e (handler)i(as)f(its)h(v)m(alue.)166 2326 y(An)f(error)g(handler)h(is)g (a)f(C)g(function)h(declared)g(as)75 2430 y Fp(typedef)23 b(int)g(\(MPI_ERRHANDLER\)\(MPICOMM,)e(int)i(*,)h(...\);)75 2534 y FE(The)13 b(\014rst)g(argumen)o(t)g(is)h(the)f(comm)o(unicator)g (in)h(use.)20 b(The)13 b(second)h(is)g(a)e(p)q(oin)o(ter)i(to)f(the)g (error)g(co)q(de)h(to)75 2591 y(b)q(e)f(returned)f(b)o(y)g(the)g(MPI)g (routine.)19 b(The)12 b(remaining)h(argumen)o(ts)e(are)h(\\)p Fp(varargs)p FE(")e(argumen)o(ts)h(whose)75 2647 y(n)o(um)o(b)q(er)j (and)f(meaning)h(is)g(implemen)o(tation-dep)q(enden)o(t.)22 b(An)13 b(implemen)o(tation)i(should)f(clearly)g(do)q(c-)75 2704 y(umen)o(t)h(these)h(parameters.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 159 164 159 163 bop 75 -100 a Ft(7.3.)34 b(ST)l(AR)l(TUP)1411 b FE(159)166 45 y(There)15 b(are)f(t)o(w)o(o)g(prede\014ned)i(error)e (handlers)i(that)e(the)h(user)f(ma)o(y)h(use)g(instead)g(of)f(writing)h (their)75 102 y(o)o(wn:)75 200 y Fm(MPI)p 152 200 13 2 v 14 w(DEF)m(A)o(UL)m(THANDLER)22 b FE(ab)q(orts)14 b(from)h(within)h(MPI.)75 299 y Fm(MPI)p 152 299 V 14 w(NOHANDLER)24 b FE(returns)15 b(the)g(error)f(v)m(alue)j(and)e(has)g (no)g(further)g(e\013ect.)75 445 y Fx(MPI)p 160 445 14 2 v 16 w(ERROR)p 323 445 V 18 w(STRING\()g(erro)o(rco)q(de,)f(len,)i (string)f(\))117 523 y Fr(IN)155 b Fx(erro)o(rco)q(de)410 b Fr(Error)15 b(co)q(de)g(returned)g(b)o(y)f(an)f(MPI)i(routine)117 601 y(IN)155 b Fx(len)533 b Fr(Length)14 b(of)g Fm(string)117 678 y Fr(OUT)108 b Fx(string)481 b Fr(T)m(ext)14 b(that)g(corresp)q (onds)i(to)e(the)g Fm(erro)o(rco)q(de)75 804 y Fp(int)23 b(MPI)p 245 804 15 2 v 17 w(ERROR)p 382 804 V 17 w(STRING\(int)f (errorcode,)h(int)g(len,)g(char)h(*string\))75 891 y(MPI)p 150 891 V 17 w(ERROR)p 287 891 V 16 w(STRING\(ERRORCODE,)e(LEN,)h (STRING,)g(IERROR\))170 948 y(INTEGER)g(ERRORCODE,)g(LEN,)g(STRING,)g (IERROR)75 1128 y Fv(7.3)59 b(Sta)n(rtup)75 1232 y FE(One)13 b(goal)g(of)f(MPI)h(is)g(to)f(ac)o(hiev)o(e)h Fs(sour)n(c)n(e)h(c)n(o)n (de)g(p)n(ortability)p FE(.)19 b(By)13 b(this)g(w)o(e)f(mean)h(that)f (a)g(program)g(writ-)75 1288 y(ten)j(using)i(MPI)e(and)g(complying)i (with)f(the)f(relev)m(an)o(t)h(language)g(standards)f(is)g(p)q(ortable) h(as)f(written,)75 1345 y(and)k(m)o(ust)e(not)h(require)i(an)o(y)e (source)g(co)q(de)h(c)o(hanges)f(when)h(mo)o(v)o(ed)f(from)g(one)g (system)g(to)g(another.)75 1401 y(This)h(explicitly)h(do)q(es)f Fs(not)j FE(sa)o(y)17 b(an)o(ything)i(ab)q(out)f(ho)o(w)f(an)h(MPI)g (program)f(is)i(started)e(or)g(launc)o(hed)75 1457 y(from)h(the)i (command)f(line,)i(nor)e(what)f(the)i(user)f(m)o(ust)f(do)i(to)e(set)h (up)h(the)f(en)o(vironmen)o(t)g(in)h(whic)o(h)75 1514 y(an)h(MPI)g(program)f(will)j(run.)38 b(Ho)o(w)o(ev)o(er,)21 b(an)h(implemen)o(tation)g(ma)o(y)e(require)i(some)f(setup)h(to)e(b)q (e)75 1570 y(p)q(erformed)c(b)q(efore)h(other)f(MPI)g(routines)g(ma)o (y)g(b)q(e)h(called.)24 b(T)l(o)16 b(pro)o(vide)g(for)g(this,)g(MPI)h (includes)h(an)75 1627 y(initialization)g(routine)e Fx(MPI)p 582 1627 14 2 v 16 w(INIT)p FE(.)75 1732 y Fx(MPI)p 160 1732 V 16 w(INIT\(\))75 1837 y Fp(int)23 b(MPI)p 245 1837 15 2 v 17 w(INIT\(int)g(argc,)g(char)g(**argv\))75 1924 y(MPI)p 150 1924 V 17 w(INIT\(IERROR\))170 1981 y(INTEGER)g(IERROR)166 2068 y FE(This)17 b(routine)g(m)o(ust)f(b)q(e)h (called)h(b)q(efore)f(an)o(y)f(other)g(MPI)g(routine.)24 b(It)17 b(m)o(ust)f(b)q(e)h(called)h(at)e(most)75 2125 y(once;)f(subsequen)o(t)h(calls)g(are)f(erroneous)g(\(see)g Fx(MPI)p 982 2125 14 2 v 16 w(INITIALIZED)p FE(\).)166 2182 y(All)j(MPI)f(programs)e(m)o(ust)i(con)o(tain)g(a)f(call)i(to)e Fx(MPI)p 1092 2182 V 16 w(init)p FE(;)i(this)f(routine)h(m)o(ust)e(b)q (e)h(called)i(b)q(efore)75 2239 y(an)o(y)d(other)g(MPI)g(routines)h (are)f(called.)24 b(The)17 b(v)o(ersion)f(for)g(ANSI)h(C)f(accepts)g (the)h Fm(a)o(rgc)f FE(and)h Fm(a)o(rgv)f FE(that)75 2295 y(are)f(pro)o(vided)h(b)o(y)f(the)g(argumen)o(ts)g(to)f Fp(main)p FE(:)75 2395 y Fp(MPI_init\()23 b(argc,)g(argv)g(\);)75 2494 y FE(The)15 b(F)l(ortran)f(v)o(ersion)i(tak)o(es)e(no)h(argumen)o (ts.)75 2599 y Fx(MPI)p 160 2599 V 16 w(FINALIZE\(\))75 2704 y Fp(int)23 b(MPI)p 245 2704 15 2 v 17 w(FINALIZE\(\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 160 165 160 164 bop 75 -100 a FE(160)532 b Ft(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fp(MPI)p 150 45 15 2 v 17 w(FINALIZE\(IERROR\))170 102 y(INTEGER)23 b(IERROR)166 188 y FE(This)17 b(routines)g(cleans)g (all)h(MPI)e(state.)23 b(Once)18 b(this)f(routine)g(is)g(called,)h(no)e (MPI)h(routine)g(\(ev)o(en)75 244 y Fx(MPI)p 160 244 14 2 v 16 w(INIT)p FE(\))d(ma)o(y)h(b)q(e)g(called.)75 348 y Fx(MPI)p 160 348 V 16 w(INITIALIZED\()e(\015ag)i(\))117 425 y Fr(OUT)108 b Fx(\015ag)518 b Fr(F)p FE(lag)15 b(is)f(true)h(if)f Fx(MPI)p 1270 425 V 16 w(INIT)g FE(has)g(b)q(een)h(called)h(and)f (false)905 482 y(otherwise.)75 606 y Fp(int)23 b(MPI)p 245 606 15 2 v 17 w(INITIALIZED\(int)f(*flag\))75 693 y(MPI)p 150 693 V 17 w(INITIALIZED\(FLAG,)f(IERROR\))170 749 y(INTEGER)i(FLAG,)h(IERROR)166 836 y FE(This)19 b(routine)f(ma)o(y) g(b)q(e)h(used)g(to)e(determine)i(whether)g Fx(MPI)p 1242 836 14 2 v 16 w(INIT)e FE(has)h(b)q(een)h(called.)31 b(It)18 b(is)h(the)75 892 y Fs(only)c FE(routine)g(that)g(ma)o(y)f(b)q (e)i(called)h(b)q(efore)e Fx(MPI)p 938 892 V 16 w(INIT)g FE(is)g(called.)75 996 y Fx(MPI)p 160 996 V 16 w(ABORT\()h(comm,)d (erro)o(rco)q(de)h(\))117 1073 y Fr(IN)155 b Fx(comm)470 b Fr(comm)o(unicator)11 b(of)j(tasks)g(to)g(ab)q(ort)117 1148 y(IN)155 b Fx(erro)o(rco)q(de)410 b Fr(error)15 b(co)q(de)g(to)e(return)j(to)d(in)o(v)o(oking)f(en)o(vironmen)o(t)75 1273 y Fp(int)23 b(MPI)p 245 1273 15 2 v 17 w(ABORT\(MPI)p 478 1273 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 1359 y(MPI)p 150 1359 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 1416 y(INTEGER)g(COMM,)h(ERRORCODE,)e(IERROR)166 1502 y FE(This)d(routine)f(mak)o(es)g(a)g(\\b)q(est)g(attempt")f(to)h(ab)q (ort)g(all)h(tasks)e(in)i(the)g(group)e(of)h Fx(comm)p FE(.)27 b(This)75 1558 y(function)21 b(do)q(es)g(not)f(require)h(that)f (the)g(in)o(v)o(oking)h(en)o(vironmen)o(t)f(tak)o(e)g(an)o(y)g(action)h (with)f(the)h(error)75 1615 y(co)q(de.)g(Ho)o(w)o(ev)o(er,)14 b(a)h(Unix)i(or)e(POSIX)h(en)o(vironmen)o(t)g(should)g(handle)h(this)f (as)f(a)g Fp(return)23 b(errorcode)75 1671 y FE(from)14 b(the)i(main)f(program)f(or)h(an)g Fp(abort\(errorcode\))p FE(.)166 1728 y(MPI)f(implemen)o(tations)h(are)f(required)h(to)e (de\014ne)i(the)f(b)q(eha)o(vior)h(of)e Fx(MPI)p 1443 1728 14 2 v 16 w(ABORT)i FE(at)f(least)g(for)f(a)75 1784 y Fp(comm)i FE(of)f Fm(MPI)p 314 1784 13 2 v 15 w(COMM)p 459 1784 V 14 w(W)o(ORLD)p FE(.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 161 166 161 165 bop 75 357 a Fw(Chapter)34 b(8)75 564 y FA(Pro\014ling)42 b(In)m(terface)75 805 y Fv(8.1)59 b(Requirements)75 907 y FE(T)l(o)15 b(meet)g(the)g(MPI)g(pro\014ling)i(in)o(terface,)e(an)g (implemen)o(tation)i(of)d(the)i(MPI)f(functions)h Fs(must)131 1001 y FE(1.)22 b(pro)o(vide)14 b(a)f(mec)o(hanism)h(through)f(whic)o (h)h(all)g(of)f(the)g(MPI)h(de\014ned)h(functions)f(ma)o(y)e(b)q(e)i (accessed)189 1057 y(with)e(a)g(name)f(shift.)20 b(Th)o(us)11 b(all)i(of)f(the)g(MPI)g(functions)h(\(whic)o(h)f(normally)g(start)f (with)h(the)g(pre\014x)189 1114 y(\\)p Fp(MPI)p 287 1114 15 2 v 16 w FE("\))j(should)h(also)f(b)q(e)h(accessable)g(with)g(the)f (pre\014x)h(\\)p Fp(PMPI)p 1300 1114 V 16 w FE(".)131 1208 y(2.)22 b(ensure)c(that)f(those)h(MPI)g(functions)h(whic)o(h)f (are)g(not)f(replaced)i(ma)o(y)f(still)h(b)q(e)g(link)o(ed)g(in)o(to)f (an)189 1265 y(executable)e(image)f(without)h(causing)f(name)h (clashes.)131 1359 y(3.)22 b(do)q(cumen)o(t)c(the)f(implemen)o(tation)i (of)e(di\013eren)o(t)h(language)g(bindings)h(of)e(the)h(MPI)g(in)o (terface)f(if)189 1415 y(they)g(are)h(la)o(y)o(ered)g(on)f(top)g(of)h (eac)o(h)f(other,)h(so)f(that)g(the)h(pro\014ler)g(dev)o(elop)q(er)h (kno)o(ws)e(whether)189 1472 y(she)d(m)o(ust)f(implemen)o(t)i(the)e (pro\014le)i(in)o(terface)f(for)f(eac)o(h)h(binding,)h(or)e(can)h (economise)h(b)o(y)e(imple-)189 1528 y(men)o(ting)i(it)h(only)f(for)g (the)g(lo)o(w)o(est)g(lev)o(el)h(routines.)75 1672 y Fv(8.2)59 b(Discussion)75 1774 y FE(The)13 b(ob)s(jectiv)o(e)g(of)g (the)g(MPI)g(pro\014ling)h(in)o(terface)g(is)f(to)f(ensure)i(that)e(it) i(is)f(relativ)o(ely)h(easy)f(for)g(authors)75 1830 y(of)18 b(pro\014ling)i(\(and)f(other)f(similar\))h(to)q(ols)g(to)f(in)o (terface)h(their)g(co)q(des)g(to)f(MPI)h(implemen)o(tations)h(on)75 1887 y(di\013eren)o(t)15 b(mac)o(hines.)166 1943 y(Since)20 b(MPI)e(is)h(a)f(mac)o(hine)h(indep)q(enden)o(t)h(standard)e(with)h (man)o(y)f(di\013eren)o(t)g(implemen)o(tations,)75 2000 y(it)e(is)h(unreasonable)g(to)e(exp)q(ect)i(that)f(the)g(authors)f(of)h (pro\014ling)h(to)q(ols)f(for)g(MPI)g(will)i(ha)o(v)o(e)d(access)i(to) 75 2056 y(the)c(source)h(co)q(de)f(whic)o(h)h(implemen)o(ts)h(MPI)e(on) g(an)o(y)g(particular)h(mac)o(hine.)20 b(It)13 b(is)h(therefore)f (necessary)75 2112 y(to)21 b(pro)o(vide)i(a)f(mec)o(hanism)g(b)o(y)g (whic)o(h)h(the)f(implemen)o(tors)h(of)f(suc)o(h)g(to)q(ols)g(can)g (collect)i(whatev)o(er)75 2169 y(p)q(erformance)15 b(information)h (they)f(wish)h Fs(without)g FE(access)f(to)g(the)g(underlying)i (implemen)o(tation.)166 2225 y(W)l(e)d(b)q(eliev)o(e)i(that)e(ha)o (ving)g(suc)o(h)g(an)h(in)o(terface)f(is)h(imp)q(ortan)o(t)e(if)i(MPI)f (is)g(to)g(b)q(e)h(attractiv)o(e)e(to)g(end)75 2282 y(users,)i(since)i (the)e(a)o(v)m(ailabilit)o(y)j(of)d(man)o(y)g(di\013eren)o(t)h(to)q (ols)f(will)i(b)q(e)g(a)e(signi\014can)o(t)h(factor)f(in)h(attracting) 75 2338 y(users)f(to)g(the)g(MPI)g(standard.)166 2395 y(The)e(pro\014ling)h(in)o(terface)e(is)h(just)g(that,)f(an)g(in)o (terface.)20 b(It)12 b(sa)o(ys)g Fs(nothing)g FE(ab)q(out)g(the)h(w)o (a)o(y)f(in)h(whic)o(h)75 2451 y(it)h(is)h(used.)20 b(There)14 b(is)h(therefore)f(no)g(attempt)f(to)g(la)o(y)h(do)o(wn)g(what)g (information)g(is)h(collected)g(through)75 2508 y(the)g(in)o(terface,)g (or)g(ho)o(w)g(the)g(collected)i(information)e(is)h(sa)o(v)o(ed,)e (\014ltered,)i(or)f(displa)o(y)o(ed.)166 2564 y(While)g(the)g(initial)h (imp)q(etus)f(for)f(the)g(dev)o(elopmen)o(t)h(of)e(this)i(in)o(terface) f(arose)g(from)f(the)h(desire)i(to)75 2621 y(p)q(ermit)g(the)f (implemen)o(tation)i(of)d(pro\014ling)j(to)q(ols,)d(it)i(is)f(clear)h (that)f(an)g(in)o(terface)g(lik)o(e)h(that)f(sp)q(eci\014ed)p 75 2661 720 2 v 127 2688 a FB(1)144 2704 y Fu(V)m(ersion)f(as)f(of)g (July)h(30,)e(1993)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 162 167 162 166 bop 75 -100 a FE(162)873 b Ft(CHAPTER)15 b(8.)35 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)75 45 y FE(ma)o(y)17 b(also)h(pro)o(v)o(e)f(useful)i(for)e(other)h(purp)q(oses,)g(suc)o(h)g (as)g(\\in)o(ternet)o(w)o(orking")f(m)o(ultiple)i(MPI)f(imple-)75 102 y(men)o(tations.)j(Since)c(all)f(that)f(is)h(de\014ned)h(is)f(an)g (in)o(terface,)f(there)h(is)g(no)f(ob)s(jection)h(to)f(its)h(b)q(eing)g (used)75 158 y(wherev)o(er)f(it)h(is)f(useful.)166 221 y(As)j(the)g(issues)h(b)q(eing)g(addressed)f(here)h(are)f(in)o (timately)h(tied)f(up)h(with)f(the)g(w)o(a)o(y)f(in)i(whic)o(h)g(ex-)75 277 y(ecutable)j(images)e(are)g(built,)j(whic)o(h)e(ma)o(y)f(di\013er)h (greatly)f(on)g(di\013eren)o(t)h(mac)o(hines,)h(the)e(examples)75 334 y(giv)o(en)c(b)q(elo)o(w)g(should)h(b)q(e)f(treated)g(solely)g(as)f (one)h(w)o(a)o(y)f(of)g(implemen)o(ting)j(the)d(ob)s(jectiv)o(e)h(of)f (the)h(MPI)75 390 y(pro\014ling)h(in)o(terface.)k(The)16 b(actual)g(requiremen)o(ts)g(made)g(of)f(an)h(implemen)o(tation)h(are)e (those)g(detailed)75 447 y(in)i(the)f(Requiremen)o(ts)i(section)e(ab)q (o)o(v)o(e,)g(the)g(whole)h(of)f(the)g(rest)g(of)f(this)i(c)o(hapter)f (is)g(only)h(presen)o(t)f(as)75 503 y(justi\014cation)g(and)f (discussion)i(of)e(the)g(logic)h(for)f(those)g(requiremen)o(ts.)166 566 y(The)i(examples)g(b)q(elo)o(w)h(sho)o(w)e(one)h(w)o(a)o(y)f(in)h (whic)o(h)h(an)f(implemen)o(tation)h(could)f(b)q(e)h(constructed)75 623 y(to)h(meet)h(the)g(requiremen)o(ts)g(on)g(a)g(Unix)h(system)e (\(there)h(are)f(doubtless)i(others)e(whic)o(h)i(w)o(ould)f(b)q(e)75 679 y(equally)d(v)m(alid\).)75 860 y Fv(8.3)59 b(Logic)20 b(of)g(the)f(design)75 974 y FE(Pro)o(vided)d(that)f(an)g(MPI)g (implemen)o(tation)i(meets)e(the)h(requiremen)o(ts)g(ab)q(o)o(v)o(e,)e (it)i(is)g(p)q(ossible)h(for)e(the)75 1030 y(implemen)o(tor)20 b(of)e(the)h(pro\014ling)h(system)f(to)f(in)o(tercept)i(all)f(of)g(the) g(MPI)g(calls)h(whic)o(h)f(are)g(made)g(b)o(y)75 1087 y(the)g(user)h(program.)30 b(She)20 b(can)f(then)h(collect)g(whatev)o (er)f(information)g(she)h(requires)g(b)q(efore)f(calling)75 1143 y(the)c(underlying)i(MPI)d(implemen)o(tation)i(\(through)f(its)g (name)f(shifted)i(en)o(try)e(p)q(oin)o(ts\))h(to)f(ac)o(hiev)o(e)i(the) 75 1200 y(desired)g(e\013ects.)75 1380 y Fv(8.4)59 b(Examples)75 1496 y Fq(8.4.1)49 b(Pro\014ler)16 b(implementation)75 1595 y FE(Supp)q(ose)i(that)d(the)i(pro\014ler)g(wishes)h(to)e(accum)o (ulate)h(the)f(total)g(n)o(um)o(b)q(er)h(of)f(times)h(the)g(MPI)p 1737 1595 14 2 v 16 w(SEND)75 1651 y(function)22 b(is)g(called,)j (along)c(with)h(the)f(total)g(elapsed)i(time)f(sp)q(en)o(t)f(in)i(the)e (function.)40 b(This)22 b(could)75 1708 y(trivially)17 b(b)q(e)f(ac)o(hiev)o(ed)g(as)f(follo)o(ws.)75 1853 y Fp(static)23 b(int)g(totalSendCount;)75 1909 y(static)g(int)g (totalTime;)75 1966 y(extern)g(int)g(now\(void\);)71 b(/*)23 b(Some)h(suitable)e(timer)i(function)e(!)i(*/)75 2079 y(int)f(MPI_Send\(void*)f(buf,)i(int)f(count,)g(MPI_Datatype)f (datatype,)h(int)h(dest,)385 2135 y(int)g(tag,)f(MPI_Comm)g(comm\))75 2192 y({)147 2248 y(int)g(tstart)g(=)h(now\(\);)147 2305 y(/*)f(Now)h(call)f(the)g(real)h(send,)f(passing)g(along)g(all)g(the)h (arguments)f(*/)147 2361 y(int)g(result)g(=)h(PMPI_Send\(buf,)e(count,) h(datatype,)f(dest,)i(tag,)f(comm\);)147 2474 y(totalSendCount++;)475 b(/*)23 b(Accumulate)g(byte)g(count)g(*/)147 2530 y(totalTime)46 b(+=)24 b(now\(\))f(-)h(tstart;)190 b(/*)23 b(and)h(time)333 b(*/)147 2643 y(return)23 b(result;)75 2700 y(})1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 163 168 163 167 bop 75 -100 a Ft(8.4.)34 b(EXAMPLES)1370 b FE(163)75 45 y Fq(8.4.2)49 b(MPI)17 b(lib)o(ra)o(ry)f(implementation)75 131 y FE(On)i(a)f(Unix)h(system,)f(in)h(whic)o(h)g(the)g(MPI)f(library) h(is)g(implemen)o(ted)h(in)f(C,)f(then)g(there)h(are)e(v)m(arious)75 187 y(p)q(ossible)22 b(options,)f(of)e(whic)o(h)i(t)o(w)o(o)e(of)g(the) h(most)g(ob)o(vious)g(are)g(presen)o(ted)g(here.)35 b(Whic)o(h)20 b(is)h(b)q(etter)75 244 y(dep)q(ends)c(on)e(whether)g(the)h(link)o(er)g (and)f(compiler)i(supp)q(ort)e(w)o(eak)g(sym)o(b)q(ols.)75 364 y Fx(Systems)h(with)g(w)o(eak)f(symb)q(ols)75 450 y FE(If)k(the)f(compiler)i(and)f(link)o(er)h(supp)q(ort)f(w)o(eak)f (external)h(sym)o(b)q(ols)f(\(e.g.)30 b(Solaris)19 b(2.x,)f(other)g (system)75 506 y(V.4)13 b(mac)o(hines\),)i(then)f(only)g(a)g(single)h (library)g(is)f(required)h(through)f(the)g(use)g(of)g Fp(#pragma)23 b(weak)13 b FE(th)o(us)75 563 y(:{)75 669 y Fp(#pragma)23 b(weak)g(MPI_EXAMPLE)g(=)g(PMPI_EXAMPLE)75 782 y(int)g(PMPI_EXAMPLE\(/*)f(appropriate)h(args)g(*/\))75 838 y({)170 895 y(/*)h(Useful)f(content)g(*/)75 951 y(})166 1057 y FE(The)d(e\013ect)g(of)g(this)g Fp(#pragma)f FE(is)i(to)e (de\014ne)i(the)f(external)h(sym)o(b)q(ol)f Fp(MPI)p 1472 1057 15 2 v 17 w(EXAMPLE)f FE(as)h(a)g(w)o(eak)75 1114 y(de\014nition,)d(this)f(means)g(that)f(the)g(link)o(er)i(will)g (not)e(complain)i(if)f(there)g(is)g(another)f(de\014nition)j(of)d(the) 75 1170 y(sym)o(b)q(ol)g(\(for)e(instance)i(in)g(the)f(pro\014ling)i (library\),)e(ho)o(w)o(ev)o(er)f(if)i(no)f(other)g(de\014nition)i (exists,)e(then)h(the)75 1226 y(link)o(er)h(will)h(use)f(the)f(w)o(eak) g(de\014nition.)75 1346 y Fx(Systems)h(without)h(w)o(eak)e(symb)q(ols) 75 1432 y FE(In)k(the)g(absence)g(of)f(w)o(eak)g(sym)o(b)q(ols)h(then)g (one)f(p)q(ossible)j(solution)e(w)o(ould)g(b)q(e)g(to)f(use)h(the)f(C)h (macro)75 1489 y(pre-pro)q(cessor)c(th)o(us)g(:{)75 1595 y Fp(#ifdef)23 b(PROFILELIB)75 1651 y(#)95 b(ifdef)23 b(__STDC__)75 1708 y(#)191 b(define)23 b(FUNCTION\(name\))f(P##name)75 1764 y(#)95 b(else)75 1821 y(#)191 b(define)23 b(FUNCTION\(name\))f (P/**/name)75 1877 y(#)95 b(endif)75 1934 y(#else)75 1990 y(#)g(define)23 b(FUNCTION\(name\))f(name)75 2047 y(#endif)166 2153 y FE(Eac)o(h)15 b(of)g(the)g(user)g(visible)j (functions)e(in)g(the)f(library)h(w)o(ould)g(then)f(b)q(e)h(declared)g (th)o(us)75 2259 y Fp(int)23 b(FUNCTION\(MPI_EXAMPLE\)\(/*)e (appropriate)h(args)i(*/\))75 2315 y({)170 2372 y(/*)g(Useful)f (content)g(*/)75 2428 y(})166 2534 y FE(The)d(same)g(source)g(\014le)h (can)g(then)f(b)q(e)h(compiled)h(to)d(pro)q(duce)i(b)q(oth)f(v)o (ersions)g(of)g(the)g(library)l(,)75 2591 y(dep)q(ending)d(on)f(the)f (state)f(of)h(the)g Fp(PROFILELIB)f FE(macro)h(sym)o(b)q(ol.)166 2647 y(It)e(is)g(required)h(that)e(the)h(standard)f(MPI)h(library)g(b)q (e)h(built)g(in)f(suc)o(h)g(a)g(w)o(a)o(y)e(that)h(the)h(inclusion)i (of)75 2704 y(MPI)g(functions)g(can)g(b)q(e)h(ac)o(hiev)o(ed)f(one)g (at)g(a)f(time.)20 b(This)c(is)f(a)f(somewhat)g(unpleasan)o(t)i (requiremen)o(t,)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 164 169 164 168 bop 75 -100 a FE(164)873 b Ft(CHAPTER)15 b(8.)35 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)75 45 y FE(since)22 b(it)f(ma)o(y)f(mean)h(that)f(eac)o(h)g(external)i(function)f(has)g(to) f(b)q(e)h(compiled)i(from)d(a)g(separate)g(\014le.)75 102 y(Ho)o(w)o(ev)o(er)14 b(this)i(is)g(necessary)g(so)f(that)g(the)g (author)g(of)g(the)h(pro\014ling)g(library)g(need)h(only)f(de\014ne)g (those)75 158 y(MPI)f(functions)g(whic)o(h)g(she)g(wishes)h(to)e(in)o (tercept,)h(references)g(to)f(an)o(y)g(others)g(b)q(eing)i(ful\014lled) h(b)o(y)e(the)75 214 y(normal)g(MPI)g(library)l(.)21 b(Therefore)15 b(the)h(link)g(step)f(can)h(lo)q(ok)f(something)h(lik)o (e)g(this)g(:{)75 321 y Fp(\045)24 b(cc)f(...)h(-lmyprof)f(-lpmpi)g (-lmpi)166 427 y FE(Here)e Fp(libmyprof.a)e FE(con)o(tains)i(the)g (pro\014ler)g(functions)h(whic)o(h)f(in)o(tercept)g(some)g(of)f(the)h (MPI)75 483 y(functions.)j Fp(libpmpi.a)15 b FE(con)o(tains)i(the)f (\\name)h(shifted")g(MPI)f(functions,)h(and)g Fp(libmpi.a)e FE(con)o(tains)75 540 y(the)g(normal)g(de\014nitions)i(of)e(the)g(MPI)h (functions.)75 683 y Fv(8.5)59 b(Multiple)20 b(levels)f(of)h (interception)75 785 y FE(The)15 b(sc)o(heme)h(giv)o(en)g(here)f(do)q (es)h(not)f(directly)h(supp)q(ort)f(the)h(nesting)g(of)e(pro\014ling)j (functions,)e(since)i(it)75 841 y(pro)o(vides)f(only)f(a)g(single)h (alternativ)o(e)g(name)f(for)f(eac)o(h)h(MPI)g(function.)21 b(Consideration)16 b(w)o(as)e(giv)o(en)i(to)75 898 y(an)f(implemen)o (tation)i(whic)o(h)f(w)o(ould)g(allo)o(w)g(m)o(ultiple)h(lev)o(els)g (of)e(call)h(in)o(terception,)h(ho)o(w)o(ev)o(er)d(w)o(e)h(w)o(ere)75 954 y(unable)j(to)e(construct)g(an)g(implemen)o(tation)i(of)e(this)h (whic)o(h)h(did)f(not)g(ha)o(v)o(e)f(the)g(follo)o(wing)i(disadv)m(an-) 75 1010 y(tages)143 1104 y FC(\017)23 b FE(assuming)15 b(a)g(particular)h(implemen)o(tation)g(language.)143 1198 y FC(\017)23 b FE(imp)q(osing)16 b(a)f(run)g(time)h(cost)f(ev)o (en)g(when)h(no)f(pro\014ling)h(w)o(as)f(taking)g(place.)75 1292 y(Since)i(one)e(of)g(the)h(ob)s(jectiv)o(es)f(of)g(MPI)h(is)f(to)g (p)q(ermit)h(e\016cien)o(t,)g(lo)o(w)f(latency)h(implemen)o(tations,)h (and)75 1348 y(it)j(is)f(not)g(the)h(business)g(of)f(a)g(standard)g(to) g(require)h(a)f(particular)h(implemen)o(tation)g(language,)g(w)o(e)75 1405 y(decided)d(to)e(accept)g(the)g(sc)o(heme)h(outlined)h(ab)q(o)o(v) o(e.)166 1461 y(Note,)f(ho)o(w)o(ev)o(er,)f(that)h(it)h(is)f(p)q (ossible)j(to)c(use)i(the)g(sc)o(heme)f(ab)q(o)o(v)o(e)g(to)g(implemen) o(t)i(a)e(m)o(ulti-lev)o(el)75 1518 y(system,)k(since)g(the)g(function) g(called)i(b)o(y)d(the)h(user)f(ma)o(y)g(call)i(man)o(y)e(di\013eren)o (t)g(pro\014ling)i(functions)75 1574 y(b)q(efore)15 b(calling)i(the)f (underlying)h(MPI)e(function.)166 1631 y(Unfortunately)22 b(suc)o(h)g(an)g(implemen)o(tation)h(ma)o(y)e(require)h(more)g(co-op)q (eration)g(b)q(et)o(w)o(een)g(the)75 1687 y(di\013eren)o(t)g (pro\014ling)i(libraries)g(than)e(is)h(required)g(for)e(the)i(single)g (lev)o(el)h(implemen)o(tation)f(detailed)75 1744 y(ab)q(o)o(v)o(e.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 165 170 165 169 bop 75 363 a Fw(Chapter)34 b(9)75 577 y FA(Initial)43 b(Implemen)m(tation)f(Subset)75 825 y Fv(9.1)59 b(Intro)r(duction)75 939 y FE(This)17 b(c)o(hapter)f(de\014nes)h(the)g(minimal)h(subset)e (of)g(MPI)g(for)g(initial)i(implemen)o(tation.)25 b(This)17 b(subset)f(is)75 996 y(b)q(eing)c(de\014ned)h(so)d(that)g(consisten)o (t)i(implemen)o(tations)g(can)f(app)q(ear)g(more)g(rapidly)l(.)20 b(It)11 b(w)o(as)f(recognized)75 1052 y(early)17 b(in)h(the)f(pro)q (cess)g(that)f(MPI)h(needed)h(to)e(app)q(ear)h(as)f(quic)o(kly)i(as)f (p)q(ossible)h(and)f(practical.)26 b(The)75 1109 y(creation)16 b(of)f(a)g(subset)h(will)h(hop)q(efully)g(allo)o(w)f(users)g(earlier)g (access)g(to)e(the)i(standard)f(and)h(still)h(allo)o(w)75 1165 y(for)e(the)g(writing)h(of)e(p)q(ortable)i(message)f(passing)g(co) q(des.)166 1228 y(This)20 b(subset)g(should)h(not)e(in)i(an)o(y)e(w)o (a)o(y)g(b)q(e)h(construed)g(as)f(a)h(limitation)h(on)e(MPI)h (implemen-)75 1285 y(tations.)35 b(It)20 b(is)h(strictly)g(the)g(minim) o(um)g(necessary)g(to)e(ha)o(v)o(e)h(an)h(initial)h(MPI)e(subset)h (conforming)75 1341 y(implemen)o(tation.)28 b(It)18 b(is)g(hop)q(ed)g (that)f(an)g(o\016cially)i(sanctioned)g(subset)e(will)i(encourage)f (and)g(allo)o(w)75 1398 y(implemen)o(tors)j(to)e(in)o(tro)q(duce)i(MPI) f(in)g(a)g(more)g(timely)g(fashion.)35 b(It)20 b(should)h(b)q(e)f (noted,)h(ho)o(w)o(ev)o(er,)75 1454 y(that)d(implemen)o(tation)h(of)f (the)g(subset)h(do)q(es)g(not)e(mak)o(e)h(an)g(implemen)o(tation)i (conform)e(with)g(MPI.)75 1510 y(The)h(subset)g(is)g(a)f(p)q(oten)o (tial)h(\014rst)g(step)f(in)i(the)f(pro)q(cess.)30 b(All)20 b(implemen)o(tations)g(shall)g(conform)e(to)75 1567 y(the)f(en)o(tire)g (standard;)f(implemen)o(tations)i(are)e(encouraged)h(to)f(do)g(the)h (full)g(standard)f(as)h(rapidly)g(as)75 1623 y(p)q(ossible.)166 1687 y(The)f(subset)g(presen)o(ted)g(is)h(consisten)o(t)f(with)g(the)g (complete)h(MPI)f(standard.)21 b(This)16 b(w)o(as)g(an)f(im-)75 1743 y(p)q(ortan)o(t)e(goal)h(so)g(the)h(additional)g(MPI)f(features)g (could)i(b)q(e)e(added)h(without)g(c)o(hanging)f(an)o(y)g(previous)75 1799 y(functionalit)o(y)g(from)e(the)h(user's)g(p)q(ersp)q(ectiv)o(e.) 20 b(Th)o(us,)13 b(users)g(can)g(b)q(e)g(assured)g(that)f(programs)g (written)75 1856 y(no)o(w)f(for)h(the)g(subset)g(will)h(run)f(without)g (mo)q(di\014cation)h(under)g(the)f(full)h(MPI)f(standard.)18 b(F)l(urthermore,)75 1912 y(using)i(additional)g(features)f(of)f(the)h (full)i(MPI)e(standard)f(in)i(the)f(future)g(will)i(not)d(require)i(c)o (hanges)75 1969 y(to)e(co)q(de)i(already)f(written.)30 b(This)20 b(requires)f(that)g(the)f(MPI)h(functions)h(ha)o(v)o(e)e (their)i(full)g(calling)g(ar-)75 2025 y(gumen)o(ts)e(in)h(the)f (subset.)28 b(T)l(o)18 b(ac)o(hiev)o(e)h(this)f(goal,)g(the)g(subset)h (will)g(generally)h(limit)f(access)f(to)g(full)75 2082 y(MPI)d(functionalit)o(y)h(through)f(complete)g(elimination)i(of)e (functions)g(or)g(limitations)h(on)f(the)g(creation)75 2138 y(mec)o(hanisms)h(for)e(certain)i(MPI)f(input)h(argumen)o(ts.)75 2320 y Fv(9.2)59 b(Criteria)21 b(and)e(Rationale)75 2434 y FE(Ha)o(ving)g(the)g(subset)g(b)q(e)h(consisten)o(t)f(with)g(the)g (en)o(tire)g(MPI)g(standard)g(w)o(as)f(considered)i(critical)h(to)75 2490 y(the)16 b(e\013ort.)22 b(In)16 b(addition)i(to)d(this,)h(there)h (w)o(ere)e(man)o(y)h(p)q(ossible)i(criteria)f(to)e(use)h(in)h (determining)h(the)75 2547 y(elemen)o(ts)d(of)e(MPI)h(to)f(include)j (in)f(the)f(subset.)20 b(The)14 b(main)g(criteria)h(used)f(w)o(ere)g (that)f(the)h(MPI)g(subset)75 2603 y(should)p 75 2661 720 2 v 127 2688 a FB(1)144 2704 y Fu(V)m(ersion)g(of)f(Oct)f(3,)h (1993)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 166 171 166 170 bop 75 -100 a FE(166)593 b Ft(CHAPTER)16 b(9.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)131 45 y FE(1.)22 b(con)o(tain)13 b(routines)h(that)e(are)h(as)g(close)h(as)f(p) q(ossible)i(to)e(curren)o(t)g(standard)g(practice)h(to)f(minimize)189 102 y(the)i(e\013ort)f(to)h(p)q(ort)f(co)q(des.)131 196 y(2.)22 b(con)o(tain)15 b(new)g(and)h(imp)q(ortan)o(t)f(MPI)g (features.)131 291 y(3.)22 b(allo)o(w)17 b(dev)o(elop)q(ers)i(to)e(b)q (e)i(able)f(to)f(ha)o(v)o(e)g(the)h(subset)g(sho)o(w)f(up)h(as)f (rapidly)i(as)e(p)q(ossible)i(while)189 347 y(still)d(meeting)g(the)f (other)g(criteria.)166 442 y(There)i(w)o(ere)g(man)o(y)f(rationales)h (for)g(these)g(criteria.)26 b(It)17 b(w)o(as)f(recognized)i(that)e(it)h (is)h(imp)q(ossible)75 498 y(to)d(come)g(up)h(with)f(an)g(ideal)i (subset,)e(and)g(compromises)h(w)o(ere)f(necessary)l(.)20 b(It)c(w)o(as)e(felt)i(that)e(curren)o(t)75 555 y(users)22 b(should)g(b)q(e)h(comfortable)e(in)h(migrating)g(to)f(MPI.)g(Th)o(us,) i(the)f(subset)f(should)i(con)o(tain)f(the)75 611 y(message)16 b(passing)h(elemen)o(ts)g(that)f(represen)o(t)g(common)g(practice)h(to) q(da)o(y)l(.)23 b(F)l(urthermore,)16 b(it)h(w)o(as)f(felt)75 667 y(that)21 b(the)h(initial)i(\(and)e(later\))f(v)o(ersion)h(of)g (MPI)g(should)g(main)o(tain)h(e\016ciency)l(.)41 b(Otherwise)23 b(users)75 724 y(could)e(su\013er)e(from)g(disenc)o(han)o(tmen)o(t)h (with)g(MPI)f(on)h(their)g(\014rst)f(usage)h(and)f(ma)o(y)g(nev)o(er)h (giv)o(e)g(the)75 780 y(standard)12 b(a)g(second)g(try)l(.)19 b(Ha)o(ving)12 b(the)h(common)e(practice)i(routines)g(in)g(the)f (subset)h(should)g(encourage)75 837 y(dev)o(elop)q(ers)k(to)f(optimize) h(these)g(routines)f(in)h(the)g(initial)h(v)o(ersion)f(of)e(MPI.)h (Also,)h(though)f(some)g(felt)75 893 y(that)e(the)g(p)q(ortabilit)o(y)h (o\013ered)f(b)o(y)g(the)h(MPI)f(subset)h(w)o(as)e(su\016cien)o(t)i(to) f(en)o(tice)h(users,)f(man)o(y)g(felt)h(that)75 950 y(the)j(subset)g (should)g(con)o(tain)g(some)g(new)g(features)f(o)o(v)o(er)g(and)h(ab)q (o)o(v)o(e)f(curren)o(t)h(common)f(practice)h(as)75 1006 y(an)i(added)h(incen)o(tiv)o(e.)36 b(The)20 b(additional)h(items)g (selected)g(w)o(ere)f(mean)o(t)f(to)g(represen)o(t)i(some)e(of)h(the)75 1063 y(signi\014can)o(t)e(new)f(features)f(of)h(MPI.)f(Balanced)i (against)f(these)g(\014rst)f(t)o(w)o(o)g(goals)h(is)g(the)g(need)h(for) e(the)75 1119 y(subset)e(to)f(sho)o(w)g(up)i(in)f(a)g(timely)h (fashion.)k(Th)o(us,)14 b(eac)o(h)g(feature)g(c)o(hosen)g(for)f (inclusion)j(in)f(the)f(subset)75 1176 y(w)o(as)21 b(deemed)h(of)f (su\016cien)o(t)h(imp)q(ortance)g(to)f(out)o(w)o(eigh)h(its)f(added)h (complexit)o(y)h(in)f(implemen)o(ting)75 1232 y(the)e(subset.)34 b(Though)20 b(some)f(functions)i(seem)f(easy)f(to)g(implemen)o(t,)j (there)e(are)g(often)f(o)o(v)o(erlo)q(ok)o(ed)75 1288 y(costs)i(in)i(testing,)f(do)q(cumen)o(tation)h(and)e(dev)o(elopmen)o (t)i(that)e(w)o(ere)g(considered)i(b)q(efore)f(a)f(feature)75 1345 y(w)o(as)d(added)h(to)f(the)h(MPI)g(subset.)31 b(Inclusion)21 b(of)d(to)q(o)g(man)o(y)g(routines)h(migh)o(t)g(lead)h(to)e(mo)q (derately)75 1401 y(e\016cien)o(t)j(implemen)o(tation)h(of)e(them)h (all)g(instead)h(of)e(a)g(v)o(ery)g(e\016cien)o(t)i(and)f(p)q(ossibly)h (more)e(useful)75 1458 y(implemen)o(tation)c(of)f(a)g(smaller)h (subset.)75 1602 y Fv(9.3)59 b(Subset)19 b(F)n(unctionalit)n(y)75 1704 y FE(The)h(sections)g(b)q(elo)o(w)g(con)o(tain)f(a)h(general)g (summary)e(and)i(sp)q(eci\014c)h(routines)f(of)f(what)g(is)h(included) 75 1760 y(in/excluded)14 b(from)e(the)g(subset)g(for)f(eac)o(h)h(c)o (hapter.)19 b(The)12 b(general)g(requiremen)o(ts)h(of)e(eac)o(h)h(c)o (hapter)g(also)75 1817 y(hold)h(for)f(the)h(subset.)19 b(As)12 b(examples,)h(the)g(message)f(en)o(v)o(elop)q(e)h(in)h (subsection)f(3.2.2,)e(the)i(seman)o(tics)f(in)75 1873 y(section)k(3.4,)f(the)h(t)o(yp)q(e)g(matc)o(hing)g(in)g(section)h(3.5) e(and)h(the)g(comm)o(unication)g(ob)s(jects)f(in)i(subsection)75 1930 y(3.8.1)d(are)g(all)j(part)d(of)h(the)g(subset.)75 2053 y Fq(9.3.1)49 b(P)o(oint)16 b(to)h(P)o(oint)f(F)o(unctionalit)o(y) 75 2139 y FE(The)j(p)q(oin)o(t)h(to)e(p)q(oin)o(t)i(routines)f(are)g (not)g(only)h(cen)o(tral)f(to)f(MPI)h(but)h(represen)o(t)f(common)f (practice)75 2195 y(to)q(da)o(y)l(.)k(As)16 b(suc)o(h,)g(the)h(ma)s (jorit)o(y)d(of)i(the)g(functionalit)o(y)h(in)g(this)g(c)o(hapter)f(is) g(included)j(in)e(the)f(subset.)75 2252 y(Deriv)o(ed)f(datat)o(yp)q(es) e(represen)o(t)h(a)g(signi\014can)o(t)h(new)f(feature)g(in)h(MPI)f(and) h(is)f(included)j(in)e(the)f(subset.)75 2308 y(Ho)o(w)o(ev)o(er,)20 b(only)h(basic)g(datat)o(yp)q(es)f(of)g(section)h(3.2.1)e(will)j(b)q(e) f(allo)o(w)o(ed)g(as)f(input)h(to)f(the)g(datat)o(yp)q(e)75 2364 y(construction)15 b(routines)h(describ)q(ed)h(in)f(section)g (3.13.1.)166 2421 y(T)l(o)e(reduce)h(the)g(di\016cult)o(y)g(of)f (implemen)o(tation,)i(sev)o(eral)e(features)g(w)o(ere)g(left)h(out)f (of)g(the)g(subset.)75 2478 y(The)j(abilit)o(y)g(to)f(test)f(for)h(m)o (ultiple)i(completions)g(of)d(messages)h(w)o(as)g(remo)o(v)o(ed.)22 b(Also,)17 b(the)f(abilit)o(y)i(to)75 2534 y(cancel)e(a)e(message)f (and)i(use)g(p)q(ersisten)o(t)g(comm)o(unication)g(ob)s(jects)e(has)h (b)q(een)i(remo)o(v)o(ed.)j(Finally)l(,)d(the)75 2590 y(routines)g(asso)q(ciated)f(with)h(ready)f(and)g(sync)o(hronous)g (comm)o(unication)h(mo)q(des)g(w)o(ere)f(excluded.)166 2647 y(The)f(table)h(b)q(elo)o(w)g(con)o(tains)f(a)g(listing)i(b)o(y)e (section)h(of)f(whic)o(h)h(routines)f(from)g(the)g(p)q(oin)o(t)h(to)e (p)q(oin)o(t)75 2704 y(comm)o(unication)j(c)o(hapter)f(are)g(included)j (and)d(excluded)i(from)e(the)g(subset.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 167 172 167 171 bop 75 -100 a Ft(9.3.)34 b(SUBSET)16 b(FUNCTIONALITY)1036 b FE(167)120 45 y(Section)62 b(Title)400 b(Included)414 b(Excluded)p 75 68 1828 2 v 120 120 a(3.2)146 b(Basic)16 b(Send)g(Op)q(eration)63 b Fx(MPI)p 905 120 14 2 v 16 w(SEND)367 b FE(||)120 189 y(3.2.1)110 b(Message)15 b(Data)217 b(All)17 b(t)o(yp)q(es)402 b(||)120 258 y(3.3)146 b(Basic)16 b(Recv)g(Op)q(eration)61 b Fx(MPI)p 905 258 V 16 w(RECV)369 b FE(||)120 327 y(3.3.1)110 b(Return)16 b(Status)213 b Fx(MPI)p 905 327 V 16 w(GET)p 1009 327 V 17 w(SOURCE)205 b FE(||)820 383 y Fx(MPI)p 905 383 V 16 w(GET)p 1009 383 V 17 w(T)l(A)o(G)820 439 y(MPI)p 905 439 V 16 w(GET)p 1009 439 V 17 w(COUNT)120 508 y FE(3.7)146 b(Comm.)19 b(Mo)q(des)202 b(Standard)400 b Fx(MPI)p 1487 508 V 16 w(RSEND)1402 565 y(MPI)p 1487 565 V 16 w(SSEND)120 634 y FE(3.8.2)110 b(Comm.)19 b(Initiation)147 b Fx(MPI)p 905 634 V 16 w(ISEND)354 b(MPI)p 1487 634 V 16 w(IRSEND)820 690 y(MPI)p 905 690 V 16 w(IRECV)i(MPI)p 1487 690 V 16 w(ISSEND)120 759 y FE(3.8.3)110 b(Comm.)19 b(Completion)102 b Fx(MPI)p 905 759 V 16 w(W)l(AIT)371 b FE(||)820 816 y Fx(MPI)p 905 816 V 16 w(TEST)120 884 y FE(3.8.3)110 b(Multiple)17 b(Completion)83 b(||)492 b Fx(MPI)p 1487 884 V 16 w(W)l(AIT)l(ANY)1402 941 y(MPI)p 1487 941 V 16 w(TEST)l(ANY)1402 997 y(MPI)p 1487 997 V 16 w(W)l(AIT)l(ALL)1402 1054 y(MPI)p 1487 1054 V 16 w(TEST)l(ALL)120 1123 y FE(3.9)146 b(Prob)q(e)15 b(&)h(Cancel)178 b Fx(MPI)p 905 1123 V 16 w(PROBE)336 b(MPI)p 1487 1123 V 16 w(CANCEL)820 1179 y(MPI)p 905 1179 V 16 w(IPROBE)323 b(MPI)p 1487 1179 V 16 w(TEST)p 1617 1179 V 17 w(CANCELLED)820 1236 y(MPI)p 905 1236 V 16 w(GET)p 1009 1236 V 17 w(COUNT)820 1292 y(MPI)p 905 1292 V 16 w(GET)p 1009 1292 V 17 w(ELEMENT)120 1361 y FE(3.10)123 b(P)o(ersisten)o(t)15 b(Comm.)139 b(||)492 b Fx(MPI)p 1487 1361 V 16 w(CREA)l(TE)p 1672 1361 V 18 w(SEND)355 1417 y FE(Ob)s(jects)894 b Fx(MPI)p 1487 1417 V 16 w(CREA)l(TE)p 1672 1417 V 18 w(RSEND)1402 1474 y(MPI)p 1487 1474 V 16 w(CREA)l(TE)p 1672 1474 V 18 w(SSEND)1402 1530 y(MPI)p 1487 1530 V 16 w(CREA)l(TE)p 1672 1530 V 18 w(RECV)1402 1587 y(MPI)p 1487 1587 V 16 w(ST)l(ART)1402 1643 y(MPI)p 1487 1643 V 16 w(COMMOBJ)p 1729 1643 V 18 w(FREE)120 1712 y FE(3.11)123 b(Send-receiv)o(e)252 b Fx(MPI)p 905 1712 V 16 w(SENDRECV)g(MPI)p 1487 1712 V 16 w(EX)o(CHANGE)120 1781 y FE(3.12)123 b(Null)17 b(pro)q(cess)252 b Fm(MPI)p 897 1781 13 2 v 15 w(PROCNULL)278 b FE(||)120 1850 y(3.13.1)87 b(Deriv)o(ed)16 b(Datat)o(yp)q(es)122 b Fx(MPI)p 905 1850 14 2 v 16 w(TYPE)p 1038 1850 V 17 w(CONTIGUOUS)62 b FE(See)16 b(text)f(ab)q(o)o(v)o(e)820 1906 y Fx(MPI)p 905 1906 V 16 w(TYPE)p 1038 1906 V 17 w(VECTOR)820 1963 y(MPI)p 905 1963 V 16 w(TYPE)p 1038 1963 V 17 w(HVECTOR)820 2019 y(MPI)p 905 2019 V 16 w(TYPE)p 1038 2019 V 17 w(INDEXED)820 2076 y(MPI)p 905 2076 V 16 w(TYPE)p 1038 2076 V 17 w(HINDEXED)820 2132 y(MPI)p 905 2132 V 16 w(TYPE)p 1038 2132 V 17 w(STRUCT)120 2201 y FE(3.13.2)87 b(Address)16 b(and)f(exten)o(t)109 b Fx(MPI)p 905 2201 V 16 w(ADDRESS)355 2258 y FE(functions)285 b Fx(MPI)p 905 2258 V 16 w(TYPE)p 1038 2258 V 17 w(EXTENT)120 2327 y FE(3.13.3)87 b(Lo)o(w)o(er-b)q(ound)16 b(and)147 b Fx(MPI)p 905 2327 V 16 w(LB)355 2383 y FE(upp)q(er-b)q(ound)212 b Fx(MPI)p 905 2383 V 16 w(UB)120 2452 y FE(3.13.4)87 b(Commit)15 b(and)g(free)158 b Fx(MPI)p 905 2452 V 16 w(TYPE)p 1038 2452 V 17 w(COMMIT)164 b FE(||)820 2508 y Fx(MPI)p 905 2508 V 16 w(TYPE)p 1038 2508 V 17 w(FREE)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 168 173 168 172 bop 75 -100 a FE(168)593 b Ft(CHAPTER)16 b(9.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)75 45 y Fq(9.3.2)49 b(Collective)17 b(Communication)g(F)o(unctionalit)o(y)75 148 y FE(As)h(with)f(p)q(oin)o(t)h(to)f(p)q(oin)o(t)h(functionalit)o(y) l(,)i(collectiv)o(e)f(comm)o(unication)f(is)g(common)g(practice)g(to)q (da)o(y)l(,)75 204 y(and)g(the)h(functionalit)o(y)g(is)g(generally)g (included)i(in)e(the)f(subset.)29 b(F)l(urthermore,)18 b(the)g(abilit)o(y)i(to)d(de-)75 261 y(\014ne)h(arbitrary)e(groups)h (in)h(the)f(MPI)g(subset,)g(whic)o(h)h(are)f(then)g(asso)q(ciated)h (with)f(a)g(comm)o(unicator,)75 317 y(allo)o(ws)k(the)g(user)f(to)g(p)q (erform)h(collectiv)o(e)h(comm)o(unication)g(op)q(erations)e(o)o(v)o (er)g(an)h(arbitrary)f(set)g(of)75 374 y(pro)q(cesses.)32 b(This)20 b(increases)f(the)h(functionalit)o(y)g(of)f(the)g(collectiv)o (e)i(comm)o(unication)e(routines)h(o)o(v)o(er)75 430 y(what)15 b(is)g(commonly)h(a)o(v)m(ailable)h(to)q(da)o(y)l(.)166 496 y(T)l(o)i(ease)h(initial)i(implemen)o(tation,)f(the)f(user)g (de\014ned)h(reduce)f(functions)h(are)e(excluded)j(from)75 552 y(the)d(subset.)33 b(Note)19 b(that)f(all)j(of)d(the)i(system)f (pro)o(vided)h(reduce)g(functions)g(listed)h(in)f(section)g(4.5.1)75 608 y(are)i(include)i(in)f(the)f(subset.)40 b(Also,)24 b(the)e(scan)g(functions)h(are)f(excluded)i(from)d(the)h(subset.)41 b(Fi-)75 665 y(nally)l(,)25 b(the)e(generalized)h(\\v")e(v)o(ersions)h (of)f(routines)h(are)f(excluded)j(from)c(the)i(subset)g(as)f(w)o(ell)h (as)75 721 y Fx(MPI)p 160 721 14 2 v 16 w(REDUCE)p 352 721 V 17 w(SCA)l(TTER)p FE(.)166 787 y(The)f(table)g(b)q(elo)o(w)g(con) o(tains)f(a)g(listing)i(b)o(y)f(section)g(of)f(whic)o(h)h(routines)g (from)f(the)g(collectiv)o(e)75 843 y(comm)o(unication)16 b(c)o(hapter)f(are)g(included)j(and)d(excluded)i(from)e(the)g(subset.) 120 960 y(Section)62 b(Title)419 b(Included)247 b(Excluded)p 75 982 1829 2 v 120 1034 a(4.3)146 b(Barrier)15 b(Sync)o(h.)227 b Fx(MPI)p 924 1034 14 2 v 16 w(BARRIER)130 b FE(||)120 1103 y(4.4.1)110 b(Broadcast)314 b Fx(MPI)p 924 1103 V 16 w(BCAST)172 b FE(||)120 1172 y(4.4.2)110 b(Gather)374 b Fx(MPI)p 924 1172 V 16 w(GA)l(THER)142 b(MPI)p 1339 1172 V 16 w(GA)l(THERV)120 1241 y FE(4.4.4)110 b(Scatter)372 b Fx(MPI)p 924 1241 V 16 w(SCA)l(TTER)119 b(MPI)p 1339 1241 V 16 w(SCA)l(TTERV)120 1310 y FE(4.4.6)110 b(All-to-all)17 b(broadcast)128 b Fx(MPI)p 924 1310 V 16 w(ALLCAST)122 b(MPI)p 1339 1310 V 16 w(ALLCASTV)120 1379 y FE(4.4.7)110 b(All-to-all)17 b(scatter-gather)44 b Fx(MPI)p 924 1379 V 16 w(ALL)l(TO)o(ALL)98 b(MPI)p 1339 1379 V 16 w(ALL)l(TO)o(ALL)-5 b(V)120 1448 y FE(4.5.1)110 b(Reduce)371 b Fx(MPI)p 924 1448 V 16 w(REDUCE)141 b(MPI)p 1339 1448 V 16 w(USER)p 1467 1448 V 17 w(REDUCE)1254 1504 y(MPI)p 1339 1504 V 16 w(USER)p 1467 1504 V 17 w(REDUCEA)839 1561 y(MPI)p 924 1561 V 16 w(ALLREDUCE)61 b(MPI)p 1339 1561 V 16 w(USER)p 1467 1561 V 17 w(ALLREDUCE)1254 1617 y(MPI)p 1339 1617 V 16 w(USER)p 1467 1617 V 17 w(ALLREDUCEA)1254 1674 y(MPI)p 1339 1674 V 16 w(REDUCE)p 1531 1674 V 17 w(SCA)l(TTER)1254 1730 y(MPI)p 1339 1730 V 16 w(USER)p 1467 1730 V 17 w(REDUCE)p 1660 1730 V 17 w(SCA)l(TTER)1254 1786 y(MPI)p 1339 1786 V 16 w(USER)p 1467 1786 V 17 w(REDUCE)p 1660 1786 V 17 w(SCA)l(TTERA)120 1855 y FE(4.5.4)110 b(Scan)421 b(||)325 b Fx(MPI)p 1339 1855 V 16 w(SCAN)1254 1912 y(MPI)p 1339 1912 V 16 w(USER)p 1467 1912 V 17 w(SCAN)1254 1968 y(MPI)p 1339 1968 V 16 w(USER)p 1467 1968 V 17 w(SCANA)75 2140 y Fq(9.3.3)49 b(Groups,)15 b(Contexts,)g(and)i(Communicato)o(rs)e(F)o (unctionalit)o(y)75 2243 y FE(Groups,)d(con)o(texts)f(and)h(comm)o (unicators)g(are)g(used)g(throughout)f(the)h(MPI)g(standard.)19 b(Though)12 b(these)75 2300 y(concepts)f(are)g(used)h(in)g(some)f(pac)o (k)m(ages)g(to)q(da)o(y)l(,)g(their)g(inclusion)j(in)e(MPI)f(represen)o (ts)g(a)g(signi\014can)o(t)h(new)75 2356 y(feature)k(o)o(v)o(er)g(what) f(is)i(generally)h(a)o(v)m(ailable)g(in)f(common)f(practice)h(to)q(da)o (y)l(.)23 b(F)l(or)16 b(these)h(reasons,)f(this)75 2413 y(functionalit)o(y)j(is)f(included)i(in)f(the)f(subset.)27 b(Ho)o(w)o(ev)o(er,)17 b(implemen)o(tation)i(of)e(groups,)h(con)o (texts)f(and)75 2469 y(comm)o(unicators)f(is)h(also)f(a)g(signi\014can) o(t)h(e\013ort.)22 b(As)16 b(a)g(compromise,)h(the)f(MPI)g(in)o (tra-comm)o(unicator)75 2525 y(functionalit)o(y)e(is)g(included)h(but)e (the)g(in)o(tra-comm)o(unicator)g(functionalit)o(y)h(is)f(excluded.)21 b(The)13 b(cac)o(hing)75 2582 y(facilit)o(y)j(w)o(as)f(included)i(to)e (supp)q(ort)g(the)h(top)q(ology)e(functionalit)o(y)j(and)e(extensions.) 166 2647 y(The)22 b(table)g(b)q(elo)o(w)g(con)o(tains)g(a)f(listing)i (b)o(y)f(section)g(of)f(whic)o(h)h(routines)g(from)f(the)h(\\Groups,)75 2704 y(Con)o(texts,)14 b(and)h(Comm)o(unicators")f(c)o(hapter)h(are)g (included)j(and)d(excluded)j(from)c(the)h(subset.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 169 174 169 173 bop 75 -100 a Ft(9.3.)34 b(SUBSET)16 b(FUNCTIONALITY)1036 b FE(169)75 45 y(Section)16 b(Title)315 b(Included)422 b(Excluded)p 75 68 1837 2 v 75 120 a(5.2.4)64 b(Prede\014ned)17 b(Comm.)39 b Fx(MPI)p 729 120 14 2 v 15 w(COMM)p 886 120 V 17 w(W)o(ORLD)172 b FE(||)75 189 y(5.3.1)64 b(Group)15 b(Accessors)79 b Fx(MPI)p 729 189 V 15 w(GROUP)p 896 189 V 18 w(SIZE)230 b FE(||)644 245 y Fx(MPI)p 729 245 V 15 w(GROUP)p 896 245 V 18 w(RANK)644 302 y(MPI)p 729 302 V 15 w(TRANSLA)l(TE)p 1000 302 V 18 w(RANKS)75 371 y FE(5.3.2)64 b(Group)15 b(Const.)138 b Fx(MPI)p 729 371 V 15 w(GROUP)p 896 371 V 18 w(UNION)182 b FE(||)644 427 y Fx(MPI)p 729 427 V 15 w(GROUP)p 896 427 V 18 w(INTERSECTION)644 483 y(MPI)p 729 483 V 15 w(GROUP)p 896 483 V 18 w(DIFFERENCE)644 540 y(MPI)p 729 540 V 15 w(GROUP)p 896 540 V 18 w(INCL)644 596 y(MPI)p 729 596 V 15 w(GROUP)p 896 596 V 18 w(EX)o(CL)644 653 y(MPI)p 729 653 V 15 w(GROUP)p 896 653 V 18 w(RANGE)p 1062 653 V 18 w(INCL)644 709 y(MPI)p 729 709 V 15 w(GROUP)p 896 709 V 18 w(RANGE)p 1062 709 V 18 w(EX)o(CL)75 778 y FE(5.3.3)64 b(Group)15 b(Destruct.)83 b Fx(MPI)p 729 778 V 15 w(GROUP)p 896 778 V 18 w(FREE)214 b FE(||)75 847 y(5.4.1)64 b(Comm.)19 b(Access.)103 b Fx(MPI)p 729 847 V 15 w(COMM)p 886 847 V 17 w(SIZE)241 b FE(||)644 904 y Fx(MPI)p 729 904 V 15 w(COMM)p 886 904 V 17 w(RANK)644 960 y(MPI)p 729 960 V 15 w(COMM)p 886 960 V 17 w(GROUP)75 1029 y FE(5.4.2)64 b(Comm.)19 b(Const.)116 b Fx(MPI)p 729 1029 V 15 w(COMM)p 886 1029 V 17 w(DUP)241 b FE(||)644 1085 y Fx(MPI)p 729 1085 V 15 w(COMM)p 886 1085 V 17 w(MAKE)644 1142 y(MPI)p 729 1142 V 15 w(COMM)p 886 1142 V 17 w(SPLIT)75 1211 y FE(5.4.3)64 b(Comm.)19 b(Destruct.)61 b Fx(MPI)p 729 1211 V 15 w(COMM)p 886 1211 V 17 w(FREE)225 b FE(||)75 1280 y(5.5.1)64 b(Comm.)19 b(Access.)103 b(||)500 b Fx(MPI)p 1319 1280 V 16 w(COMM)p 1477 1280 V 16 w(ST)l(A)l(T)1234 1336 y(MPI)p 1319 1336 V 16 w(COMM)p 1477 1336 V 16 w(MERGE)75 1405 y FE(5.5.2)64 b(In)o(terComm.)19 b(Const.)h(||)500 b Fx(MPI)p 1319 1405 V 16 w(INTERCOMM)p 1609 1405 V 17 w(MAKE)264 1461 y FE(&)16 b(Destr.)797 b Fx(MPI)p 1319 1461 V 16 w(INTERCOMM)p 1609 1461 V 17 w(ST)l(ART)1234 1518 y(MPI)p 1319 1518 V 16 w(INTERCOMM)p 1609 1518 V 17 w(FINISH)75 1587 y FE(5.5.3)64 b(Name)15 b(Service)141 b(||)500 b Fx(MPI)p 1319 1587 V 16 w(INTERCOMM)p 1609 1587 V 17 w(NAME)1234 1643 y(MPI)p 1319 1643 V 16 w(INTERCOMM)p 1609 1643 V 17 w(NAME)p 1755 1643 V 16 w(ST)l(ART)75 1712 y FE(5.6.1)64 b(F)l(unctionalit)o(y)148 b(||)500 b Fx(MPI)p 1319 1712 V 16 w(A)l(TTRIBUTE)p 1584 1712 V 17 w(ALLOC)1234 1769 y(MPI)p 1319 1769 V 16 w(A)l(TTRIBUTE)p 1584 1769 V 17 w(FREE)1234 1825 y(MPI)p 1319 1825 V 16 w(GET)p 1423 1825 V 17 w(A)l(TTRIBUTE)p 1689 1825 V 17 w(KEY)1234 1882 y(MPI)p 1319 1882 V 16 w(SET)p 1418 1882 V 17 w(A)l(TTRIBUTE)1234 1938 y(MPI)p 1319 1938 V 16 w(TEST)p 1449 1938 V 16 w(A)l(TTRIBUTE)1234 1994 y(MPI)p 1319 1994 V 16 w(DELETE)p 1505 1994 V 16 w(A)l(TTRIBUTE)75 2275 y Fq(9.3.4)49 b(T)l(op)q(ology)19 b(F)o(unctionalit)o(y)75 2401 y FE(The)d(capabilit)o(y)h(presen)o(ted)f(in)g(the)g(pro)q(cess)f (top)q(ology)h(c)o(hapter)f(represen)o(ts)g(a)h(new)f(functionalit)o(y) i(in)75 2457 y(MPI)i(and)g(is)h(not)f(generally)h(common)f(practice)h (to)q(da)o(y)l(.)31 b(T)l(o)19 b(mak)o(e)f(this)i(a)o(v)m(ailable)h(in) f(the)f(subset,)75 2514 y(the)h(functionalit)o(y)i(asso)q(ciated)f (with)f(Cartesian)h(top)q(ologies)f(is)h(included)i(but)e(the)f (general)h(graph)75 2570 y(top)q(ologies)16 b(are)f(excluded.)166 2647 y(The)22 b(table)h(b)q(elo)o(w)g(con)o(tains)f(a)g(listing)i(b)o (y)e(section)h(of)f(whic)o(h)h(routines)f(from)g(the)g(top)q(ology)75 2704 y(c)o(hapter)15 b(are)g(included)j(and)d(excluded)i(from)e(the)g (subset.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 170 175 170 174 bop 75 -100 a FE(170)593 b Ft(CHAPTER)16 b(9.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)120 45 y FE(Section)62 b(Title)404 b(Included)273 b(Excluded)p 75 68 1644 2 v 120 120 a(6.5.4)110 b(Lo)o(w{lev)o(el)16 b(top)q(ol.)k(func.)58 b Fx(MPI)p 909 120 14 2 v 16 w(MAP)p 1024 120 V 16 w(CART)109 b(MPI)p 1350 120 V 16 w(MAP)p 1465 120 V 17 w(GRAPH)120 189 y Fz(??)155 b FE(High{lev)o(el)17 b(top)q(ol.)j(func.)46 b Fx(MPI)p 909 189 V 16 w(CART)224 b(MPI)p 1350 189 V 16 w(GRAPH)120 258 y FE(6.5.1)110 b(T)l(op)q(ol.)20 b(inquiry)d(func.)99 b Fx(MPI)p 909 258 V 16 w(INQMAP)824 314 y(MPI)p 909 314 V 16 w(INQCARTDIM)60 b(MPI)p 1350 314 V 16 w(INQGRAPHDIMS)824 371 y(MPI)p 909 371 V 16 w(INQCART)146 b(MPI)p 1350 371 V 16 w(INQGRAPH)824 427 y(MPI)p 909 427 V 16 w(INQRANK)824 483 y(MPI)p 909 483 V 16 w(INQLOC)120 552 y FE(6.5.3)110 b(P)o(artitioning)16 b(of)e(Cart.)88 b Fx(MPI)p 909 552 V 16 w(P)l(ARTC)824 609 y(MPI)p 909 609 V 16 w(SHIFT)p 1052 609 V 16 w(ID)824 665 y(MPI)p 909 665 V 16 w(MAKDIM)75 798 y Fq(9.3.5)49 b(Language)18 b(Binding)75 887 y FE(The)c(subset)f(shall)i(con)o(tain)f (the)f(F)l(ortran)f(77)h(and)h(C)f(language)h(bindings.)21 b(All)15 b(the)e(routines)h(included)75 944 y(in)i(the)f(subset)h(will) h(conform)d(to)h(the)g(rules)h(stated)f(in)h(c)o(hapter)f(2)g(and)g (annex)g(A.)75 1076 y Fq(9.3.6)49 b(MPI)17 b(Environmental)g (Management)75 1166 y FE(The)i(abilit)o(y)i(for)d(the)i(user)f(to)g (mak)o(e)f(inquiries)k(of)c(the)i(MPI)f(system)g(is)g(an)g(imp)q(ortan) o(t)g(feature)g(in)75 1222 y(p)q(ortable)g(programming.)28 b(F)l(or)18 b(this)g(reason)g(most)g(of)g(the)g(functions)h(are)f (included)j(in)e(the)f(subset.)75 1279 y(Ho)o(w)o(ev)o(er,)11 b(the)i(abilit)o(y)g(for)f(the)g(user)h(to)e(sp)q(ecify)j(an)e(error)g (handler)h(is)g(not)e(included)k(in)e(the)g(subset)f(nor)75 1335 y(is)18 b(the)f(abilit)o(y)h(to)f(unpac)o(k)g(deriv)o(ed)h(datat)o (yp)q(es.)25 b(The)17 b(\014rst)g(is)h(consisten)o(t)f(with)g(not)g (including)j(user)75 1392 y(called)d(functions)f(in)g(the)f(subset)h (and)f(the)h(second)g(is)f(of)g(minimal)i(impact)f(since)g(deriv)o(ed)h (datat)o(yp)q(es)75 1448 y(can)e(only)h(ha)o(v)o(e)f(basic)h(datat)o (yp)q(es)e(in)i(them.)166 1507 y(The)e(table)h(b)q(elo)o(w)g(con)o (tains)f(a)g(listing)i(b)o(y)e(section)h(of)f(whic)o(h)h(routines)f (from)g(the)g(MPI)h(En)o(viron-)75 1563 y(men)o(tal)g(Managemen)o(t)f (c)o(hapter)h(are)g(included)j(and)e(excluded)h(from)d(the)h(subset.)75 1679 y(Section)h(Title)214 b(Included)513 b(Excluded)p 75 1702 1883 2 v 75 1754 a(7.1)100 b(Implem.)21 b(Info.)35 b Fx(MPI)p 628 1754 14 2 v 16 w(GET)p 732 1754 V 17 w(V)l(ALID)p 876 1754 V 16 w(T)l(A)o(G)p 978 1754 V 17 w(RANGE)75 1823 y FE(7.1.2)64 b(Bu\013ering)127 b Fx(MPI)p 628 1823 V 16 w(GET)p 732 1823 V 17 w(BUFFER)p 918 1823 V 17 w(P)l(ARAMS)543 1880 y(MPI)p 628 1880 V 16 w(SUGGEST)p 843 1880 V 18 w(BUFFER)p 1030 1880 V 17 w(P)l(ARAMS)543 1936 y(MPI)p 628 1936 V 16 w(USER)p 756 1936 V 17 w(SPECIFIES)p 987 1936 V 17 w(BUFFER)75 2005 y FE(7.2)100 b(Error)14 b(handling)j Fx(MPI)p 628 2005 V 16 w(GET)p 732 2005 V 17 w(ERRORMODE)543 2061 y(MPI)p 628 2061 V 16 w(SET)p 727 2061 V 17 w(ERRORMODE)203 b(MPI)p 1309 2061 V 16 w(SET)p 1408 2061 V 17 w(ERRHANDLER)543 2118 y(MPI)p 628 2118 V 16 w(ERROR)p 791 2118 V 18 w(STRING)75 2187 y Fz(??)109 b FE(\\Unpac)o(king")734 b Fx(MPI)p 1309 2187 V 16 w(TYPE)p 1442 2187 V 17 w(QUERY)p 1609 2187 V 18 w(CONSTRUCTOR)75 2256 y FE(7.3)100 b(Startup)157 b Fx(MPI)p 628 2256 V 16 w(INIT)543 2312 y(MPI)p 628 2312 V 16 w(FINALIZE)543 2369 y(MPI)p 628 2369 V 16 w(INITIALIZED)543 2425 y(MPI)p 628 2425 V 16 w(ABORT)75 2558 y Fq(9.3.7)49 b(Pro\014ling)18 b(F)o(unctionalit)o(y)75 2647 y FE(Pro\014ling)12 b(is)f(a)g(capabilit)o(y)h(whic)o(h)g(will)h(b)q(e)e(of)g(great)f (utilit)o(y)i(in)g(MPI)f(for)f(determining)j(the)e(p)q(erformance)75 2704 y(of)16 b(eac)o(h)h(implemen)o(tation.)26 b(F)l(urthermore,)16 b(though)g(implemen)o(tation)j(of)d(the)h(pro\014ling)h(capabilities) 1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 171 176 171 175 bop 75 -100 a Ft(9.4.)34 b(SUBSET)16 b(TESTING)1222 b FE(171)75 45 y(doubles)19 b(the)f(n)o(um)o(b)q(er)g(of)f(routines)h (supplied,)j(these)d(routines)g(can)g(easily)h(b)q(e)f(generated)g (from)f(the)75 102 y(routines)11 b(required)h(elsewhere)g(in)g(the)f (subset.)18 b(Giv)o(en)11 b(these)g(considerations,)i(the)d(subset)h (will)i(con)o(tain)75 158 y(the)i(shado)o(w)g(routines)h(\\PMPI)p 635 158 14 2 v 16 w(")e(for)h(eac)o(h)g(routine)h(included)i(in)e(the)f (subset.)75 301 y Fv(9.4)59 b(Subset)19 b(T)-5 b(esting)75 403 y FE(The)14 b(test)g(suite)g(is)h(exp)q(ected)g(to)f(also)f (include)k(a)d(subset)g(whic)o(h)h(will)g(v)o(erify)g(the)f (functionalit)o(y)h(of)f(this)75 459 y(subset.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 172 177 172 176 bop 75 379 a FA(Biblio)q(graph)m(y)98 604 y FE([1])21 b(V.)14 b(Bala)h(and)f(S.)h(Kipnis.)20 b(Pro)q(cess)14 b(groups:)19 b(a)14 b(mec)o(hanism)h(for)f(the)g(co)q(ordination)i(of)e (and)g(com-)168 661 y(m)o(unication)i(among)e(pro)q(cesses)i(in)f(the)g (Ven)o(us)h(collectiv)o(e)g(comm)o(unication)g(library)l(.)k(T)l(ec)o (hnical)168 717 y(rep)q(ort,)15 b(IBM)g(T.)g(J.)g(W)l(atson)g(Researc)o (h)g(Cen)o(ter,)g(Octob)q(er)g(1992.)k(Preprin)o(t.)98 820 y([2])i(V.)h(Bala,)h(S.)e(Kipnis,)k(L.)d(Rudolph,)j(and)c(Marc)g (Snir.)40 b(Designing)23 b(e\016cien)o(t,)h(scalable,)g(and)168 876 y(p)q(ortable)17 b(collectiv)o(e)i(comm)o(unication)e(libraries.)26 b(T)l(ec)o(hnical)18 b(rep)q(ort,)e(IBM)h(T.)f(J.)g(W)l(atson)g(Re-)168 933 y(searc)o(h)f(Cen)o(ter,)g(Octob)q(er)h(1992.)i(Preprin)o(t.)98 1035 y([3])j(A.)14 b(Beguelin,)j(J.)d(Dongarra,)e(A.)i(Geist,)h(R.)f (Manc)o(hek,)g(and)g(V.)g(Sunderam.)19 b(Visualization)e(and)168 1092 y(debugging)g(in)f(a)f(heterogeneous)g(en)o(vironmen)o(t.)20 b Fs(IEEE)15 b(Computer)p FE(,)h(26\(6\):88{95,)11 b(June)17 b(1993.)98 1194 y([4])k(Luc)14 b(Bomans)f(and)h(Rolf)f(Hemp)q(el.)18 b(The)c(Argonne/GMD)e(macros)g(in)i(F)o(OR)l(TRAN)g(for)e(p)q(ortable) 168 1251 y(parallel)20 b(programming)e(and)h(their)f(implemen)o(tation) i(on)e(the)h(In)o(tel)g(iPSC/2.)29 b Fs(Par)n(al)r(lel)19 b(Com-)168 1307 y(puting)p FE(,)c(15:119{132,)d(1990.)98 1410 y([5])21 b(R.)14 b(Butler)h(and)f(E.)f(Lusk.)18 b(User's)13 b(guide)i(to)e(the)h(P4)g(programming)f(system.)k(T)l(ec)o (hnical)f(Rep)q(ort)168 1466 y(TM-ANL{92/17,)e(Argonne)h(National)h (Lab)q(oratory)l(,)e(1992.)98 1569 y([6])21 b(Rob)q(ert)i(Calkin,)i (Rolf)f(Hemp)q(el,)h(Hans-Christian)f(Hopp)q(e,)g(and)f(P)o(eter)f (Wypior.)42 b(P)o(ortable)168 1626 y(programming)17 b(with)g(the)g (parmacs)g(message{passing)f(library)l(.)27 b Fs(Par)n(al)r(lel)17 b(Computing,)h(sp)n(e)n(cial)168 1682 y(issue)e(on)g(message{p)n (assing)f(interfac)n(es)p FE(,)e(to)i(app)q(ear.)98 1785 y([7])21 b(S.)c(Chittor)g(and)h(R.)f(J.)g(En)o(b)q(o)q(dy)l(.)27 b(P)o(erformance)16 b(ev)m(aluation)j(of)d(mesh{connected)i(w)o (ormhole{)168 1841 y(routed)f(net)o(w)o(orks)e(for)i(in)o(terpro)q (cessor)f(comm)o(unication)i(in)f(m)o(ulticomputers.)25 b(In)17 b Fs(Pr)n(o)n(c)n(e)n(e)n(dings)168 1898 y(of)g(the)f(1990)i (Sup)n(er)n(c)n(omputing)e(Confer)n(enc)n(e)p FE(,)c(pages)j(647{656,)e (1990.)98 2000 y([8])21 b(S.)f(Chittor)g(and)g(R.)g(J.)f(En)o(b)q(o)q (dy)l(.)35 b(Predicting)21 b(the)f(e\013ect)g(of)f(mapping)i(on)e(the)h (comm)o(unica-)168 2057 y(tion)g(p)q(erformance)g(of)f(large)g(m)o (ulticomputers.)34 b(In)20 b Fs(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)k(the)f (1991)h(International)168 2113 y(Confer)n(enc)n(e)15 b(on)h(Par)n(al)r(lel)f(Pr)n(o)n(c)n(essing,)f(vol.)i(II)f(\(Softwar)n (e\))p FE(,)f(pages)h(I)q(I{1)h({)f(I)q(I{4,)g(1991.)98 2216 y([9])21 b(J.)e(Dongarra,)f(A.)g(Geist,)h(R.)g(Manc)o(hek,)g(and)g (V.)f(Sunderam.)31 b(In)o(tegrated)18 b(PVM)g(framew)o(ork)168 2272 y(supp)q(orts)c(heterogeneous)g(net)o(w)o(ork)f(computing.)18 b Fs(Computers)e(in)e(Physics)p FE(,)f(7\(2\):166{75,)e(April)168 2329 y(1993.)75 2432 y([10])21 b(J.)c(J.)g(Dongarra,)f(R.)h(Hemp)q(el,) i(A.)e(J.)g(G.)f(Hey)l(,)i(and)f(D.)f(W.)h(W)l(alk)o(er.)25 b(A)17 b(prop)q(osal)g(for)g(a)g(user-)168 2488 y(lev)o(el,)c(message)d (passing)h(in)o(terface)g(in)h(a)e(distributed)i(memory)e(en)o (vironmen)o(t.)j(T)l(ec)o(hnical)g(Rep)q(ort)168 2544 y(TM-12231,)g(Oak)j(Ridge)g(National)g(Lab)q(oratory)l(,)e(F)l(ebruary) h(1993.)75 2647 y([11])21 b(Edin)o(burgh)g(P)o(arallel)f(Computing)g (Cen)o(tre,)g(Univ)o(ersit)o(y)g(of)f(Edin)o(burgh.)34 b Fs(CHIMP)19 b(Conc)n(epts)p FE(,)168 2704 y(June)e(1991.)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 173 178 173 177 bop 75 -100 a Ft(BIBLIOGRAPHY)1371 b FE(173)75 45 y([12])21 b(Edin)o(burgh)c(P)o(arallel)g(Computing)f(Cen)o(tre,)f (Univ)o(ersit)o(y)h(of)g(Edin)o(burgh.)22 b Fs(CHIMP)15 b(V)m(ersion)h(1.0)168 102 y(Interfac)n(e)p FE(,)e(Ma)o(y)g(1992.)75 195 y([13])21 b(D.)d(F)l(eitelson.)28 b(Comm)o(unicators:)d(Ob)s (ject-based)18 b(m)o(ultipart)o(y)g(in)o(teractions)g(for)g(parallel)h (pro-)168 252 y(gramming.)j(T)l(ec)o(hnical)c(Rep)q(ort)f(91-12,)e (Dept.)g(Computer)h(Science,)i(The)e(Hebrew)g(Univ)o(ersit)o(y)168 308 y(of)f(Jerusalem,)h(No)o(v)o(em)o(b)q(er)f(1991.)75 402 y([14])21 b(G.)14 b(A.)g(Geist,)g(M.)g(T.)f(Heath,)h(B.)g(W.)g(P)o (eyton,)f(and)i(P)l(.)f(H.)g(W)l(orley)l(.)19 b(A)14 b(user's)g(guide)h(to)f(PICL:)g(a)168 459 y(p)q(ortable)f(instrumen)o (ted)f(comm)o(unication)h(library)l(.)i(T)l(ec)o(hnical)f(Rep)q(ort)e (TM-11616,)f(Oak)h(Ridge)168 515 y(National)k(Lab)q(oratory)l(,)e (Octob)q(er)i(1990.)75 609 y([15])21 b(nCUBE)16 b(Corp)q(oration.)j Fs(nCUBE)c(2)i(Pr)n(o)n(gr)n(ammers)e(Guide,)i(r2.0)p FE(,)f(Decem)o(b)q(er)f(1990.)75 703 y([16])21 b(H.)c(M)q(\177)-24 b(uhlen)o(b)q(ein)19 b(O.)d(Kr\177)-23 b(amer.)24 b(Mapping)16 b(strategies)g(in)i(message{based)e(m)o(ultipro)q(cessor)h(sys-)168 759 y(tems.)j Fs(Par)n(al)r(lel)15 b(Computing)p FE(,)g(9:213{225,)d (1989.)75 853 y([17])21 b(P)o(arasoft)14 b(Corp)q(oration.)22 b Fs(Expr)n(ess)16 b(V)m(ersion)g(1.0:)22 b(A)17 b(Communic)n(ation)g (Envir)n(onment)e(for)j(Par-)168 909 y(al)r(lel)e(Computers)p FE(,)f(1988.)75 1003 y([18])21 b(P)o(aul)c(Pierce.)23 b(The)16 b(NX/2)g(op)q(erating)g(system.)22 b(In)17 b Fs(Pr)n(o)n(c)n(e)n(e)n(dings)e(of)i(the)g(Thir)n(d)g(Confer)n(enc)n(e) e(on)168 1060 y(Hyp)n(er)n(cub)n(e)g(Concurr)n(ent)f(Computers)i(and)f (Applic)n(ations)p FE(,)e(pages)h(384{390.)d(A)o(CM)j(Press,)f(1988.)75 1153 y([19])21 b(A.)14 b(Skjellum)i(and)e(A.)g(Leung.)19 b(Zip)q(co)q(de:)h(a)14 b(p)q(ortable)h(m)o(ulticomputer)f(comm)o (unication)h(library)168 1210 y(atop)g(the)f(reactiv)o(e)h(k)o(ernel.) 20 b(In)15 b(D.)f(W.)g(W)l(alk)o(er)h(and)g(Q.)g(F.)f(Stout,)g (editors,)g Fs(Pr)n(o)n(c)n(e)n(e)n(dings)g(of)i(the)168 1266 y(Fifth)21 b(Distribute)n(d)h(Memory)f(Concurr)n(ent)g(Computing)g (Confer)n(enc)n(e)p FE(,)e(pages)i(767{776.)d(IEEE)168 1323 y(Press,)d(1990.)75 1417 y([20])21 b(A.)12 b(Skjellum,)i(S.)e (Smith,)h(C.)e(Still,)j(A.)d(Leung,)i(and)g(M.)e(Morari.)i(The)g(Zip)q (co)q(de)g(message)f(passing)168 1473 y(system.)20 b(T)l(ec)o(hnical)d (rep)q(ort,)d(La)o(wrence)i(Liv)o(ermore)f(National)h(Lab)q(oratory)l (,)e(Septem)o(b)q(er)i(1992.)75 1567 y([21])21 b(D.)12 b(W)l(alk)o(er.)k(Standards)c(for)g(message)g(passing)g(in)i(a)e (distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec)o(h-)168 1623 y(nical)i(Rep)q(ort)f(TM-12147,)d(Oak)i(Ridge)h(National)g(Lab)q (oratory)l(,)e(August)h(1992.)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 174 179 174 178 bop 75 357 a Fw(Annex)33 b(A)75 565 y FA(Language)41 b(Binding)75 807 y Fv(A.1)59 b(Intro)r(duction)75 909 y FE(W)l(e)12 b(summarize)g(the)g(sp)q(eci\014c)i(bindings)f(for)e(b)q (oth)h(F)l(ortran)f(and)h(C.)f(W)l(e)h(presen)o(t)g(\014rst)f(the)h(C)f (bindings,)75 966 y(then)16 b(the)f(F)l(ortran)f(bindings.)22 b(Listings)16 b(are)f(alphab)q(etical)i(within)g(c)o(hapter.)75 1112 y Fv(A.2)59 b(De\014ned)19 b(Constants)g(fo)n(r)i(C)f(and)f(F)n(o) n(rtran)75 1215 y FE(These)e(are)g(required)h(de\014ned)h(constan)o (ts,)d(to)g(b)q(e)i(de\014ned)h(in)e(the)h(\014les)g Fp(mpi.h)e FE(\(for)g(C\))h(and)g Fp(mpif.h)75 1271 y FE(\(for)d(F)l(ortran\).)75 1381 y Fp(/*)24 b(return)f(codes)g(*/)75 1438 y(MPI_SUCCESS)75 1551 y(/*)h(assorted)e(constants)h(\(both)g(C)h (and)f(Fortran\))g(*/)75 1607 y(MPI_BOTTOM)75 1664 y(MPI_PROCNULL)75 1720 y(MPI_ANY_SOURCE)75 1777 y(MPI_ANY_TAG)75 1833 y(MPI_UNDEFINED)75 1889 y(MPI_UNDEFINED_RANK)75 1946 y(MPI_UB)75 2002 y(MPI_LB)75 2115 y(/*)h(constant)e(array)i(sizes)f(\(Fortran\))f(*/)75 2172 y(MPI_STATUS_SIZE)75 2285 y(/*)i(attributes)e(\(C)i(and)f (Fortran\))g(*/)75 2341 y(MPI_UNDEFINED_ATTR)75 2398 y(MPI_FOUND)75 2454 y(MPI_NOT_FOUND)75 2510 y(MPI_ATTR_COPY_FREELY)75 2567 y(MPI_ATTR_COPY_NEVER)75 2623 y(MPI_ATTRCOPY_CALLBACK)p 75 2661 720 2 v 127 2688 a FB(1)144 2704 y Fu(V)m(ersion)14 b(as)f(of)g(Octob)q(er)g(13,)g(1993)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 175 180 175 179 bop 75 -100 a Ft(A.2.)29 b(DEFINED)15 b(CONST)l(ANTS)h(F)o(OR)f (C)g(AND)g(F)o(OR)l(TRAN)590 b FE(175)75 102 y Fp(/*)24 b(Error-handling)e(specifiers)g(\(C)i(and)f(Fortran\))g(*/)75 158 y(MPI_ERRORSFATAL)75 214 y(MPI_ERRORSRETURN)75 271 y(MPI_ERRORSCALLUSERROUTINE)75 384 y(/*)h(Pre-defined)e(error)h (handlers)g(\(C)g(and)h(Fortran\))f(*/)75 440 y(MPI_NOHANDLER)75 497 y(MPI_DEFAULTHANDLER)75 610 y(/*)h(Maximum)e(sizes)i(for)f(strings) g(*/)75 666 y(MPI_MAX_INTERCOMM_NAME_SIZ)o(E)75 779 y(/*)h(elementary)e (datatypes)h(\(C\))g(*/)75 835 y(MPI_CHAR)75 892 y(MPI_SHORT)75 948 y(MPI_INT)75 1005 y(MPI_LONG)75 1061 y(MPI_FLOAT)75 1118 y(MPI_DOUBLE)75 1174 y(MPI_BYTE)75 1231 y(MPI_UNSIGNED_CHAR)75 1287 y(MPI_UNSIGNED_SHORT)75 1344 y(MPI_UNSIGNED_LONG)75 1400 y(MPI_UNSIGNED_INT)75 1513 y(/*)h(elementary)e(datatypes)h (\(Fortran\))f(*/)75 1569 y(MPI_INTEGER)75 1626 y(MPI_REAL)75 1682 y(MPI_DOUBLE_PRECISION)75 1739 y(MPI_COMPLEX)75 1795 y(MPI_DOUBLE_COMPLEX)75 1852 y(MPI_LOGICAL)75 1908 y(MPI_CHARACTER)75 1965 y(MPI_BYTE)75 2077 y(/*)i(reserved)e (communicators)g(\(C)i(and)f(Fortran\))g(*/)75 2134 y(MPI_COMM_WORLD)75 2247 y(/*)h(collective)e(operations)h(\(C)g(and)h(Fortran\))e(*/)75 2303 y(MPI_MAX)75 2360 y(MPI_MIN)75 2416 y(MPI_SUM)75 2473 y(MPI_PROD)75 2529 y(MPI_AND)75 2586 y(MPI_OR)75 2642 y(MPI_XOR)75 2698 y(MPI_MAXLOC)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 176 181 176 180 bop 75 -100 a FE(176)987 b Ft(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fp(MPI_MINLOC)75 158 y(/*)24 b(topologies)e(\(C)i(and)f(Fortran\))g(*/)75 214 y(MPI_REORDER)75 271 y(MPI_NOREORDER)75 327 y(MPI_GRAPHTOP)75 384 y(MPI_CARTTOP)75 440 y(MPI_NOTOP)75 497 y(MPI_PER)75 553 y(MPI_NOPER)166 698 y FE(The)15 b(follo)o(wing)h(are)f(de\014ned)i (C)e(t)o(yp)q(e)g(de\014nitions,)i(also)e(included)j(in)e(the)f(\014le) h Fp(mpi.h)p FE(.)75 787 y Fp(/*)24 b(opaque)f(types)g(\(C\))g(*/)75 843 y(MPI_Persistence)75 900 y(MPI_Mode)75 956 y(MPI_Op)75 1013 y(MPI_Comm_name)75 1069 y(MPI_Addr)75 1126 y(MPI_Errmode)75 1239 y(/*)h(assorted)e(opaque)h(structures)g(\(C\))g(*/)75 1295 y(MPI_Status)75 1352 y(MPI_Group)75 1408 y(MPI_Communicator)75 1464 y(MPI_Datatype)75 1521 y(MPI_Comm_request)75 1577 y(MPI_Attribute)75 1634 y(MPI_Intercomm_request)75 1747 y(/*)h(Function)e(types)i(\(C\))f(*/)75 1803 y(MPI_Errhandler)75 1944 y Fv(A.3)59 b(C)20 b(bindings)g(fo)n(r)h(P)n(oint-to-P)n(oint)e (Communication)75 2045 y FE(These)d(are)e(presen)o(ted)i(here)g(in)g (the)f(order)g(of)g(their)g(app)q(earance)h(in)g(the)g(c)o(hapter.)75 2102 y Fp(int)23 b(MPI)p 245 2102 15 2 v 17 w(Send\(void*)g(buf,)g(int) g(count,)h(MPI)p 979 2102 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 2158 y(int)h(tag,)f(MPI)p 683 2158 V 17 w(Comm)g(comm\))75 2245 y(int)g(MPI)p 245 2245 V 17 w(Recv\(void*)g(buf,)g(int)g(count,)h (MPI)p 979 2245 V 16 w(Datatype)f(datatype,)g(int)g(source,)393 2301 y(int)h(tag,)f(MPI)p 683 2301 V 17 w(Comm)g(comm,)g(MPI)p 1034 2301 V 17 w(Status)g(*status\))75 2388 y(int)g(MPI)p 245 2388 V 17 w(Get)p 334 2388 V 17 w(source\(MPI)p 591 2388 V 16 w(Status)g(status,)g(int)g(*source\))75 2474 y(int)g(MPI)p 245 2474 V 17 w(Get)p 334 2474 V 17 w(tag\(MPI)p 519 2474 V 16 w(Status)g(status,)g(int)h(*tag\))75 2561 y(int)f(MPI)p 245 2561 V 17 w(Get)p 334 2561 V 17 w(count\(MPI)p 567 2561 V 16 w(Status)g(status,)g(MPI)p 1013 2561 V 17 w(Datatype)f(datatype,)h(int)h(*count\))75 2647 y(int)f(MPI)p 245 2647 V 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2647 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2704 y(int)h(tag,)f(MPI)p 683 2704 V 17 w(Comm)g(comm\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 177 182 177 181 bop 75 -100 a Ft(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b FE(177)75 45 y Fp(int)23 b(MPI)p 245 45 15 2 v 17 w(Ssend\(void*)g(buf,)g(int)g (count,)g(MPI)p 1002 45 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 102 y(int)h(tag,)f(MPI)p 683 102 V 17 w(Comm)g(comm\))75 191 y(int)g(MPI)p 245 191 V 17 w(Isend\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 191 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 247 y(int)h(tag,)f(MPI)p 683 247 V 17 w(Comm)g(comm,)g(MPI)p 1034 247 V 17 w(Comm)p 1147 247 V 17 w(request)g(*request\))75 336 y(int)g(MPI)p 245 336 V 17 w(Irsend\(void*)f(buf,)i(int)f(count,)g (MPI)p 1026 336 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 392 y(int)g(tag,)f(MPI)p 683 392 V 17 w(Comm)g(comm,)g(MPI)p 1034 392 V 17 w(Comm)p 1147 392 V 17 w(request)g(*request\))75 481 y(int)g(MPI)p 245 481 V 17 w(Issend\(void*)f(buf,)i(int)f(count,)g (MPI)p 1026 481 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 538 y(int)g(tag,)f(MPI)p 683 538 V 17 w(Comm)g(comm,)g(MPI)p 1034 538 V 17 w(Comm)p 1147 538 V 17 w(request)g(*request\))75 627 y(int)g(MPI)p 245 627 V 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 627 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 683 y(int)h(tag,)f(MPI)p 683 683 V 17 w(Comm)g(comm,)g(MPI)p 1034 683 V 17 w(Comm)p 1147 683 V 17 w(request)g(*request\))75 772 y(int)g(MPI)p 245 772 V 17 w(Wait\(MPI)p 454 772 V 16 w(Comm)p 566 772 V 17 w(request)g(request,)g(MPI)p 1061 772 V 16 w(Status)g(*status\))75 861 y(int)g(MPI)p 245 861 V 17 w(Test\(MPI)p 454 861 V 16 w(Comm)p 566 861 V 17 w(request)g(request,)g(int)g(flag,)g(MPI)p 1299 861 V 17 w(Status)g(*status\))75 950 y(int)g(MPI)p 245 950 V 17 w(Waitany\(int)g(count,)g(MPI)p 788 950 V 16 w(Comm)p 900 950 V 17 w(request)g(*array)p 1252 950 V 16 w(of)p 1316 950 V 17 w(requests,)g(int)g(*index,)393 1007 y(MPI)p 468 1007 V 17 w(Status)g(*status\))75 1095 y(int)g(MPI)p 245 1095 V 17 w(Testany\(int)g(count,)g(MPI)p 788 1095 V 16 w(Comm)p 900 1095 V 17 w(request)g(*array)p 1252 1095 V 16 w(of)p 1316 1095 V 17 w(requests,)g(int)g(*index,)393 1152 y(int)h(*flag,)f(MPI)p 731 1152 V 16 w(Status)g(*status\))75 1241 y(int)g(MPI)p 245 1241 V 17 w(Waitall\(int)g(count,)g(MPI)p 788 1241 V 16 w(Comm)p 900 1241 V 17 w(request)g(*array)p 1252 1241 V 16 w(of)p 1316 1241 V 17 w(requests,)393 1297 y(MPI)p 468 1297 V 17 w(Status)g(*array)p 796 1297 V 16 w(of)p 860 1297 V 17 w(statuses\))75 1386 y(int)g(MPI)p 245 1386 V 17 w(Testall\(int)g(count,)g(MPI)p 788 1386 V 16 w(Comm)p 900 1386 V 17 w(request)g(*array)p 1252 1386 V 16 w(of)p 1316 1386 V 17 w(requests,)g(int)g(*flag,)393 1443 y(MPI)p 468 1443 V 17 w(Status)g(*array)p 796 1443 V 16 w(of)p 860 1443 V 17 w(statuses\))75 1532 y(int)g(MPI)p 245 1532 V 17 w(Iprobe\(int)g(source,)g(int)g(tag,)g(MPI)p 1002 1532 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 1588 y(MPI)p 468 1588 V 17 w(Status)g(*status\))75 1677 y(int)g(MPI)p 245 1677 V 17 w(Probe\(int)g(source,)g(int)g(tag,)h(MPI)p 979 1677 V 16 w(Comm)g(comm,)f(MPI)p 1330 1677 V 17 w(Status)g (*status\))75 1766 y(int)g(MPI)p 245 1766 V 17 w(Cancel\(MPI)p 502 1766 V 16 w(Comm)p 614 1766 V 17 w(request)g(request\))75 1855 y(int)g(MPI)p 245 1855 V 17 w(Test)p 358 1855 V 17 w(cancelled\(MPI)p 687 1855 V 15 w(Status)g(status,)g(int)h(*flag\)) 75 1944 y(int)f(MPI)p 245 1944 V 17 w(Create)p 406 1944 V 17 w(send\(void*)f(buf,)h(int)h(count,)f(MPI)p 1139 1944 V 17 w(Datatype)f(datatype,)h(int)h(dest,)393 2000 y(int)g(tag,)f(MPI)p 683 2000 V 17 w(Comm)g(comm,)g(MPI)p 1034 2000 V 17 w(Comm)p 1147 2000 V 17 w(request)g(*request\))75 2089 y(int)g(MPI)p 245 2089 V 17 w(Create)p 406 2089 V 17 w(rsend\(void*)f(buf,)h(int)h(count,)f(MPI)p 1163 2089 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 2146 y(int)h(tag,)f(MPI)p 683 2146 V 17 w(Comm)g(comm,)g(MPI)p 1034 2146 V 17 w(Comm)p 1147 2146 V 17 w(request)g(*request\))75 2235 y(int)g(MPI)p 245 2235 V 17 w(Create)p 406 2235 V 17 w(ssend\(void*)f(buf,)h(int)h(count,)f(MPI)p 1163 2235 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 2291 y(int)h(tag,)f(MPI)p 683 2291 V 17 w(Comm)g(comm,)g(MPI)p 1034 2291 V 17 w(Comm)p 1147 2291 V 17 w(request)g(*request\))75 2380 y(int)g(MPI)p 245 2380 V 17 w(Create)p 406 2380 V 17 w(recv\(void*)f(buf,)h(int)h(count,)f(MPI)p 1139 2380 V 17 w(Datatype)f(datatype,)h(int)h(source,)393 2437 y(int)g(tag,)f(MPI)p 683 2437 V 17 w(Comm)g(comm,)g(MPI)p 1034 2437 V 17 w(Comm)p 1147 2437 V 17 w(request)g(*request\))75 2526 y(int)g(MPI)p 245 2526 V 17 w(Start\(MPI)p 478 2526 V 16 w(Comm)p 590 2526 V 17 w(request)g(request\))75 2615 y(int)g(MPI)p 245 2615 V 17 w(Startall\(int)f(count,)h(MPI)p 811 2615 V 17 w(Comm)p 924 2615 V 17 w(request)g(*array)p 1276 2615 V 16 w(of)p 1340 2615 V 17 w(requests\))75 2704 y(int)g(MPI)p 245 2704 V 17 w(Commreq)p 430 2704 V 16 w(free\(MPI)p 638 2704 V 17 w(Comm)p 751 2704 V 16 w(request)g(*request\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 178 183 178 182 bop 75 -100 a FE(178)987 b Ft(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fp(int)23 b(MPI)p 245 45 15 2 v 17 w(Sendrecv\(void)f(*sendbuf,)h(int)g(sendcount,)g(MPI) p 1265 45 V 17 w(Datatype)f(sendtype,)393 102 y(int)i(dest,)f(int)g (sendtag,)g(void)g(*recvbuf,)g(int)g(recvcount,)393 158 y(MPI)p 468 158 V 17 w(Datatype)g(recvtype,)f(int)i(source,)f(MPI)p 1297 158 V 16 w(Datatype)g(recvtag,)393 214 y(MPI)p 468 214 V 17 w(Comm)g(comm,)g(MPI)p 819 214 V 17 w(Status)g(*status\))75 301 y(int)g(MPI)p 245 301 V 17 w(Sendrecv)p 454 301 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g(MPI)p 1258 301 V 17 w(Datatype)g(datatype,)393 357 y(int)h(dest,)f(int)g(sendtag,) g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 357 V 16 w(Comm)h(comm,)393 414 y(MPI)p 468 414 V 17 w(Status)f(*status\))75 500 y(int)g(MPI)p 245 500 V 17 w(Type)p 358 500 V 17 w(contiguous\(int)f(count,)h(MPI)p 972 500 V 17 w(Datatype)f(oldtype,) 393 557 y(MPI)p 468 557 V 17 w(Datatype)h(*newtype\))75 643 y(int)g(MPI)p 245 643 V 17 w(Type)p 358 643 V 17 w(vector\(int)f(count,)h(MPI)p 876 643 V 17 w(Datatype)g(oldtype,)g (int)g(stride,)393 700 y(int)h(blocklength,)e(MPI)p 874 700 V 17 w(Datatype)g(*newtype\))75 786 y(int)h(MPI)p 245 786 V 17 w(Type)p 358 786 V 17 w(hvector\(int)f(count,)h(MPI)p 900 786 V 17 w(Datatype)g(oldtype,)g(int)g(stride,)393 843 y(int)h(blocklength,)e(MPI)p 874 843 V 17 w(Datatype)g(*newtype\)) 75 929 y(int)h(MPI)p 245 929 V 17 w(Type)p 358 929 V 17 w(indexed\(int)f(count,)h(MPI)p 900 929 V 17 w(Datatype)g(oldtype,) 393 985 y(int)h(*array)p 636 985 V 16 w(of)p 700 985 V 17 w(displacements,)e(int)h(*array)p 1314 985 V 17 w(of)p 1379 985 V 17 w(blocklengths,)393 1042 y(MPI)p 468 1042 V 17 w(Datatype)g(*newtype\))75 1128 y(int)g(MPI)p 245 1128 V 17 w(Type)p 358 1128 V 17 w(hindexed\(int)f(count,)h(MPI)p 924 1128 V 17 w(Datatype)g(oldtype,)393 1185 y(int)h(*array)p 636 1185 V 16 w(of)p 700 1185 V 17 w(displacements,)e(int)h(*array)p 1314 1185 V 17 w(of)p 1379 1185 V 17 w(blocklengths,)393 1241 y(MPI)p 468 1241 V 17 w(Datatype)g(*newtype\))75 1328 y(int)g(MPI)p 245 1328 V 17 w(Type)p 358 1328 V 17 w(struct\(int)f(count,)h(MPI)p 876 1328 V 17 w(Datatype)g(*array)p 1252 1328 V 16 w(of)p 1316 1328 V 17 w(types,)393 1384 y(int)h(*array)p 636 1384 V 16 w(of)p 700 1384 V 17 w(displacements,)e (int)h(*array)p 1314 1384 V 17 w(of)p 1379 1384 V 17 w(blocklengths,)393 1441 y(MPI)p 468 1441 V 17 w(Datatype)g(*newtype\)) 75 1527 y(int)g(MPI)p 245 1527 V 17 w(Address\(void*)f(location,)h(MPI) p 907 1527 V 17 w(ADDR)g(address\))75 1614 y(int)g(MPI)p 245 1614 V 17 w(Type)p 358 1614 V 17 w(extent\(MPI)p 615 1614 V 16 w(Datatype)f(datatype,)h(int)h(*extent\))75 1700 y(int)f(MPI)p 245 1700 V 17 w(Type)p 358 1700 V 17 w(size\(MPI)p 567 1700 V 16 w(Datatype)g(datatype,)f(int)i(*size\)) 75 1786 y(int)f(MPI)p 245 1786 V 17 w(Type)p 358 1786 V 17 w(count\(MPI)p 591 1786 V 16 w(Datatype)g(datatype,)f(int)i (*count\))75 1873 y(int)f(MPI)p 245 1873 V 17 w(Type)p 358 1873 V 17 w(commit\(MPI)p 615 1873 V 16 w(Datatype)f(datatype\))75 1959 y(int)h(MPI)p 245 1959 V 17 w(Type)p 358 1959 V 17 w(free\(MPI)p 567 1959 V 16 w(Datatype)g(datatype\))75 2046 y(int)g(MPI)p 245 2046 V 17 w(Get)p 334 2046 V 17 w(elements\(MPI)p 639 2046 V 16 w(Status)g(status,)f(MPI)p 1084 2046 V 17 w(Datatype)h(datatype,)g(int)g(*count\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 179 184 179 183 bop 75 -100 a Ft(A.4.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (COLLECTIVE)g(COMMUNICA)l(TION)475 b FE(179)75 45 y Fv(A.4)59 b(C)20 b(Bindings)g(fo)n(r)h(Collective)e(Communication)75 155 y Fp(int)k(MPI)p 245 155 15 2 v 17 w(BARRIER\(MPI)p 526 155 V 16 w(Comm)g(comm)h(\))75 246 y(int)47 b(MPI)p 269 246 V 17 w(BCAST\(void*)22 b(buf,)i(int)f(count,)g(MPI)p 1026 246 V 17 w(Datatype)g(datatype,)f(int)i(root,)393 302 y(MPI)p 468 302 V 17 w(Comm)f(comm)h(\))75 393 y(int)f(MPI)p 245 393 V 17 w(GATHER\(void*)f(sendbuf,)h(int)h(sendcnt,)e(MPI)p 1169 393 V 17 w(Datatype)h(sendtype,)393 450 y(void*)g(recvbuf,)g(int)h (recvcount,)e(MPI)p 1184 450 V 17 w(Datatype,)g(int)i(root,)393 506 y(MPI)p 468 506 V 17 w(Comm)f(comm\))75 597 y(int)g(MPI)p 245 597 V 17 w(GATHERV\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 597 V 17 w(Datatype)h(sendtype,)393 654 y(void*)g(recvbuf,)g(int)h (*displs,)e(int)i(*recvcounts,)e(MPI)p 1542 654 V 17 w(Datatype,)393 710 y(int)i(root,)f(MPI)p 707 710 V 17 w(Comm)g(comm\))75 801 y(int)g(MPI)p 245 801 V 17 w(SCATTER\(void*)f (sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 801 V 17 w(Datatype)h (sendtype,)393 857 y(void*)g(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 857 V 17 w(Datatype,)g(int)i(root,)393 914 y(MPI)p 468 914 V 17 w(Comm)f(comm\))75 1005 y(int)g(MPI)p 245 1005 V 17 w(SCATTERV\(void*)f(sendbuf,)h(int)g(*displs,)g(int)h (*sendcounts,)393 1061 y(MPI)p 468 1061 V 17 w(Datatype)f(sendtype,)f (void*)i(recvbuf,)e(int)i(recvcount,)393 1118 y(MPI)p 468 1118 V 17 w(Datatype,)f(int)g(root,)g(MPI)p 1034 1118 V 17 w(Comm)g(comm\))75 1208 y(int)g(MPI)p 245 1208 V 17 w(ALLGATHER\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1289 1208 V 17 w(Datatype)f(sendtype,)393 1265 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1265 V 17 w(Datatype,)g(MPI)p 1511 1265 V 17 w(Comm)i(comm\))75 1356 y(int)f(MPI)p 245 1356 V 17 w(ALLGATHERV\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1356 V 16 w(Datatype)g(sendtype,)393 1412 y(void*)g(recvbuf,)g (int)h(*displs,)e(int)i(*recvcounts,)e(MPI)p 1542 1412 V 17 w(Datatype,)393 1469 y(MPI)p 468 1469 V 17 w(Comm)h(comm\))75 1559 y(int)g(MPI)p 245 1559 V 17 w(ALLTOALL\(void*)f(sendbuf,)h(int)g (sendcount,)g(MPI)p 1265 1559 V 17 w(Datatype)f(sendtype,)393 1616 y(void*)h(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 1616 V 17 w(Datatype,)g(MPI)p 1511 1616 V 17 w(Comm)i(comm\))75 1707 y(int)f(MPI)p 245 1707 V 17 w(ALLTOALLV\(void*)f(sendbuf,)h(int)g (*sdispls,)g(int)g(*sendcounts,)393 1763 y(MPI)p 468 1763 V 17 w(Datatype)g(sendtype,)f(void*)i(recvbuf,)e(int)i(*rdispls,) 393 1820 y(int)g(*recvcounts,)e(MPI)p 874 1820 V 17 w(Datatype,)g(MPI)p 1201 1820 V 17 w(Comm)h(comm\))75 1911 y(int)g(MPI)p 245 1911 V 17 w(REDUCE\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)h (count,)393 1967 y(MPI)p 468 1967 V 17 w(Datatype)f(datatype,)f(void)i (op,)f(int)h(root,)f(MPI)p 1464 1967 V 17 w(Comm)g(comm\))75 2058 y(int)g(MPI)p 245 2058 V 17 w(USER)p 358 2058 V 17 w(REDUCE\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g(count,)393 2114 y(MPI)p 468 2114 V 17 w(Datatype)g(datatype,)f(void)i(function,)e (int)i(root,)f(MPI)p 1607 2114 V 17 w(Comm)g(comm\))75 2205 y(int)g(MPI)p 245 2205 V 17 w(USER)p 358 2205 V 17 w(REDUCEA\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g(count,)393 2262 y(MPI)p 468 2262 V 17 w(Datatype)g(datatype,)f(void)i(function,)e (int)i(root,)f(MPI)p 1607 2262 V 17 w(Comm)g(comm\))75 2353 y(int)g(MPI)p 245 2353 V 17 w(ALLREDUCE\(void*)f(sendbuf,)h(void*) g(recvbuf,)g(int)g(count,)393 2409 y(MPI)p 468 2409 V 17 w(Datatype)g(datatype,)f(void)i(op,)f(MPI)p 1225 2409 V 17 w(Comm)g(comm\))75 2500 y(int)g(MPI)p 245 2500 V 17 w(USER)p 358 2500 V 17 w(ALLREDUCE\(void*)f(sendbuf,)g(void*)i (recvbuf,)e(int)i(count,)393 2556 y(MPI)p 468 2556 V 17 w(Datatype)f(datatype,)f(void)i(function,)e(MPI)p 1368 2556 V 17 w(Comm)h(comm\))166 2647 y FE(e)p Fp(int)h(MPI)p 357 2647 V 16 w(USER)p 469 2647 V 17 w(ALLREDUCEA\(void*)e(sendbuf,)g (void*)i(recvbuf,)e(int)i(count,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)f(datatype,)f(void)i(function,)e(MPI)p 1368 2704 V 17 w(Comm)h(comm\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 180 185 180 184 bop 75 -100 a FE(180)987 b Ft(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fp(int)23 b(MPI)p 245 45 15 2 v 17 w(REDUCE)p 406 45 V 17 w(SCATTER\(void*)f(sendbuf,)g (void*)i(recvbuf,)e(int)i(*recvcounts,)393 102 y(MPI)p 468 102 V 17 w(Datatype)f(datatype,)f(void)i(op,)f(MPI)p 1225 102 V 17 w(Comm)g(comm\))75 190 y(int)g(MPI)p 245 190 V 17 w(SCAN\(void*)g(sendbuf,)f(void*)i(recvbuf,)e(int)i(count,)393 247 y(MPI)p 468 247 V 17 w(Datatype)f(datatype,)f(void)i(op,)f(MPI)p 1225 247 V 17 w(Comm)g(comm)h(\))75 335 y(int)f(MPI)p 245 335 V 17 w(USER)p 358 335 V 17 w(SCAN\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)h(count,)393 392 y(MPI)p 468 392 V 17 w(Datatype)f(datatype,)f(void)i(function,)e(MPI)p 1368 392 V 17 w(Comm)h(comm\))75 480 y(int)g(MPI)p 245 480 V 17 w(USER)p 358 480 V 17 w(SCANA\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)g(count,)393 537 y(MPI)p 468 537 V 17 w(Datatype)g(datatype,)f(void)i(function,)e(MPI)p 1368 537 V 17 w(Comm)h(comm\))75 722 y Fv(A.5)59 b(C)20 b(Bindings)g(fo)n(r) h(Groups,)e(Contexts,)f(and)h(Communicato)n(rs)75 827 y Fp(int)k(MPI)p 245 827 V 17 w(Group)p 382 827 V 17 w(size\(MPI)p 591 827 V 16 w(Group)g(group,)g(int)g(*size\))75 916 y(int)g(MPI)p 245 916 V 17 w(Group)p 382 916 V 17 w(rank\(MPI)p 591 916 V 16 w(Group)g(group,)g(int)g(*rank\))75 1004 y(int)g(MPI)p 245 1004 V 17 w(Group)p 382 1004 V 17 w(translate)p 615 1004 V 16 w(ranks)g(\(MPI)p 870 1004 V 16 w(Group)h(group1,)f(int)g(n,)h(int)f(*ranks1,)393 1061 y(MPI)p 468 1061 V 17 w(group)g(group2,)g(int)g(*ranks2\))75 1149 y(int)g(MPI)p 245 1149 V 17 w(Group)p 382 1149 V 17 w(union\(MPI)p 615 1149 V 16 w(group)g(group1,)g(MPI)p 1037 1149 V 17 w(group)g(group2,)g(MPI)p 1460 1149 V 16 w(Group)h(*newgroup\))75 1238 y(int)f(MPI)p 245 1238 V 17 w(Group)p 382 1238 V 17 w(intersection\(MPI)p 783 1238 V 15 w(group)g(group1,)g(MPI)p 1204 1238 V 17 w(group)g(group2,) 393 1294 y(MPI)p 468 1294 V 17 w(Group)g(*newgroup\))75 1383 y(int)g(MPI)p 245 1383 V 17 w(Group)p 382 1383 V 17 w(difference\(MPI)p 735 1383 V 15 w(group)g(group1,)g(MPI)p 1156 1383 V 17 w(group)g(group2,)393 1439 y(MPI)p 468 1439 V 17 w(Group)g(*newgroup\))75 1528 y(int)g(MPI)p 245 1528 V 17 w(Group)p 382 1528 V 17 w(incl\(MPI)p 591 1528 V 16 w(group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1528 V 17 w(Group)h(*newgroup\))75 1617 y(int)g(MPI)p 245 1617 V 17 w(Group)p 382 1617 V 17 w(excl\(MPI)p 591 1617 V 16 w(group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1617 V 17 w(Group)h(*newgroup\))75 1705 y(int)g(MPI)p 245 1705 V 17 w(Group)p 382 1705 V 17 w(range)p 519 1705 V 16 w(incl\(MPI)p 727 1705 V 16 w(group)g(group,)h(int)f(n,)h(int)f (**ranges,)393 1762 y(MPI)p 468 1762 V 17 w(Group)g(*newgroup\))75 1850 y(int)g(MPI)p 245 1850 V 17 w(Group)p 382 1850 V 17 w(range)p 519 1850 V 16 w(excl\(MPI)p 727 1850 V 16 w(group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 1907 y(MPI)p 468 1907 V 17 w(Group)g(*newgroup\))75 1995 y(int)g(MPI)p 245 1995 V 17 w(Group)p 382 1995 V 17 w(free\(MPI)p 591 1995 V 16 w(group)g(*group\))75 2084 y(int)g(MPI)p 245 2084 V 17 w(Comm)p 358 2084 V 17 w(size\(MPI)p 567 2084 V 16 w(Comm)g(comm,)g(int)h(*size\))75 2172 y(int)f(MPI)p 245 2172 V 17 w(Comm)p 358 2172 V 17 w(rank\(MPI)p 567 2172 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 2261 y(int)f(MPI)p 245 2261 V 17 w(Comm)p 358 2261 V 17 w(group\(MPI)p 591 2261 V 16 w(Comm)g(comm,)g(MPI)p 941 2261 V 17 w(group)g(*group\))75 2349 y(int)g(MPI)p 245 2349 V 17 w(Comm)p 358 2349 V 17 w(dup\(MPI)p 543 2349 V 16 w(Comm)g(comm,)h(MPI)p 894 2349 V 16 w(Comm)g(*newcomm\))75 2438 y(int)f(MPI)p 245 2438 V 17 w(Comm)p 358 2438 V 17 w(make\(MPI)p 567 2438 V 16 w(Comm)g(comm,)g(MPI)p 917 2438 V 17 w(group)g(group,)g(MPI)p 1316 2438 V 17 w(Comm)h(*newcomm\))75 2526 y(int)f(MPI)p 245 2526 V 17 w(Comm)p 358 2526 V 17 w(split\(MPI)p 591 2526 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g(key,)h(MPI)p 1419 2526 V 16 w(Comm)g(*newcomm\))75 2615 y(int)f(MPI)p 245 2615 V 17 w(Comm)p 358 2615 V 17 w(free\(MPI)p 567 2615 V 16 w(Comm)g(*comm\))75 2704 y(int)g(MPI)p 245 2704 V 17 w(Comm)p 358 2704 V 17 w(test)p 471 2704 V 16 w(inter\(MPI)p 703 2704 V 16 w(Comm)h(comm,)f(int)g(*flag\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 181 186 181 185 bop 75 -100 a Ft(A.6.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(PR)o (OCESS)g(TOPOLOGIES)677 b FE(181)75 45 y Fp(int)23 b(MPI)p 245 45 15 2 v 17 w(Intercomm)p 478 45 V 16 w(start\(MPI)p 710 45 V 16 w(Comm)g(local)p 965 45 V 17 w(comm,)g(int)h(local)p 1341 45 V 16 w(leader,)393 102 y(MPI)p 468 102 V 17 w(Comm)f(peer)p 700 102 V 17 w(comm,)g(int)h(remote)p 1100 102 V 16 w(leader,)f(int)g (tag,)393 158 y(MPI)p 468 158 V 17 w(Intercomm)p 701 158 V 16 w(request)g(*inter)p 1052 158 V 16 w(request\))75 245 y(int)g(MPI)p 245 245 V 17 w(Intercomm)p 478 245 V 16 w(finish\(MPI)p 734 245 V 16 w(Intercomm)p 966 245 V 16 w(request)g(*inter)p 1317 245 V 16 w(request,)393 302 y(MPI)p 468 302 V 17 w(Comm)g(*newcomm\))75 389 y(int)g(MPI)p 245 389 V 17 w(Intercomm)p 478 389 V 16 w(merge\(MPI)p 710 389 V 16 w(Comm)g(intercomm,)g(int)g(key,)h(MPI)p 1395 389 V 16 w(Comm)g(*newintracomm\))75 477 y(int)f(MPI)p 245 477 V 17 w(Intercomm)p 478 477 V 16 w(name)p 590 477 V 17 w(start\(MPI)p 823 477 V 16 w(Comm)g(comm,)g(char)h(*name,)393 533 y(MPI)p 468 533 V 17 w(Intercomm)p 701 533 V 16 w(request)f(*inter) p 1052 533 V 16 w(request\))75 620 y(int)g(MPI)p 245 620 V 17 w(Attr)p 358 620 V 17 w(get)p 447 620 V 17 w(key\(copy)p 656 620 V 16 w(fn,)g(delete)p 911 620 V 17 w(fn,)g(int)g(keyval,)g (void*)h(extra)p 1573 620 V 16 w(state\))75 708 y(int)f(MPI)p 245 708 V 17 w(Attr)p 358 708 V 17 w(free)p 471 708 V 16 w(key\(int)g(keyval\))75 795 y(int)g(MPI)p 245 795 V 17 w(Attr)p 358 795 V 17 w(put)p 447 795 V 17 w(value\(MPI)p 680 795 V 16 w(Comm)g(comm,)g(int)h(keyval,)e(void*)i(attribute)p 1604 795 V 16 w(val\))75 882 y(int)f(MPI)p 245 882 V 17 w(Attr)p 358 882 V 17 w(get)p 447 882 V 17 w(value\(MPI)p 680 882 V 16 w(Comm)g(comm,)g(int)h(keyval,)e(void*)i(attribute)p 1604 882 V 16 w(val,)393 939 y(int)g(*found\))75 1026 y(int)f(MPI)p 245 1026 V 17 w(Attr)p 358 1026 V 17 w(delete)p 519 1026 V 16 w(value\(MPI)p 751 1026 V 16 w(Comm)g(comm,)h(int)f (keyval\))75 1205 y Fv(A.6)59 b(C)20 b(Bindings)g(fo)n(r)h(Pro)r(cess)e (T)-5 b(op)r(ologies)75 1326 y(A.7)59 b(C)20 b(bindings)g(fo)n(r)h (Environmental)d(Inquiry)75 1429 y Fp(int)23 b(MPI)p 245 1429 V 17 w(GET)p 334 1429 V 17 w(VALID)p 471 1429 V 16 w(TAG)p 559 1429 V 17 w(RANGE\(int)g(*low,)g(int)h(*high\))75 1517 y(int)f(MPI)p 245 1517 V 17 w(GET)p 334 1517 V 17 w(PROCESSOR)p 567 1517 V 16 w(NAME\(int)g(len,)g(char)g(*name\))75 1604 y(int)g(MPI)p 245 1604 V 17 w(GET)p 334 1604 V 17 w(HOST\(MPI)p 543 1604 V 16 w(Comm)g(comm,)h(int)f(*rank\))75 1691 y(int)g(MPI)p 245 1691 V 17 w(GET)p 334 1691 V 17 w(BUFFER)p 495 1691 V 16 w(PARAMS\(int)g(*numsend,)f(int)i(*sizesend,)e (int)i(*numrecv,)393 1748 y(int)g(*sizerecv,)e(int)i(*totalnumsednd,)d (int)j(*totalsizesen,)393 1804 y(int)g(*totalnumrecv,)e(int)h (*totalsizerecv\))75 1892 y(int)g(MPI)p 245 1892 V 17 w(SUGGEST)p 430 1892 V 16 w(BUFFER)p 590 1892 V 17 w(PARAMS\(int)f (numsend,)h(int)h(sizesend,)e(int)i(numrecv,)393 1948 y(int)g(sizerecv,)e(int)i(totalnumsend,)e(int)h(totalsizesen,)393 2005 y(int)h(totalnumrecv,)e(int)h(totalsizerecv,)f(int)i(*flag\))75 2092 y(int)f(MPI)p 245 2092 V 17 w(USER)p 358 2092 V 17 w(SPECIFIES)p 591 2092 V 16 w(BUFFER\(MPI)p 847 2092 V 16 w(Comm)g(comm,)g(void*)g(buffer,)g(int)h(size\))75 2179 y(int)f(MPI)p 245 2179 V 17 w(GET)p 334 2179 V 17 w(ERRORMODE\(MPI)p 663 2179 V 15 w(Comm)h(comm,)f(MPI)p 1013 2179 V 17 w(Errmode)g(mode\))75 2267 y(int)g(MPI)p 245 2267 V 17 w(SET)p 334 2267 V 17 w(ERRORMODE\(MPI)p 663 2267 V 15 w(Comm)h(comm,)f(MPI)p 1013 2267 V 17 w(Errmode)g(mode\)) 75 2354 y(int)g(MPI)p 245 2354 V 17 w(SET)p 334 2354 V 17 w(ERRHANDLER\(MPI)p 687 2354 V 15 w(Comm)h(comm,)f(MPI)p 1037 2354 V 17 w(Errhandler)f(handler\))75 2441 y(int)h(MPI)p 245 2441 V 17 w(ERROR)p 382 2441 V 17 w(STRING\(int)f(errorcode,)h(int) g(len,)g(char)h(*string\))75 2529 y(int)f(MPI)p 245 2529 V 17 w(INIT\(int)g(argc,)g(char)g(**argv\))75 2616 y(int)g(MPI)p 245 2616 V 17 w(FINALIZE\(\))75 2704 y(int)g(MPI)p 245 2704 V 17 w(INITIALIZED\(int)f(*flag\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 182 187 182 186 bop 75 -100 a FE(182)987 b Ft(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fp(int)23 b(MPI)p 245 45 15 2 v 17 w(ABORT\(MPI)p 478 45 V 16 w(Comm)g(comm,)h(int)f (errorcode\))75 223 y Fv(A.8)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r) i(P)n(oint-to-P)n(oint)f(Communication)75 326 y Fp(MPI)p 150 326 V 17 w(SEND\(BUF,)i(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f (IERROR\))170 383 y()g(BUF\(*\))170 439 y(INTEGER)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 527 y(MPI)p 150 527 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g (COMM,)g(STATUS,)g(IERROR\))170 583 y()g(BUF\(*\))170 640 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS\(MPI)p 1463 640 V 16 w(STATUS)p 1623 640 V 17 w(SIZE\),)170 696 y(IERROR)75 783 y(MPI)p 150 783 V 17 w(GET)p 239 783 V 17 w(SOURCE\(STATUS,)e(SOURCE,)i(IERROR\))170 840 y(INTEGER)g(STATUS\(MPI)p 604 840 V 16 w(STATUS)p 764 840 V 17 w(SIZE\),)g(SOURCE,)f(IERROR)75 927 y(MPI)p 150 927 V 17 w(GET)p 239 927 V 17 w(TAG\(STATUS,)g(TAG,)h(IERROR\))170 984 y(INTEGER)g(STATUS\(MPI)p 604 984 V 16 w(STATUS)p 764 984 V 17 w(SIZE\),)g(TAG,)g(IERROR)75 1071 y(MPI)p 150 1071 V 17 w(GET)p 239 1071 V 17 w(COUNT\(STATUS,)f(DATATYPE,)g (COUNT,)h(IERROR\))170 1127 y(INTEGER)g(STATUS\(MPI)p 604 1127 V 16 w(STATUS)p 764 1127 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)75 1215 y(MPI)p 150 1215 V 17 w(RSEND\(BUF,)f(COUNT,)h (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)h(IERROR\))170 1271 y()f(BUF\(*\))170 1328 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g (TAG,)h(COMM,)f(IERROR)75 1415 y(MPI)p 150 1415 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)h(IERROR\)) 170 1471 y()f(BUF\(*\))170 1528 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(IERROR)75 1615 y(MPI)p 150 1615 V 17 w(ISEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)h (REQUEST,)e(IERROR\))170 1672 y()h(BUF\(*\))170 1728 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 1815 y(MPI)p 150 1815 V 17 w(IRSEND\(BUF,)g(COUNT,)h (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1872 y()g(BUF\(*\))170 1928 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 2016 y(MPI)p 150 2016 V 17 w(ISSEND\(BUF,)g(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g (COMM,)g(REQUEST,)g(IERROR\))170 2072 y()g(BUF\(*\))170 2128 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 2216 y(MPI)p 150 2216 V 17 w(IRECV\(BUF,)g(COUNT,)h (DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2272 y()g(BUF\(*\))170 2329 y(INTEGER)g(COUNT,)g(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 2416 y(MPI)p 150 2416 V 17 w(WAIT\(REQUEST,)f(STATUS,)h(IERROR\))170 2472 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 2472 V 16 w(STATUS)p 979 2472 V 16 w(SIZE\),)g(IERROR)75 2560 y(MPI)p 150 2560 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g(IERROR\))170 2616 y(INTEGER)g(REQUEST,)g(FLAG,)g(STATUS\(MPI)p 962 2616 V 16 w(STATUS)p 1122 2616 V 16 w(SIZE\),)h(IERROR)75 2704 y(MPI)p 150 2704 V 17 w(WAITANY\(COUNT,)e(ARRAY)p 645 2704 V 16 w(OF)p 709 2704 V 17 w(REQUESTS,)h(INDEX,)g(STATUS,)g (IERROR\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 183 188 183 187 bop 75 -100 a Ft(A.8.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o (OR)g(POINT-TO-POINT)h(COMMUNICA)l(TION)185 b FE(183)170 45 y Fp(INTEGER)23 b(COUNT,)g(ARRAY)p 651 45 15 2 v 17 w(OF)p 716 45 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 45 V 16 w(STATUS)p 1610 45 V 16 w(SIZE\),)170 102 y(IERROR)75 190 y(MPI)p 150 190 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 190 V 16 w(OF)p 709 190 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g (STATUS,)g(IERROR\))170 247 y(INTEGER)g(COUNT,)g(ARRAY)p 651 247 V 17 w(OF)p 716 247 V 17 w(REQUESTS\(*\),)f(INDEX,)h(FLAG,)170 303 y(STATUS\(MPI)p 413 303 V 16 w(STATUS)p 573 303 V 17 w(SIZE\),)g(IERROR)75 392 y(MPI)p 150 392 V 17 w(WAITALL\(COUNT,)f (ARRAY)p 645 392 V 16 w(OF)p 709 392 V 17 w(REQUESTS,)h(ARRAY)p 1085 392 V 16 w(OF)p 1149 392 V 17 w(STATUSES,)g(IERROR\))170 448 y(INTEGER)g(COUNT,)g(ARRAY)p 651 448 V 17 w(OF)p 716 448 V 17 w(REQUESTS\(*\),)170 504 y(ARRAY)p 293 504 V 17 w(OF)p 358 504 V 17 w(STATUSES\(MPI)p 663 504 V 15 w(STATUS)p 822 504 V 17 w(SIZE,*\),)g(IERROR)75 593 y(MPI)p 150 593 V 17 w(TESTALL\(COUNT,)f(ARRAY)p 645 593 V 16 w(OF)p 709 593 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 593 V 16 w(OF)p 1292 593 V 17 w(STATUSES,)g(IERROR\))170 649 y(INTEGER)g(COUNT,)g(ARRAY)p 651 649 V 17 w(OF)p 716 649 V 17 w(REQUESTS\(*\),)f(FLAG,)170 706 y(ARRAY)p 293 706 V 17 w(OF)p 358 706 V 17 w(STATUSES\(MPI)p 663 706 V 15 w(STATUS)p 822 706 V 17 w(SIZE,*\),)h(IERROR)75 794 y(MPI)p 150 794 V 17 w(IPROBE\(SOURCE,)f(TAG,)h(COMM,)g(FLAG,)g (STATUS,)g(IERROR\))170 851 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(FLAG,) g(STATUS\(MPI)p 1201 851 V 16 w(STATUS)p 1361 851 V 16 w(SIZE\),)g(IERROR)75 939 y(MPI)p 150 939 V 17 w(PROBE\(SOURCE,)f(TAG,) h(COMM,)g(STATUS,)g(IERROR\))170 996 y(INTEGER)g(SOURCE,)g(TAG,)h (COMM,)f(STATUS\(MPI)p 1058 996 V 16 w(STATUS)p 1218 996 V 16 w(SIZE\),)g(IERROR)75 1084 y(MPI)p 150 1084 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 1141 y(INTEGER)h(REQUEST,)g (IERROR)75 1229 y(MPI)p 150 1229 V 17 w(TEST)p 263 1229 V 16 w(CANCELLED\(STATUS,)f(FLAG,)h(IERROR\))170 1286 y(INTEGER)g(STATUS\(MPI)p 604 1286 V 16 w(STATUS)p 764 1286 V 17 w(SIZE\),)g(FLAG,)g(IERROR)75 1374 y(MPI)p 150 1374 V 17 w(CREATE)p 311 1374 V 16 w(SEND\(BUF,)g(COUNT,)g (DATATYPE,)f(DEST,)i(TAG,)f(COMM,)g(REQUEST,)g(IERROR\))170 1431 y(INTEGER)g(REQUEST,)g(BUF,)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h (COMM,)f(REQUEST,)g(IERROR)75 1519 y(MPI)p 150 1519 V 17 w(CREATE)p 311 1519 V 16 w(RSEND\(BUF,)g(COUNT,)g(DATATYPE,)f(DEST,) h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 1576 y(INTEGER)g(BUF,)h (COUNT,)f(DATATYPE,)f(DEST,)i(TAG,)f(COMM,)g(REQUEST,)g(IERROR)75 1664 y(MPI)p 150 1664 V 17 w(CREATE)p 311 1664 V 16 w(SSEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 1721 y(INTEGER)g(BUF,)h(COUNT,)f(DATATYPE,)f(DEST,)i(TAG,)f(COMM,)g (REQUEST,)g(IERROR)75 1809 y(MPI)p 150 1809 V 17 w(CREATE)p 311 1809 V 16 w(RECV\(BUF,)g(COUNT,)g(DATATYPE,)f(SOURCE,)h(TAG,)h (COMM,)f(REQUEST,)g(IERROR\))170 1866 y(INTEGER)g(BUF,)h(COUNT,)f (DATATYPE,)f(SOURCE,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR)75 1954 y(MPI)p 150 1954 V 17 w(START\(REQUEST,)f(IERROR\))170 2011 y(INTEGER)h(REQUEST,)g(IERROR)75 2099 y(MPI)p 150 2099 V 17 w(STARTALL\(COUNT,)f(ARRAY)p 669 2099 V 16 w(OF)p 733 2099 V 17 w(REQUESTS,)h(IERROR\))170 2156 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2156 V 17 w(OF)p 716 2156 V 17 w(REQUESTS\(*\),)f(IERROR)75 2244 y(MPI)p 150 2244 V 17 w(COMMOBJ)p 335 2244 V 16 w(FREE\(REQUEST,)g(IERROR\))170 2301 y(INTEGER)h(REQUEST,)g(IERROR)75 2389 y(MPI)p 150 2389 V 17 w(SENDRECV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(DEST,)i (SENDTAG,)e(RECVBUF,)393 2446 y(RECVCOUNT,)h(RECVTYPE,)f(SOURCE,)h (RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))170 2502 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 2559 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h(RECVTYPE,)170 2615 y(SOURCE,)g(RECVTAG,)g(COMM,)g(STATUS\(MPI)p 962 2615 V 16 w(STATUS)p 1122 2615 V 16 w(SIZE\),)h(IERROR)75 2704 y(MPI)p 150 2704 V 17 w(SENDRECV)p 359 2704 V 16 w(REPLACE\(BUF,)e(COUNT,)h(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g (RECVTAG,)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 184 189 184 188 bop 75 -100 a FE(184)987 b Ft(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)393 45 y Fp(COMM,)23 b(STATUS,)g(IERROR\)) 170 102 y()g(BUF\(*\))170 158 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)g(COMM,)170 214 y(STATUS\(MPI)p 413 214 15 2 v 16 w(STATUS)p 573 214 V 17 w(SIZE\),)g(IERROR)75 316 y(MPI)p 150 316 V 17 w(TYPE)p 263 316 V 16 w(CONTIGUOUS\(COUNT,)f(OLDTYPE,)h(NEWTYPE,)f(IERROR\))170 373 y(INTEGER)h(COUNT,)g(OLDTYPE,)g(NEWTYPE,)g(IERROR)75 475 y(MPI)p 150 475 V 17 w(TYPE)p 263 475 V 16 w(VECTOR\(COUNT,)f (OLDTYPE,)h(STRIDE,)g(BLOCKLENGTH,)f(NEWTYPE,)h(IERROR\))170 531 y(INTEGER)g(COUNT,)g(OLDTYPE,)g(STRIDE,)g(BLOCKLENGTH,)f(NEWTYPE,)h (IERROR)75 633 y(MPI)p 150 633 V 17 w(TYPE)p 263 633 V 16 w(HVECTOR\(COUNT,)f(OLDTYPE,)h(STRIDE,)g(BLOCKLENGTH,)f(NEWTYPE,)h (IERROR\))170 689 y(INTEGER)g(COUNT,)g(OLDTYPE,)g(STRIDE,)g (BLOCKLENGTH,)f(NEWTYPE,)h(IERROR)75 791 y(MPI)p 150 791 V 17 w(TYPE)p 263 791 V 16 w(INDEXED\(COUNT,)f(OLDTYPE,)h(ARRAY)p 972 791 V 17 w(OF)p 1037 791 V 17 w(DISPLACEMENTS,)393 848 y(ARRAY)p 516 848 V 17 w(OF)p 581 848 V 17 w(BLOCKLENGTHS,)f (NEWTYPE,)g(IERROR\))170 904 y(INTEGER)h(COUNT,)g(OLDTYPE,)g(ARRAY)p 866 904 V 17 w(OF)p 931 904 V 17 w(DISPLACEMENTS\(*\),)170 960 y(ARRAY)p 293 960 V 17 w(OF)p 358 960 V 17 w(BLOCKLENGTHS\(*\),)f (NEWTYPE,)g(IERROR)75 1062 y(MPI)p 150 1062 V 17 w(TYPE)p 263 1062 V 16 w(HINDEXED\(COUNT,)g(OLDTYPE,)h(ARRAY)p 996 1062 V 16 w(OF)p 1060 1062 V 17 w(DISPLACEMENTS,)393 1119 y(ARRAY)p 516 1119 V 17 w(OF)p 581 1119 V 17 w(BLOCKLENGTHS,)f (NEWTYPE,)g(IERROR\))170 1175 y(INTEGER)h(COUNT,)g(OLDTYPE,)g(ARRAY)p 866 1175 V 17 w(OF)p 931 1175 V 17 w(DISPLACEMENTS\(*\),)170 1232 y(ARRAY)p 293 1232 V 17 w(OF)p 358 1232 V 17 w(BLOCKLENGTHS\(*\),) f(NEWTYPE,)g(IERROR)75 1334 y(MPI)p 150 1334 V 17 w(TYPE)p 263 1334 V 16 w(STRUCT\(COUNT,)g(ARRAY)p 733 1334 V 17 w(OF)p 798 1334 V 17 w(TYPES,)h(ARRAY)p 1102 1334 V 16 w(OF)p 1166 1334 V 17 w(DISPLACEMENTS,)393 1390 y(ARRAY)p 516 1390 V 17 w(OF)p 581 1390 V 17 w(BLOCKLENGTHS,)f(NEWTYPE,)g (IERROR\))170 1446 y(INTEGER)h(COUNT,)g(ARRAY)p 651 1446 V 17 w(OF)p 716 1446 V 17 w(TYPES,)g(ARRAY)p 1020 1446 V 16 w(OF)p 1084 1446 V 17 w(DISPLACEMENTS\(*\),)170 1503 y(ARRAY)p 293 1503 V 17 w(OF)p 358 1503 V 17 w(BLOCKLENGTHS\(*\),) f(NEWTYPE,)g(IERROR)75 1605 y(MPI)p 150 1605 V 17 w(ADDRESS\(LOCATION,) f(ADDRESS,)i(IERROR\))170 1661 y()g(LOCATION\(*\))170 1718 y(INTEGER)g(ADDRESS,)g(IERROR)75 1819 y(MPI)p 150 1819 V 17 w(TYPE)p 263 1819 V 16 w(EXTENT\(DATATYPE,)f(EXTENT,)h (IERROR\))170 1876 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR)75 1978 y(MPI)p 150 1978 V 17 w(TYPE)p 263 1978 V 16 w(SIZE\(DATATYPE,)f (SIZE,)i(IERROR\))170 2034 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)75 2136 y(MPI)p 150 2136 V 17 w(TYPE)p 263 2136 V 16 w(COUNT\(DATATYPE,)f (COUNT,)h(IERROR\))170 2192 y(INTEGER)g(DATATYPE,)g(COUNT,)g(IERROR)75 2294 y(MPI)p 150 2294 V 17 w(TYPE)p 263 2294 V 16 w(COMMIT\(DATATYPE,)f (IERROR\))170 2351 y(INTEGER)h(DATATYPE,)g(IERROR)75 2453 y(MPI)p 150 2453 V 17 w(TYPE)p 263 2453 V 16 w(FREE\(DATATYPE,)f (IERROR\))170 2509 y(INTEGER)h(DATATYPE,)g(IERROR)75 2611 y(MPI)p 150 2611 V 17 w(GET)p 239 2611 V 17 w(ELEMENTS\(STATUS,)e (DATATYPE,)i(COUNT,)g(IERROR\))170 2667 y(INTEGER)g(STATUS\(MPI)p 604 2667 V 16 w(STATUS)p 764 2667 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 185 190 185 189 bop 75 -100 a Ft(A.9.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o (OR)g(COLLECTIVE)g(COMMUNICA)l(TION)280 b FE(185)75 45 y Fv(A.9)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Collective)e (Communication)75 156 y Fp(MPI)p 150 156 15 2 v 17 w(BARRIER\(COMM\)) 170 212 y(INTEGER)k(COMM,)h(IERROR)75 303 y(MPI)p 150 303 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g(ROOT,)g(COMM\))170 360 y(INTEGER)g(BUFFER,)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR) 75 451 y(MPI)p 150 451 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g (SENDTYPE,)h(RECVBUF,)f(RECVCOUNT,)h(RECVTYPE,)393 507 y(ROOT,)g(COMM\))170 564 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 620 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g (ROOT,)g(COMM,)g(IERROR)75 711 y(MPI)p 150 711 V 17 w (GATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f(DISPLS,)h (RECVCOUNTS,)393 768 y(RECVTYPE,)g(ROOT,)g(COMM\))170 824 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 880 y(INTEGER)g (SENDCOUNT,)g(SENDTYPE,)f(DISPLS,)h(RECVCOUNTS,)g(RECVTYPE,)f(ROOT,)i (COMM,)170 937 y(IERROR)75 1028 y(MPI)p 150 1028 V 17 w(SCATTER\(SENDBUF,)d(SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f(RECVCOUNT,)h (RECVTYPE,)393 1084 y(ROOT,)g(COMM\))170 1141 y()g(SENDBUF\(*\),) g(RECVBUF\(*\))170 1197 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f (RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR)75 1288 y(MPI)p 150 1288 V 17 w(SCATTERV\(SENDBUF,)e(DISPLS,)i(SENDCOUNTS,)g (SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)393 1345 y(RECVTYPE,)g(ROOT,)g (COMM\))170 1401 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1458 y(INTEGER)g(DISPLS,)g(SENDCOUNTS,)g(SENDTYPE,)f(RECVCOUNT,)h (RECVTYPE,)f(ROOT,)i(COMM,)170 1514 y(IERROR)75 1605 y(MPI)p 150 1605 V 17 w(ALLGATHER\(SENDBUF,)d(SENDCOUNT,)i(SENDTYPE,)f (RECVBUF,)h(RECVCOUNT,)g(RECVTYPE,)393 1662 y(COMM\))170 1718 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1775 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)75 1866 y(MPI)p 150 1866 V 17 w(ALLGATHERV\(SENDBUF,)e (SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h(DISPLS,)g(RECVCOUNTS,)393 1922 y(RECVTYPE,)g(COMM\))170 1979 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2035 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(DISPLS,)h (RECVCOUNTS,)g(RECVTYPE,)f(COMM,)i(IERROR)75 2126 y(MPI)p 150 2126 V 17 w(ALLTOALL\(SENDBUF,)d(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,) h(RECVCOUNT,)g(RECVTYPE,)393 2183 y(COMM\))170 2239 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 2296 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g(IERROR)75 2387 y(MPI)p 150 2387 V 17 w(ALLTOALLV\(SENDBUF,)e(SDISPLS,)i (SENDCOUNTS,)f(SENDTYPE,)h(RECVBUF,)g(RDISPLS,)393 2443 y(RECVCOUNTS,)g(RECVTYPE,)f(COMM\))170 2500 y()h(SENDBUF\(*\),)g (RECVBUF\(*\))170 2556 y(INTEGER)g(SDISPLS,)g(SENDCOUNTS,)f(SENDTYPE,)h (RDISPLS,)g(RECVCOUNTS,)f(RECVTYPE,)170 2613 y(COMM,)i(IERROR)75 2704 y(MPI)p 150 2704 V 17 w(REDUCE\(SENDBUF,)e(RECVBUF,)g(COUNT,)h (DATATYPE,)g(OP,)g(ROOT,)h(COMM\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 186 191 186 190 bop 75 -100 a FE(186)987 b Ft(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Fp()23 b(SENDBUF\(*\),)g (RECVBUF\(*\))170 102 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(ROOT,)h (COMM,)f(IERROR)75 191 y(MPI)p 150 191 15 2 v 17 w(USER)p 263 191 V 16 w(REDUCE\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)g (FUNCTION,)f(ROOT,)h(COMM\))170 247 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 304 y(EXTERNAL)g(FUNCTION)170 360 y(INTEGER)g(COUNT,) g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)75 450 y(MPI)p 150 450 V 17 w(USER)p 263 450 V 16 w(REDUCEA\(SENDBUF,)f(RECVBUF,)h(COUNT,) g(DATATYPE,)g(FUNCTION,)f(ROOT,)h(COMM\))170 506 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 563 y(EXTERNAL)g(FUNCTION)170 619 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)75 708 y(MPI)p 150 708 V 17 w(ALLREDUCE\(SENDBUF,)e(RECVBUF,)i(COUNT,)g (DATATYPE,)g(OP,)g(COMM\))170 765 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 821 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)75 911 y(MPI)p 150 911 V 17 w(USER)p 263 911 V 16 w(ALLREDUCE\(SENDBUF,)e(RECVBUF,)h(COUNT,)g(DATATYPE,)f(FUNCTION,) h(COMM\))170 967 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1024 y(EXTERNAL)g(FUNCTION)170 1080 y(INTEGER)g(COUNT,)g(DATATYPE,)g (COMM,)g(IERROR)75 1169 y(MPI)p 150 1169 V 17 w(USER)p 263 1169 V 16 w(ALLREDUCEA\(SENDBUF,)f(RECVBUF,)g(COUNT,)h(DATATYPE,)g (FUNCTION,)g(COMM\))170 1226 y()g(SENDBUF\(*\),)g(RECVBUF\(*\)) 170 1282 y(EXTERNAL)g(FUNCTION)170 1339 y(INTEGER)g(COUNT,)g(DATATYPE,) g(COMM,)g(IERROR)75 1428 y(MPI)p 150 1428 V 17 w(REDUCE)p 311 1428 V 16 w(SCATTER\(SENDBUF,)f(RECVBUF,)h(RECVCOUNTS,)f(DATATYPE,) h(OP,)g(COMM\))170 1485 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1541 y(INTEGER)g(RECVCOUNTS,)g(DATATYPE,)f(OP,)i(COMM,)f(IERROR)75 1630 y(MPI)p 150 1630 V 17 w(SCAN\(SENDBUF,)f(RECVBUF,)h(COUNT,)g (DATATYPE,)f(OP,)i(COMM\))170 1687 y()f(SENDBUF\(*\),)g (RECVBUF\(*\))170 1743 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)75 1833 y(MPI)p 150 1833 V 17 w(USER)p 263 1833 V 16 w(SCAN\(SENDBUF,)e(RECVBUF,)h(COUNT,)g(DATATYPE,)g(FUNCTION,)g (COMM\))170 1889 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1946 y(EXTERNAL)g(FUNCTION)170 2002 y(INTEGER)g(COUNT,)g(DATATYPE,)g (COMM,)g(IERROR)75 2091 y(MPI)p 150 2091 V 17 w(USER)p 263 2091 V 16 w(SCANA\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)g (FUNCTION,)f(COMM\))170 2148 y()h(SENDBUF\(*\),)g(RECVBUF\(*\)) 170 2204 y(EXTERNAL)g(FUNCTION)170 2261 y(INTEGER)g(COUNT,)g(DATATYPE,) g(COMM,)g(IERROR)75 2451 y Fv(A.10)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Groups,)f(Contexts,)d(and)j(Communicato)n(rs)75 2558 y Fp(MPI)p 150 2558 V 17 w(GROUP)p 287 2558 V 16 w(SIZE\(GROUP,)j(SIZE,)g(IERROR\))170 2614 y(INTEGER)g(GROUP,)g(SIZE,)h (IERROR)75 2704 y(MPI)p 150 2704 V 17 w(GROUP)p 287 2704 V 16 w(RANK\(GROUP,)f(RANK,)g(IERROR\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 187 192 187 191 bop 75 -100 a Ft(A.10.)27 b(F)o(OR)l(TRAN)11 b(BINDINGS)g(F)o(OR)g(GR)o(OUPS,)f(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)p FE(187)170 45 y Fp(INTEGER)23 b(GROUP,)g(RANK,)h(IERROR)75 136 y(MPI)p 150 136 15 2 v 17 w(GROUP)p 287 136 V 16 w(TRANSLATE)p 519 136 V 16 w(RANKS\(GROUP1,)e(N,)i(RANKS1,)f(GROUP2,)g (RANKS2,)g(IERROR\))170 193 y(INTEGER)g(GROUP1,)g(N,)h(RANKS1,)f (GROUP2,)g(RANKS2,)g(IERROR)75 284 y(MPI)p 150 284 V 17 w(GROUP)p 287 284 V 16 w(UNION\(GROUP1,)f(GROUP2,)h(NEWGROUP,)g (IERROR\))170 340 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 432 y(MPI)p 150 432 V 17 w(GROUP)p 287 432 V 16 w (INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\))170 488 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 579 y(MPI)p 150 579 V 17 w(GROUP)p 287 579 V 16 w(DIFFERENCE\(GROUP1,)f (GROUP2,)h(NEWGROUP,)f(IERROR\))170 636 y(INTEGER)h(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 727 y(MPI)p 150 727 V 17 w(GROUP)p 287 727 V 16 w(INCL\(GROUP,)g(N,)g(RANKS,)g(NEWGROUP,)g(IERROR\))170 784 y(INTEGER)g(GROUP,)g(N,)h(RANKS,)f(NEWGROUP,)g(IERROR)75 875 y(MPI)p 150 875 V 17 w(GROUP)p 287 875 V 16 w(EXCL\(GROUP,)g(N,)g (RANKS,)g(NEWGROUP,)g(IERROR\))170 931 y(INTEGER)g(GROUP,)g(N,)h (RANKS,)f(NEWGROUP,)g(IERROR)75 1023 y(MPI)p 150 1023 V 17 w(GROUP)p 287 1023 V 16 w(RANGE)p 423 1023 V 17 w(INCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1079 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 1170 y(MPI)p 150 1170 V 17 w(GROUP)p 287 1170 V 16 w(RANGE)p 423 1170 V 17 w(EXCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1227 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 1318 y(MPI)p 150 1318 V 17 w(GROUP)p 287 1318 V 16 w(FREE\(GROUP,)g (IERROR\))170 1374 y(INTEGER)g(GROUP,)g(IERROR)75 1466 y(MPI)p 150 1466 V 17 w(COMM)p 263 1466 V 16 w(SIZE\(COMM,)g(SIZE,)g (IERROR\))170 1522 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 1613 y(MPI)p 150 1613 V 17 w(COMM)p 263 1613 V 16 w(RANK\(COMM,)g (RANK,)g(IERROR\))170 1670 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 1761 y(MPI)p 150 1761 V 17 w(COMM)p 263 1761 V 16 w(GROUP\(COMM,)g (GROUP,)g(IERROR\))170 1817 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)75 1909 y(MPI)p 150 1909 V 17 w(COMM)p 263 1909 V 16 w(DUP\(COMM,)g (NEWCOMM,)g(IERROR\))170 1965 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR)75 2056 y(MPI)p 150 2056 V 17 w(COMM)p 263 2056 V 16 w(MAKE\(COMM,)h (GROUP,)g(NEWCOMM,)g(IERROR\))170 2113 y(INTEGER)g(COMM,)h(GROUP,)f (NEWCOMM,)f(IERROR)75 2204 y(MPI)p 150 2204 V 17 w(COMM)p 263 2204 V 16 w(SPLIT\(COMM,)h(COLOR,)g(KEY,)g(NEWCOMM,)g(IERROR\))170 2261 y(INTEGER)g(COMM,)h(COLOR,)f(KEY,)g(NEWCOMM,)g(IERROR)75 2352 y(MPI)p 150 2352 V 17 w(COMM)p 263 2352 V 16 w(FREE\(COMM,)g (IERROR\))170 2408 y(INTEGER)g(COMM,)h(IERROR)75 2499 y(MPI)p 150 2499 V 17 w(COMM)p 263 2499 V 16 w(TEST)p 375 2499 V 17 w(INTER\(COMM,)e(FLAG,)i(IERROR\))170 2556 y(INTEGER)f(COMM,)h(FLAG,)f(IERROR)75 2647 y(MPI)p 150 2647 V 17 w(INTERCOMM)p 383 2647 V 16 w(START\(LOCAL)p 663 2647 V 15 w(COMM,)h(LOCAL)p 942 2647 V 16 w(LEADER,)f(PEER)p 1245 2647 V 17 w(COMM,)g(REMOTE)p 1549 2647 V 16 w(LEADER,)g(TAG,)393 2704 y(NEW)p 468 2704 V 17 w(COMM,)g(INTER)p 748 2704 V 17 w(REQUEST,)f(IERROR\))-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 188 193 188 192 bop 75 -100 a FE(188)987 b Ft(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Fp(INTEGER)23 b(LOCAL)p 484 45 15 2 v 17 w(COMM,)g(LOCAL)p 764 45 V 17 w(LEADER,)f(PEER)p 1067 45 V 17 w(COMM,)h(REMOTE)p 1371 45 V 17 w(LEADER,)g(TAG,)170 102 y(INTER)p 293 102 V 17 w(REQUEST,)g(IERROR)75 189 y(MPI)p 150 189 V 17 w(INTERCOMM)p 383 189 V 16 w(FINISH\(INTER)p 687 189 V 15 w(REQUEST,)g(IERROR\))170 245 y(INTEGER)g(INTER)p 484 245 V 17 w(REQUEST,)g(NEWCOMM,)f(IERROR)75 332 y(MPI)p 150 332 V 17 w(INTERCOMM)p 383 332 V 16 w(MERGE\(INTERCOMM,)f(KEY,)j (NEWINTERCOMM,)e(IERROR\))170 388 y(INTEGER)h(INTERCOMM,)g(KEY,)g (NEWINTRACOMM,)f(IERROR)75 475 y(MPI)p 150 475 V 17 w(INTERCOMM)p 383 475 V 16 w(NAME)p 495 475 V 16 w(START\(COMM,)h(NAME,)g(NEW)p 1013 475 V 17 w(COMM,)g(INTER)p 1293 475 V 16 w(REQUEST,)g(IERROR\))170 532 y(INTEGER)g(COMM,)h(NAME,)f(NEW)p 723 532 V 16 w(COMM,)h(INTER)p 1003 532 V 16 w(REQUEST,)f(IERROR)75 619 y(MPI)p 150 619 V 17 w(ATTR)p 263 619 V 16 w(GET)p 351 619 V 17 w(KEY\(COPY)p 560 619 V 16 w(FN,)h(DELETE)p 816 619 V 16 w(FN,)g(KEYVAL,)e(EXTRA)p 1238 619 V 17 w(STATE,)h(IERROR\))170 675 y(INTEGER)g(COPY)p 460 675 V 17 w(FN,)g(DELETE)p 716 675 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1139 675 V 17 w(STATE,)g(IERROR)75 762 y(MPI)p 150 762 V 17 w(ATTR)p 263 762 V 16 w(FREE)p 375 762 V 17 w(KEY\(KEYVAL,)f (IERROR\))170 819 y(INTEGER)h(KEYVAL,)g(IERROR)75 906 y(MPI)p 150 906 V 17 w(ATTR)p 263 906 V 16 w(PUT)p 351 906 V 17 w(VALUE\(COMM,)g(KEYVAL,)f(ATTRIBUTE)p 1061 906 V 16 w(VAL,)i(IERROR\))170 962 y(INTEGER)f(COMM,)h(KEYVAL,)e (ATTRIBUTE)p 914 962 V 16 w(VAL,)i(IERROR)75 1049 y(MPI)p 150 1049 V 17 w(ATTR)p 263 1049 V 16 w(GET)p 351 1049 V 17 w(VALUE\(COMM,)f(KEYVAL,)f(ATTRIBUTE)p 1061 1049 V 16 w(VAL,)i(FOUND,)f(IERROR\))170 1105 y(INTEGER)g(COMM,)h(KEYVAL,)e (ATTRIBUTE)p 914 1105 V 16 w(VAL,)i(FOUND,)f(IERROR)75 1192 y(MPI)p 150 1192 V 17 w(ATTR)p 263 1192 V 16 w(DELETE)p 423 1192 V 17 w(VALUE\(COMM,)f(KEYVAL,)h(IERROR\))170 1249 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)75 1425 y Fv(A.11)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Pro)r(cess)e(T)-5 b(op)r(ologies)75 1545 y(A.12)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n (r)i(Environmental)d(Inquiry)75 1648 y Fp(MPI)p 150 1648 V 17 w(GET)p 239 1648 V 17 w(VALID)p 376 1648 V 16 w(TAG)p 464 1648 V 17 w(RANGE\(LOW,)k(HIGH,)h(IERROR\))170 1704 y(INTEGER)g(LOW,)h(HIGH,)f(IERROR)75 1791 y(MPI)p 150 1791 V 17 w(GET)p 239 1791 V 17 w(PROCESSOR)p 472 1791 V 15 w(NAME\(LEN,)g(NAME,)g(IERROR\))170 1848 y(INTEGER)g(LEN,)h(NAME,) f(IERROR)75 1935 y(MPI)p 150 1935 V 17 w(GET)p 239 1935 V 17 w(HOST\(COMM,)f(RANK,)h(IERROR\))170 1991 y(INTEGER)g(COMM,)h (RANK,)f(IERROR)75 2078 y(MPI)p 150 2078 V 17 w(GET)p 239 2078 V 17 w(BUFFER)p 400 2078 V 16 w(PARAMS\(NUMSEND,)f(SIZESEND,)g (NUMRECV,)h(SIZERECV,)g(TOTALNUMSEDND,)393 2135 y(TOTALSIZESEN,)f (TOTALNUMRECV,)g(TOTALSIZERECV,)g(IERROR\))170 2191 y(INTEGER)h (NUMSEND,)g(SIZESEND,)g(NUMRECV,)g(SIZERECV,)f(TOTALNUMSEDND,)170 2247 y(TOTALSIZESEN,)h(TOTALNUMRECV,)f(TOTALSIZERECV,)g(IERROR)75 2334 y(MPI)p 150 2334 V 17 w(SUGGEST)p 335 2334 V 16 w(BUFFER)p 495 2334 V 16 w(PARAMS\(NUMSEND,)g(SIZESEND,)h(NUMRECV,)g (SIZERECV,)393 2391 y(TOTALNUMSEND,)f(TOTALSIZESEN,)g(TOTALNUMRECV,)g (TOTALSIZERECV,)g(FLAG,)393 2447 y(IERROR\))170 2504 y(INTEGER)h(NUMSEND,)g(SIZESEND,)g(NUMRECV,)g(SIZERECV,)f (TOTALNUMSEND,)170 2560 y(TOTALSIZESEN,)h(TOTALNUMRECV,)f (TOTALSIZERECV,)g(FLAG,)h(IERROR)75 2647 y(MPI)p 150 2647 V 17 w(USER)p 263 2647 V 16 w(SPECIFIES)p 495 2647 V 16 w(BUFFER\(COMM,)g(BUFFER,)f(SIZE,)i(IERROR\))170 2704 y()f(SENDBUF\(*\))1967 46 y FB(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 189 194 189 193 bop 75 -100 a Ft(A.12.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(ENVIR)o(ONMENT)l(AL)g(INQUIR)l(Y)364 b FE(189)170 45 y Fp(INTEGER)23 b(COMM,)h(BUFFER,)e(SIZE,)i(IERROR)75 132 y(MPI)p 150 132 15 2 v 17 w(GET)p 239 132 V 17 w(ERRORMODE\(COMM,)d (MODE,)j(IERROR\))170 188 y(INTEGER)f(COMM,)h(MODE,)f(IERROR)75 274 y(MPI)p 150 274 V 17 w(SET)p 239 274 V 17 w(ERRORMODE\(COMM,)e (MODE,)j(IERROR\))170 331 y(INTEGER)f(COMM,)h(MODE,)f(IERROR)75 417 y(MPI)p 150 417 V 17 w(SET)p 239 417 V 17 w(ERRHANDLER\(COMM,)e (HANDLER,)i(IERROR\))170 474 y(INTEGER)g(COMM,)h(HANDLER,)e(IERROR)75 560 y(MPI)p 150 560 V 17 w(ERROR)p 287 560 V 16 w(STRING\(ERRORCODE,)g (LEN,)h(STRING,)g(IERROR\))170 617 y(INTEGER)g(ERRORCODE,)g(LEN,)g (STRING,)g(IERROR)75 703 y(MPI)p 150 703 V 17 w(INIT\(IERROR\))170 760 y(INTEGER)g(IERROR)75 846 y(MPI)p 150 846 V 17 w (FINALIZE\(IERROR\))170 903 y(INTEGER)g(IERROR)75 989 y(MPI)p 150 989 V 17 w(INITIALIZED\(FLAG,)e(IERROR\))170 1045 y(INTEGER)i(FLAG,)h(IERROR)75 1132 y(MPI)p 150 1132 V 17 w(ABORT\(COMM,)e(ERRORCODE,)h(IERROR\))170 1188 y(INTEGER)g(COMM,)h(ERRORCODE,)e(IERROR)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 190 195 190 194 bop 75 283 a FA(MPI)40 b(F)-10 b(unction)40 b(Index)75 490 y FE(MPI)p 167 490 14 2 v 16 w(ABOR)l(T,)16 b(160)75 547 y(MPI)p 167 547 V 16 w(ADDRESS,)g(53)75 604 y(MPI)p 167 604 V 16 w(ALLGA)l(THER,)g(88)75 661 y(MPI)p 167 661 V 16 w(ALLGA)l(THER)-5 b(V,)16 b(89)75 717 y(MPI)p 167 717 V 16 w(ALLREDUCE,)g(98)75 774 y(MPI)p 167 774 V 16 w(ALL)l(TO)o(ALL,)h(90)75 831 y(MPI)p 167 831 V 16 w(ALL)l(TO)o(ALL)-5 b(V,)17 b(91)75 887 y(MPI)p 167 887 V 16 w(A)l(TTR)p 312 887 V 17 w(DELETE)p 518 887 V 16 w(V)-5 b(ALUE,)16 b(124)75 944 y(MPI)p 167 944 V 16 w(A)l(TTR)p 312 944 V 17 w(FREE)p 454 944 V 17 w(KEY,)f(123)75 1001 y(MPI)p 167 1001 V 16 w(A)l(TTR)p 312 1001 V 17 w(GET)p 429 1001 V 16 w(KEY,)g(122)75 1057 y(MPI)p 167 1057 V 16 w(A)l(TTR)p 312 1057 V 17 w(GET)p 429 1057 V 16 w(V)-5 b(ALUE,)15 b(123)75 1114 y(MPI)p 167 1114 V 16 w(A)l(TTR)p 312 1114 V 17 w(PUT)p 427 1114 V 16 w(V)-5 b(ALUE,)16 b(123)75 1217 y(MPI)p 167 1217 V 16 w(BARRIER,)h(72)75 1273 y(MPI)p 167 1273 V 16 w(BCAST,)e(74)75 1376 y(MPI)p 167 1376 V 16 w(CANCEL,)g(39)75 1432 y(MPI)p 167 1432 V 16 w(CAR)l(T)p 312 1432 V 17 w(COORD,)h(146)75 1489 y(MPI)p 167 1489 V 16 w(CAR)l(T)p 312 1489 V 17 w(RANK,)g(146)75 1546 y(MPI)p 167 1546 V 16 w(CAR)l(T)p 312 1546 V 17 w(SHIFT,)f(148)75 1603 y(MPI)p 167 1603 V 16 w(CAR)l(T)p 312 1603 V 17 w(SUB,)g(149)75 1659 y(MPI)p 167 1659 V 16 w(COMM)p 335 1659 V 16 w(DUP)l(,)g(114)75 1716 y(MPI)p 167 1716 V 16 w(COMM)p 335 1716 V 16 w(FREE,)g(116)75 1773 y(MPI)p 167 1773 V 16 w(COMM)p 335 1773 V 16 w(GR)o(OUP)l(,)g(114) 75 1829 y(MPI)p 167 1829 V 16 w(COMM)p 335 1829 V 16 w(INIT,)h(69)75 1886 y(MPI)p 167 1886 V 16 w(COMM)p 335 1886 V 16 w(MAKE,)f(115)75 1943 y(MPI)p 167 1943 V 16 w(COMM)p 335 1943 V 16 w(RANK,)h(114)75 1999 y(MPI)p 167 1999 V 16 w(COMM)p 335 1999 V 16 w(SIZE,)f(114)75 2056 y(MPI)p 167 2056 V 16 w(COMM)p 335 2056 V 16 w(SPLIT,)h(115)75 2113 y(MPI)p 167 2113 V 16 w(COMM)p 335 2113 V 16 w(TEST)p 473 2113 V 16 w(INTER,)g(118)75 2170 y(MPI)p 167 2170 V 16 w(COMMOBJ)p 425 2170 V 17 w(FREE,)f(43)75 2226 y(MPI)p 167 2226 V 16 w(CREA)l(TE)p 374 2226 V 17 w(RECV,)g(42)75 2283 y(MPI)p 167 2283 V 16 w(CREA)l(TE)p 374 2283 V 17 w(RSEND,)g(41)75 2340 y(MPI)p 167 2340 V 16 w(CREA)l(TE)p 374 2340 V 17 w(SEND,)g(40)75 2396 y(MPI)p 167 2396 V 16 w(CREA)l(TE)p 374 2396 V 17 w(SSEND,)g(41)75 2499 y(MPI)p 167 2499 V 16 w(ERR)o(OR)p 347 2499 V 18 w(STRING,)h(159)75 2601 y(MPI)p 167 2601 V 16 w(FINALIZE,)g(159)75 2704 y(MPI)p 167 2704 V 16 w(GA)l(THER,)f(74)1048 490 y(MPI)p 1140 490 V 16 w(GA)l(THER)-5 b(V,)15 b(75)1048 547 y(MPI)p 1140 547 V 16 w(GET)p 1256 547 V 16 w(BUFFER)p 1462 547 V 16 w(P)l(ARAMS,)h(154)1048 604 y(MPI)p 1140 604 V 16 w(GET)p 1256 604 V 16 w(CAR)l(T,)f(146)1048 661 y(MPI)p 1140 661 V 16 w(GET)p 1256 661 V 16 w(CAR)l(TDIM,)g(146)1048 718 y(MPI)p 1140 718 V 16 w(GET)p 1256 718 V 16 w(COUNT,)g(21)1048 775 y(MPI)p 1140 775 V 16 w(GET)p 1256 775 V 16 w(ELEMENTS,)g(58)1048 832 y(MPI)p 1140 832 V 16 w(GET)p 1256 832 V 16 w(ERR)o(ORMODE,)h(158) 1048 889 y(MPI)p 1140 889 V 16 w(GET)p 1256 889 V 16 w(GRAPH,)f(145)1048 946 y(MPI)p 1140 946 V 16 w(GET)p 1256 946 V 16 w(GRAPHDIMS,)g(145)1048 1003 y(MPI)p 1140 1003 V 16 w(GET)p 1256 1003 V 16 w(HOST,)g(153)1048 1060 y(MPI)p 1140 1060 V 16 w(GET)p 1256 1060 V 16 w(PR)o(OCESSOR)p 1552 1060 V 18 w(NAME,)f(152)1048 1117 y(MPI)p 1140 1117 V 16 w(GET)p 1256 1117 V 16 w(SOUR)o(CE,)h(20)1048 1174 y(MPI)p 1140 1174 V 16 w(GET)p 1256 1174 V 16 w(T)l(A)o(G,)f(20)1048 1231 y(MPI)p 1140 1231 V 16 w(GET)p 1256 1231 V 16 w(V)-5 b(ALID)p 1414 1231 V 17 w(T)l(A)o(G)p 1529 1231 V 16 w(RANGE,)15 b(152)1048 1288 y(MPI)p 1140 1288 V 16 w(GRAPH)p 1324 1288 V 17 w(NEIGHBORS,)h(147)1048 1345 y(MPI)p 1140 1345 V 16 w(GRAPH)p 1324 1345 V 17 w(NNEIGHBORS,)g(147)1048 1402 y(MPI)p 1140 1402 V 16 w(GR)o(OUP)p 1324 1402 V 17 w(DIFFERENCE,)e(110)1048 1459 y(MPI)p 1140 1459 V 16 w(GR)o(OUP)p 1324 1459 V 17 w(EX)o(CL,)g(111)1048 1515 y(MPI)p 1140 1515 V 16 w(GR)o(OUP)p 1324 1515 V 17 w(FREE,)g(113)1048 1572 y(MPI)p 1140 1572 V 16 w(GR)o(OUP)p 1324 1572 V 17 w(INCL,)h(111)1048 1629 y(MPI)p 1140 1629 V 16 w(GR)o(OUP)p 1324 1629 V 17 w(INTERSECTION,)h(110)1048 1686 y(MPI)p 1140 1686 V 16 w(GR)o(OUP)p 1324 1686 V 17 w(RANGE)p 1509 1686 V 16 w(EX)o(CL,)f(112)1048 1743 y(MPI)p 1140 1743 V 16 w(GR)o(OUP)p 1324 1743 V 17 w(RANGE)p 1509 1743 V 16 w(INCL,)h(112)1048 1800 y(MPI)p 1140 1800 V 16 w(GR)o(OUP)p 1324 1800 V 17 w(RANK,)f(109)1048 1857 y(MPI)p 1140 1857 V 16 w(GR)o(OUP)p 1324 1857 V 17 w(SIZE,)g(108)1048 1914 y(MPI)p 1140 1914 V 16 w(GR)o(OUP)p 1324 1914 V 17 w(TRANSLA)l(TE)p 1622 1914 V 17 w(RANKS,)c(109)1048 1971 y(MPI)p 1140 1971 V 16 w(GR)o(OUP)p 1324 1971 V 17 w(UNION,)k(109)1048 2078 y(MPI)p 1140 2078 V 16 w(INIT,)h(159)1048 2135 y(MPI)p 1140 2135 V 16 w(INITIALIZED,)g(160)1048 2192 y(MPI)p 1140 2192 V 16 w(INTER)o(COMM)p 1454 2192 V 16 w(FINISH,)g(120)1048 2249 y(MPI)p 1140 2249 V 16 w(INTER)o(COMM)p 1454 2249 V 16 w(MER)o(GE\(in)o(tercomm,k)o(ey)l (,newin)o(tracomm,)1214 2305 y(120)1048 2362 y(MPI)p 1140 2362 V 16 w(INTER)o(COMM)p 1454 2362 V 16 w(NAME)p 1611 2362 V 17 w(ST)l(AR)l(T,)f(121)1048 2419 y(MPI)p 1140 2419 V 16 w(INTER)o(COMM)p 1454 2419 V 16 w(ST)l(AR)l(T,)h(119) 1048 2476 y(MPI)p 1140 2476 V 16 w(IPR)o(OBE,)g(38)1048 2533 y(MPI)p 1140 2533 V 16 w(IRECV,)g(31)1048 2590 y(MPI)p 1140 2590 V 16 w(IRSEND,)g(30)1048 2647 y(MPI)p 1140 2647 V 16 w(ISEND,)f(30)1048 2704 y(MPI)p 1140 2704 V 16 w(ISSEND,)g(31)941 2828 y(190)p eop %%Page: 191 196 191 195 bop 75 -100 a Ft(MPI)15 b(F)l(unction)h(Index)1328 b FE(191)75 45 y(MPI)p 167 45 14 2 v 16 w(MAKE)p 325 45 V 17 w(CAR)l(T,)15 b(142)75 102 y(MPI)p 167 102 V 16 w(MAKE)p 325 102 V 17 w(DIMS,)g(143)75 158 y(MPI)p 167 158 V 16 w(MAKE)p 325 158 V 17 w(GRAPH,)g(143)75 215 y(MPI)p 167 215 V 16 w(MAP)p 290 215 V 16 w(CAR)l(T,)h(149)75 272 y(MPI)p 167 272 V 16 w(MAP)p 290 272 V 16 w(GRAPH,)f(150)75 373 y(MPI)p 167 373 V 16 w(PR)o(OBE,)h(39)75 474 y(MPI)p 167 474 V 16 w(RECV,)f(19)75 531 y(MPI)p 167 531 V 16 w(REDUCE,)g(92)75 587 y(MPI)p 167 587 V 16 w(REDUCE)p 380 587 V 17 w(SCA)l(TTER,)g(99)75 644 y(MPI)p 167 644 V 16 w(RSEND,)h(27)75 745 y(MPI)p 167 745 V 16 w(SCAN,)f(100)75 802 y(MPI)p 167 802 V 16 w(SCA)l(TTER,)h(83)75 859 y(MPI)p 167 859 V 16 w(SCA)l(TTER)-5 b(V,)16 b(84)75 915 y(MPI)p 167 915 V 16 w(SEND,)f(17)75 972 y(MPI)p 167 972 V 16 w(SENDRECV,)g(44)75 1028 y(MPI)p 167 1028 V 16 w(SENDRECV)p 439 1028 V 17 w(REPLA)o(CE,)g(45)75 1085 y(MPI)p 167 1085 V 16 w(SET)p 272 1085 V 17 w(ERRHANDLER,)h(158)75 1142 y(MPI)p 167 1142 V 16 w(SET)p 272 1142 V 17 w(ERR)o(ORMODE,)g(158) 75 1198 y(MPI)p 167 1198 V 16 w(SSEND,)f(28)75 1255 y(MPI)p 167 1255 V 16 w(ST)l(AR)l(T,)h(42)75 1312 y(MPI)p 167 1312 V 16 w(ST)l(AR)l(T)l(ALL,)h(42)75 1368 y(MPI)p 167 1368 V 16 w(SUGGEST)p 403 1368 V 16 w(BUFFER)p 609 1368 V 17 w(P)l(ARAMS,)10 b(155)75 1470 y(MPI)p 167 1470 V 16 w(TEST,)15 b(32)75 1526 y(MPI)p 167 1526 V 16 w(TEST)p 305 1526 V 17 w(CANCELLED,)g(40)75 1583 y(MPI)p 167 1583 V 16 w(TEST)l(ALL,)h(36)75 1639 y(MPI)p 167 1639 V 16 w(TEST)l(ANY,)f(35)75 1696 y(MPI)p 167 1696 V 16 w(TOPO)p 317 1696 V 17 w(ST)l(A)l(TUS,)h(145)75 1753 y(MPI)p 167 1753 V 16 w(TYPE)p 312 1753 V 17 w(COMMIT,)e(56)75 1809 y(MPI)p 167 1809 V 16 w(TYPE)p 312 1809 V 17 w(CONTIGUOUS,)h(48)75 1866 y(MPI)p 167 1866 V 16 w(TYPE)p 312 1866 V 17 w(COUNT,)g(54)75 1923 y(MPI)p 167 1923 V 16 w(TYPE)p 312 1923 V 17 w(EXTENT,)f(54)75 1979 y(MPI)p 167 1979 V 16 w(TYPE)p 312 1979 V 17 w(FREE,)g(56)75 2036 y(MPI)p 167 2036 V 16 w(TYPE)p 312 2036 V 17 w(HINDEXED,)g(51)75 2093 y(MPI)p 167 2093 V 16 w(TYPE)p 312 2093 V 17 w(HVECTOR,)h(49)75 2149 y(MPI)p 167 2149 V 16 w(TYPE)p 312 2149 V 17 w(INDEXED,)f(50)75 2206 y(MPI)p 167 2206 V 16 w(TYPE)p 312 2206 V 17 w(SIZE,)h(54)75 2263 y(MPI)p 167 2263 V 16 w(TYPE)p 312 2263 V 17 w(STR)o(UCT,)f(52)75 2319 y(MPI)p 167 2319 V 16 w(TYPE)p 312 2319 V 17 w(VECTOR,)h(48)75 2420 y(MPI)p 167 2420 V 16 w(USER)p 306 2420 V 17 w(ALLREDUCE,)h(98)75 2477 y(MPI)p 167 2477 V 16 w(USER)p 306 2477 V 17 w(ALLREDUCEA,)g(99)75 2534 y(MPI)p 167 2534 V 16 w(USER)p 306 2534 V 17 w(REDUCE,)f(95)75 2590 y(MPI)p 167 2590 V 16 w(USER)p 306 2590 V 17 w(REDUCEA,)g(97)75 2647 y(MPI)p 167 2647 V 16 w(USER)p 306 2647 V 17 w(SCAN,)h(101)75 2704 y(MPI)p 167 2704 V 16 w(USER)p 306 2704 V 17 w(SCANA,)g(101)1048 45 y(MPI)p 1140 45 V 16 w(USER)p 1279 45 V 17 w(SPECIFIES)p 1534 45 V 17 w(BUFFER,)f(156)1048 143 y(MPI)p 1140 143 V 16 w(W)-5 b(AIT,)15 b(32)1048 200 y(MPI)p 1140 200 V 16 w(W)-5 b(AIT)l(ALL,)16 b(36)1048 256 y(MPI)p 1140 256 V 16 w(W)-5 b(AIT)l(ANY,)15 b(35)-32 46 y FB(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .