%!PS-Adobe-2.0 %%Creator: dvips 5.516 Copyright 1986, 1993 Radical Eye Software %%Title: mpi-report.dvi %%CreationDate: Sat Aug 7 13:55:03 1993 %%Pages: 128 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips mpi-report %DVIPSSource: TeX output 1993.08.07:1354 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /hsize 8.5 72 mul N /landplus90{false}def /@rigin{isls{[0 landplus90{1 -1}{-1 1} ifelse 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div hsize mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{ statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{/nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{/sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0]N df-tail}B /E{ pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get} B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]}if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{cc 1 add D }B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin /SDict 200 dict N SDict begin /@SpecialDefaults{/hs 612 N /vs 792 N /ho 0 N /vo 0 N /hsc 1 N /vsc 1 N /ang 0 N /CLIP 0 N /rwiSeen false N /rhiSeen false N /letter{}N /note{}N /a4{}N /legal{}N}B /@scaleunit 100 N /@hscale{@scaleunit div /hsc X}B /@vscale{@scaleunit div /vsc X}B /@hsize{/hs X /CLIP 1 N}B /@vsize{/vs X /CLIP 1 N}B /@clip{ /CLIP 2 N}B /@hoffset{/ho X}B /@voffset{/vo X}B /@angle{/ang X}B /@rwi{ 10 div /rwi X /rwiSeen true N}B /@rhi{10 div /rhi X /rhiSeen true N}B /@llx{/llx X}B /@lly{/lly X}B /@urx{/urx X}B /@ury{/ury X}B /magscale true def end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N /note{}N /legal{} N /od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto} }{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}N /txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N /cp {pop pop showpage pm restore}N end}if}if}N /normalscale{Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale}if 0 setgray} N /psfts{S 65781.76 div N}N /startTexFig{/psf$SavedState save N userdict maxlength dict begin /magscale false def normalscale currentpoint TR /psf$ury psfts /psf$urx psfts /psf$lly psfts /psf$llx psfts /psf$y psfts /psf$x psfts currentpoint /psf$cy X /psf$cx X /psf$sx psf$x psf$urx psf$llx sub div N /psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR /showpage{}N /erasepage{}N /copypage{}N /p 3 def @MacSetUp}N /doclip{ psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N /endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin /SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count /ocount X /dcount countdictstack N}N /@setspecial {CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR }{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if /showpage{}N /erasepage{}N /copypage{}N newpath }N /@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{ end}repeat grestore SpecialSave restore end}N /@defspecial{SDict begin} N /@fedspecial{end}B /li{lineto}B /rl{rlineto}B /rc{rcurveto}B /np{ /SaveX currentpoint /SaveY X N 1 setlinecap newpath}N /st{stroke SaveX SaveY moveto}N /fil{fill SaveX SaveY moveto}N /ellipse{/endangle X /startangle X /yrad X /xrad X /savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet TeXDict begin 40258437 52099154 1000 300 300 (/amd/iliamna/projects/dsrg/C/staff/otto/papers/Editor/MPI/draft5/mpi-report.dvi) @start /Fa 7 88 df<387CFEFEFE7C380000000000000000387CFEFEFE7C3807167C95 10>58 D<000070000000007000000000F800000000F800000000F800000001FC00000001 FC00000003FE00000003FE00000003FE00000006FF000000067F0000000E7F8000000C3F 8000000C3F800000183FC00000181FC00000381FE00000300FE00000300FE00000600FF0 00006007F00000E007F80000FFFFF80000FFFFF800018001FC00018001FC00038001FE00 030000FE00030000FE000600007F000600007F00FFE00FFFF8FFE00FFFF825227EA12A> 65 D<0003FE0040001FFFC0C0007F00F1C001F8003FC003F0000FC007C00007C00FC000 03C01F800003C03F000001C03F000001C07F000000C07E000000C07E000000C0FE000000 00FE00000000FE00000000FE00000000FE00000000FE00000000FE00000000FE000FFFFC 7E000FFFFC7F00001FC07F00001FC03F00001FC03F00001FC01F80001FC00FC0001FC007 E0001FC003F0001FC001FC003FC0007F80E7C0001FFFC3C00003FF00C026227DA12C>71 D73 D78 D82 D87 D E /Fb 3 120 df<01E007100C1018083810701070607F80E000E000E000E000E000E0 086010602030C01F000D127B9113>101 D<3C3C002646004687004707008E07008E0700 0E07000E07001C0E001C0E001C0E001C1C00381C40381C40383840383880701900300E00 12127C9117>110 D<1E01832703874703874703838707018707010E07010E07011C0E02 1C0E021C0E021C0E04180C04181C04181C081C1C100C263007C3C018127C911C>119 D E /Fc 1 1 df0 D E /Fd 1 106 df<040C0000000000705898983030606464683006127E910B>105 D E /Fe 5 118 df<03F80FFC1C1C380870006000E000E000E000E00060007000380E1C 1E0FFC03F00F107E8F14>99 D110 D<07C01FF03C78701C701CE00EE00EE00EE00EE00EE00E701C783C3C781FF007C00F107E 8F14>I<030007000700070007007FFCFFFC07000700070007000700070007000700070E 070E070E070C03FC00F00F157F9414>116 DI E /Ff 9 90 df<0000180000300000600000E00000C0000180000380000700000600 000E00000C00001C0000380000380000700000700000E00000E00001E00001C00001C000 0380000380000380000780000700000700000F00000E00000E00001E00001E00001E0000 1C00001C00003C00003C00003C00003C0000380000780000780000780000780000780000 780000780000780000700000F00000F00000F00000F00000F00000F00000F00000F00000 F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000 F00000F00000F00000F00000F00000F00000700000780000780000780000780000780000 7800007800007800003800003C00003C00003C00003C00001C00001C00001E00001E0000 1E00000E00000E00000F000007000007000007800003800003800003800001C00001C000 01E00000E00000E000007000007000003800003800001C00000C00000E00000600000700 0003800001800000C00000E0000060000030000018157C768121>32 DI<00001C00003C0000F80001E00003C000 0780000F00000E00001E00003C00003C00003C0000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000F00000F00000F00001 E00001E00003C0000380000700000E00001C0000780000E00000E000007800001C00000E 000007000003800003C00001E00001E00000F00000F00000F00000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 7800007800007800007800007800007800007800007800007800007800007800003C0000 3C00003C00001E00000E00000F000007800003C00001E00000F800003C00001C167C7B81 21>40 D<0018007800F001E003C007800F001F001E003E003C007C007C007800F800F800 F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800 F800F800F8000D25707E25>56 D58 D<007C007C007C007C007C007C 007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C 00F800F800F800F001F001E003E003C0078007000E001C003800F000C000F00038001C00 0E000700078003C003E001E001F000F000F800F800F8007C007C007C007C007C007C007C 007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C0E4D 798025>60 D62 D80 D89 D E /Fg 1 1 df0 D E /Fh 2 111 df<3E0006000C000C000C000C001800187018B819383230340038003E00630063106310 6310C320C1C00D147E9312>107 D<30F8590C4E0C9C0C980C180C180C30183019303130 316032601C100D7F8C15>110 D E /Fi 2 50 df<0F0030C0606060604020C030C030C0 30C030C030C030C030C030C03040206060606030C00F000C137E9211>48 D<0C001C00EC000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C 00FFC00A137D9211>I E /Fj 7 111 df<60F0F06004047C830C>58 D<60F0F0701010101020204080040C7C830C>I<0000038000000F0000003C000000F000 0003C000000F0000003C000000F0000003C000000F0000003C000000F0000000F0000000 3C0000000F00000003C0000000F00000003C0000000F00000003C0000000F00000003C00 00000F000000038019187D9520>I62 D<3F00070007000E000E000E000E001C001C001C001C0039E03A 303C1838187018701C701C701CE038E038E038E030E070E060E0C061C023001E000E1D7E 9C12>98 D<01C003C003C001800000000000000000000000001C00270047004700870087 000E000E001C001C001C003800388038807080710032001C000A1C7E9B0E>105 D<381F004E61804681C04701C08F01C08E01C00E01C00E01C01C03801C03801C03801C07 00380710380710380E10380E2070064030038014127E9119>110 D E /Fk 8 116 df<60F0F070101020204040040A7D830A>59 D<003E000C000C000C00 0C0018001800180018073018F0307060706060C060C060C06080C080C480C4C1C446C838 700F177E9612>100 D<0300038003000000000000000000000000001C00240046004600 8C000C0018001800180031003100320032001C0009177F960C>105 D<00180038001000000000000000000000000001C0022004300430086000600060006000 C000C000C000C001800180018001806300E300C60078000D1D80960E>I<383C1E0044C6 630047028100460301008E0703000C0603000C0603000C060600180C0600180C0620180C 0C20180C0C4030180440301807801B0E7F8D1F>109 D<383C0044C6004702004602008E 06000C06000C06000C0C00180C00180C40181840181880300880300F00120E7F8D15>I< 38F04518463846308C000C000C000C001800180018001800300030000D0E7F8D10>114 D<07C00C201870187038001E000FC003E000606060E060C0C0C1803F000C0E7E8D10>I E /Fl 5 62 df<0102040C1818303070606060E0E0E0E0E0E0E0E0E0E060606070303018 180C04020108227D980E>40 D<8040203018180C0C0E0606060707070707070707070706 06060E0C0C18183020408008227E980E>I<07C018303018701C600C600CE00EE00EE00E E00EE00EE00EE00EE00EE00E600C600C701C30181C7007C00F157F9412>48 D<03000700FF000700070007000700070007000700070007000700070007000700070007 00070007007FF00C157E9412>I61 D E /Fm 3 55 df0 D<020002000200C218F2783AE00F800F803AE0F278C2180200020002000D0E7E8E12>3 D<0003000300060006000C000C00180018003000300060006000C000C001800180030003 00060006000C000C00180018003000300060006000C0004000101E7B9600>54 D E /Fn 51 122 df<00E001E0038007000E001C001C0038003800700070007000E000E0 00E000E000E000E000E000E000E000700070007000380038001C001C000E000700038001 E000E00B217A9C16>40 DI<387C7E7E3E0E1E1C78F060070B798416>44 D<70F8F8F8700505788416>46 D<70F8F8F870000000000000000070F8F8F87005127891 16>58 D<00E00001F00001F00001B00001B00003B80003B80003B800031800071C00071C 00071C00071C00071C000E0E000E0E000FFE000FFE001FFF001C07001C07001C07007F1F C0FF1FE07F1FC013197F9816>65 D<7FF800FFFE007FFF001C0F001C07801C03801C0380 1C03801C07801C07001FFF001FFE001FFE001C1F001C03801C03C01C01C01C01C01C01C0 1C01C01C03C01C07807FFF80FFFF007FFC0012197F9816>I<01F18007FB800FFF801F0F 803C0780380380700380700380F00000E00000E00000E00000E00000E00000E00000E000 00F000007003807003803803803C07001F0F000FFE0007FC0001F00011197E9816>I<7F F800FFFE007FFF001C0F001C07801C03C01C01C01C01C01C01E01C00E01C00E01C00E01C 00E01C00E01C00E01C00E01C00E01C01C01C01C01C03C01C07801C0F807FFF00FFFE007F F8001319809816>I<7FFFC0FFFFC07FFFC01C01C01C01C01C01C01C01C01C00001C0000 1C1C001C1C001FFC001FFC001FFC001C1C001C1C001C00001C00E01C00E01C00E01C00E0 1C00E07FFFE0FFFFE07FFFE013197F9816>II<03E30007FF000F FF001E1F003C0F00380700700700700700F00000E00000E00000E00000E00000E03F80E0 7FC0E03F80F00700700700700700380F003C0F001E1F000FFF0007F70003E70012197E98 16>I<7F1FC0FFBFE07F1FC01C07001C07001C07001C07001C07001C07001C07001FFF00 1FFF001FFF001C07001C07001C07001C07001C07001C07001C07001C07001C07007F1FC0 FFBFE07F1FC013197F9816>II<7F0F E0FF8FF07F0FE01C07801C0F001C0E001C1C001C3C001C78001CF0001CE0001DF0001FF0 001FF8001F38001E1C001C1C001C0E001C0E001C07001C07001C03807F07E0FF8FF07F07 E01419809816>75 DII<7E1FC0FF3FE07F 1FC01D07001D87001D87001D87001DC7001DC7001CC7001CC7001CE7001CE7001CE7001C 67001C67001C77001C77001C37001C37001C37001C17007F1F00FF9F007F0F0013197F98 16>I<1FFC003FFE007FFF00780F00F00780E00380E00380E00380E00380E00380E00380 E00380E00380E00380E00380E00380E00380E00380E00380F00780F00780780F007FFF00 3FFE001FFC0011197E9816>I<7FF800FFFE007FFF001C0F801C03801C03C01C01C01C01 C01C01C01C03C01C03801C0F801FFF001FFE001FF8001C00001C00001C00001C00001C00 001C00001C00007F0000FF80007F000012197F9816>I<7FE000FFF8007FFC001C1E001C 0F001C07001C07001C07001C07001C0F001C1E001FFC001FF8001FFC001C1C001C0E001C 0E001C0E001C0E001C0E201C0E701C0E707F07E0FF87E07F03C014197F9816>82 D<07E3001FFF003FFF00781F00F00700E00700E00700E00000F000007800003F80001FF0 0007FC0000FE00000F00000700000380000380600380E00380E00700F80F00FFFE00FFFC 00C7F00011197E9816>I<7FFFE0FFFFE0FFFFE0E0E0E0E0E0E0E0E0E0E0E0E000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00007FC000FFE0007FC0013197F9816>I<7F07F0FF8FF87F07F01C01C01C01C0 1C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C0 1C01C01C01C00E03800E038007070007FF0003FE0000F8001519809816>II<7F1F807F3F807F1F800E1E000E1C00073C0007380003B80003F00001F00001 E00000E00001E00001F00003F00003B80007B800071C00071C000E0E000E0E001C07007F 1FC0FF1FE07F1FC013197F9816>88 DI91 D93 D<1FE0003FF0007FF80078 3C00300E00000E00000E0003FE001FFE003E0E00700E00E00E00E00E00E00E00783E007F FFE03FE7E00F83E013127E9116>97 D<7E0000FE00007E00000E00000E00000E00000E00 000E3E000EFF000FFF800F83C00F00E00E00E00E00700E00700E00700E00700E00700E00 700E00E00F01E00F83C00FFF800EFF00063C001419809816>I<03F80FFC1FFE3C1E780C 7000E000E000E000E000E000F000700778073E0E1FFC0FF803F010127D9116>I<003F00 007F00003F0000070000070000070000070003C7000FF7001FFF003C1F00780F00700700 E00700E00700E00700E00700E00700E00700700F00700F003C1F001FFFE00FE7F007C7E0 14197F9816>I<03E00FF81FFC3C1E780E7007E007FFFFFFFFFFFFE000E000700778073C 0F1FFE0FFC03F010127D9116>I<001F00007F8000FF8001E78001C30001C00001C0007F FF00FFFF00FFFF0001C00001C00001C00001C00001C00001C00001C00001C00001C00001 C00001C00001C0003FFE007FFF003FFE0011197F9816>I<03E3C007F7E00FFFE01C1CC0 380E00380E00380E00380E00380E001C1C000FF8001FF0001BE0003800001800001FFC00 1FFF003FFF807803C0E000E0E000E0E000E0E000E07001C07C07C03FFF800FFE0003F800 131C7F9116>I<7E0000FE00007E00000E00000E00000E00000E00000E3C000EFE000FFF 000F87800F03800E03800E03800E03800E03800E03800E03800E03800E03800E03800E03 807FC7F0FFE7F87FC7F01519809816>I<018003C003C0018000000000000000007FC07F C07FC001C001C001C001C001C001C001C001C001C001C001C001C07FFFFFFF7FFF101A7D 9916>I<003000780078003000000000000000001FF81FF81FF800380038003800380038 003800380038003800380038003800380038003800380038003800386070F0F0FFE07FC0 3F800D237E9916>I108 DI<7E3C00FEFE007FFF000F87800F03800E03800E03800E03800E03 800E03800E03800E03800E03800E03800E03807FC7F0FFE7F87FC7F01512809116>I<03 E0000FF8001FFC003C1E00780F00700700E00380E00380E00380E00380E00380F0078070 0700780F003C1E001FFC000FF80003E00011127E9116>I<7E3E00FEFF007FFF800F83C0 0F00E00E00E00E00700E00700E00700E00700E00700E00700E00E00F01E00F83C00FFF80 0EFF000E3C000E00000E00000E00000E00000E00000E00007FC000FFE0007FC000141B80 9116>I114 D<0FEC3FFC7FFCF03CE01CE01C70007F801FF007F8003C600EE00EF00EF81EFFFCFFF8C7 E00F127D9116>I<0300000700000700000700000700007FFF00FFFF00FFFF0007000007 000007000007000007000007000007000007010007038007038007038007870003FE0001 FC0000F80011177F9616>I<7E1F80FE3F807E1F800E03800E03800E03800E03800E0380 0E03800E03800E03800E03800E03800E03800E0F800FFFF007FBF803E3F01512809116> I<7F1FC0FF1FE07F1FC01C07001E0F000E0E000E0E000E0E00071C00071C00071C00071C 0003B80003B80003B80001F00001F00000E00013127F9116>I<7F1FC07F3FC07F1FC00F 1C00073C0003B80003F00001F00000E00001E00001F00003B800073C00071C000E0E007F 1FC0FF3FE07F1FC013127F9116>120 D<7F1FC0FF9FE07F1FC01C07000E07000E0E000E 0E00070E00071C00071C00039C00039C0003980001B80001B80000F00000F00000F00000 E00000E00000E00001C00079C0007BC0007F80003F00003C0000131B7F9116>I E /Fo 68 123 df<007F07F001FF1FF003FF3FF007807800070070000F00F0000F00F000 0F00F0000F00F0000F00F0000F00F0000F00F000FFF8FF80FFF8FF80FFF8FF800F00F000 0F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F000 0F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0001C20809F1B>11 D<00F8F001F8F003F8F00780F00700F00F00F00F00F00F00F00F00F00F00F00F00F00F00 F0FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00 F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F01420809F18>13 D<007000E001C00380078007000E001E001E003C003C003C0078007800780078007000F0 00F000F000F000F000F000F000F000F000F000F000F000700078007800780078003C003C 003C001E001E000E0007000780038001C000E000700C2E7EA112>40 DI<018001C001800180 C183E187F99F7DBE1FF807E007E01FF87DBEF99FE187C1830180018001C0018010147DA1 17>I<000600000006000000060000000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000FFFFFFF0FFFFFFF000 060000000600000006000000060000000600000006000000060000000600000006000000 06000000060000000600000006000000060000000600001C207D9A23>I<787878781830 306060E0050A7D830D>III<0001 00030003000600060006000C000C000C00180018001800300030003000600060006000C0 00C000C00180018001800300030003000600060006000C000C000C001800180018003000 30003000600060006000C000C000C000102D7DA117>I<03F00007F8001FFE001E1E003C 0F00380700780780780780700380F003C0F003C0F003C0F003C0F003C0F003C0F003C0F0 03C0F003C0F003C0F003C0F003C0F003C0F003C07807807807807807803807003C0F001E 1E001FFE000FFC0003F00012207E9E17>I<00C001C00FC0FFC0FFC0F3C003C003C003C0 03C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C0 03C0FFFEFFFEFFFE0F1F7C9E17>I<07F0000FFC001FFE00383F00700F00600780E00780 E003C04003C04003C00003C00003C00003C0000780000780000F00000E00001C00003C00 00780000E00001C0000380000700000E00001C0000380000700000FFFFC0FFFFC0FFFFC0 121F7E9E17>I<03F0000FFC001FFE003C1F00780F003007802007800007800007800007 80000F00000F00003E0003FC0003F80003FC00001E00000F000007800003800003C00003 C00003C00003C08003C0C003C0C00780700F807C1F003FFE000FFC0003F00012207E9E17 >I<003E00003E00005E00005E0000DE0001DE00019E00039E00039E00079E00071E000F 1E000E1E001E1E003C1E003C1E00781E00781E00F01E00FFFFF0FFFFF0FFFFF0001E0000 1E00001E00001E00001E00001E00001E00001E00141E7F9D17>I61 D<001F0000001F0000003F8000003B8000003B8000007BC00000 73C0000071C00000F1E00000E1E00000E0E00001E0F00001E0F00001C0F00003C0780003 C078000380780007803C0007803C0007003C000FFFFE000FFFFE000FFFFE001E000F001E 000F003C000F803C0007803C000780780007C0780003C0780003C0F00003E01B207F9F1E >65 DI<001F C000FFF801FFFC03E03C07800C0F00001E00003E00003C00007C00007800007800007800 00F00000F00000F00000F00000F00000F00000F00000F000007800007800007800007C00 003C00003E00001E00000F000207800E03E03E01FFFC00FFF0001FC017227DA01D>IIII<001FE000FFF801FFFE03E03E07800E0F 00001E00003E00003C00007C0000780000780000780000F00000F00000F00000F00000F0 0000F00000F003FEF003FE7803FE78001E78001E7C001E3C001E3E001E1E001E0F001E07 801E03E03E01FFFE00FFF8001FC017227DA01E>III75 DI II<003F0000 00FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C000F0078000780 7800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0 F00003C0F00003C0F80007C07800078078000780780007803C000F003C000F001E001E00 1F003E000F807C0007E1F80003FFF00000FFC000003F00001A227DA021>II<003F000000FFC00003FF F00007E1F8000F807C001F003E001E001E003C000F003C000F0078000780780007807800 0780F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F000 03C0F00003C07800078078000780781E07803C0F0F003C0F8F001E079E001F03FE000F83 FC0007E1F80003FFF00000FFF800003F780000007C0000003E0000001E0000001F000000 0F801A277DA021>II<01FC0007FF800FFFC01F03C03C00C03C00007800007800007800007800007800 007C00003C00003F00001FE0000FFC0007FE0001FF00003F800007C00003C00003E00001 E00001E00001E00001E00001E00001C0C003C0F007C0FC0F807FFF001FFE0003F8001322 7EA019>IIIII<780007807C000F003E001F001E001E000F003C000F807C0007807800 03C0F00003E1F00001F1E00000F3C000007FC000007F8000003F0000001F0000001E0000 003F0000007F8000007FC00000F3C00001F1E00001E0F00003C0F80007C0780007803C00 0F003E001F001E001E000F003C000F807C0007C0780003C0F00003E01B207F9F1E>III I93 D<07E03FF87FFC701E401F000F000F000F003F 07FF1FFF7E0FF80FF00FF00FF00FF83F7FFF3FEF1F8F10147E9316>97 DI<03F00FFC 1FFE3E0E3C0278007800F000F000F000F000F000F000780078003C013E0F1FFF0FFE03F0 10147E9314>I<0007800007800007800007800007800007800007800007800007800007 8000078000078007C7800FF7801FFF803E1F807C0780780780F80780F00780F00780F007 80F00780F00780F00780F00780780780780F803E1F801FFF800FF78007C78011207E9F17 >I<03F0000FFC001FFE003E1F003C0700780700700380FFFF80FFFF80FFFF80F00000F0 0000F000007000007800003C01003E07001FFF0007FE0001F80011147F9314>I<007E01 FE03FE078007000F000F000F000F000F000F000F00FFF0FFF0FFF00F000F000F000F000F 000F000F000F000F000F000F000F000F000F000F000F000F000F20809F0E>I<03E0F00F FFF01FFFF03E3E003C1E00780F00780F00780F00780F00780F003C1E003E3E001FFC003F F80033E0003000003800003FFE003FFF801FFFC03FFFE07803F0F000F0F000F0F000F0F8 01F07E07E03FFFC00FFF0003FC00141E7F9317>III<01E001E001E001E00000000000 000000000000000000000001E001E001E001E001E001E001E001E001E001E001E001E001 E001E001E001E001E001E001E001E001E001E001E001E001E0C3C0FFC0FF803F000B2983 9F0C>IIIII<01F80007FE001FFF 803F0FC03C03C07801E07801E0F000F0F000F0F000F0F000F0F000F0F000F07801E07801 E03C03C03F0FC01FFF8007FE0001F80014147F9317>II114 D<07F01FFC3FFC780C780078007800 7C003FC01FF00FF803F8007C003C003CC03CF07CFFF87FF00FC00E147F9311>I<1E001E 001E001E001E001E00FFF0FFF0FFF01E001E001E001E001E001E001E001E001E001E001E 001E001E001E201FF00FF007C00C1A7F9910>IIII<7801E07C03C03E07801E0F000F0F00079E0003FC0003F80001F80000F00001 F00001F80003FC00079E000F0F000E0F001E07803C03C07801E0F801F01414809315>I< F003C0F003C07807807807807C07803C0F003C0F001E0F001E1E000E1E000F1C000F1C00 073C0007380003B80003B80003B00001F00001F00000E00000E00001C00001C00001C000 0380000780007F00007E00007C0000121D7F9315>I<7FFF7FFF7FFF003E003C007800F8 00F001E003E007C007800F001F001E003C007C00FFFFFFFFFFFF10147F9314>I E /Fp 85 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8F870000000000070F8F8F870 051C779B18>33 D<030600078F00078F00078F00078F00078F00078F007FFFC0FFFFE0FF FFE07FFFC00F1E000F1E000F1E000F1E000F1E000F1E007FFFC0FFFFE0FFFFE07FFFC01E 3C001E3C001E3C001E3C001E3C001E3C000C1800131C7E9B18>35 D<3803007C07807C0780EE0F80EE0F00EE0F00EE1F00EE1E00EE1E00EE3E007C3C007C3C 00387C0000780000780000F80000F00001F00001E00001E00003E00003C00003C00007C0 000783800787C00F87C00F0EE00F0EE01F0EE01E0EE01E0EE03E0EE03C07C03C07C01803 8013247E9F18>37 D<01C00007E0000FF0000E70001C38001C38001C38001C38001C73F0 1C73F01CE3F00FE3800FC7000F87000F07001F0E003F0E007B8E0073DC00E1DC00E0F800 E0F800E07070E0787070FC707FFFE03FCFE00F03C0141C7F9B18>I<387C7C7E3E0E0E0E 1C1C38F8F0C0070E789B18>I<007000F001E003C007800F001E001C0038003800700070 0070007000E000E000E000E000E000E000E000E0007000700070007000380038001C001E 000F00078003C001F000F000700C24799F18>I<6000F00078003C001E000F0007800380 01C001C000E000E000E000E00070007000700070007000700070007000E000E000E000E0 01C001C0038007800F001E003C007800F00060000C247C9F18>I<01C00001C00001C000 01C000C1C180F1C780F9CF807FFF001FFC0007F00007F0001FFC007FFF00F9CF80F1C780 C1C18001C00001C00001C00001C00011147D9718>I<00600000F00000F00000F00000F0 0000F00000F00000F0007FFFC0FFFFE0FFFFE07FFFC000F00000F00000F00000F00000F0 0000F00000F00000600013147E9718>I<1C3E7E7F3F1F070E1E7CF860080C788518>I<7F FF00FFFF80FFFF807FFF0011047D8F18>I<3078FCFC78300606778518>I<000300000780 000780000F80000F00001F00001E00001E00003E00003C00007C0000780000780000F800 00F00001F00001E00003E00003C00003C00007C0000780000F80000F00000F00001F0000 1E00003E00003C00003C00007C0000780000F80000F00000F0000060000011247D9F18> I<01F00007FC000FFE001F1F001C07003803807803C07001C07001C0E000E0E000E0E000 E0E000E0E000E0E000E0E000E0E000E0E000E0F001E07001C07001C07803C03803801C07 001F1F000FFE0007FC0001F000131C7E9B18>I<01800380038007800F803F80FF80FB80 438003800380038003800380038003800380038003800380038003800380038003807FFC FFFE7FFC0F1C7B9B18>I<03F0000FFE003FFF007C0F807003C0E001C0F000E0F000E060 00E00000E00000E00001C00001C00003C0000780000F00001E00003C0000780000F00001 E00007C0000F80001E00E03C00E07FFFE0FFFFE07FFFE0131C7E9B18>I<07F8001FFE00 3FFF007807807803C07801C03001C00001C00003C0000380000F0003FF0003FE0003FF00 0007800003C00001C00000E00000E00000E0F000E0F000E0F001C0F003C07C07803FFF00 1FFE0003F800131C7E9B18>I<001F00003F0000770000770000E70001E70001C7000387 000787000707000E07001E07003C0700380700780700F00700FFFFF8FFFFF8FFFFF80007 00000700000700000700000700000700007FF000FFF8007FF0151C7F9B18>I<1FFF803F FF803FFF803800003800003800003800003800003800003800003800003BF8003FFE003F FF003C07801803C00001C00000E00000E06000E0F000E0F000E0E001C07003C07C0F803F FF001FFC0003F000131C7E9B18>I<007E0001FF0007FF800F83C01E03C01C03C0380180 380000700000700000E1F800E7FE00FFFF00FE0780F803C0F001C0F000E0E000E0F000E0 7000E07000E07000E03801C03C03C01E07800FFF0007FE0001F800131C7E9B18>II<03F0000FFC001FFE003C0F0078078070 0380E001C0E001C0E001C0E001E0E001E07001E07803E03C0FE01FFFE00FFEE003F0E000 00E00001C00001C00001C0300380780780780F00783E003FFC001FF00007C000131C7E9B 18>57 D<3078FCFC783000000000000000003078FCFC78300614779318>I<183C7E7E3C 180000000000000000183C7E7E3E1E0E1C3C78F060071A789318>I<000300000780001F 80003F00007E0001FC0003F00007E0001FC0003F00007E0000FC0000FC00007E00003F00 001FC00007E00003F00001FC00007E00003F00001F8000078000030011187D9918>I<7F FFC0FFFFE0FFFFE0FFFFE0000000000000000000000000FFFFE0FFFFE0FFFFE07FFFC013 0C7E9318>I<600000F00000FC00007E00003F00001FC00007E00003F00001FC00007E00 003F00001F80001F80003F00007E0001FC0003F00007E0001FC0003F00007E0000FC0000 F0000060000011187D9918>I<00700000F80000F80000D80000D80001DC0001DC0001DC 00018C00038E00038E00038E00038E000306000707000707000707000707000FFF800FFF 800FFF800E03800E03801C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>65 DI<00F8E003FEE007FFE00F07E01E03E03C 01E03800E07000E07000E0700000E00000E00000E00000E00000E00000E00000E00000E0 00007000007000E07000E03800E03C00E01E01C00F07C007FF8003FE0000F800131C7E9B 18>I<7FF800FFFE007FFF001C0F801C03C01C03C01C01E01C00E01C00E01C00F01C0070 1C00701C00701C00701C00701C00701C00701C00701C00F01C00E01C00E01C01E01C01C0 1C03C01C0F807FFF00FFFE007FF800141C7F9B18>III<01F1C003FDC00FFFC01F0FC0 1C03C03803C03801C07001C07001C0700000E00000E00000E00000E00000E00000E00FF0 E01FF0E00FF07001C07001C07003C03803C03803C01C07C01F0FC00FFFC003FDC001F1C0 141C7E9B18>I<7F07F0FF8FF87F07F01C01C01C01C01C01C01C01C01C01C01C01C01C01 C01C01C01C01C01FFFC01FFFC01FFFC01C01C01C01C01C01C01C01C01C01C01C01C01C01 C01C01C01C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>I<7FFF00FFFF807FFF0001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C0007FFF00FFFF807F FF00111C7D9B18>I<7F07F0FF87F87F07F01C03C01C07801C07001C0E001C1E001C3C00 1C38001C70001CF0001DF0001DF0001FB8001FB8001F1C001E1C001C0E001C0E001C0700 1C07001C03801C03801C01C07F03F0FF87F87F03F0151C7F9B18>75 D<7FE000FFE0007FE0000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00700E00700E00700E00 700E00707FFFF0FFFFF07FFFF0141C7F9B18>II<7E07F0FF0FF87F07F01D81C01D81C01D81C01DC1C01CC1C01CC1C01CE1C01CE1C0 1CE1C01C61C01C71C01C71C01C31C01C39C01C39C01C39C01C19C01C19C01C1DC01C0DC0 1C0DC01C0DC07F07C0FF87C07F03C0151C7F9B18>I<0FF8003FFE007FFF00780F007007 00F00780E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E003 80E00380E00380E00380E00380E00380F00780700700780F007FFF003FFE000FF800111C 7D9B18>II<0FF8003FFE007FFF00780F00 700700F00780E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380 E00380E00380E00380E00380E1E380E1E380F0E78070F700787F007FFF003FFE000FFC00 001C00001E00000E00000F0000070000070011227D9B18>I<7FF800FFFE007FFF001C0F 801C03801C03C01C01C01C01C01C01C01C03C01C03801C0F801FFF001FFE001FFE001C0F 001C07001C03801C03801C03801C03801C03801C039C1C039C1C039C7F01F8FF81F87F00 F0161C7F9B18>I<03F3801FFF803FFF807C0F80700780E00380E00380E00380E0000070 00007800003F00001FF00007FE0000FF00000F800003C00001C00000E00000E06000E0E0 00E0E001E0F001C0F80780FFFF80FFFE00E7F800131C7E9B18>I<7FFFF8FFFFF8FFFFF8 E07038E07038E07038E07038007000007000007000007000007000007000007000007000 00700000700000700000700000700000700000700000700000700000700007FF0007FF00 07FF00151C7F9B18>IIII<7F8FE07F9F E07F8FE00E07000F0700070E00078E00039C0003DC0001F80001F80000F00000F0000070 0000F00000F80001F80001DC00039E00038E00070F000707000E07800E03801E03C07F07 F0FF8FF87F07F0151C7F9B18>II<3FFFE0 7FFFE07FFFE07001C07003C0700780700700000F00001E00001C00003C00007800007000 00F00001E00001C00003C0000780000700000F00001E00E01C00E03C00E07800E07000E0 FFFFE0FFFFE0FFFFE0131C7E9B18>II<600000F00000F00000F8000078 00007C00003C00003C00003E00001E00001F00000F00000F00000F800007800007C00003 C00003C00003E00001E00001F00000F00000F800007800007800007C00003C00003E0000 1E00001E00001F00000F00000F8000078000078000030011247D9F18>I I<7FFF00FFFF80FFFF807FFF0011047D7F18>95 D<061E3E387070E0E0E0F8FC7C7C3807 0E789E18>I<1FE0003FF8007FFC00781E00300E0000070000070000FF0007FF001FFF00 7F0700780700E00700E00700E00700F00F00781F003FFFF01FFBF007E1F014147D9318> I<7E0000FE00007E00000E00000E00000E00000E00000E00000E3E000EFF800FFFC00FC1 E00F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00700F00700F80 E00FC1E00FFFC00EFF80063E00151C809B18>I<01FE0007FF001FFF803E078038030070 0000700000E00000E00000E00000E00000E00000E000007000007001C03801C03E03C01F FF8007FF0001FC0012147D9318>I<001F80003F80001F80000380000380000380000380 00038003E3800FFB801FFF803C1F80380F80700780700380E00380E00380E00380E00380 E00380E00380700780700780380F803C1F801FFFF00FFBF803E3F0151C7E9B18>I<01F0 0007FC001FFE003E0F00380780700380700380E001C0E001C0FFFFC0FFFFC0FFFFC0E000 007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<001F80007FC000 FFE000E1E001C0C001C00001C00001C0007FFFC0FFFFC0FFFFC001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0007FFF007F FF007FFF00131C7F9B18>I<01E1F007FFF80FFFF81E1E301C0E00380700380700380700 3807003807001C0E001E1E001FFC001FF80039E0003800001C00001FFE001FFFC03FFFE0 7801F0700070E00038E00038E00038E000387800F07E03F01FFFC00FFF8001FC00151F7F 9318>I<7E0000FE00007E00000E00000E00000E00000E00000E00000E3E000EFF800FFF C00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00 E00E00E00E00E07FC3FCFFE7FE7FC3FC171C809B18>I<03800007C00007C00007C00003 80000000000000000000000000007FC000FFC0007FC00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C000FFFF00FFFF80FF FF00111D7C9C18>I<0038007C007C007C003800000000000000000FFC1FFC0FFC001C00 1C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C6038F078FFF07FE03F800E277E9C18>II<7FE000FFE0007FE00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E0007FFFC0FFFFE07FFFC0131C7E9B18>I<7CE0E000FFFBF8007FFFF800 1F1F1C001E1E1C001E1E1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C00 1C1C1C001C1C1C001C1C1C001C1C1C001C1C1C007F1F1F00FFBFBF807F1F1F0019148193 18>I<7E3E00FEFF807FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E0 0E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFFE7FE7FC3FC1714809318>I<01F0 000FFE001FFF003E0F803803807001C07001C0E000E0E000E0E000E0E000E0E000E0F001 E07001C07803C03C07803E0F801FFF000FFE0001F00013147E9318>I<7E3E00FEFF807F FFC00FC1E00F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00700F 00700F80E00FC1E00FFFC00EFF800E3E000E00000E00000E00000E00000E00000E00000E 00007FC000FFE0007FC000151E809318>I<01E38007FB801FFF803E1F80380F80700780 700780E00380E00380E00380E00380E00380E00380700780700780380F803C1F801FFF80 0FFB8003E380000380000380000380000380000380000380000380003FF8003FF8003FF8 151E7E9318>I<7F87E0FF9FF07FBFF803F87803F03003E00003C00003C0000380000380 000380000380000380000380000380000380000380007FFE00FFFF007FFE0015147F9318 >I<07F7003FFF007FFF00780F00E00700E00700E007007C00007FE0001FFC0003FE0000 1F00600780E00380E00380F00380F80F00FFFF00FFFC00E7F00011147D9318>I<018000 0380000380000380000380007FFFC0FFFFC0FFFFC0038000038000038000038000038000 0380000380000380000380000380400380E00380E00380E001C1C001FFC000FF80003E00 13197F9818>I<7E07E0FE0FE07E07E00E00E00E00E00E00E00E00E00E00E00E00E00E00 E00E00E00E00E00E00E00E00E00E00E00E01E00F03E007FFFC03FFFE01FCFC1714809318 >I<7F8FF0FF8FF87F8FF01E03C00E03800E03800E0380070700070700070700038E0003 8E00038E00038E0001DC0001DC0001DC0000F80000F80000700015147F9318>II<7F8FF07F9FF07F8F F0070700078E00039E0001DC0001F80000F80000700000F00000F80001DC00039E00038E 000707000F07807F8FF0FF8FF87F8FF015147F9318>I<7F8FF0FF8FF87F8FF00E01C00E 03800E0380070380070700070700038700038600038E0001CE0001CE0000CC0000CC0000 DC0000780000780000780000700000700000700000F00000E00079E0007BC0007F80003F 00001E0000151E7F9318>I<3FFFF07FFFF07FFFF07001E07003C0700780000F00001E00 003C0000F80001F00003C0000780000F00701E00703C0070780070FFFFF0FFFFF0FFFFF0 14147F9318>I<0007E0001FE0007FE000780000E00000E00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00001E0007FC000FF8000FF80007FC00001E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000007800007F E0001FE00007E013247E9F18>I<7C0000FF0000FFC00003C00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000F000007FC0003FE0003FE000 7FC000F00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00003C000FFC000FF00007C000013247E9F18>125 D E /Fq 60 124 df<000078007C7800FC7801FC7803C0000380000780000780000780000780000780 00078000078000078000FFFC78FFFC78FFFC780780780780780780780780780780780780 780780780780780780780780780780780780780780780780780780780780780780780780 78078078152480A31A>12 D<787878781830306060E0050A7CA20E>39 D45 DI<01F00007FC000FFE001F 1F003C07803C07807803C07803C07803C07001C0F001E0F001E0F001E0F001E0F001E0F0 01E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E07803C07803C078 03C07803C03C07803C07801F1F000FFE0007FC0001F00013237EA118>48 D<00C001C007C0FFC0FFC0FBC003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C003C003C003C003C003C003C0FFFFFFFFFFFF10227C A118>I<03F0000FFC001FFE003C1F003007807007C06003C0E003E0C001E04001E04001 E00001E00001E00001E00003C00003C0000780000780000F00001E00003C0000780000F0 0001E00001C0000380000700000E00001C0000380000700000FFFFE0FFFFE0FFFFE01322 7EA118>I<01F00007FC001FFF003E0F003807807003C02003C02003C00003C00003C000 03C0000780000780000F00001E0003FC0003F80003FE00000F000007800003C00003C000 01E00001E00001E00001E00001E08001E0C003C0E003C07007803C0F801FFF000FFC0003 F00013237EA118>I<001F00001F00002F00002F00006F0000EF0000CF0001CF0001CF00 038F00038F00078F00070F000F0F000E0F001E0F003C0F003C0F00780F00780F00F00F00 FFFFF8FFFFF8FFFFF8000F00000F00000F00000F00000F00000F00000F00000F00000F00 15217FA018>I<3FFF803FFF803FFF803C00003C00003C00003C00003C00003C00003C00 003C00003C00003CF8003FFE003FFF003F0F803E07803C03C03803C00001E00001E00001 E00001E00001E00001E00001E04003C04003C0E003C07007807C1F003FFE000FFC0003F0 0013227EA018>I<007E0001FF0003FF0007C1000F00001E00001E00003C00003C000078 000078000078F800F1FE00F7FF00FF0F80FC0780F803C0F803C0F801C0F001E0F001E0F0 01E0F001E0F001E07001E07001E07801E07803C03803C03C03801C07801F0F000FFE0007 FC0001F00013237EA118>II<01F00007FC000FFE001E0F003C07803C07807803C07803C07803 C07803C07803C03803803C07801E0F000F1E0007FC0003F8000FFE001E0F003C07807803 C07803C0F001E0F001E0F001E0F001E0F001E0F001E07803C07803C03C07803E0F801FFF 0007FC0001F00013237EA118>I<01F00007FC000FFE001E0F003C070038078078038078 03C0F003C0F001C0F001E0F001E0F001E0F001E0F001E0F001E07003E07803E07803E03C 07E03E1FE01FFDE00FF1E003E3C00003C00003C0000380000780000700000F00101E0018 3C003FF8001FF00007C00013237EA118>I<001F0000001F0000003F8000003F8000003B 8000007BC0000073C0000071C00000F1E00000F1E00000E0E00001E0F00001E0F00001C0 F00003C0780003C078000380780007803C0007803C0007003C000F001E000F001E000FFF FE001FFFFF001FFFFF001C000F003C0007803C00078038000780780003C0780003C07000 03C0F00001E0F00001E0E00001E01B237EA220>65 DI<000FF000007FFC0000FF FF0001F01F0003C00700078000000F0000001E0000003E0000003C0000003C0000007800 00007800000078000000F0000000F0000000F0000000F0000000F0000000F0000000F000 0000F0000000F00000007800000078000000780000003C0000003C0000003E0000001E00 00000F0000000780008003C0038001F00F8000FFFF00007FFC00000FF00019257DA31F> IIII<000FF000003FFE0000FFFF8001F80F8003E0038007800000 0F0000001E0000001E0000003C0000003C000000780000007800000078000000F0000000 F0000000F0000000F0000000F0000000F0000000F000FFC0F000FFC0F000FFC0780003C0 780003C0780003C03C0003C03C0003C01E0003C01E0003C00F0003C0078003C003E003C0 01F807C000FFFFC0003FFF00000FF8001A257DA321>III 76 DII<001FC000007FF00001FFFC0003F07E0007C01F000F800F801F00 07C01E0003C03C0001E03C0001E0780000F0780000F0780000F070000070F0000078F000 0078F0000078F0000078F0000078F0000078F0000078F0000078F0000078780000F07800 00F0780000F07C0001F03C0001E03E0003E01E0003C01F0007C00F800F8007C01F0003F0 7E0001FFFC00007FF000001FC0001D257DA324>II82 D<00FE0003FFC007FFE00F81E01E00603C00003C00007800007800007800007800007800 007C00003C00003F00001FC0000FFC0007FF0001FF80003FC00007E00001F00000F00000 F8000078000078000078000078000078000078C000F0E000F0F801E07E07C03FFF800FFF 0001FC0015257EA31B>III87 D<07E01FF83FFC381E201E000F00 0F000F000F00FF07FF1FFF3E0F780FF00FF00FF00FF00FF83F7FFF3FEF1F8F10167E9517 >97 DI<01FC0007FF000FFF801F03803C0180780000780000700000F00000F000 00F00000F00000F00000F000007800007800007800003C00401F03C00FFFC007FF8001FC 0012167E9516>I<0003C00003C00003C00003C00003C00003C00003C00003C00003C000 03C00003C00003C00003C003E3C00FFBC01FFFC03F0FC03C07C07803C07803C0F003C0F0 03C0F003C0F003C0F003C0F003C0F003C0F003C07803C07803C03C07C03E0FC01FFFC00F FBC003E3C012237EA219>I<03F00007FC001FFE003E0F003C0780780380780380F001C0 FFFFC0FFFFC0FFFFC0F00000F00000F000007000007800007800003C00801F07800FFF80 07FF0001F80012167E9516>I<003F00FF01FF03C0038007800780078007800780078007 800780FFF8FFF8FFF8078007800780078007800780078007800780078007800780078007 8007800780078007800780102380A20F>I<01F07807FFF80FFFF81F1F001E0F003C0780 3C07803C07803C07803C07801E0F001F1F000FFE001FFC0019F0003800003800003C0000 1FFE001FFFC01FFFE03FFFF07801F07800F8F00078F00078F00078F000787800F03E03E0 1FFFC00FFF8001FC0015217F9518>III<00F000F000F000F000 000000000000000000000000000000000000F000F000F000F000F000F000F000F000F000 F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F080 F0E3E0FFE07FC01F000C2D83A20D>IIIII<01FC0007FF000FFF 801F07C03C01E07800F07800F0700070F00078F00078F00078F00078F00078F000787800 F07800F07C01F03E03E01F07C00FFF8007FF0001FC0015167F9518>II<03E3C007FBC00FFFC01F0FC0 3C07C07C03C07803C0F803C0F003C0F003C0F003C0F003C0F003C0F003C0F803C07803C0 7803C03C07C03F0FC01FFFC00FFBC003E3C00003C00003C00003C00003C00003C00003C0 0003C00003C00003C00003C012207E9519>II<07F0 1FFC3FFE3C0E7806780078007C003F003FF01FF80FFC01FE001F000F000F000FC00FF81E FFFE3FFC0FF010167F9513>I<0F000F000F000F000F000F00FFF8FFF8FFF80F000F000F 000F000F000F000F000F000F000F000F000F000F000F000F080F1C07FC07F803E00E1C7F 9B12>IIII<7801F07C01E03E03C01E07C00F0780078F0007DE0003FC0001FC0000F8 0000700000F80001FC0003DC00039E00078F000F07801E07801E03C03C01E07800F0F800 F81516809516>II<7FFFC07FFFC07FFFC0000F80000F00001E00003E00007C0000780000F00001F000 03E00003C0000780000F80001F00001E00003C00007C0000FFFFC0FFFFC0FFFFC012167F 9515>II E /Fr 53 123 df<00FC1F8001FC3F8003 FC7F800700E0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C000FF E1FC00FFE1FC000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E 01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C00019 1D809C18>11 D<01F1C003F1C007F1C00701C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C0FFF1C0FFF1C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0121D809C16>13 D<00E001C0038007000E000E001C001C003800380038007000700070007000E000E000E0 00E000E000E000E000E000E000E000E000E00070007000700070003800380038001C001C 000E000E000700038001C000E00B2A7E9E10>40 DI<018001C0018001806186F99F7DBE1FF807E007E01FF87DBEF99F618601800180 01C0018010127E9E15>I<00060000000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000FFFFFFE0FFFFFFE000 060000000600000006000000060000000600000006000000060000000600000006000000 0600000006000000060000000600001B1C7E9720>II< FFE0FFE00B0280890E>I<00010003000600060006000C000C000C001800180018003000 3000300060006000C000C000C0018001800180030003000300060006000C000C000C0018 0018001800300030003000600060006000C000C00010297E9E15>47 D<07C01FF03878701C601EE00EC00F400F400700070007000F000E001E001C003C007800 F001E001C0038007000E001C0038007000FFFFFFFF101C7E9B15>50 D<003C00007C00005C0000DC0001DC00019C00039C00039C00071C00071C000E1C000E1C 001C1C003C1C00381C00781C00701C00F01C00FFFFC0FFFFC0001C00001C00001C00001C 00001C00001C00001C00121B7F9A15>52 D<001C0000003E0000003E0000002E00000067 00000067000000E7800000C7800000C3800001C3C0000183C0000181C0000381E0000381 E0000700F0000700F0000600F0000E0078000FFFF8000FFFF8001C003C001C003C001800 3C0038001E0038001E0070001F0070000F0070000F00E0000780191D7F9C1C>65 DI<003FC000FFF003C0F00780300F 00001E00003C00003C0000780000780000780000F00000F00000F00000F00000F00000F0 0000F00000F00000F000007800007800007800003C00003C00001E00000F000807801803 C07800FFF0003F80151F7D9D1B>I II<003F8001FFF003C0F80780380F 00181E00003C00003C0000780000780000780000F00000F00000F00000F00000F00000F0 0000F007F8F007F8F000387800387800387800383C00383C00381E00380F003807803803 C0F801FFF0003F80151F7D9D1C>71 DI I76 DII<003F000001FFE00003FFF00007C0F8000F807C001E001E003E001F003C000F007800 07807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F000 03C0F00003C0F80007C078000780780007807C000F803C000F003E001F001F003E000F80 7C0007C0F80003FFF00001FFE000003F00001A1F7E9D1F>II82 D<03F8000FFE001C0F00380700700300600000E00000E00000E00000E00000F000007800 007F00003FE0001FFC0007FE0001FF00001F800007800003C00003C00001C00001C00001 C00001C0C00180E00380F007007C0E001FFC0007F000121F7E9D17>IIII<78000E007C001E003C003C001E0038000F0070000F00F000 0781E00003C1C00001C3C00001E7800000F70000007E0000003E0000003C0000003C0000 007E00000077000000E7800001E3800003C1C0000381E0000700F0000F00F8000E007800 1C003C003C003E0078001F0070000F00F0000F80191D7F9C1C>88 DI<0FC03FF07FF87038401C001C001C00FC0FFC3FFC 781CE01CE01CE01CF07C7FFC7FDC3F1C0E127E9114>97 DI<07E00FF81FFC3C1C70047000E000E000E000E0 00E000E000700070043C1C1FFC0FF807E00E127E9112>I<000E000E000E000E000E000E 000E000E000E000E000E0F8E1FEE3FFE7C3E700E700EE00EE00EE00EE00EE00EE00EF00E 701E7C3E3FFE1FEE0F8E0F1D7E9C15>I<07C01FE03FF078787018601CFFFCFFFCFFFCE0 00E000E000700070043C1C3FFC1FF807E00E127E9112>I<00FC01FC03FC07000E000E00 0E000E000E000E000E00FFE0FFE00E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E1D809C0D>I<03C3C00FFFC01FFFC01C3800381C00381C0038 1C00381C00381C001C38001FF8001FF0003BC0003800003800001FFC001FFF003FFF8070 0780E001C0E001C0E001C0F003C07C0F803FFF001FFE0007F800121B7F9115>III108 DII<03F0000FFC001FFE003C0F00780780700380E001C0 E001C0E001C0E001C0E001C0F003C07003807807803C0F001FFE000FFC0003F00012127F 9115>II114 D<1FC03FF07FF0F030E000E000F0007F003FC01FE000F0003800388038F078FFF07FE01F C00D127F9110>I<1C001C001C001C001C001C00FFE0FFE01C001C001C001C001C001C00 1C001C001C001C001C001C001C201FF00FF007C00C187F970F>III<7003807807003C0E001C1C000E1C0007380003F00001E00001C00001E00003 F0000738000E18000E1C001C0E00380700700380F003C01212809113>120 DI<7FFC7FFC7FFC007800F000E0 01E003C0038007000F001E001C003C007800FFFCFFFCFFFC0E127F9112>I E /Fs 50 122 df<000100020004000800100020006000C0018001800300070006000E00 0C001C0018003800380030007000700060006000E000E000C000C000C000C000C000C000 C000C000C000C000C000C000C0004000600060002000100010000800102E79A113>40 D<0010000008000004000006000002000003000003000003000001000001800001800001 800001800001800001800001800003800003800003800003000003000003000007000007 00000600000600000E00000C00000C00001C0000180000380000300000700000600000E0 0000C0000180000100000300000600000C0000180000300000600000800000112E80A113 >I<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I< 70F8F8F0E005057B840E>I<000F800030E000E07001C0700380300380380700380F0078 0F00780E00781E00781E00703C00F03C00F03C00F03C00F07801E07801E07801E07801C0 7003C0F003C0F00380F00780F00700700700700E00701C003038001870000FC000151F7C 9D17>48 D<000200020006000E003C00DC031C001C003800380038003800700070007000 7000E000E000E000E001C001C001C001C003800380038003800780FFF80F1E7B9D17>I< 001F000061800080E00100E00200700220700420700410700820F00820F00820F00840E0 0881E00703C0000380000700000C00001800006000008000030000040000080040100040 1000802001807E030047FF0041FE0080FC00807800141F7C9D17>I<001F0000718000C0 C00180C00380E00700E00F00E00F01E01E01E01E01E01E01E01E01C01C03C01C03C01C03 C01C07C01C0F800C0F8006378003C700000F00000E00000E00001C00601C00F03800F070 00E0600080C0004380003E0000131F7B9D17>57 D<070F1F1F0E00000000000000000000 70F8F8F0E008147B930E>I<00000200000006000000060000000E0000001E0000001E00 00003F0000002F0000004F0000004F0000008F0000010F0000010F0000020F0000020F00 00040F00000C0F0000080F0000100F0000100F0000200F80003FFF800040078000C00780 0080078001000780010007800200078002000780060007801E000F80FF807FF81D207E9F 22>65 D<01FFFFC0001E00F0001E0078001E0038001E003C003C003C003C003C003C003C 003C003C0078007800780078007800F0007801E000F0078000FFFE0000F00F8000F003C0 01E001C001E001E001E001E001E001E003C001E003C001E003C001E003C001C0078003C0 0780078007800F0007801E000F007800FFFFE0001E1F7D9E20>I<0000FE020007818600 1C004C0038003C0060003C00C0001C01C0001803800018070000180F0000181E0000101E 0000103C0000003C00000078000000780000007800000078000000F0000000F0000000F0 000000F0000000F00000807000008070000080700001003800010038000200180004000C 001800060020000381C00000FE00001F217A9F21>I<01FFFF80001E00E0001E0070001E 0038001E001C003C001C003C000E003C000E003C000E0078000E0078000E0078000E0078 000E00F0001E00F0001E00F0001E00F0001E01E0003C01E0003C01E0003C01E0007803C0 007003C0007003C000E003C001C0078001C00780038007800E0007801C000F007000FFFF C0001F1F7D9E22>I<01FFFFFE001E001C001E000C001E0004001E0004003C0004003C00 04003C0004003C00040078080800780800007808000078180000F0300000FFF00000F030 0000F0300001E0200001E0200001E0200001E0001003C0002003C0002003C0004003C000 40078000800780018007800100078007000F001F00FFFFFE001F1F7D9E1F>I<01FFFFFC 001E0038001E0018001E0008001E0008003C0008003C0008003C0008003C000800780010 00780800007808000078080000F0100000F0300000FFF00000F0300001E0200001E02000 01E0200001E0200003C0000003C0000003C0000003C00000078000000780000007800000 078000000F800000FFF800001E1F7D9E1E>I<0000FC040007030C001C00980030007800 E0007801C000380380003003800030070000300E0000301E0000201E0000203C0000003C 00000078000000780000007800000078000000F0000000F000FFF0F0000780F0000780F0 000F0070000F0070000F0070000F0070001E0038001E0018003E001C002E000E00CC0003 83040000FC00001E217A9F23>I<01FFF3FFE0001F003E00001E003C00001E003C00001E 003C00003C007800003C007800003C007800003C007800007800F000007800F000007800 F000007800F00000F001E00000FFFFE00000F001E00000F001E00001E003C00001E003C0 0001E003C00001E003C00003C007800003C007800003C007800003C007800007800F0000 07800F000007800F000007800F00000F801F0000FFF1FFE000231F7D9E22>I<01FFF000 1F00001E00001E00001E00003C00003C00003C00003C0000780000780000780000780000 F00000F00000F00000F00001E00001E00001E00001E00003C00003C00003C00003C00007 80000780000780000780000F8000FFF800141F7D9E12>I<01FE00007FC0001E0000FC00 001E0000F80000170001780000170001780000270002F00000270004F00000270004F000 00270008F00000470009E00000470011E00000470021E00000470021E00000870043C000 00838043C00000838083C00000838083C000010381078000010382078000010382078000 0103840780000203840F00000203880F00000203900F00000203900F00000401E01E0000 0401E01E00000401C01E00000C01801E00001C01803E0000FF8103FFC0002A1F7D9E29> 77 D<01FFFF80001E00E0001E0070001E0038001E003C003C003C003C003C003C003C00 3C003C0078007800780078007800F0007800E000F003C000F00F0000FFFC0000F0000001 E0000001E0000001E0000001E0000003C0000003C0000003C0000003C000000780000007 80000007800000078000000F800000FFF000001E1F7D9E1F>80 D<0007E040001C18C000 3005800060038000C0038001C00180018001000380010003800100038001000380000003 C0000003C0000003F8000001FF800001FFE000007FF000001FF0000001F8000000780000 007800000038000000380020003800200038002000300060007000600060006000E00070 00C000E8038000C606000081F800001A217D9F1A>83 D<0FFFFFF01E0780E01807802010 07802020078020200F0020600F0020400F0020400F0020801E0040001E0000001E000000 1E0000003C0000003C0000003C0000003C00000078000000780000007800000078000000 F0000000F0000000F0000000F0000001E0000001E0000001E0000001E0000003E00000FF FF00001C1F789E21>I<7FFC1FF807C003C00780010007800100078001000F0002000F00 02000F0002000F0002001E0004001E0004001E0004001E0004003C0008003C0008003C00 08003C00080078001000780010007800100078001000F0002000F0002000F0002000F000 4000F0004000700080007001000030020000380400000C18000007E000001D20779E22> II<00F1800389C00707 800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E20F01C 40F01C40703C40705C40308C800F070013147C9317>97 D<07803F800700070007000700 0E000E000E000E001C001C001CF01D0C3A0E3C0E380F380F700F700F700F700FE01EE01E E01EE01CE03CE038607060E031C01F0010207B9F15>I<007E0001C1000300800E07801E 07801C07003C0200780000780000780000F00000F00000F00000F00000F0000070010070 020030040018380007C00011147C9315>I<0000780003F8000070000070000070000070 0000E00000E00000E00000E00001C00001C000F1C00389C00707800E03801C03803C0380 380700780700780700780700F00E00F00E00F00E00F00E20F01C40F01C40703C40705C40 308C800F070015207C9F17>I<007C01C207010E011C013C013802780C7BF07C00F000F0 00F000F0007000700170023804183807C010147C9315>I<00007800019C00033C00033C 000718000700000700000E00000E00000E00000E00000E0001FFE0001C00001C00001C00 001C00003800003800003800003800003800007000007000007000007000007000007000 00E00000E00000E00000E00000C00001C00001C0000180003180007B0000F30000660000 3C00001629829F0E>I<003C6000E27001C1E00380E00700E00F00E00E01C01E01C01E01 C01E01C03C03803C03803C03803C03803C07003C07001C0F001C17000C2E0003CE00000E 00000E00001C00001C00301C00783800F0700060E0003F8000141D7E9315>I<01E0000F E00001C00001C00001C00001C000038000038000038000038000070000070000071E0007 63000E81800F01C00E01C00E01C01C03801C03801C03801C038038070038070038070038 0E10700E20700C20701C20700C40E00CC060070014207D9F17>I<00C001E001E001C000 000000000000000000000000000E003300230043804300470087000E000E000E001C001C 001C003840388030807080310033001C000B1F7C9E0E>I<0001800003C00003C0000380 000000000000000000000000000000000000000000003C00004600008700008700010700 010700020E00000E00000E00000E00001C00001C00001C00001C00003800003800003800 00380000700000700000700000700000E00000E00030E00079C000F180006300003C0000 1228829E0E>I<01E0000FE00001C00001C00001C00001C0000380000380000380000380 000700000700000703C00704200E08E00E11E00E21E00E40C01C80001D00001E00001FC0 0038E000387000387000383840707080707080707080703100E03100601E0013207D9F15 >I<03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C00 1C0038003800380038007000700070007100E200E200E200E200640038000A207C9F0C> I<1C0F80F0002630C318004740640C004780680E004700700E004700700E008E00E01C00 0E00E01C000E00E01C000E00E01C001C01C038001C01C038001C01C038001C01C0708038 038071003803806100380380E10038038062007007006600300300380021147C9325>I< 1C0F802630C04740604780604700704700708E00E00E00E00E00E00E00E01C01C01C01C0 1C01C01C03843803883803083807083803107003303001C016147C931A>I<007C0001C3 000301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F007 80F00700700F00700E0030180018700007C00013147C9317>I<01C1E002621804741C04 781C04701E04701E08E01E00E01E00E01E00E01E01C03C01C03C01C03C01C03803807803 80700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C 0000FFC000171D809317>I<00F0400388C00705800E03801C03803C0380380700780700 780700780700F00E00F00E00F00E00F00E00F01C00F01C00703C00705C0030B8000F3800 00380000380000700000700000700000700000E00000E0000FFE00121D7C9315>I<1C1E 002661004783804787804707804703008E00000E00000E00000E00001C00001C00001C00 001C000038000038000038000038000070000030000011147C9313>I<00FC030206010C 030C070C060C000F800FF007F803FC003E000E700EF00CF00CE008401020601F8010147D 9313>I<018001C0038003800380038007000700FFF007000E000E000E000E001C001C00 1C001C003800380038003820704070407080708031001E000C1C7C9B0F>I<0E00C03300 E02301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C07 101C0E20180E20180E201C1E200C264007C38014147C9318>I<0E03803307802307C043 83C04301C04700C08700800E00800E00800E00801C01001C01001C01001C02001C02001C 04001C04001C08000E300003C00012147C9315>I<0E00C1C03300E3C02301C3E04381C1 E04301C0E04701C060870380400E0380400E0380400E0380401C0700801C0700801C0700 801C0701001C0701001C0602001C0F02000C0F04000E13080003E1F0001B147C931E>I< 0383800CC4401068E01071E02071E02070C040E00000E00000E00000E00001C00001C000 01C00001C040638080F38080F38100E5810084C60078780013147D9315>I<0E00C03300 E02301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C07 001C0E00180E00180E001C1E000C3C0007DC00001C00001C00003800F03800F07000E060 00C0C0004380003E0000131D7C9316>I E /Ft 73 124 df<007E1F0001C1B1800303E3 C00703C3C00E03C1800E01C0000E01C0000E01C0000E01C0000E01C0000E01C000FFFFFC 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0007F87FC001A1D80 9C18>11 D<007E0001C1800301800703C00E03C00E01800E00000E00000E00000E00000E 0000FFFFC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C07F87F8151D809C17>I<007FC001C1C00303C0 0703C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFFFC00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C07FCFF8151D809C17>I<003F07E00001C09C18000380F018000701F03C000E01E0 3C000E00E018000E00E000000E00E000000E00E000000E00E000000E00E00000FFFFFFFC 000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C00 0E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E 00E01C000E00E01C007FC7FCFF80211D809C23>I<6060F0F0F8F8686808080808080810 1010102020404080800D0C7F9C15>34 D<60F0F8680808081010204080050C7C9C0C>39 D<004000800100020006000C000C0018001800300030007000600060006000E000E000E0 00E000E000E000E000E000E000E000E000E000600060006000700030003000180018000C 000C00060002000100008000400A2A7D9E10>I<800040002000100018000C000C000600 060003000300038001800180018001C001C001C001C001C001C001C001C001C001C001C0 01C0018001800180038003000300060006000C000C00180010002000400080000A2A7E9E 10>I<01800180018001804182F18F399C0FF003C003C00FF0399CF18F41820180018001 80018010127E9E15>I<60F0F0701010101020204080040C7C830C>44 DI<60F0F06004047C830C>I<00010003000600060006000C000C 000C0018001800180030003000300060006000C000C000C0018001800180030003000300 060006000C000C000C00180018001800300030003000600060006000C000C00010297E9E 15>I<03C00C301818300C300C700E60066006E007E007E007E007E007E007E007E007E0 07E007E007E007E00760066006700E300C300C18180C3007E0101D7E9B15>I<03000700 3F00C7000700070007000700070007000700070007000700070007000700070007000700 0700070007000700070007000F80FFF80D1C7C9B15>I<07C01830201C400C400EF00FF8 0FF807F8077007000F000E000E001C001C00380070006000C00180030006010C01180110 023FFE7FFEFFFE101C7E9B15>I<07E01830201C201C781E780E781E381E001C001C0018 0030006007E00030001C001C000E000F000F700FF80FF80FF80FF00E401C201C183007E0 101D7E9B15>I<000C00000C00001C00003C00003C00005C0000DC00009C00011C00031C 00021C00041C000C1C00081C00101C00301C00201C00401C00C01C00FFFFC0001C00001C 00001C00001C00001C00001C00001C0001FFC0121C7F9B15>I<4000007FFF807FFF007F FF0040020080040080040080080000100000100000200000600000400000C00000C00001 C00001800001800003800003800003800003800007800007800007800007800007800007 8000030000111D7E9B15>55 D<03C00C301818300C700C600EE006E006E007E007E007E0 07E0076007700F300F18170C2707C700060006000E300C780C78187010203030C00F8010 1D7E9B15>57 D<60F0F0600000000000000000000060F0F06004127C910C>I<60F0F060 0000000000000000000060F0F0701010101020204080041A7C910C>I<7FFFFFC0FFFFFF E00000000000000000000000000000000000000000000000000000000000000000FFFFFF E07FFFFFC01B0C7E8F20>61 D<0FE03038401CE00EF00EF00EF00E000C001C0030006000 C0008001800100010001000100010001000000000000000000000003000780078003000F 1D7E9C14>63 D<000600000006000000060000000F0000000F0000000F00000017800000 178000001780000023C0000023C0000023C0000041E0000041E0000041E0000080F00000 80F0000180F8000100780001FFF80003007C0002003C0002003C0006003E0004001E0004 001E000C001F001E001F00FF80FFF01C1D7F9C1F>65 DI<001F808000E0618001801980070007800E0003801C0003801C0001803800 0180780000807800008070000080F0000000F0000000F0000000F0000000F0000000F000 0000F0000000F0000000700000807800008078000080380000801C0001001C0001000E00 0200070004000180080000E03000001FC000191E7E9C1E>II< FFFFFC0F003C0F000C0F00040F00040F00060F00020F00020F02020F02000F02000F0200 0F06000FFE000F06000F02000F02000F02000F02010F00010F00020F00020F00020F0006 0F00060F000C0F003CFFFFFC181C7E9B1C>II<001F808000E0618001801980070007800E0003801C0003801C000180380001807800 00807800008070000080F0000000F0000000F0000000F0000000F0000000F0000000F000 FFF0F0000F80700007807800078078000780380007801C0007801C0007800E0007800700 0B800180118000E06080001F80001C1E7E9C21>III75 DIII<003F 800000E0E0000380380007001C000E000E001C0007003C00078038000380780003C07800 03C0700001C0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0F000 01E0700001C0780003C0780003C0380003803C0007801C0007000E000E0007001C000380 380000E0E000003F80001B1E7E9C20>II< FFFF00000F01E0000F0078000F003C000F001C000F001E000F001E000F001E000F001E00 0F001C000F003C000F0078000F01E0000FFF00000F03C0000F00E0000F00F0000F007800 0F0078000F0078000F0078000F0078000F0078000F0078100F0078100F0038100F003C20 FFF01C20000007C01C1D7E9B1F>82 D<07E0801C1980300580700380600180E00180E000 80E00080E00080F00000F800007C00007FC0003FF8001FFE0007FF0000FF80000F800007 C00003C00001C08001C08001C08001C0C00180C00180E00300D00200CC0C0083F800121E 7E9C17>I<7FFFFFC0700F01C0600F00C0400F0040400F0040C00F0020800F0020800F00 20800F0020000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00 00000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00 00001F800003FFFC001B1C7F9B1E>II87 D89 D<08081010202040404040808080808080B0B0F8F8 787830300D0C7A9C15>92 D<1FC000307000783800781C00301C00001C00001C0001FC00 0F1C00381C00701C00601C00E01C40E01C40E01C40603C40304E801F870012127E9115> 97 DI<07E00C3018783078703060 00E000E000E000E000E000E00060007004300418080C3007C00E127E9112>I<003F0000 070000070000070000070000070000070000070000070000070000070003E7000C170018 0F00300700700700600700E00700E00700E00700E00700E00700E0070060070070070030 0700180F000C370007C7E0131D7E9C17>I<03E00C301818300C700E6006E006FFFEE000 E000E000E00060007002300218040C1803E00F127F9112>I<00F8018C071E061E0E0C0E 000E000E000E000E000E00FFE00E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E007FE00F1D809C0D>I<00038003C4C00C38C01C3880181800381C00 381C00381C00381C001818001C38000C300013C0001000003000001800001FF8001FFF00 1FFF803003806001C0C000C0C000C0C000C06001803003001C0E0007F800121C7F9215> II<18003C003C0018000000000000 000000000000000000FC001C001C001C001C001C001C001C001C001C001C001C001C001C 001C001C001C00FF80091D7F9C0C>I<00C001E001E000C0000000000000000000000000 00000FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E060E0F0C0F1C061803E000B25839C0D>IIIII<03F0000E1C00180600300300700380600180 E001C0E001C0E001C0E001C0E001C0E001C06001807003803003001806000E1C0003F000 12127F9115>II<03C1000C3300180B00300F0070070070 0700E00700E00700E00700E00700E00700E00700600700700700300F00180F000C370007 C700000700000700000700000700000700000700000700003FE0131A7E9116>II<1F9030704030C010C010E010F8007F803FE00FF000F880388018C018C018E010 D0608FC00D127F9110>I<04000400040004000C000C001C003C00FFE01C001C001C001C 001C001C001C001C001C001C101C101C101C101C100C100E2003C00C1A7F9910>IIII<7F8FF00F03800F030007020003840001C8 0001D80000F00000700000780000F800009C00010E00020E000607000403801E07C0FF0F F81512809116>II<7FFC70386038407040F040E041C003 C0038007000F040E041C043C0C380870087038FFF80E127F9112>I I E /Fu 36 122 df<78FCFCFEFE7A0202040408083040070E7D850D>44 D<78FCFCFCFC7806067D850D>46 D<78FCFCFCFC7800000000000078FCFCFCFC7806127D 910D>58 D<00038000000380000007C0000007C0000007C000000FE000000FE000001FF0 00001BF000001BF0000031F8000031F8000061FC000060FC0000E0FE0000C07E0000C07E 0001803F0001FFFF0003FFFF8003001F8003001F8006000FC006000FC00E000FE00C0007 E0FFC07FFEFFC07FFE1F1C7E9B24>65 D<001FE02000FFF8E003F80FE007C003E00F8001 E01F0000E03E0000E03E0000607E0000607C000060FC000000FC000000FC000000FC0000 00FC000000FC000000FC000000FC0000007C0000607E0000603E0000603E0000C01F0000 C00F80018007C0030003F80E0000FFFC00001FE0001B1C7D9B22>67 DIII73 D76 DII80 D<7FFFFFE07FFFFFE0781F81E0701F80E0601F8060E01F8070C01F8030C01F8030C01F80 30C01F8030001F8000001F8000001F8000001F8000001F8000001F8000001F8000001F80 00001F8000001F8000001F8000001F8000001F8000001F8000001F8000001F800007FFFE 0007FFFE001C1C7E9B21>84 D<0FF8001C1E003E0F803E07803E07C01C07C00007C0007F C007E7C01F07C03C07C07C07C0F807C0F807C0F807C0780BC03E13F80FE1F815127F9117 >97 DI<03FC000E0E001C1F003C1F 00781F00780E00F80000F80000F80000F80000F80000F800007800007801803C01801C03 000E0E0003F80011127E9115>I<000FF0000FF00001F00001F00001F00001F00001F000 01F00001F00001F00001F001F9F00F07F01C03F03C01F07801F07801F0F801F0F801F0F8 01F0F801F0F801F0F801F07801F07801F03C01F01C03F00F0FFE03F9FE171D7E9C1B>I< 01FC000F07001C03803C01C07801C07801E0F801E0F801E0FFFFE0F80000F80000F80000 7800007C00603C00601E00C00F038001FC0013127F9116>I<007F0001E38003C7C00787 C00F87C00F83800F80000F80000F80000F80000F8000FFF800FFF8000F80000F80000F80 000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80007FF8 007FF800121D809C0F>I<03F8F00E0F381E0F381C07303C07803C07803C07803C07801C 07001E0F000E0E001BF8001000001800001800001FFF001FFFC00FFFE01FFFF07801F8F0 0078F00078F000787000707800F01E03C007FF00151B7F9118>II<1E003F003F003F003F001E000000000000000000000000 00FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F001F00FFE0FF E00B1E7F9D0E>I 108 DII<01FC000F07801C01C03C01E07800F07800F0F8 00F8F800F8F800F8F800F8F800F8F800F87800F07800F03C01E01E03C00F078001FC0015 127F9118>II114 D<1FD830786018E018E018F000FF807FE07FF01FF807FC007CC01C C01CE01CE018F830CFC00E127E9113>I<0300030003000300070007000F000F003FFCFF FC1F001F001F001F001F001F001F001F001F001F0C1F0C1F0C1F0C0F08079803F00E1A7F 9913>IIIIII E /Fv 40 90 df<387C7C7E3A040404080810204080070E789F0D>39 D<1C3E7E7E3A0202040408081020C0070E7D840D>44 DI<3078 F8787005057C840D>I<007E0001C3000301800701C00E00C00E00E01C00E01C00E03C01 E03801E07801E07801E07801E07801E07801E0F003C0F003C0F003C0F003C0F003C0F003 80F00780E00780E00700E00700E00E00600E00701C003038003870000FC000131F7C9D17 >48 D<000C001C00FC0F380038003800380038003800700070007000700070007000E000 E000E000E000E000E001C001C001C001C001C001C0038003C0FFFE0F1E7C9D17>I<003F 8000C1E00100F00200780400780400780F007C0F807C0F807C0F00780600780000F80000 F00001E00001C0000380000700000E00001C0000380000600000C0000180000300200600 200800401000403FFFC07FFF80FFFF80161E7E9D17>I<007F000183C00201E00400F007 00F00F00F00F01F00F01F00001E00001E00003C0000380000700000E0000F800000E0000 07000007800007C00003C00007C03007C07807C0F807C0F807C0F00780800F00400E0020 1C0018780007E000141F7D9D17>I<0000600000600000E00001C00003C00005C0000DC0 0009C00011C000238000438000C380008380010380020380040700080700180700100700 200700400700FFFFF0000E00000E00000E00000E00000E00001C00001E0001FFE0141E7E 9D17>I<01803001FFE003FFC003FF0003FC000200000200000200000400000400000400 00047C000587000603800C01800801C00001C00001E00001E00001E00001E07003C0F803 C0F003C0E00380800780400700400E00201C0018700007C000141F7D9D17>I<000F8000 704000C0200180E00301E00701E00E00C01E00001C00003C000038000078F800790E007A 07007C0300F80380F80380F003C0F003C0F003C0F003C0F00780E00780E00780E00700E0 0F00600E00701C0030180018700007C000131F7C9D17>I<2000003FFFE07FFFC07FFF80 400100C00200800200800400000800001000002000004000004000008000018000030000 0300000700000600000E00000E00001E00001C00001C00003C00003C00003C0000780000 780000780000300000131F799D17>I<003F0000C1C00100600200600400300C00300C00 300C00300C00600E00600F80C00FC18007F60003FC0001FC0001FF00063F800C0F801007 C03003C06001C06000C0C000C0C000C0C000C0C00080C0010060030030040018180007E0 00141F7D9D17>I<007E0001C3000301800601C00E01C01C00C03C00E03C00E03C01E078 01E07801E07801E07801E07803E07803E03803C03807C01C0BC00C13C003E38000078000 0780000700000E00600E00F01C00F01800E0300080600041C0003F0000131F7C9D17>I< 07E01838201C401C701CF03CF03C603C0038007000E001C0018003000200060004000400 08000800080008000000000000000000000030007800F800780070000E20799F15>63 D<0000100000001800000038000000380000007800000078000000FC000001BC0000013C 0000033C0000023C0000063C0000043E0000081E0000081E0000101E0000101E0000201E 0000200F0000400F0000400F0000FFFF0000800F0001000F800100078002000780020007 8004000780040007800C0007C03E0007C0FF807FFC1E207E9F22>65 D<07FFFF00007C01C0003C01E0003C00F0007800F8007800F8007800F8007800F8007800 F8007800F000F001F000F001E000F003C000F00F8000FFFE0000F00F0001E007C001E003 C001E003E001E001E001E001E001E001E003C001E003C003E003C003E003C003C003C007 C003C00F8007800F0007803E00FFFFF0001D1F7E9E20>I<0001F808000E061800380138 007000F801E0007803C0007007800030078000300F0000301F0000301E0000303E000020 3C0000007C0000007C0000007C0000007C000000F8000000F8000000F8000000F8000000 F80000007800004078000080780000803C0000803C0001001C0002000E00020006000C00 0300100001C0E000003F00001D217B9F21>I<07FFFF00007C01E0003C00F0003C007800 78003C0078003C0078001E0078001E0078001E0078001F00F0001F00F0001F00F0001F00 F0001F00F0001F00F0001F01E0001E01E0003E01E0003E01E0003E01E0003C01E0007C03 C0007803C000F003C000F003C001E003C003C003C0078007800F0007803C00FFFFE00020 1F7E9E23>I<07FFFFF8007C0078003C0038003C00180078001800780008007800080078 0008007800080078080800F0100000F0100000F0100000F0300000FFF00000F0700001E0 200001E0200001E0200001E0200001E0000801E0001003C0001003C0001003C0002003C0 002003C0006003C000C0078001C0078007C0FFFFFF801D1F7E9E1F>I<07FFFFF8007C00 78003C0038003C001800780018007800080078000800780008007800080078000800F010 0000F0100000F0100000F0300000F0700000FFF00001E0600001E0200001E0200001E020 0001E0200001E0000003C0000003C0000003C0000003C0000003C0000003C00000078000 0007C00000FFFE00001D1F7E9E1E>I<0001FC04000F030C003C009C0070007C00E0003C 01C0003803800018078000180F0000181F0000181E0000183E0000103C0000007C000000 7C0000007C0000007C000000F8000000F8000000F8007FFCF80003E0780001E0780001E0 780003C0780003C03C0003C03C0003C01C0003C00E0007C007000B800380118001E06080 003F80001E217B9F24>I<07FFC7FFC0007C00F800003C007800003C007800007800F000 007800F000007800F000007800F000007800F000007800F00000F001E00000F001E00000 F001E00000F001E00000FFFFE00000F001E00001E003C00001E003C00001E003C00001E0 03C00001E003C00001E003C00003C007800003C007800003C007800003C007800003C007 800003C007800007800F000007C00F8000FFF8FFF800221F7E9E22>I<07FFE0007C0000 3C00003C0000780000780000780000780000780000780000F00000F00000F00000F00000 F00000F00001E00001E00001E00001E00001E00001E00003C00003C00003C00003C00003 C00003C00007800007C000FFFC00131F7F9E10>I<00FFFC0007C00003C00003C0000780 000780000780000780000780000780000F00000F00000F00000F00000F00000F00001E00 001E00001E00001E00001E00001E00003C00303C00783C00F83C00F83800F07800807000 40E00021C0001F000016207D9E17>I<07FFE07FE0007C001F00003C000C00003C001800 00780010000078004000007800800000780100000078020000007804000000F008000000 F010000000F060000000F0F0000000F1F0000000F278000001E478000001E878000001F0 3C000001E03C000001E01E000001E01E000003C00F000003C00F000003C00F000003C007 800003C007800003C003C000078003C00007C007E000FFFC3FFC00231F7E9E23>I<07FF F000007E0000003C0000003C000000780000007800000078000000780000007800000078 000000F0000000F0000000F0000000F0000000F0000000F0000001E0000001E0000001E0 000001E0000001E0008001E0010003C0010003C0010003C0030003C0020003C0060003C0 060007801E0007807C00FFFFFC00191F7E9E1C>I<07FC0000FFC0007C0000F800003C00 017800003C00017800004E0002F000004E0002F000004E0004F000004E0004F000004E00 08F000004E0008F00000870011E00000870011E00000870021E00000870021E000008700 41E00000838041E00001038083C00001038083C00001038103C00001038203C0000101C2 03C0000101C403C0000201C40780000201C80780000201C80780000201D00780000200F0 0780000600E00780000600E00F00000F00C00F8000FFE0C1FFF8002A1F7E9E2A>I<07FC 01FFC0003E003E00003E001800003E001800004F001000004F0010000047801000004780 10000043C010000043C010000083C020000081E020000081E020000080F020000080F020 000080782000010078400001007C400001003C400001003C400001001E400001001E4000 02000F800002000F800002000F800002000780000200078000060003800006000300000F 00010000FFE0010000221F7E9E22>I<0003F800001E0E000038070000E0038001C001C0 03C001E0078000E00F0000F00F0000F01E0000F01E0000F83E0000F83C0000F87C0000F8 7C0000F87C0000F87C0000F8F80001F0F80001F0F80001F0F80001F0F80003E0780003E0 780003C0780007C07C0007803C000F003C001E001E001C000E0038000700F00003C3C000 00FE00001D217B9F23>I<07FFFF00007C03C0003C01E0003C00F0007800F0007800F800 7800F8007800F8007800F8007800F000F001F000F001E000F003C000F0078000F00F0000 FFF80001E0000001E0000001E0000001E0000001E0000001E0000003C0000003C0000003 C0000003C0000003C0000003C000000780000007C00000FFFC00001D1F7E9E1F>I<0003 F800001E0E000038070000F0038001E001C003C001E0078001E00F0000F00F0000F01F00 00F01E0000F83E0000F83C0000F87C0000F87C0000F87C0000F87C0000F8F80001F0F800 01F0F80001F0F80001F0F80003E0780003E0780003C0780007C0781E07803C210F003C40 9E001E409C000E80F8000740F00003C1C04000FEC0400000E0400000E0800000E1800000 FF800000FF000000FF0000007E0000003C001D297B9F23>I<07FFFC00007C0700003C03 C0003C01E0007801E0007801F0007801F0007801F0007801F0007801E000F003E000F003 C000F0078000F00F0000F03C0000FFF00001E0300001E0380001E01C0001E01C0001E01C 0001E01E0003C03E0003C03E0003C03E0003C03E0003C03E0003C03E0207803E0407C01F 04FFFC0F18000003E01F207E9E21>I<003F040060CC01803C03801C03001C0700180600 080E00080E00080E00080E00000F00000F80000FE00007FE0003FF8001FFC0007FE00007 E00001E00000E00000F00000F04000E04000E04000E04000E06000C0600180E00380F803 00C60C0081F80016217D9F19>I<3FFFFFF03C0780F03007803060078030400F0010400F 0010C00F0010800F0010800F0010800F0010001E0000001E0000001E0000001E0000001E 0000001E0000003C0000003C0000003C0000003C0000003C0000003C0000007800000078 00000078000000780000007800000078000000F0000001F800007FFFE0001C1F7A9E21> IIII<03FFC0FFC0007F007E00003E003800001E003000001E002000000F004000000F0080 00000F81000000078200000007C600000003C400000003E800000001F000000001F00000 0000F000000000F800000000F8000000017C000000023C000000043C0000000C1E000000 081E000000101F000000200F000000400F800000C0078000008007C000010003C0000700 03E0001F8007E000FFE01FFE00221F7F9E22>II E /Fw 62 124 df<00000F80003F8F80007F8F8000FF8F8001FF8F8003 E0000003C0000007C0000007C0000007C0000007C0000007C0000007C0000007C0000007 C0000007C00000FFFF8F80FFFF8F80FFFF8F8007C00F8007C00F8007C00F8007C00F8007 C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007 C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007 C00F8007C00F80192B7FAA20>12 D39 D45 DI<007F000001FFC00007 FFF0000FFFF8000FC1F8001F007C003F007E003E003E003C001E007C001F007C001F007C 001F0078000F00F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8 000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F8078000F007C 001F007C001F007C001F003E003E003E003E003F007E001F80FC000FC1F8000FFFF80007 FFF00001FFC000007F000019297EA71E>48 D<00180000380000F80007F800FFF800FFF8 00F8F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F8007FFFF07FFFF07FFFF014287BA71E >I<00FE0003FFC007FFE00FFFF01F03F83C00FC38007E78003E70003EF0001FF0001F60 001F20001F00001F00001F00001F00003E00003E00007C00007C0000F80001F00001E000 03C0000780000F00001E00003C0000780000F00001E00003C0000780000F00001E00003C 00007FFFFF7FFFFF7FFFFF7FFFFF18287EA71E>I<007F000001FFC00007FFF0000FFFF8 001FC1F8003E007C003C003E0078003E0038003E0010003E0000003E0000003E0000003C 0000007C000000FC000001F8000007F00000FFE00000FFC00000FFE00000FFF0000001FC 0000007C0000003E0000001F0000001F0000000F8000000F8000000F8000000F8000000F 8040000F8060001F00F0001F00F8003F007E007E003F81FC001FFFF8000FFFF00003FFE0 00007F000019297EA71E>I<0003F0000007F0000005F000000DF000000DF000001DF000 0039F0000039F0000079F0000079F00000F1F00000F1F00001E1F00003E1F00003E1F000 07C1F00007C1F0000F81F0000F81F0001F01F0001F01F0003E01F0007C01F0007C01F000 F801F000FFFFFF80FFFFFF80FFFFFF80FFFFFF800001F0000001F0000001F0000001F000 0001F0000001F0000001F0000001F0000001F0000001F00019277EA61E>I<3FFFFC3FFF FC3FFFFC3FFFFC3E00003E00003E00003E00003E00003E00003E00003E00003E00003E00 003E3F003EFFC03FFFE03FFFF03FE1F83F807C3F003E3E003E00003E00001F00001F0000 1F00001F00001F00001F00001F20001F60003E70003EF8007C7C00FC3F03F81FFFF00FFF E007FF8000FE0018287EA61E>I<000FF000003FFC0000FFFC0001FFFC0003F80C0007E0 00000FC000000F8000001F0000001E0000003E0000003C0000007C0000007C0000007C3F E000F8FFF000F9FFF800FBFFFC00FF807E00FF003E00FE003F00FC001F00FC001F00FC00 0F80F8000F80F8000F80F8000F80F8000F8078000F807C000F807C000F807C000F003E00 1F003E001F001F003E001F807C000FC1FC0007FFF80003FFF00001FFC000007F00001929 7EA71E>II<007F000001FFC00007FFF000 0FFFF8001FC1FC003F007E003E003E007E003F007C001F007C001F007C001F007C001F00 7C001F003E003E003E003E001F007C000FC1F80007FFF00003FFE00003FFE0000FFFF800 1FC1FC003F007E003E003E007C001F007C001F00F8000F80F8000F80F8000F80F8000F80 F8000F80F8000F807C001F007C001F007E003F003F007E001FC1FC000FFFF80007FFF000 03FFE000007F000019297EA71E>I<007F000001FFC00003FFE0000FFFF0000FC1F8001F 007C003E007C007C003E007C001E007C001F00F8001F00F8001F00F8000F00F8000F80F8 000F80F8000F80F8000F80F8001F807C001F807C001F807E003F803E007F803F00FF801F FFEF800FFFCF8007FF8F8003FE1F0000001F0000001F0000001E0000003E0000003E0000 007C0000007C000000F8001801F0001E07E0003FFFC0001FFF80000FFE000003F8000019 297EA71E>I<03FC001FFF807FFFC0FFFFE0FC07F07001F04000F80000F80000F80000F8 0000F80001F80001F00007F0000FE0001FC0001F80003E00003E00007C0000780000F800 00F00000F00000F00000F00000F00001F00001F00001F000000000000000000000000000 00000000000000000001F00001F00001F00001F00001F000152A7DA91C>63 D<0001F000000003F800000003F800000007FC00000007BC00000007BC0000000F3E0000 000F1E0000000F1E0000001F1F0000001E1F0000001E0F0000003E0F8000003C0F800000 3C078000007C07C000007807C00000F803E00000F803E00000F003E00001F001F00001F0 01F00001E001F00003E000F80003E000F80003C000F80007FFFFFC0007FFFFFC000FFFFF FE000F80003E000F80003E001F00003F001F00001F001E00001F003E00000F803E00000F 803C00000F807C000007C07C000007C078000007C0F8000003E0F8000003E0232A7EA928 >65 DI<0001 FF00000FFFE0003FFFF8007FFFF800FE01F801F8003003F0001007C000000F8000001F80 00001F0000003E0000003E0000007E0000007C0000007C0000007C000000F8000000F800 0000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F80000007C00 00007C0000007C0000007E0000003E0000003E0000001F0000001F8000000F80000007C0 000003F0000401F8001C00FE00FC007FFFFC003FFFF8000FFFE00001FF001E2C7CAA26> IIII<0001FF00000FFFE0 003FFFFC007FFFFE00FF01FE01F8003E03F0000C07C000000FC000001F8000001F000000 3F0000003E0000007E0000007C0000007C0000007C000000F8000000F8000000F8000000 F8000000F8000000F8000000F8000000F8001FFEF8001FFEF8001FFE7C001FFE7C00003E 7C00003E7E00003E3E00003E3F00003E1F00003E1F80003E0FC0003E07C0003E03F0003E 01F8003E00FF00FE007FFFFE003FFFFC000FFFE00001FF001F2C7CAA28>III76 DII<0001FC0000000FFF8000003FFFE000007FFFF00001FE03FC0003F800FE0007E000 3F0007C0001F000F80000F801F000007C01F000007C03E000003E03E000003E07C000001 F07C000001F07C000001F078000000F0F8000000F8F8000000F8F8000000F8F8000000F8 F8000000F8F8000000F8F8000000F8F8000000F8F8000000F8F8000000F87C000001F07C 000001F07C000001F07E000003F03E000003E03F000007E01F000007C01F80000FC00FC0 001F8007E0003F0007F0007F0003F800FE0001FE03FC0000FFFFF800003FFFE000000FFF 80000001FC0000252C7DAA2C>II82 D<007FC00001FFF80007FFFE000FFFFF001FC07F003F000F007E0006007C0000 007C000000F8000000F8000000F8000000F8000000F8000000FC0000007E0000007F0000 003F8000001FF800000FFF800007FFE00003FFF80000FFFC00000FFE000000FF0000003F 0000001F8000000F8000000FC0000007C0000007C0000007C0000007C0000007C0000007 C000000F8060000F80F0001F00FC003F00FF80FE007FFFFC001FFFF80007FFE00000FF80 001A2C7DAA21>IIIII<01FE000FFF803FFFC03FFFE03C03 F03001F00001F80000F80000F80000F80000F80000F8007FF807FFF81FFFF83FE0F87F00 F8FC00F8F800F8F800F8F800F8FC01F87E07F87FFFF83FFFF81FFCF80FE0F8151B7E9A1D >97 DI<007FC001FFF007FF FC0FFFFC1FC07C1F00083E00007C00007C00007C0000F80000F80000F80000F80000F800 00F80000F800007C00007C00007E00003E00001F000C1FC07C0FFFFC07FFFC01FFF0007F 80161B7E9A1B>I<00003E00003E00003E00003E00003E00003E00003E00003E00003E00 003E00003E00003E00003E00003E00003E00FC3E03FF3E07FFFE0FFFFE1FC1FE3F007E3E 003E7C003E7C003EFC003EF8003EF8003EF8003EF8003EF8003EF8003EF8003EFC003E7C 003E7C003E3E007E3F00FE1FC1FE0FFFFE07FFBE03FF3E00FC3E172A7EA91F>I<007E00 03FF8007FFC00FFFE01F83F03F00F03E00787C00787C003878003CFFFFFCFFFFFCFFFFFC FFFFFCF80000F80000F800007800007C00007C00003E00003F000C1FC07C0FFFFC07FFFC 01FFF0007F80161B7E9A1B>I<001FC0007FC000FFC001FFC003F00003E00007C00007C0 0007C00007C00007C00007C00007C00007C00007C000FFFE00FFFE00FFFE0007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C000122A7FA912 >I<00F8078003FE7FC00FFFFFC01FFFFFC01F07C0003E03E0003E03E0007C01F0007C01 F0007C01F0007C01F0007C01F0007C01F0003E03E0003E03E0001F07C0001FFFC0003FFF 80003BFE000038F8000078000000780000003C0000003FFFC0003FFFF8001FFFFC001FFF FE003FFFFF007C007F00F8001F80F8000F80F8000F80F8000F80FC001F807E003F003F80 FE003FFFFE000FFFF80007FFF00000FF80001A287E9A1E>III<007C007C007C007C00 7C0000000000000000000000000000000000000000007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C007C40FCF1F8FFF8FFF07FE00F800E3683A910>III< F83F003F00F8FFC0FFC0FBFFE3FFE0FFFFF7FFF0FF83F783F0FE01FE01F8FC00FC00F8FC 00FC00F8FC00FC00F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800 F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800F8 00F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8251B7B9A30>II<007F000001FFC00007FFF0000FFFF8001FC1FC003F00 7E003E003E007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F80F800 0F80F8000F80F8000F807C001F007C001F007E003F003E003E003F007E001FC1FC000FFF F80007FFF00001FFC000007F0000191B7E9A1E>I I<00FC3E03FF3E07FFBE0FFFFE1FC1FE3F80FE3E007E7E003E7C003E7C003EF8003EF800 3EF8003EF8003EF8003EF8003EF8003EFC003E7C003E7E003E3E007E3F00FE1FC1FE1FFF FE0FFFBE03FF3E01F83E00003E00003E00003E00003E00003E00003E00003E00003E0000 3E00003E00003E00003E17277E9A1F>II<03FC001FFF803FFFC07FFFC07C07C0F80080F80000F80000F80000FC0000 7F80007FF8003FFE001FFF0007FF8000FFC0000FE00007E00003E00003E04003E0E007E0 FC0FC0FFFFC07FFF801FFE0003F800131B7E9A17>I<07C00007C00007C00007C00007C0 0007C00007C000FFFFC0FFFFC0FFFFC007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 4007E1C003FFE003FFE001FF8000FC0013227FA116>IIII<7C000FC03E001F803F001F001F803E000F807C0007C0FC0003E0F800 01F1F00001FBE00000FFC000007FC000003F8000001F0000001F0000003F8000007FC000 00FBC00000F3E00001F1F00003E0F80007C07C000F807C000F803E001F001F003E000F80 7E000FC0FC0007E01B1B809A1C>IIII E /Fx 17 117 df<0001FF8000000FFFF000003FFFFC0000FF81FF0001FE007F8003FC00 3FC007F8001FE00FF8001FF00FF0000FF01FF0000FF81FF0000FF83FF0000FFC3FE00007 FC3FE00007FC7FE00007FE7FE00007FE7FE00007FE7FE00007FE7FE00007FEFFE00007FF FFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFF E00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE0 0007FFFFE00007FFFFE00007FFFFE00007FF7FE00007FE7FE00007FE7FE00007FE7FE000 07FE7FE00007FE3FE00007FC3FF0000FFC3FF0000FFC1FF0000FF81FF0000FF80FF0000F F00FF8001FF007F8001FE003FC003FC001FE007F8000FF81FF00003FFFFC00000FFFF000 0001FF800028397CB731>48 D<00001E000000003E00000000FE00000003FE0000003FFE 0000FFFFFE0000FFFFFE0000FFFFFE0000FFCFFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE00007FFFFFFFC07FFFFFFFC07FFFFFFFC0 7FFFFFFFC0223879B731>I<0007FE0000007FFFE00001FFFFF80003FFFFFE0007F01FFF 000F8007FF801F0001FFC03E0000FFE07F8000FFF07FC0007FF0FFE0007FF8FFF0003FF8 FFF0003FFCFFF0003FFCFFF0003FFCFFF0003FFC7FE0003FFC3FC0003FFC1F80003FFC00 00003FFC0000003FF80000007FF80000007FF00000007FF0000000FFE0000000FFC00000 01FF80000001FF00000003FE00000007FC00000007F80000000FF00000001FC00000003F 800000007F00000000FC00000001F800000001F0003C0003E0003C0007C0003C000F0000 78001E000078003C00007800780000F800F00000F801FFFFFFF803FFFFFFF007FFFFFFF0 0FFFFFFFF01FFFFFFFF03FFFFFFFF07FFFFFFFF0FFFFFFFFF0FFFFFFFFE0FFFFFFFFE0FF FFFFFFE026387BB731>I<0003FF8000001FFFF000007FFFFE0000FE03FF0001F000FF80 03C000FFC00780007FE00FF0007FF00FF8007FF01FFC007FF81FFE007FF81FFE007FF81F FE007FF81FFE007FF81FFE007FF80FFC007FF007F8007FF003F0007FF0000000FFE00000 00FFC0000001FF80000001FF00000003FE00000007FC0000001FF000000FFFC000000FFF 8000000FFFF800000003FE00000000FF800000007FE00000003FF00000003FF80000003F FC0000001FFC0000001FFE0000001FFE0200001FFF1FC0001FFF3FE0001FFF7FF0001FFF 7FF0001FFFFFF8001FFFFFF8001FFFFFF8001FFEFFF8001FFEFFF0001FFE7FF0003FFC7F E0003FFC3FC0003FF81F80007FF01FE000FFE007FC03FFC003FFFFFF0001FFFFFE00003F FFF0000007FF800028397CB731>I<00000007C0000000000FC0000000000FC000000000 1FC0000000003FC0000000007FC000000000FFC000000000FFC000000001FFC000000003 FFC000000007FFC00000000FFFC00000000FFFC00000001EFFC00000003CFFC00000007C FFC0000000F8FFC0000000F0FFC0000001E0FFC0000003C0FFC0000007C0FFC000000F80 FFC000000F00FFC000001E00FFC000003C00FFC000007C00FFC00000F800FFC00000F000 FFC00001E000FFC00003C000FFC00007C000FFC0000F8000FFC0000F0000FFC0001E0000 FFC0003C0000FFC0007C0000FFC000F80000FFC000FFFFFFFFFFC0FFFFFFFFFFC0FFFFFF FFFFC0FFFFFFFFFFC0000001FFC000000001FFC000000001FFC000000001FFC000000001 FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000007FF FFFFC00007FFFFFFC00007FFFFFFC00007FFFFFFC02A377DB631>I<04000000C00F8000 07C00FF8007FC00FFFFFFF800FFFFFFF000FFFFFFE000FFFFFFC000FFFFFF8000FFFFFF0 000FFFFFE0000FFFFF80000FFFFE00000FFFF800000F800000000F800000000F80000000 0F800000000F800000000F800000000F800000000F800000000F81FF00000F8FFFE0000F BFFFF8000FFE03FE000FF000FF000FC000FF800F80007FC00F00007FE00700007FF00000 003FF00000003FF80000003FF80000003FF80000003FFC0000003FFC0600003FFC1F8000 3FFC3FC0003FFC7FE0003FFCFFE0003FFCFFF0003FFCFFF0003FFCFFF0003FF8FFE0003F F8FFE0003FF87FC0007FF07F00007FF03C00007FE03E0000FFC01F0000FF800FC003FF00 07F00FFE0003FFFFFC0001FFFFF000007FFFC000000FFC000026397BB731>I<00000FF8 0000007FFF000003FFFF80000FFC07C0001FE001E0003FC001F0007F0007F000FF000FF0 01FE001FF803FC003FF807FC003FF80FFC003FF80FF8003FF81FF8001FF01FF8000FE03F F80007C03FF00000003FF00000007FF00000007FF00000007FF00000007FF07FF000FFF0 FFFE00FFF1F7FF00FFF3807F80FFF6003FE0FFFE001FF0FFFC001FF0FFFC000FF8FFF800 0FFCFFF8000FFCFFF8000FFEFFF8000FFEFFF0000FFEFFF0000FFFFFF0000FFFFFF0000F FF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF3FF0000FFF3FF0000FFE 3FF0000FFE1FF0000FFE1FF8000FFC0FF8000FFC0FF8001FF807FC001FF003FC001FF001 FE003FE000FF80FFC0007FFFFF00001FFFFE000007FFF8000000FFC00028397CB731>I< 1E00000000001F00000000001FF0000000001FFFFFFFFFC01FFFFFFFFFC01FFFFFFFFFC0 3FFFFFFFFFC03FFFFFFFFF803FFFFFFFFF003FFFFFFFFE003FFFFFFFFC003FFFFFFFF800 3FFFFFFFF0007FFFFFFFF0007C000007E0007C00000FC0007800001F80007800001F0000 7800003E0000F000007E0000F00000FC0000F00001F80000000003F00000000003E00000 000007E0000000000FC0000000000F80000000001F80000000003F80000000003F000000 00007F00000000007F0000000000FF0000000000FE0000000001FE0000000001FE000000 0003FE0000000003FE0000000003FE0000000007FC0000000007FC0000000007FC000000 000FFC000000000FFC000000000FFC000000000FFC000000000FFC000000001FFC000000 001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000 001FFC000000001FFC000000000FF80000000007F00000000003E00000002A3B7BB931> I<0001FF8000000FFFF800003FFFFE00007F00FF0000F8003F8001F0000FC003E0000FE0 07C00007F007C00007F00FC00003F80FC00003F81FC00003F81FC00003F81FE00003F81F F00003F81FF80003F81FFC0007F01FFF0007F01FFFC007E00FFFF00FE00FFFFC1FC007FF FE3F8007FFFFFF0003FFFFFC0001FFFFF80000FFFFFE00007FFFFF00003FFFFF80003FFF FFE000FFFFFFF001FDFFFFF007F07FFFF80FE01FFFFC1FC007FFFC3F8001FFFE3F8000FF FE7F00003FFF7F00000FFFFE000003FFFE000001FFFE000000FFFE000000FFFE0000007F FE0000007FFE0000007EFE0000007E7F0000007E7F000000FC3F800000FC3FC00001F81F E00003F00FF00007E007FE007FC003FFFFFF8000FFFFFE00003FFFF8000003FF80002839 7CB731>I<0001FF8000001FFFF000007FFFFC0000FF81FE0001FE007F0007FC003F800F F8003FC00FF8001FE01FF8001FF03FF0001FF03FF0000FF87FF0000FF87FF0000FFCFFF0 000FFCFFF0000FFCFFF0000FFEFFF0000FFEFFF0000FFEFFF0000FFEFFF0000FFEFFF000 0FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0001FFF7FF0001FFF3FF0001FFF3FF0001F FF1FF0003FFF0FF8003FFF07F8007FFF03FC006FFF01FE01CFFF00FFEF8FFF007FFF0FFF 000FFE0FFE0000000FFE0000000FFE0000000FFE0000000FFC0000000FFC03E0001FFC07 F0001FF80FF8001FF81FFC001FF01FFC003FF01FFC003FE01FFC003FC01FF8007FC00FF0 00FF800FE001FF0007C003FE0007F01FFC0003FFFFF00001FFFFC000007FFF0000000FF8 000028397CB731>I<0003FF800180001FFFF00380007FFFFC078001FFFFFF0F8003FE00 FF9F8007F0000FFF800FE00003FF801FC00001FF803F8000007F803F8000007F807F0000 003F807F0000001F807F0000001F80FF0000000F80FF0000000F80FF0000000F80FF8000 000780FF8000000780FFC000000780FFE000000780FFF8000000007FFE000000007FFFF0 0000007FFFFF0000003FFFFFF800003FFFFFFF00001FFFFFFFC0000FFFFFFFF00007FFFF FFF80003FFFFFFFC0001FFFFFFFE00007FFFFFFF00003FFFFFFF800007FFFFFF8000007F FFFFC0000007FFFFC00000003FFFE000000003FFE000000000FFF0000000007FF0000000 003FF0700000001FF0F00000001FF0F00000001FF0F00000000FF0F00000000FF0F80000 000FF0F80000000FE0F80000000FE0FC0000000FE0FC0000001FC0FE0000001FC0FF0000 001F80FFC000003F80FFF000007F00FFFC0001FE00FCFFC007FC00F87FFFFFF800F01FFF FFE000E003FFFF8000C0003FFC00002C3D7BBB37>83 D<0000FFF000000FFFFF00003FFF FF8000FFC01FC001FF003FE003FC007FF007FC007FF00FF8007FF01FF0007FF01FF0003F E03FF0003FE03FF0001FC07FE00007007FE00000007FE0000000FFE0000000FFE0000000 FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00000007FE00000007F E00000007FF00000003FF00000003FF00000001FF00000781FF80000780FF80000F007FC 0000F003FE0001E001FF8007C000FFE01F80003FFFFF00000FFFFC000000FFC00025267D A52C>99 D<0001FFC000000FFFF800003FFFFE0000FF80FF0001FE003F8007FC001FC00F F8000FE00FF8000FF01FF00007F03FF00007F83FF00007F87FE00007F87FE00003FC7FE0 0003FC7FE00003FCFFE00003FCFFFFFFFFFCFFFFFFFFFCFFFFFFFFFCFFE0000000FFE000 0000FFE0000000FFE00000007FE00000007FE00000007FE00000003FE00000003FF00000 3C1FF000003C1FF000003C0FF800007807FC0000F803FE0001F001FF0007E000FFC03FC0 003FFFFF000007FFFC000000FFE00026267DA52D>101 D<01F00007FC000FFE000FFE00 1FFF001FFF001FFF001FFF001FFF000FFE000FFE0007FC0001F000000000000000000000 00000000000000000000000000000000000000000000FF00FFFF00FFFF00FFFF00FFFF00 07FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF00 03FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF00 03FF0003FF0003FF0003FF0003FF00FFFFF8FFFFF8FFFFF8FFFFF8153D7DBC1B>105 D<00FE007FC000FFFE01FFF800FFFE07FFFC00FFFE0F03FE00FFFE1C01FF0007FE3001FF 8003FE6000FF8003FEE000FFC003FEC000FFC003FF8000FFC003FF8000FFC003FF8000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFFFFFFFC3FFF FFFFFFFC3FFFFFFFFFFC3FFFFF30267CA537>110 D<0000FFC00000000FFFFC0000003F FFFF000000FFC0FFC00001FE001FE00007FC000FF80007F80007F8000FF00003FC001FF0 0003FE003FF00003FF003FE00001FF007FE00001FF807FE00001FF807FE00001FF807FE0 0001FF80FFE00001FFC0FFE00001FFC0FFE00001FFC0FFE00001FFC0FFE00001FFC0FFE0 0001FFC0FFE00001FFC0FFE00001FFC0FFE00001FFC07FE00001FF807FE00001FF807FE0 0001FF803FF00003FF003FF00003FF001FF00003FE000FF80007FC000FF80007FC0007FC 000FF80003FE001FF00000FFC0FFC000003FFFFF0000000FFFFC00000001FFE000002A26 7DA531>I<0007800000078000000780000007800000078000000F8000000F8000000F80 00000F8000001F8000001F8000003F8000003F8000007F800000FF800001FF800007FF80 001FFFFFF0FFFFFFF0FFFFFFF0FFFFFFF001FF800001FF800001FF800001FF800001FF80 0001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF80 0001FF800001FF800001FF800001FF800001FF800001FF803C01FF803C01FF803C01FF80 3C01FF803C01FF803C01FF803C01FF803C00FF807800FFC078007FC070003FE0E0001FFF C00007FF800001FF001E377EB626>116 D E /Fy 10 104 df0 D<70F8F8F87005057C8D0D>I<01800180018001800180C183F18F399C0FF003 C003C00FF0399CF18FC1830180018001800180018010147D9417>3 D<03C00FF01C38300C60066006C003C003C003C003C003C00360066006300C1C380FF003 C010127D9317>14 D<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFFFFFFFF7FFE7F FE3FFC1FF80FF003C010127D9317>I<000000C0000003C000000F0000003C000000F000 0003C00000070000001C00000078000001E00000078000001E00000078000000E0000000 780000001E0000000780000001E0000000780000001C0000000700000003C0000000F000 00003C0000000F00000003C0000000C00000000000000000000000000000000000000000 00000000000000007FFFFF80FFFFFFC01A247C9C23>20 D<000000040000000002000000 0002000000000100000000008000000000400000000020FFFFFFFFFCFFFFFFFFFC000000 002000000000400000000080000000010000000002000000000200000000040026107D92 2D>33 D<400002C00006C00006C00006C00006C00006C00006C00006C00006C00006C000 06C00006C00006C00006C00006C00006C00006C00006C00006C00006C0000660000C6000 0C3000181C00700F01E003FF8000FE00171C7D9A1E>91 D<000F0038006000E001C001C0 01C001C001C001C001C001C001C001C001C001C001C001C001C0038007001E00F8001E00 0700038001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C000E0 00600038000F102D7DA117>102 DI E /Fz 1 98 df<001800001800001800003C00003C00004E00004E00004E0000 87000087000187800103800103800201C00201C003FFC00400E00400E008007008007018 00703C0078FE01FF18177F961C>97 D E /FA 10 58 df<1F00318060C04040C060C060 C060C060C060C060C060C060404060C031801F000B107F8F0F>48 D<0C003C00CC000C000C000C000C000C000C000C000C000C000C000C000C00FF8009107E 8F0F>I<1F00618040C08060C0600060006000C00180030006000C00102020207FC0FFC0 0B107F8F0F>I<1F00218060C060C000C0008001800F00008000400060C060C060804060 801F000B107F8F0F>I<0300030007000F000B001300330023004300C300FFE003000300 030003001FE00B107F8F0F>I<20803F002C002000200020002F00308020400060006000 60C06080C061801F000B107F8F0F>I<0780184030C060C06000C000CF00F080E040C060 C060C060406060C030801F000B107F8F0F>I<40007FE07FC08080808001000200040004 000C0008000800180018001800180018000B117E900F>I<1F00318060C060C060C07180 3F000F00338061C0C060C060C060404060801F000B107F8F0F>I<1F00318060C0C040C0 60C060C06040E021E01E600060004060C0608043003E000B107F8F0F>I E /FB 29 123 df<70F8F8F87005057C840D>58 D<70F8FCFC7404040408081010204006 0E7C840D>I<000001C00000078000001E00000078000001E00000078000000E00000038 000000F0000003C000000F0000003C000000F0000000F00000003C0000000F00000003C0 000000F0000000380000000E0000000780000001E0000000780000001E00000007800000 01C01A1A7C9723>I62 D<000002000000060000000E0000000E0000001E0000001F 0000002F0000002F0000004F0000008F0000008F0000010F0000010F0000020F0000040F 0000040F0000080F8000080780001007800020078000200780007FFF8000400780008007 80018007800100078002000780020007C0040003C00C0003C01E0007C0FF807FFC1E207E 9F22>65 D<00FFFFE000000F007800000F001C00000F000E00000F000700001E00070000 1E000380001E000380001E000380003C000380003C000380003C000380003C0003800078 00078000780007800078000780007800078000F0000F0000F0000F0000F0000E0000F000 1E0001E0001C0001E0003C0001E000380001E000700003C000E00003C001C00003C00380 0003C007000007803C0000FFFFF00000211F7E9E26>68 D<0FFFFFFC1E03C0381803C018 1003C0082003C00820078008600780084007800840078008800F0010000F0000000F0000 000F0000001E0000001E0000001E0000001E0000003C0000003C0000003C0000003C0000 0078000000780000007800000078000000F0000000F0000000F0000000F0000001F00000 7FFFC0001E1F7F9E1B>84 D<00F1800389C00707800E03801C03803C0380380700780700 780700780700F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F0780 14147E9318>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C 3A0E3C0E380F380F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F00 10207E9F14>I<0000780003F80000700000700000700000700000E00000E00000E00000 E00001C00001C000F1C00389C00707800E03801C03803C03803807007807007807007807 00F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F078015207E9F18 >100 D<007C01C207010E011C013C013802780C7BF07C00F000F000F000F00070007001 70023004183807C010147E9315>I<00007C0000CE00019E00039E00030C000700000700 000700000700000E00000E00000E0000FFF0000E00000E00001C00001C00001C00001C00 001C0000380000380000380000380000380000700000700000700000700000700000E000 00E00000E00000E00000C00001C000318000798000F300006200003C000017297E9F16> I<001E3000713800E0F001C0700380700780700700E00F00E00F00E00F00E01E01C01E01 C01E01C01E01C01E03801E03800E07800E0B8006170001E700000700000700000E00000E 00300E00781C00F038006070003FC000151D809316>I<00E001E001E000C00000000000 0000000000000000000E00130023804380438043808700070007000E000E001C001C001C 20384038403840388019000E000B1F7E9E10>105 D<0000C00001E00001E00001C00000 00000000000000000000000000000000000000001E000063000043800083800103800103 80020700000700000700000700000E00000E00000E00000E00001C00001C00001C00001C 0000380000380000380000380000700000700030700078E000F1C0006380003E00001328 819E13>I<01E0000FE00001C00001C00001C00001C00003800003800003800003800007 00000700000701E00706100E08700E10F00E20F00E40601C80001D00001E00001FC00038 7000383800383800381C20703840703840703840701880E01880600F0014207E9F18>I< 03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C001C00 38003800380038007000700070007100E200E200E200E200640038000A207E9F0E>I<1E 07C07C00231861860023A032030043C03403004380380380438038038087007007000700 700700070070070007007007000E00E00E000E00E00E000E00E00E000E00E01C101C01C0 1C201C01C038201C01C038401C01C0184038038018801801800F0024147E9328>I<1E07 802318C023A06043C0704380704380708700E00700E00700E00700E00E01C00E01C00E01 C00E03821C03841C07041C07081C03083803101801E017147E931B>I<007C0001C30003 01800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780F0 0700700F00700E0030180018700007C00013147E9316>I<03C1E004621804741C08781C 08701E08701E10E01E00E01E00E01E00E01E01C03C01C03C01C03C01C038038078038070 0380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C0000 FFC000171D819317>I<007C018203010603060706060E00078007F803FC01FE001F0007 7007F006F006E004400820301FC010147E9315>115 D<00C000E001C001C001C001C003 800380FFF8038007000700070007000E000E000E000E001C001C001C001C103820382038 20384018800F000D1C7F9B10>I<0F00601180702180E021C0E041C0E04380E08381C007 01C00701C00701C00E03800E03800E03800E03840E07080C07080C07080E0F1006131003 E1E016147E931A>I<0F01801183C02183E021C1E041C0E0438060838040070040070040 0700400E00800E00800E00800E01000E01000C02000E04000E040006180001E00013147E 9316>I<0F006060118070F02180E0F821C0E07841C0E0384380E0188381C0100701C010 0701C0100701C0100E0380200E0380200E0380200E0380400E0380400E0380800E078080 060781000709860001F078001D147E9321>I<03C1C00C62201034701038F02038F02038 6040700000700000700000700000E00000E00000E00000E02061C040F1C040F1C080E2C0 80446300383C0014147E931A>I<0F00601180702180E021C0E041C0E04380E08381C007 01C00701C00701C00E03800E03800E03800E03800E07000C07000C07000E0F00061E0003 EE00000E00000E00001C0078180078380070700060600021C0001F0000141D7E9316>I< 01E02003F04007F8C00C1F8008010000020000040000080000100000600000C000010000 0200000400800801001003003F060061FC0040F80080700013147E9315>I E /FC 88 128 df<001F83E000F06E3001C078780380F8780300F0300700700007007000 0700700007007000070070000700700007007000FFFFFF80070070000700700007007000 070070000700700007007000070070000700700007007000070070000700700007007000 0700700007007000070070000700700007007000070070007FE3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E00700000700000700000700000700000700 00FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700 E00700E00700E00700E00700E00700E00700E00700E07FC3FE1720809F19>I<003FE000 E0E001C1E00381E00700E00700E00700E00700E00700E00700E00700E00700E0FFFFE007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E07FE7FE1720809F19>I<001F81F80000F04F04 0001C07C06000380F80F000300F00F000700F00F00070070000007007000000700700000 070070000007007000000700700000FFFFFFFF0007007007000700700700070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007000700700700070070070007007007000700700700070070 07007FE3FE3FF02420809F26>I<001F81FF0000F06F070001C07C0F000380F80F000300 F00700070070070007007007000700700700070070070007007007000700700700070070 0700FFFFFFFF000700700700070070070007007007000700700700070070070007007007 000700700700070070070007007007000700700700070070070007007007000700700700 070070070007007007000700700700070070070007007007007FE3FE3FF02420809F26> I<7038F87CFC7EFC7E743A0402040204020804080410081008201040200F0E7E9F17>34 D<70F8FCFC74040404080810102040060E7C9F0D>39 D<0020004000800100020006000C 000C00180018003000300030007000600060006000E000E000E000E000E000E000E000E0 00E000E000E000E0006000600060007000300030003000180018000C000C000600020001 000080004000200B2E7DA112>I<800040002000100008000C0006000600030003000180 0180018001C000C000C000C000E000E000E000E000E000E000E000E000E000E000E000E0 00C000C000C001C001800180018003000300060006000C00080010002000400080000B2E 7DA112>I<01800180018001800180C183F18F399C0FF003C003C00FF0399CF18FC18301 80018001800180018010147DA117>I<0006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 00060000FFFFFFF0FFFFFFF0000600000006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 1C207D9A23>I<70F8FCFC74040404080810102040060E7C840D>II<70F8F8F87005057C840D>I<000100030003000600060006000C000C000C0018001800 1800300030003000600060006000C000C000C00180018001800300030003000600060006 000C000C000C00180018001800300030003000600060006000C000C000C000102D7DA117 >I<03F0000E1C001C0E00180600380700700380700380700380700380F003C0F003C0F0 03C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C070038070 03807003807807803807001806001C0E000E1C0003F000121F7E9D17>I<018003800F80 F38003800380038003800380038003800380038003800380038003800380038003800380 038003800380038003800380038007C0FFFE0F1E7C9D17>I<03F0000C1C00100E002007 00400780800780F007C0F803C0F803C0F803C02007C00007C0000780000780000F00000E 00001C0000380000700000600000C0000180000300000600400C00401800401000803FFF 807FFF80FFFF80121E7E9D17>I<03F0000C1C00100E00200F00780F8078078078078038 0F80000F80000F00000F00000E00001C0000380003F000003C00000E00000F0000078000 07800007C02007C0F807C0F807C0F807C0F00780400780400F00200E001C3C0003F00012 1F7E9D17>I<000600000600000E00000E00001E00002E00002E00004E00008E00008E00 010E00020E00020E00040E00080E00080E00100E00200E00200E00400E00C00E00FFFFF0 000E00000E00000E00000E00000E00000E00000E0000FFE0141E7F9D17>I<1803001FFE 001FFC001FF8001FE00010000010000010000010000010000010000011F000161C00180E 001007001007800003800003800003C00003C00003C07003C0F003C0F003C0E003804003 80400700200600100E000C380003E000121F7E9D17>I<007C000182000701000E03800C 07801C0780380300380000780000700000700000F1F000F21C00F40600F80700F80380F8 0380F003C0F003C0F003C0F003C0F003C07003C07003C07003803803803807001807000C 0E00061C0001F000121F7E9D17>I<4000007FFFC07FFF807FFF80400100800200800200 80040000080000080000100000200000200000400000400000C00000C00001C000018000 038000038000038000038000078000078000078000078000078000078000078000030000 121F7D9D17>I<03F0000C0C001006003003002001806001806001806001807001807803 003E03003F06001FC8000FF00003F80007FC000C7E00103F00300F806003804001C0C001 C0C000C0C000C0C000C0C000806001802001001002000C0C0003F000121F7E9D17>I<03 F0000E18001C0C00380600380700700700700380F00380F00380F003C0F003C0F003C0F0 03C0F003C07007C07007C03807C0180BC00E13C003E3C000038000038000038000070030 0700780600780E00700C002018001070000FC000121F7E9D17>I<70F8F8F87000000000 00000000000070F8F8F87005147C930D>I<70F8F8F8700000000000000000000070F0F8 F878080808101010202040051D7C930D>I<7FFFFFE0FFFFFFF000000000000000000000 00000000000000000000000000000000000000000000FFFFFFF07FFFFFE01C0C7D9023> 61 D<07000F800F800F8007000000000000000000000002000200020002000200020006 00060004000C001C00380078007000F018F03CF03CF01CF008701838200FC00E207D9615 >I<0FC0307040384038E03CF03CF03C603C0038007000E000C001800180010003000200 020002000200020002000000000000000000000007000F800F800F8007000E207D9F15> I<000100000003800000038000000380000007C0000007C0000007C0000009E0000009E0 000009E0000010F0000010F0000010F00000207800002078000020780000403C0000403C 0000403C0000801E0000801E0000FFFE0001000F0001000F0001000F0002000780020007 8002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F22>65 DI<000FC040007030C001C009C0 038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078000040 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E0070001E0 0F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C0001E0 3C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000781820 000FE0001E217D9F24>III<0FFFC0007C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C00F83C 00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007001E00 07803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001F0F800 01F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003E03C00 03C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80001C21 7D9F23>II<001F800000F0F000 01C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0 780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0 F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E2047000F204F00 07A03E0001E0380000F0F010001FB01000003010000038300000387000003FF000001FE0 00001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E00080E000 80E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003 C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F8 0012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010 800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7FF83FF8 0FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C1000003C2000 003E4000001E4000000F8000000F8000000780000003C0000007E0000005E0000009F000 0018F8000010780000207C0000603C0000401E0000801F0001800F0001000780020007C0 070003C01F8007E0FFE01FFE1F1F7F9E22>II<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80800F00 001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C004078004 0F80040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8161F7D9E1C>II<080410082010201040204020804080408040B85CFC7E FC7E7C3E381C0F0E7B9F17>II<08102020404080 8080B8FCFC7C38060E7D9F0D>96 D<1FE000303000781800781C00300E00000E00000E00 000E0000FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E10781E10386720 0F83C014147E9317>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00700E00700E00780E00 780E00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC300083E001520 7F9F19>I<03F80E0C1C1E381E380C70007000F000F000F000F000F000F0007000700038 0138011C020E0C03F010147E9314>I<000380003F800003800003800003800003800003 8000038000038000038000038000038003E380061B801C07803803803803807003807003 80F00380F00380F00380F00380F00380F003807003807003803803803807801C07800E1B 8003E3F815207E9F19>I<03F0000E1C001C0E00380700380700700700700380F00380F0 0380FFFF80F00000F00000F000007000007000003800801800800C010007060001F80011 147F9314>I<007C00C6018F038F07060700070007000700070007000700FFF007000700 07000700070007000700070007000700070007000700070007000700070007007FF01020 809F0E>I<0000E003E3300E3C301C1C30380E00780F00780F00780F00780F00780F0038 0E001C1C001E380033E0002000002000003000003000003FFE001FFF800FFFC03001E060 0070C00030C00030C00030C000306000603000C01C038003FC00141F7F9417>I<0E0000 FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E3E00 0E43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16207F9F19>I<1C001E003E001E00 1C000000000000000000000000000E007E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E00FFC00A1F809E0C>I<00E001F001F001F000E000 0000000000000000000000007007F000F000700070007000700070007000700070007000 70007000700070007000700070007000700070007000706070F060F0C061803F000C2882 9E0E>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E0FF00E03C00E03000E02000E04000E08000E10000E30000E70000EF8000F38 000E1C000E1E000E0E000E07000E07800E03800E03C00E03E0FFCFF815207F9F18>I<0E 00FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B20809F0C>I<0E1F 01F000FE618618000E81C81C000F00F00E000F00F00E000E00E00E000E00E00E000E00E0 0E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E 000E00E00E000E00E00E000E00E00E000E00E00E00FFE7FE7FE023147F9326>I<0E3E00 FE43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16147F9319>I<01F800070E001C03 803801C03801C07000E07000E0F000F0F000F0F000F0F000F0F000F0F000F07000E07000 E03801C03801C01C0380070E0001F80014147F9317>I<0E3E00FEC3800F01C00F00E00E 00E00E00F00E00700E00780E00780E00780E00780E00780E00780E00700E00F00E00E00F 01E00F01C00EC3000E3E000E00000E00000E00000E00000E00000E00000E00000E0000FF E000151D7F9319>I<03E0800619801C05803C0780380380780380700380F00380F00380 F00380F00380F00380F003807003807803803803803807801C0B800E138003E380000380 000380000380000380000380000380000380000380003FF8151D7E9318>I<0E78FE8C0F 1E0F1E0F0C0E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00F 147F9312>I<1F9030704030C010C010C010E00078007F803FE00FF00070803880188018 C018C018E030D0608F800D147E9312>I<020002000200060006000E000E003E00FFF80E 000E000E000E000E000E000E000E000E000E000E000E080E080E080E080E080610031001 E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E03C00603C0030DC001F1FC16147F93 19>III<7FC3FC0F01E00701C007018003810001C20000E40000EC 00007800003800003C00007C00004E000087000107000303800201C00601E01E01E0FF07 FE1714809318>II<3FFF380E200E 201C40384078407000E001E001C00380078007010E011E011C0338027006700EFFFE1014 7F9314>II<30307878F87C787830300E057C9E17>127 D E /FD 67 123 df<000FF07F00007FF9FF8000F83FC7C001E07F8FC003E07F0FC007C0 7F0FC007C03F078007C01F000007C01F000007C01F000007C01F000007C01F0000FFFFFF F800FFFFFFF80007C01F000007C01F000007C01F000007C01F000007C01F000007C01F00 0007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F0000 07C01F000007C01F000007C01F00003FF8FFF0003FF8FFF0002220809F1F>11 D<000FE000007FF80000F81C0001E07C0003E07C0007C07C0007C07C0007C0380007C000 0007C0000007C0000007C1FE00FFFFFE00FFFFFE0007C03E0007C03E0007C03E0007C03E 0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E 0007C03E0007C03E0007C03E003FF9FFC03FF9FFC01A20809F1D>I<3807007C0F80FE1F C0FF1FE0FF1FE07F0FE03B07600300600300600600C00600C00E01C00C0180180300700E 0020040013107E9F1B>34 D<0018007000E001C00380038007000E000E001E001C003C00 3C007800780078007800F800F000F000F000F000F000F000F000F000F000F80078007800 780078003C003C001C001E000E000E0007000380038001C000E0007000180D2D7DA114> 40 DI<387CFEFFFF7F3B03 0306060E0C18702008107C860F>44 DI<01FC0007FF 001F07C01E03C03E03E07C01F07C01F07C01F0FC01F8FC01F8FC01F8FC01F8FC01F8FC01 F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F87C01F07C01F07C01F03E03E01E03 C01F8FC007FF0001FC00151D7E9C1A>48 D<00E00001E0000FE000FFE000F3E00003E000 03E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E000 03E00003E00003E00003E00003E00003E00003E00003E00003E000FFFF80FFFF80111D7C 9C1A>I<07F0001FFE00383F007C1F80FE0FC0FE0FC0FE0FE0FE07E07C07E03807E0000F E0000FC0000FC0001F80001F00003E0000780000F00000E00001C0000380600700600E00 601C00E01FFFC03FFFC07FFFC0FFFFC0FFFFC0131D7D9C1A>I<01FC0007FF000E0F801E 0FC03F07E03F07E03F07E03F07E01E0FC0000FC0000F80001F0001FC0001FC00000F8000 07C00003E00003F00003F83803F87C03F8FE03F8FE03F8FE03F0FC03F07807E03C0FC01F FF8003FC00151D7E9C1A>I<0001C00003C00007C00007C0000FC0001FC0003BC00073C0 0063C000C3C00183C00383C00703C00E03C00C03C01803C03803C07003C0E003C0FFFFFE FFFFFE0007C00007C00007C00007C00007C00007C000FFFE00FFFE171D7F9C1A>I<3803 803FFF803FFF003FFE003FFC003FF0003F800030000030000030000030000033F80037FE 003C1F00380F801007C00007C00007E00007E07807E0FC07E0FC07E0FC07E0FC07C0780F C0600F80381F001FFC0007F000131D7D9C1A>I<003F0001FFC007E0E00F81E01F03F01E 03F03E03F07C03F07C01E07C0000FC1000FCFF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC 01F8FC01F8FC01F87C01F87C01F87C01F83C01F03E01F01E03E00F07C007FF8001FE0015 1D7E9C1A>I<6000007FFFF87FFFF87FFFF07FFFE07FFFC0E00180C00300C00300C00600 000C0000180000380000380000780000700000F00000F00001F00001F00001F00001F000 03F00003F00003F00003F00003F00003F00001E00000C000151E7D9D1A>I<01FC0007FF 000F07801E03C01C01E03C01E03C01E03E01E03F01E03FC3C01FE3801FFF000FFE0007FF 8007FFC01FFFE03C3FF0780FF07803F8F001F8F000F8F00078F00078F000707800707C00 E03E03C00FFF8003FC00151D7E9C1A>I<01FC000FFF001F07803E03C07C03E07C01E0FC 01F0FC01F0FC01F0FC01F8FC01F8FC01F8FC01F87C03F87C03F83E05F81FFDF807F9F800 41F80001F03C01F07E01F07E03E07E03E07E07C03C0780381F001FFC0007F000151D7E9C 1A>I<0000E000000000E000000001F000000001F000000001F000000003F800000003F8 00000006FC00000006FC0000000EFE0000000C7E0000000C7E000000183F000000183F00 0000303F800000301F800000701FC00000600FC00000600FC00000C007E00000FFFFE000 01FFFFF000018003F000018003F000030001F800030001F800060001FC00060000FC000E 0000FE00FFE00FFFE0FFE00FFFE0231F7E9E28>65 DI<0007FC02003FFF0E00FE03DE03F000FE07E0003E0FC0001E1F8000 1E3F00000E3F00000E7F0000067E0000067E000006FE000000FE000000FE000000FE0000 00FE000000FE000000FE0000007E0000007E0000067F0000063F0000063F00000C1F8000 0C0FC0001807E0003803F0007000FE01C0003FFF800007FC001F1F7D9E26>II II<0007FC0200003FFF0E0000FE 03DE0003F000FE0007E0003E000FC0001E001F80001E003F00000E003F00000E007F0000 06007E000006007E00000600FE00000000FE00000000FE00000000FE00000000FE000000 00FE003FFFE0FE003FFFE07E00007E007E00007E007F00007E003F00007E003F00007E00 1F80007E000FC0007E0007E0007E0003F000FE0000FE01FE00003FFF8E000007FC060023 1F7D9E29>III75 DIII<001FF80000FFFF0001F81F8007E0 07E00FC003F01F8001F81F0000F83F0000FC7F0000FE7E00007E7E00007EFE00007FFE00 007FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007F7E00007E7F00 00FE7F0000FE3F0000FC3F8001FC1F8001F80FC003F007E007E001F81F8000FFFF00001F F800201F7D9E27>II<001FF800 00FFFF0001F81F8007E007E00FC003F01F8001F81F8001F83F0000FC7F0000FE7F0000FE 7E00007EFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007F FE00007F7E00007E7E00007E7F0000FE3F0000FC3F87C1FC1F8FE1F80FD833F007F83FE0 01F81F8000FFFF00001FFE0300001E0300001F0700000FFF00000FFF000007FE000007FE 000003FC000001F8000000F020287D9E27>I I<03FC080FFF381E03F83800F8700078700038F00038F00018F00018F80000FC00007FC0 007FFE003FFF801FFFE00FFFF007FFF000FFF80007F80000FC00007C00003CC0003CC000 3CC0003CE00038E00078F80070FE01E0E7FFC081FF00161F7D9E1D>I<7FFFFFFC7FFFFF FC7C07E07C7007E01C6007E00C6007E00CE007E00EC007E006C007E006C007E006C007E0 060007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0 000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E00003FFFF C003FFFFC01F1E7E9D24>III88 DI<3FFFFF803FFFFF803F803F003E007F0038 007E003800FC007001FC007001F8006003F0006007F0006007E000000FC000001FC00000 1F8000003F0000007F0000007E000000FC000001FC018001F8018003F0018007F0018007 E003800FC003801FC003001F8007003F000F007F001F007E007F00FFFFFF00FFFFFF0019 1F7D9E20>I<0400800E01C0180300300600700E00600C00600C00C01800C01800DC1B80 FE1FC0FF1FE0FF1FE07F0FE03E07C01C038013107B9F1B>92 D<07FC001FFF003F0F803F 07C03F03E03F03E00C03E00003E0007FE007FBE01F03E03C03E07C03E0F803E0F803E0F8 03E0FC05E07E0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007FF801F0FC03E0FC03E0FC07C0FC07C 0300FC0000FC0000FC0000FC0000FC0000FC00007C00007E00003E00603F00C01F81C007 FF0001FC0013147E9317>I<0007F80007F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F801F8F80FFEF81F83F83E01F87E00F87C00F87C00F8FC00F8 FC00F8FC00F8FC00F8FC00F8FC00F87C00F87C00F87E00F83E01F81F07F80FFEFF03F8FF 18207E9F1D>I<01FE0007FF800F83C01E01E03E00F07C00F07C00F8FC00F8FFFFF8FFFF F8FC0000FC0000FC00007C00007C00003E00181E00180F807007FFE000FF8015147F9318 >I<001F8000FFC001F3E003E7E003C7E007C7E007C3C007C00007C00007C00007C00007 C000FFFC00FFFC0007C00007C00007C00007C00007C00007C00007C00007C00007C00007 C00007C00007C00007C00007C00007C00007C0003FFC003FFC0013207F9F10>I<01FC3C 07FFFE0F079E1E03DE3E03E03E03E03E03E03E03E03E03E01E03C00F07800FFF0009FC00 1800001800001C00001FFF800FFFF007FFF81FFFFC3C007C70003EF0001EF0001EF0001E 78003C78003C3F01F80FFFE001FF00171E7F931A>II<1C003E007F007F007F003E001C000000000000 00000000000000FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F 001F001F001F00FFE0FFE00B217EA00E>I107 DIII<01FF0007FFC01F83F03E00F83E00F87C007C7C007CFC007EFC007EFC 007EFC007EFC007EFC007E7C007C7C007C3E00F83E00F81F83F007FFC001FF0017147F93 1A>II<01F81807FE381F87783F01 F83E01F87E00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8FC00F87C00F87E00F87E00 F83F01F81F87F80FFEF803F8F80000F80000F80000F80000F80000F80000F80000F80007 FF0007FF181D7E931C>II<0FE63FFE701E600EE006E006F800FFC07FF83FFC1FFE03FE001FC007C007 E007F006F81EFFFCC7F010147E9315>I<01800180018003800380038007800F803F80FF FCFFFC0F800F800F800F800F800F800F800F800F800F800F860F860F860F860F8607CC03 F801F00F1D7F9C14>IIIIII<3FFFE03FFFE03C07C0380F80701F80603F00603E00607C 0000F80001F80003F00003E06007C0600F80601F80E03F00C03E01C07C03C0FFFFC0FFFF C013147F9317>I E /FE 37 122 df<00000007FF800000000001FFFFF0000000000FFF FFFC000000003FFFFFFE00000000FFFC00FF00000001FFC0003F80000007FF00007FC000 000FFE0001FFC000001FFC0001FFE000001FF80003FFE000003FF00003FFE000003FF000 03FFE000007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE00003FFE000 007FE00001FFC000007FE00000FF8000007FE000003E0000007FE00000000000007FE000 00000000007FE00000000000007FE00000000000007FE00000000000007FE00000000000 007FE0003FFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFF FFFFF000FFFFFFFFFFFFF000007FF00000FFF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF0003FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE0 3FFFFFE03FFFFFE03FFFFFE03FFFFFE03B487EC742>12 D<07C0001FF0003FFC007FFC00 7FFE00FFFE00FFFF00FFFF00FFFF80FFFF80FFFF807FFF807FFF803FFF801FFF8007E780 000780000780000F00000F00000F00000F00001E00001E00003C00003C0000780000F800 00F00001F00003E00007C0000F80001F00001E00000C00001124788F21>44 D<0000000003E000000000000000000007F000000000000000000007F000000000000000 00000FF80000000000000000000FF80000000000000000000FF80000000000000000001F FC0000000000000000001FFC0000000000000000003FFE0000000000000000003FFE0000 000000000000007FFF0000000000000000007FFF0000000000000000007FFF0000000000 00000000FFFF800000000000000000FFFF800000000000000001FFFFC000000000000000 01FFFFC00000000000000001FFFFC00000000000000003FFFFE00000000000000003EFFF E00000000000000007EFFFF00000000000000007CFFFF00000000000000007C7FFF00000 00000000000FC7FFF8000000000000000F83FFF8000000000000001F83FFFC0000000000 00001F03FFFC000000000000001F01FFFC000000000000003F01FFFE000000000000003E 00FFFE000000000000007E00FFFF000000000000007C007FFF00000000000000FC007FFF 80000000000000F8007FFF80000000000000F8003FFF80000000000001F8003FFFC00000 00000001F0001FFFC0000000000003F0001FFFE0000000000003E0001FFFE00000000000 03E0000FFFE0000000000007E0000FFFF0000000000007C00007FFF000000000000FC000 07FFF800000000000F800007FFF800000000000F800003FFF800000000001F800003FFFC 00000000001F000001FFFC00000000003FFFFFFFFFFE00000000003FFFFFFFFFFE000000 00007FFFFFFFFFFF00000000007FFFFFFFFFFF00000000007FFFFFFFFFFF0000000000FC 0000007FFF8000000000F80000003FFF8000000001F80000003FFFC000000001F0000000 3FFFC000000001F00000001FFFC000000003F00000001FFFE000000003E00000000FFFE0 00000007E00000000FFFF000000007C00000000FFFF000000007C000000007FFF0000000 0FC000000007FFF80000000F8000000003FFF80000001F8000000003FFFC0000001F0000 000003FFFC0000007FC000000001FFFC0000FFFFFFC00007FFFFFFFF80FFFFFFC00007FF FFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF 8051487CC75A>65 DI<000000003FFE00000E0000000FFFFFC0001E0000007FFFFFF8003E0000 03FFFFFFFE00FE00000FFFFFFFFF81FE00003FFFF800FFC3FE0000FFFF80000FF7FE0001 FFFC000003FFFE0007FFF0000001FFFE000FFFC00000007FFE001FFF800000003FFE003F FF000000001FFE007FFE000000000FFE00FFFC0000000007FE01FFF80000000007FE03FF F00000000003FE03FFF00000000001FE07FFE00000000001FE07FFE00000000000FE0FFF C00000000000FE0FFFC000000000007E1FFFC000000000007E1FFF8000000000007E3FFF 8000000000007E3FFF8000000000003E3FFF8000000000003E7FFF8000000000003E7FFF 0000000000003E7FFF000000000000007FFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF000000000000007FFF 000000000000007FFF000000000000007FFF000000000000007FFF8000000000003E3FFF 8000000000003E3FFF8000000000003E3FFF8000000000003E1FFF8000000000003E1FFF C000000000003E0FFFC000000000007C0FFFC000000000007C07FFE000000000007C07FF E00000000000F803FFF00000000000F803FFF00000000001F801FFF80000000001F000FF FC0000000003E0007FFE0000000007E0003FFF000000000FC0001FFF800000001F80000F FFC00000003F000007FFF0000000FE000001FFFC000001FC000000FFFF80000FF8000000 3FFFF8007FF00000000FFFFFFFFFC000000003FFFFFFFF00000000007FFFFFFC00000000 000FFFFFE00000000000003FFE000000474979C756>I69 DI<000000003FFE00000E000000000FFFFFC0001E000000007FFFFFF8003E00000003FF FFFFFE00FE0000000FFFFFFFFF81FE0000003FFFF800FFC3FE000000FFFF80000FF7FE00 0001FFFC000003FFFE000007FFF0000001FFFE00000FFFC00000007FFE00001FFF800000 003FFE00003FFF000000001FFE00007FFE000000000FFE0000FFFC0000000007FE0001FF F80000000007FE0003FFF00000000003FE0003FFF00000000001FE0007FFE00000000001 FE0007FFE00000000000FE000FFFC00000000000FE000FFFC000000000007E001FFFC000 000000007E001FFF8000000000007E003FFF8000000000007E003FFF8000000000003E00 3FFF8000000000003E007FFF8000000000003E007FFF0000000000003E007FFF00000000 000000007FFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF 0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF000000000000 0000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000 000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000007FFFFFFFFE 7FFF0000007FFFFFFFFE7FFF0000007FFFFFFFFE7FFF8000007FFFFFFFFE7FFF8000007F FFFFFFFE3FFF8000000000FFFE003FFF8000000000FFFE003FFF8000000000FFFE001FFF 8000000000FFFE001FFFC000000000FFFE000FFFC000000000FFFE000FFFC000000000FF FE0007FFE000000000FFFE0007FFE000000000FFFE0003FFF000000000FFFE0003FFF800 000000FFFE0001FFF800000000FFFE0000FFFC00000000FFFE00007FFE00000000FFFE00 003FFF00000000FFFE00001FFF80000000FFFE00000FFFE0000001FFFE000007FFF00000 03FFFE000001FFFC000007FFFE000000FFFF80001FFFFE0000003FFFFC00FFC7FE000000 0FFFFFFFFF83FE00000003FFFFFFFE00FE000000007FFFFFF8003E000000000FFFFFE000 0E00000000003FFE000000004F4979C75D>I73 D76 DI80 D<00007FF00007000007FFFF000F00001FFFFFC01F00 007FFFFFF03F0000FFFFFFFC7F0003FFC00FFEFF0007FE0000FFFF0007FC00003FFF000F F000001FFF001FF0000007FF001FE0000003FF003FE0000001FF003FC0000001FF007FC0 000000FF007FC00000007F007FC00000007F00FFC00000003F00FFC00000003F00FFC000 00003F00FFE00000003F00FFE00000001F00FFF00000001F00FFF80000001F00FFFC0000 001F00FFFF00000000007FFFC0000000007FFFFE000000007FFFFFE00000003FFFFFFF00 00003FFFFFFFF000001FFFFFFFFC00001FFFFFFFFF00000FFFFFFFFFC00007FFFFFFFFE0 0007FFFFFFFFF00003FFFFFFFFF80000FFFFFFFFFC00007FFFFFFFFE00003FFFFFFFFE00 000FFFFFFFFF000001FFFFFFFF0000003FFFFFFF80000001FFFFFF800000000FFFFFC000 000000FFFFC0000000001FFFC0000000000FFFE00000000007FFE00000000003FFE07800 000001FFE0F800000000FFE0F800000000FFE0F8000000007FE0F8000000007FE0F80000 00007FE0FC000000007FE0FC000000007FC0FC000000007FC0FE000000007FC0FE000000 007F80FF00000000FF80FF80000000FF80FFC0000000FF00FFE0000001FE00FFF8000003 FE00FFFE000007FC00FFFF80000FF800FFFFFC007FF000FE3FFFFFFFE000FC0FFFFFFF80 00F803FFFFFF0000F0007FFFF80000E00003FFC00000334979C742>83 D<3FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFF FFFFFFFF003FFFFFFFFFFFFFFFFF003FFF0003FFF8003FFF007FF80003FFF80007FF807F E00003FFF80001FF807FC00003FFF80000FF807F800003FFF800007F807F000003FFF800 003F807F000003FFF800003F807E000003FFF800001F807E000003FFF800001F807E0000 03FFF800000F807C000003FFF800000F807C000003FFF800000F807C000003FFF800000F 807C000003FFF800000F80FC000003FFF800000FC0F8000003FFF8000007C0F8000003FF F8000007C0F8000003FFF8000007C0F8000003FFF8000007C0F8000003FFF8000007C000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 00000000000003FFF80000000000000003FFF80000000000000003FFF800000000000000 03FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8000000 0000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF80000000000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 00000000000003FFF80000000000000003FFF80000000000000003FFF800000000000000 03FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8000000 0000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF80000000000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 0000000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FF FFFFFFFFF800000003FFFFFFFFFFF800004A467CC553>I<0007FFFC000000007FFFFFC0 000001FFFFFFF8000003FFFFFFFE000007FE001FFF000007FF0003FFC0000FFF8001FFE0 000FFF8000FFF0000FFF80007FF0000FFF80007FF8000FFF80007FF80007FF00003FFC00 07FF00003FFC0003FE00003FFC0000F800003FFC00000000003FFC00000000003FFC0000 0000003FFC00000000003FFC00000007FFFFFC000000FFFFFFFC000007FFFFFFFC00003F FFE03FFC0000FFFE003FFC0003FFF0003FFC0007FFC0003FFC000FFF00003FFC001FFE00 003FFC003FFC00003FFC007FF800003FFC007FF800003FFC00FFF000003FFC00FFF00000 3FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000007FFC007FF80000FF FC007FF80001EFFC003FFC0003EFFC003FFF0007CFFF000FFFC03F8FFFF807FFFFFF07FF FC01FFFFFC03FFFC007FFFF001FFFC0003FF80007FF8362E7DAD3A>97 D<007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0 0000000000FFFFC0000000000003FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00FFC0000 0001FFC07FFFC0000001FFC3FFFFF0000001FFCFFFFFFC000001FFDFF00FFF000001FFFF 8003FF800001FFFE0001FFC00001FFF800007FE00001FFF000007FF00001FFE000003FF8 0001FFE000001FFC0001FFE000001FFC0001FFE000001FFE0001FFE000001FFE0001FFE0 00000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF 8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE0 00000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF 8001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000001FFE0001FFE0 00001FFE0001FFE000001FFC0001FFE000001FFC0001FFE000003FF80001FFF000003FF8 0001FFF800007FF00001FFFC0000FFE00001FFFE0001FFC00001FFBF0007FF800001FF1F E01FFE000001FE0FFFFFFC000001FC03FFFFF0000001F800FFFF80000001F0001FF80000 0039487CC742>I<00001FFFC0000000FFFFF8000007FFFFFE00001FFFFFFF80007FFC00 FFC000FFE001FFC001FFC003FFE003FF8003FFE007FF0003FFE00FFE0003FFE00FFE0003 FFE01FFC0001FFC01FFC0001FFC03FFC0000FF803FFC00003E007FF8000000007FF80000 00007FF800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF80000 0000FFF800000000FFF800000000FFF800000000FFF800000000FFF8000000007FF80000 00007FF8000000007FFC000000003FFC000000003FFC000000001FFC000000F81FFE0000 00F80FFE000000F80FFF000001F007FF800003F003FFC00007E001FFE0000FC000FFF000 1F80007FFE00FF00001FFFFFFE000007FFFFF8000000FFFFE00000001FFE00002D2E7CAD 35>I<00000000007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000 000000FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000000FFC01 FFC0000000FFFF81FFC0000007FFFFE1FFC000001FFFFFF9FFC000007FFC03FFFFC00000 FFF0007FFFC00001FFC0001FFFC00003FF80000FFFC00007FF000007FFC0000FFE000003 FFC0000FFE000003FFC0001FFC000003FFC0001FFC000003FFC0003FFC000003FFC0003F FC000003FFC0007FF8000003FFC0007FF8000003FFC0007FF8000003FFC000FFF8000003 FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FF F8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003 FFC0007FF8000003FFC0007FF8000003FFC0007FF8000003FFC0003FF8000003FFC0003F FC000003FFC0003FFC000003FFC0001FFC000003FFC0001FFC000003FFC0000FFE000007 FFC00007FF00000FFFC00003FF00001FFFC00001FFC0003FFFC00000FFE000FFFFE00000 7FF807FBFFFF80001FFFFFF3FFFF800007FFFFC3FFFF800001FFFF03FFFF8000001FF803 FFFF8039487CC742>I<00001FFE00000001FFFFE0000007FFFFF800001FFFFFFE00007F FC07FF0000FFE001FF8001FFC0007FC003FF80003FE007FF00003FF00FFE00001FF01FFE 00000FF81FFC00000FF83FFC00000FFC3FFC000007FC7FFC000007FC7FF8000007FC7FF8 000007FE7FF8000007FEFFF8000007FEFFF8000007FEFFFFFFFFFFFEFFFFFFFFFFFEFFFF FFFFFFFEFFFFFFFFFFFCFFF800000000FFF800000000FFF800000000FFF8000000007FF8 000000007FF8000000007FFC000000003FFC000000003FFC000000003FFC0000001C1FFE 0000003E0FFE0000003E07FF0000007E07FF000000FC03FF800001F801FFC00003F0007F F0001FE0003FFE00FFC0001FFFFFFF800007FFFFFE000000FFFFF80000000FFF80002F2E 7DAD36>I<00000000001F8000007FF000FFE00007FFFF03FFF0001FFFFFC7FFF0007FFF FFFFC7F800FFE03FFE0FF801FF800FFC0FF803FF0007FE0FF807FE0003FF07F007FE0003 FF07F00FFC0001FF81C00FFC0001FF80000FFC0001FF80001FFC0001FFC0001FFC0001FF C0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0000FFC0001FF80 000FFC0001FF80000FFC0001FF800007FE0003FF000007FE0003FF000003FF0007FE0000 01FF800FFC000000FFE03FF8000001FFFFFFF0000001DFFFFFC0000003C7FFFF00000003 C07FF000000007C0000000000007C0000000000007C0000000000007C0000000000007E0 000000000007F0000000000007F8000000000007FFFFFFF0000007FFFFFFFF000003FFFF FFFFE00003FFFFFFFFF80001FFFFFFFFFE0001FFFFFFFFFF0000FFFFFFFFFF80007FFFFF FFFF8003FFFFFFFFFFC00FFFFFFFFFFFC01FF800001FFFE03FE0000001FFE07FC0000000 7FF07FC00000003FF0FF800000001FF0FF800000001FF0FF800000001FF0FF800000001F F0FF800000001FF07FC00000003FE07FC00000003FE03FE00000007FC03FF0000000FFC0 1FFC000003FF800FFF00000FFF0003FFF000FFFC0000FFFFFFFFF000003FFFFFFFC00000 07FFFFFE000000003FFFC0000035447DAE3B>103 D<007FC00000000000FFFFC0000000 0000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FF C0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000 000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FF C0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000 000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FF C0000000000001FFC0000000000001FFC001FFC0000001FFC00FFFF8000001FFC03FFFFE 000001FFC0FFFFFF000001FFC1FC07FF800001FFC3E003FFC00001FFC7C001FFC00001FF CF0001FFE00001FFDE0000FFE00001FFDC0000FFE00001FFFC0000FFF00001FFF80000FF F00001FFF00000FFF00001FFF00000FFF00001FFF00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF000FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFF FFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B487CC742>I<00FC0001FE0003 FF0007FF800FFFC01FFFE01FFFE01FFFE01FFFE01FFFE01FFFE00FFFC007FF8003FF0001 FE0000FC0000000000000000000000000000000000000000000000000000000000000000 0000007FC0FFFFC0FFFFC0FFFFC0FFFFC0FFFFC003FFC001FFC001FFC001FFC001FFC001 FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001 FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001 FFC001FFC001FFC001FFC001FFC001FFC0FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF18497CC8 20>I<007FC000000000FFFFC000000000FFFFC000000000FFFFC000000000FFFFC00000 0000FFFFC00000000003FFC00000000001FFC00000000001FFC00000000001FFC0000000 0001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC000000000 01FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001 FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FF C00000000001FFC0007FFFF801FFC0007FFFF801FFC0007FFFF801FFC0007FFFF801FFC0 007FFFF801FFC0000FFE0001FFC00007F00001FFC0000FE00001FFC0003FC00001FFC000 7F800001FFC000FE000001FFC001FC000001FFC007F8000001FFC00FF0000001FFC01FC0 000001FFC03F80000001FFC0FF00000001FFC1FE00000001FFC3FF00000001FFCFFF8000 0001FFDFFFC0000001FFFFFFC0000001FFFFFFE0000001FFFFFFF0000001FFFCFFF80000 01FFF87FFC000001FFE03FFC000001FFC01FFE000001FFC01FFF000001FFC00FFF800001 FFC007FFC00001FFC003FFC00001FFC001FFE00001FFC001FFF00001FFC000FFF80001FF C0007FFC0001FFC0003FFC0001FFC0001FFE0001FFC0000FFF0001FFC0000FFF8001FFC0 000FFFC0FFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFFFFFFFF80 7FFFFF38487CC73F>107 D<007FC000FFFFC000FFFFC000FFFFC000FFFFC000FFFFC000 03FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000FFFFFF80FFFFFF80 FFFFFF80FFFFFF80FFFFFF8019487CC720>I<007FC001FFC00000FFE00000FFFFC00FFF F80007FFFC0000FFFFC03FFFFE001FFFFF0000FFFFC0FFFFFF007FFFFF8000FFFFC1FC07 FF80FE03FFC000FFFFC3E003FFC1F001FFE00003FFC7C001FFC3E000FFE00001FFCF0001 FFE78000FFF00001FFDE0000FFEF00007FF00001FFDC0000FFEE00007FF00001FFFC0000 FFFE00007FF80001FFF80000FFFC00007FF80001FFF00000FFF800007FF80001FFF00000 FFF800007FF80001FFF00000FFF800007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF800FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07F FFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF05C2E7CAD63 >I<007FC001FFC00000FFFFC00FFFF80000FFFFC03FFFFE0000FFFFC0FFFFFF0000FFFF C1FC07FF8000FFFFC3E003FFC00003FFC7C001FFC00001FFCF0001FFE00001FFDE0000FF E00001FFDC0000FFE00001FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FF F00000FFF00001FFF00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF000FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFF FFE0FFFFFFC07FFFFFE03B2E7CAD42>I<00000FFF0000000000FFFFF000000007FFFFFE 0000001FFFFFFF8000003FFC03FFC00000FFE0007FF00001FF80001FF80003FF00000FFC 0007FE000007FE000FFE000007FF000FFC000003FF001FFC000003FF803FFC000003FFC0 3FF8000001FFC03FF8000001FFC07FF8000001FFE07FF8000001FFE07FF8000001FFE0FF F8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8 000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF07FF800 0001FFE07FF8000001FFE07FF8000001FFE07FF8000001FFE03FFC000003FFC03FFC0000 03FFC01FFC000003FF801FFE000007FF800FFE000007FF0007FF00000FFE0003FF80001F FC0001FFC0003FF80000FFE0007FF000007FFC03FFE000001FFFFFFF80000007FFFFFE00 000000FFFFF0000000000FFF000000342E7DAD3B>I<007FC00FFC000000FFFFC07FFFC0 0000FFFFC3FFFFF00000FFFFCFFFFFFC0000FFFFDFF01FFF0000FFFFFF8007FF800003FF FE0001FFC00001FFF80000FFE00001FFF00000FFF00001FFE000007FF80001FFE000003F FC0001FFE000003FFC0001FFE000003FFE0001FFE000001FFE0001FFE000001FFF0001FF E000001FFF0001FFE000001FFF0001FFE000000FFF0001FFE000000FFF8001FFE000000F FF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FF E000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000F FF0001FFE000001FFF0001FFE000001FFF0001FFE000001FFE0001FFE000001FFE0001FF E000003FFC0001FFE000003FFC0001FFE000007FF80001FFF000007FF80001FFF80000FF F00001FFFC0001FFE00001FFFE0003FFC00001FFFF0007FF800001FFFFE03FFE000001FF EFFFFFFC000001FFE3FFFFF0000001FFE0FFFF80000001FFE01FF800000001FFE0000000 000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FF E0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000 000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FF E00000000000FFFFFFC000000000FFFFFFC000000000FFFFFFC000000000FFFFFFC00000 0000FFFFFFC00000000039427CAD42>I<00000FFC0003C0000000FFFF0007C0000007FF FFC00FC000001FFFFFF01FC000003FFE03F81FC00000FFF000FC3FC00001FFE0003E7FC0 0003FFC0001F7FC00007FF80001FFFC0000FFF00000FFFC0000FFF000007FFC0001FFE00 0007FFC0001FFE000003FFC0003FFC000003FFC0003FFC000001FFC0007FFC000001FFC0 007FFC000001FFC0007FF8000001FFC000FFF8000001FFC000FFF8000001FFC000FFF800 0001FFC000FFF8000001FFC000FFF8000001FFC000FFF8000001FFC000FFF8000001FFC0 00FFF8000001FFC000FFF8000001FFC000FFF8000001FFC0007FF8000001FFC0007FFC00 0001FFC0007FFC000001FFC0003FFC000001FFC0003FFC000001FFC0003FFE000003FFC0 001FFE000003FFC0001FFE000007FFC0000FFF00000FFFC00007FF80001FFFC00003FF80 003FFFC00001FFC0007FFFC00000FFF000FFFFC000007FFC07FBFFC000001FFFFFE3FFC0 000007FFFFC3FFC0000001FFFF03FFC00000001FF803FFC0000000000003FFC000000000 0003FFC0000000000003FFC0000000000003FFC0000000000003FFC0000000000003FFC0 000000000003FFC0000000000003FFC0000000000003FFC0000000000003FFC000000000 0003FFC0000000000003FFC0000000000003FFC0000000000003FFC0000000000003FFC0 0000000001FFFFFF8000000001FFFFFF8000000001FFFFFF8000000001FFFFFF80000000 01FFFFFF8039427CAD3F>I<00FF803F8000FFFF80FFF000FFFF83FFFC00FFFF87FFFE00 FFFF8FC3FF00FFFF8F07FF0003FF9E0FFF8001FFBC0FFF8001FFB80FFF8001FFF80FFF80 01FFF00FFF8001FFF007FF0001FFF007FF0001FFE003FE0001FFE000F80001FFE0000000 01FFE000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC0000000FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000 292E7CAD31>I<000FFF00E0007FFFF3E001FFFFFFE007FFFFFFE00FF800FFE01FC0001F E03F80000FE03F000007E07F000003E07F000003E0FF000003E0FF000003E0FF800003E0 FFC0000000FFF0000000FFFE000000FFFFF800007FFFFFC0007FFFFFF0003FFFFFFC001F FFFFFF000FFFFFFF8007FFFFFFC003FFFFFFE000FFFFFFF0003FFFFFF00003FFFFF80000 1FFFF8000000FFFC0000001FFC7800000FFCF8000007FCF8000003FCFC000003FCFC0000 03FCFE000003F8FE000003F8FF000003F8FF800007F0FFC0000FF0FFF0001FE0FFFC00FF C0FFFFFFFF80FC7FFFFE00F81FFFF800E003FF8000262E7CAD2F>I<0001F000000001F0 00000001F000000001F000000001F000000001F000000003F000000003F000000003F000 000007F000000007F000000007F00000000FF00000000FF00000001FF00000003FF00000 003FF00000007FF0000001FFF0000003FFF000000FFFFFFFC0FFFFFFFFC0FFFFFFFFC0FF FFFFFFC0FFFFFFFFC000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FF F0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0 000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF000 0000FFF0000000FFF0000000FFF0000000FFF001F000FFF001F000FFF001F000FFF001F0 00FFF001F000FFF001F000FFF001F000FFF001F000FFF001F0007FF001E0007FF803E000 3FF803E0003FFC07C0001FFE0F80000FFFFF800007FFFE000001FFFC0000001FF0002442 7EC12E>I<007FE000003FF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF0 00FFFFE0007FFFF000FFFFE0007FFFF00003FFE00001FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00001FFF00001FFE00001FFF00001FFE0 0001FFF00001FFE00003FFF00000FFE00007FFF00000FFE0000F7FF000007FE0001F7FF0 00007FF0003E7FF800003FFC00FC7FFFE0001FFFFFF87FFFE00007FFFFE07FFFE00001FF FF807FFFE000003FFE007FFFE03B2E7CAD42>III<7FFFFF801FFFFF007FFFFF80 1FFFFF007FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF00007FF80001FF0000 007FFC0001FC0000003FFE0003F80000001FFF0007F00000000FFF0007E000000007FF80 0FC000000003FFC01F8000000003FFE03F8000000001FFF07F0000000000FFF8FE000000 00007FF9FC00000000003FFFF800000000003FFFF000000000001FFFE000000000000FFF C0000000000007FFC0000000000003FFC0000000000001FFE0000000000001FFF0000000 000001FFF8000000000003FFFC000000000003FFFE000000000007FFFE00000000000FEF FF00000000001FCFFF80000000003F87FFC0000000007F03FFE000000000FE01FFE00000 0001FC00FFF000000001F8007FF800000003F0007FFC00000007F0003FFE0000000FE000 1FFF0000001FC0000FFF0000003F800007FF800000FF800007FFC000FFFFF8003FFFFFC0 FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC03A2E7EAD 3F>I<7FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807F FFFFC000FFFF8000FFF000000FE00000FFF800000FC00000FFF800000FC000007FFC0000 0F8000007FFC00001F8000003FFC00001F0000003FFE00003F0000001FFE00003E000000 1FFF00007E0000000FFF00007C0000000FFF8000FC00000007FF8000F800000007FFC001 F800000003FFC001F000000003FFE003F000000003FFE003F000000001FFF003E0000000 01FFF007E000000000FFF007C000000000FFF80FC0000000007FF80F80000000007FFC1F 80000000003FFC1F00000000003FFE3F00000000001FFE3E00000000001FFF7E00000000 000FFF7C00000000000FFFFC00000000000FFFFC000000000007FFF8000000000007FFF8 000000000003FFF0000000000003FFF0000000000001FFE0000000000001FFE000000000 0000FFC0000000000000FFC00000000000007F800000000000007F800000000000003F00 0000000000003F000000000000003F000000000000003E000000000000007E0000000000 00007C00000000000000FC000000001F8000F8000000003FC001F8000000007FE001F000 000000FFF003F000000000FFF003E000000000FFF007E000000000FFF00FC000000000FF F01F8000000000FFF03F80000000007FE07F00000000007F43FE00000000003FFFF80000 0000001FFFF0000000000007FFC0000000000001FE00000000000039427EAD3F>I E /FF 47 122 df<0000C018000000C018000000C0180000018030000001803000000180 3000000180300000030060000003006000000300600000030060000003006000000600C0 00000600C000000600C000000600C000000C018000FFFFFFFFC0FFFFFFFFC00018030000 0018030000001803000000180300000030060000003006000000300600000030060000FF FFFFFFC0FFFFFFFFC000600C000000C018000000C018000000C018000000C01800000180 300000018030000001803000000180300000030060000003006000000300600000030060 00000600C000000600C000000600C00000222D7DA229>35 D<70F8FCFC74040404040808 10102040060F7C840E>44 DI<70F8F8F87005057C840E>I<01F0 00071C000C06001803003803803803807001C07001C07001C07001C0F001E0F001E0F001 E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E07001 C07001C07001C07803C03803803803801C07000C0600071C0001F00013227EA018>48 D<008003800F80F380038003800380038003800380038003800380038003800380038003 80038003800380038003800380038003800380038003800380038007C0FFFE0F217CA018 >I<03F8000C1E001007002007804007C07807C07803C07807C03807C000078000078000 0700000F00000E0000380003F000001C00000F000007800007800003C00003C00003E020 03E07003E0F803E0F803E0F003C04003C0400780200780100F000C1C0003F00013227EA0 18>51 D<000200000600000E00000E00001E00001E00002E00004E00004E00008E00008E 00010E00020E00020E00040E00040E00080E00100E00100E00200E00200E00400E00800E 00FFFFF8000E00000E00000E00000E00000E00000E00000E00001F0001FFF015217FA018 >I<1000801E07001FFF001FFE001FF80013E00010000010000010000010000010000010 000010F800130E001407001803801003800001C00001C00001E00001E00001E00001E070 01E0F001E0F001E0E001C08001C04003C04003802007001006000C1C0003F00013227EA0 18>I<007E0001C1000300800601C00E03C01C03C0180180380000380000780000700000 700000F0F800F30C00F40600F40300F80380F801C0F001C0F001E0F001E0F001E0F001E0 F001E07001E07001E07001E03801C03801C01803801C03000C0600070C0001F00013227E A018>I<4000006000007FFFE07FFFC07FFFC0400080C001008001008002008002000004 0000080000080000100000300000200000600000600000600000E00000C00000C00001C0 0001C00001C00001C00003C00003C00003C00003C00003C00003C00003C00003C0000180 0013237DA118>I<01F800060E000803001001802001802000C06000C06000C06000C070 00C07801803E01003F02001FC4000FF80003F80003FC00067F00083F80100F803007C060 01C06000E0C000E0C00060C00060C00060C000606000406000C03000801803000E0E0003 F00013227EA018>I<01F000060C000C0600180700380380700380700380F001C0F001C0 F001C0F001E0F001E0F001E0F001E0F001E07001E07003E03803E01805E00C05E00619E0 03E1E00001C00001C00001C0000380000380300300780700780600700C00201800103000 0FC00013227EA018>I<0001800000018000000180000003C0000003C0000003C0000005 E0000005E000000DF0000008F0000008F0000010F800001078000010780000203C000020 3C0000203C0000401E0000401E0000401E0000800F0000800F0000FFFF00010007800100 0780030007C0020003C0020003C0040003E0040001E0040001E00C0000F00C0000F03E00 01F8FF800FFF20237EA225>65 D<0007E0100038183000E0063001C00170038000F00700 00F00E0000701E0000701C0000303C0000303C0000307C0000107800001078000010F800 0000F8000000F8000000F8000000F8000000F8000000F8000000F8000000780000007800 00107C0000103C0000103C0000101C0000201E0000200E000040070000400380008001C0 010000E0020000381C000007E0001C247DA223>67 D69 DI73 D77 DI80 D82 D<03F0200C0C601802603001 E07000E0600060E00060E00060E00020E00020E00020F00000F000007800007F00003FF0 001FFE000FFF0003FF80003FC00007E00001E00000F00000F00000708000708000708000 70800070C00060C00060E000C0F000C0C80180C6070081FC0014247DA21B>I<7FFFFFF8 7807807860078018400780084007800840078008C007800C800780048007800480078004 800780040007800000078000000780000007800000078000000780000007800000078000 000780000007800000078000000780000007800000078000000780000007800000078000 00078000000780000007800000078000000FC00003FFFF001E227EA123>I<0FE0001838 003C0C003C0E0018070000070000070000070000FF0007C7001E07003C07007807007007 00F00708F00708F00708F00F087817083C23900FC1E015157E9418>97 D<0E0000FE00001E00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E1F000E61C00E80600F00300E00380E003C0E001C0E001E0E001E0E00 1E0E001E0E001E0E001E0E001E0E001C0E003C0E00380F00700C80600C41C0083F001723 7FA21B>I<01FE000703000C07801C0780380300780000700000F00000F00000F00000F0 0000F00000F00000F000007000007800403800401C00800C010007060001F80012157E94 16>I<0000E0000FE00001E00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000E001F8E00704E00C02E01C01E03800E07800E07000E0F000E0F000E0 F000E0F000E0F000E0F000E0F000E07000E07800E03800E01801E00C02E0070CF001F0FE 17237EA21B>I<01FC000707000C03801C01C03801C07801E07000E0F000E0FFFFE0F000 00F00000F00000F00000F000007000007800203800201C00400E008007030000FC001315 7F9416>I<003C00C6018F038F030F070007000700070007000700070007000700FFF807 000700070007000700070007000700070007000700070007000700070007000700070007 807FF8102380A20F>I<00007001F198071E180E0E181C07001C07003C07803C07803C07 803C07801C07001C07000E0E000F1C0019F0001000001000001800001800001FFE000FFF C00FFFE03800F0600030400018C00018C00018C000186000306000303800E00E038003FE 0015217F9518>I<0E0000FE00001E00000E00000E00000E00000E00000E00000E00000E 00000E00000E00000E00000E00000E1F800E60C00E80E00F00700F00700E00700E00700E 00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E 0070FFE7FF18237FA21B>I<1C001E003E001E001C000000000000000000000000000000 00000E00FE001E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E00FFC00A227FA10E>I<01C003E003E003E001C00000000000000000000000 000000000001E00FE001E000E000E000E000E000E000E000E000E000E000E000E000E000 E000E000E000E000E000E000E000E000E000E000E060E0F0C0F18061803E000B2C82A10F >I<0E0000FE00001E00000E00000E00000E00000E00000E00000E00000E00000E00000E 00000E00000E00000E03FC0E01F00E01C00E01800E02000E04000E08000E10000E38000E F8000F1C000E1E000E0E000E07000E07800E03C00E01C00E01E00E00F00E00F8FFE3FE17 237FA21A>I<0E00FE001E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E00FFE00B237FA20E>I<0E1FC07F00FE60E183801E807201C00F003C00E00F003C00E0 0E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E 003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E00 3800E0FFE3FF8FFE27157F942A>I<0E1F80FE60C01E80E00F00700F00700E00700E0070 0E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E0070 0E0070FFE7FF18157F941B>I<01FC000707000C01801800C03800E0700070700070F000 78F00078F00078F00078F00078F00078F000787000707800F03800E01C01C00E03800707 0001FC0015157F9418>I<0E1F00FE61C00E80600F00700E00380E003C0E001C0E001E0E 001E0E001E0E001E0E001E0E001E0E001E0E003C0E003C0E00380F00700E80E00E41C00E 3F000E00000E00000E00000E00000E00000E00000E00000E00000E0000FFE000171F7F94 1B>I<0E3CFE461E8F0F0F0F060F000E000E000E000E000E000E000E000E000E000E000E 000E000E000F00FFF010157F9413>114 D<0F8830786018C018C008C008E008F0007F80 3FE00FF001F8003C801C800C800CC00CC008E018D0308FC00E157E9413>I<0200020002 0002000600060006000E001E003E00FFF80E000E000E000E000E000E000E000E000E000E 000E000E040E040E040E040E040E040708030801F00E1F7F9E13>I<0E0070FE07F01E00 F00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00 700E00700E00F00E00F006017003827800FC7F18157F941B>III121 D E /FG 20 118 df45 D68 D73 D77 D80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0380001E078 0000E0700000E070000060F0000060F0000060F0000020F0000020F0000020F8000020F8 0000007C0000007E0000003F0000003FC000001FF800000FFF800007FFF80003FFFC0000 FFFF00000FFF800000FFC000001FE0000007E0000003F0000001F0000000F0000000F800 0000F88000007880000078800000788000007880000078C0000078C0000070E00000F0E0 0000E0F00000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF0001D337CB125 >83 D<00FE00000303C0000C00E00010007000100038003C003C003E001C003E001E003E 001E0008001E0000001E0000001E0000001E00000FFE0000FC1E0003E01E000F801E001F 001E003E001E003C001E007C001E00F8001E04F8001E04F8001E04F8003E04F8003E0478 003E047C005E043E008F080F0307F003FC03E01E1F7D9E21>97 D<003F8000E060038018 0700040F00041E001E1C003E3C003E7C003E7C0008780000F80000F80000F80000F80000 F80000F80000F80000F80000F800007800007C00007C00003C00011E00011E00020F0002 07000403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E0 000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0 000001E0000001E0000001E0000001E0000001E0000001E0001F81E000F061E001C019E0 078005E00F0003E00E0003E01E0001E03C0001E03C0001E07C0001E0780001E0F80001E0 F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0780001E0 780001E03C0001E03C0001E01C0001E01E0003E00E0005E0070009E0038011F000E061FF 003F81FF20327DB125>I<003F800000E0E0000380380007003C000E001E001E001E001C 000F003C000F007C000F0078000F8078000780F8000780F8000780FFFFFF80F8000000F8 000000F8000000F8000000F8000000F8000000780000007C0000003C0000003C0000801E 0000800E0001000F0002000780020001C00C0000F03000001FC000191F7E9E1D>I<0007 E0001C1000383800707C00E07C01E07C01C03803C00003C00003C00003C00003C00003C0 0003C00003C00003C00003C00003C00003C000FFFFC0FFFFC003C00003C00003C00003C0 0003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C0 0003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00007E0007FFF 007FFF0016327FB114>I<000000F0007F030801C1C41C0380E81C070070080F0078001E 003C001E003C003E003E003E003E003E003E003E003E003E003E003E003E001E003C001E 003C000F007800070070000780E00009C1C000087F000018000000180000001800000018 000000180000001C0000000E0000000FFFF80007FFFF0003FFFF800E000FC0180001E030 0000F070000070E0000038E0000038E0000038E0000038E0000038700000707000007038 0000E01C0001C00700070001C01C00003FE0001E2F7E9F21>I<07000F801F801F800F80 0700000000000000000000000000000000000000000000000780FF80FF800F8007800780 078007800780078007800780078007800780078007800780078007800780078007800780 07800780078007800FC0FFF8FFF80D307EAF12>105 D<0780FE001FC000FF83078060F0 00FF8C03C18078000F9001E2003C0007A001E4003C0007A000F4001E0007C000F8001E00 07C000F8001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E0007 8000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E000780 00F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000 F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0 001E000FC001F8003F00FFFC1FFF83FFF0FFFC1FFF83FFF0341F7E9E38>109 D<0780FE0000FF83078000FF8C03C0000F9001E00007A001E00007A000F00007C000F000 07C000F000078000F000078000F000078000F000078000F000078000F000078000F00007 8000F000078000F000078000F000078000F000078000F000078000F000078000F0000780 00F000078000F000078000F000078000F000078000F000078000F000078000F0000FC001 F800FFFC1FFF80FFFC1FFF80211F7E9E25>I<001FC00000F0780001C01C00070007000F 0007801E0003C01C0001C03C0001E03C0001E0780000F0780000F0780000F0F80000F8F8 0000F8F80000F8F80000F8F80000F8F80000F8F80000F8F80000F8780000F07C0001F03C 0001E03C0001E01E0003C01E0003C00F00078007800F0001C01C0000F07800001FC0001D 1F7E9E21>I<0783E0FF8C18FF907C0F907C07A07C07C03807C00007C00007C000078000 078000078000078000078000078000078000078000078000078000078000078000078000 0780000780000780000780000780000780000FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030E00010E00010E00010F00010F800007E00 003FF0001FFF000FFFC003FFE0003FF00001F80000F880003C80003C80001CC0001CC000 1CE0001CE00018F00038F00030CC0060C301C080FE00161F7E9E1A>I<00400000400000 400000400000400000C00000C00000C00001C00001C00003C00007C0000FC0001FFFE0FF FFE003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C01003C01003C01003C01003C01003C01003C01003 C01001C02001E02000E0400078C0001F00142C7FAB19>I<078000F000FF801FF000FF80 1FF0000F8001F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0 00078000F000078000F000078000F000078000F000078000F000078000F000078001F000 078001F000078001F000038002F00003C004F00001C008F800007030FF80001FC0FF8021 1F7E9E25>I E /FH 5 85 df<00000000C00000000000E00000000001E00000000003E0 0000000003E00000000007E00000000007E0000000000FE0000000000FE0000000001FE0 000000001FE00000000037E00000000067E00000000067E000000000C7E000000000C7F0 0000000183F00000000183F00000000303F00000000703F00000000603F00000000C03F0 0000000C03F00000001803F00000001803F00000003003F00000003003F00000006003F0 000000C003F0000000C003F00000018003F00000018003F8000003FFFFF8000003FFFFF8 0000060001F800000E0001F800000C0001F80000180001F80000180001F80000300001F8 0000300001F80000600001F80000E00001F80000C00001F80001C00001F80001C00001F8 0007C00001FC001FC00003FC00FFF8007FFFE0FFF8007FFFE02B327BB135>65 D<000FFFFFFE0000000FFFFFFF800000007F000FE00000007E0003F00000007E0000F800 00007E0000FC0000007E00007C000000FC00003E000000FC00003E000000FC00003F0000 00FC00001F000001F800001F000001F800001F800001F800001F800001F800001F800003 F000001F800003F000001F800003F000001F800003F000001F800007E000003F800007E0 00003F800007E000003F800007E000003F80000FC000003F00000FC000007F00000FC000 007F00000FC000007F00001F8000007E00001F800000FE00001F800000FE00001F800000 FC00003F000001FC00003F000001F800003F000001F800003F000003F000007E000003E0 00007E000007E000007E00000FC000007E00000F800000FC00001F800000FC00003F0000 00FC00007E000000FC0000FC000001F80001F0000001F80003E0000001F8000FC0000003 F8007F000000FFFFFFFC000000FFFFFFE000000031317BB036>68 D<000FFFFFFFFC000FFFFFFFFC00007F0001FC00007E00007C00007E00003C00007E0000 3C00007E0000180000FC0000180000FC0000180000FC0000180000FC0000180001F80000 180001F80000180001F80000180001F80000180003F00080100003F00180000003F00180 000003F00180000007E00300000007E00300000007E00700000007E01F0000000FFFFE00 00000FFFFE0000000FC01E0000000FC00E0000001F800C0000001F800C0000001F800C00 00001F800C0000003F00180000003F00080000003F00000000003F00000000007E000000 00007E00000000007E00000000007E0000000000FC0000000000FC0000000000FC000000 0000FC0000000001F80000000001F80000000001F80000000003F800000000FFFFF00000 00FFFFF00000002E317BB02F>70 D<000FFFFFF000000FFFFFFE0000007F003F8000007E 000FC000007E0007E000007E0003F000007E0001F80000FC0001F80000FC0001F80000FC 0001F80000FC0001F80001F80003F80001F80003F80001F80003F80001F80003F00003F0 0007F00003F00007E00003F0000FC00003F0000FC00007E0001F000007E0007E000007E0 00FC000007E007F000000FFFFFC000000FFFFF0000000FC00F8000000FC003C000001F80 03E000001F8001F000001F8001F000001F8001F800003F0001F800003F0001F800003F00 01F800003F0001F800007E0003F800007E0003F800007E0003F000007E0003F00000FC00 07F00000FC0007F00000FC0007F00800FC0007F00C01F80007F01801F80007F01801F800 03F03003F80003F030FFFFE001F0E0FFFFE000FFC0000000003F002E327BB034>82 D<07FFFFFFFFF00FFFFFFFFFF00FC00FE003F01E000FC000F01C000FC000E018000FC000 E038000FC0006030001F8000E030001F8000E060001F8000C060001F8000C060003F0000 C0C0003F0000C0C0003F0000C0C0003F0000C080007E00008000007E00000000007E0000 0000007E0000000000FC0000000000FC0000000000FC0000000000FC0000000001F80000 000001F80000000001F80000000001F80000000003F00000000003F00000000003F00000 000003F00000000007E00000000007E00000000007E00000000007E0000000000FC00000 00000FC0000000000FC0000000000FC0000000001F80000000001F80000000001F800000 00001F80000000003F00000000003F00000000003F0000000000FF00000000FFFFFF0000 00FFFFFF0000002C3173B033>84 D E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 795 908 a FH(D)26 b(R)g(A)f(F)h(T)225 999 y FG(Do)r(cumen)n(t) 20 b(for)i(a)f(Standard)g(Message-P)n(assing)f(In)n(terface)621 1194 y FF(Message)c(P)o(assing)h(In)o(terface)e(F)l(orum)814 1320 y(August)i(7,)f(1993)87 1378 y(This)g(w)o(ork)g(w)o(as)h(supp)q (orted)g(b)o(y)f(ARP)l(A)g(and)g(NSF)g(under)g(con)o(tract)g(n)o(um)o (b)q(er)f(###,)g(b)o(y)g(the)192 1436 y(National)h(Science)f(F)l (oundation)i(Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q (erativ)o(e)76 1494 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o (y)e(the)h(Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)654 1552 y(through)f(Esprit)f(pro)s(ject)g(P6643.)p eop %%Page: 1 2 1 1 bop 75 377 a FE(Con)m(ten)m(ts)75 645 y FD(Ac)o(kno)o(wledgmen)o (ts)1362 b(1)75 747 y(1)42 b(In)o(tro)q(duction)19 b(to)f(MPI)1230 b(2)143 804 y FC(1.1)46 b(Ov)o(erview)16 b(and)f(Goals)38 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(2)143 860 y(1.2)46 b(Who)15 b(Should)h(Use)f(This)h(Standard?)32 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(3)143 917 y(1.3)46 b(What)14 b(Platforms)h(Are)g(T)l(argets)f(F)l(or)h(Implemen)o(tation?) 38 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(3)143 973 y(1.4)46 b(What)14 b(Is)i(Included)h(In)f(The)g (Standard?)42 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(3)143 1030 y(1.5)46 b(What)14 b(Is)i(Not)e(Included)k(In)e(The)f(Standard?)23 b FB(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)91 b FC(4)143 1086 y(1.6)46 b(Organization)16 b(of)e(this)i(Do)q(cumen)o(t)34 b FB(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(4)75 1188 y FD(2)42 b(MPI)17 b(T)l(erms)f(and)i(Con)o(v)o(en)o (tions)1042 b(5)143 1245 y FC(2.1)46 b(Do)q(cumen)o(t)15 b(Con)o(v)o(en)o(tions)e FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)91 b FC(5)143 1302 y(2.2)46 b(Pro)q(cedure)16 b(Sp)q(eci\014cation)44 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)91 b FC(5)143 1358 y(2.3)46 b(Seman)o(tical)16 b(T)l(erms)f FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(6)143 1415 y(2.4)46 b(Data)14 b(T)o(yp)q(es)30 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)91 b FC(6)248 1471 y(2.4.1)50 b(Opaque)16 b(ob)s(jects)21 b FB(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(6)248 1528 y(2.4.2)50 b(Arra)o(y)14 b(of)h(handles)35 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(7)248 1584 y(2.4.3)50 b(State)14 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(7)248 1641 y(2.4.4)50 b(Named)15 b(constan)o(ts)23 b FB(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(7)248 1697 y(2.4.5)50 b(Choice)20 b FB(:)i(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(8)143 1754 y(2.5)46 b(Pro)q(cesses)40 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(8)143 1810 y(2.6)46 b(Error)14 b(Handling)32 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(9)75 1912 y FD(3)42 b(P)o(oin)o(t)17 b(to)h(P)o(oin)o(t)g(Comm)o(unication) 975 b(11)143 1969 y FC(3.1)46 b(In)o(tro)q(duction)15 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FC(11)143 2025 y(3.2)46 b(Basic)16 b(send)f(op)q(eration) 27 b FB(:)c(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(11)248 2082 y(3.2.1)50 b(Message)15 b(data)26 b FB(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(12)248 2138 y(3.2.2)50 b(Message)15 b(en)o(v)o(elop)q(e)j FB(:)k(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(13)143 2195 y(3.3)46 b(Basic)16 b(receiv)o(e)g(op)q(eration)h FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(14)248 2251 y(3.3.1)50 b(Return)16 b(status)j FB(:)k(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(15)143 2308 y(3.4)46 b(Seman)o(tics)15 b(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)40 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b FC(16)143 2364 y(3.5)46 b(Data)14 b(T)o(yp)q(e)h(Matc)o(hing)22 b FB(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)69 b FC(18)143 2421 y(3.6)46 b(Data)14 b(con)o(v)o(ersion)g FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b FC(20)143 2477 y(3.7)46 b(Comm)o(unication)15 b(Mo)q(des)i FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(22)143 2534 y(3.8)46 b(Non)o(blo)q(c)o(king)16 b(comm)o(unication) j FB(:)j(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(24)248 2591 y(3.8.1)50 b(Comm)o(unication)16 b(Ob)s(jects)23 b FB(:)g(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(25)248 2647 y(3.8.2)50 b(Comm)o(unication)16 b(initiation)32 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FC(25)248 2704 y(3.8.3)50 b(Comm)o(unication)16 b(Completion)i FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(26)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 3 2 2 bop 248 45 a FC(3.8.4)50 b(Multiple)17 b(Completions)41 b FB(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(28)143 103 y(3.9)46 b(Prob)q(e)15 b(and)g(Cancel)g FB(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(31)143 162 y(3.10)23 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(ob)s(jects)i FB(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(34)143 220 y(3.11)23 b(Send-receiv)o(e)d FB(:)i(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(37)143 278 y(3.12)23 b(Null)16 b(pro)q(cesses)i FB(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(39)143 337 y(3.13)23 b(Deriv)o(ed)15 b(datat)o(yp)q(es)42 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(40)248 395 y(3.13.1)27 b(Datat)o(yp)q(e)14 b(constructors)27 b FB(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FC(41)248 453 y(3.13.2)27 b(Additional)17 b(functions)30 b FB(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(46)248 511 y(3.13.3)27 b(Use)16 b(of)e(general)i(datat)o(yp)q(es)f (in)h(comm)o(unication)36 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)69 b FC(48)248 570 y(3.13.4)27 b(Examples)33 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(49)248 628 y(3.13.5)27 b(Correct)15 b(use)g(of)g(addresses)26 b FB(:)d(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(54)143 686 y(3.14)23 b(Univ)o(ersal)16 b(comm)o(unication)g(functions)24 b FB(:)f(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b FC(55)248 745 y(3.14.1)27 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(ob)s(jects)e FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FC(56)248 803 y(3.14.2)27 b(Non)o(blo)q(c)o(king)17 b(comm)o(unication)e(initiation)31 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(56)248 861 y(3.14.3)27 b(Comm)o(unication)16 b(completion)31 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(57)248 920 y(3.14.4)27 b(Blo)q(c)o(king)17 b(comm)o(unication)g FB(:)23 b(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b FC(57)248 978 y(3.14.5)27 b(Prob)q(e)16 b(and)f(cancel)23 b FB(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(57)248 1036 y(3.14.6)27 b(Return)16 b(status)j FB(:)k(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(57)248 1094 y(3.14.7)27 b(send-receiv)o(e)17 b(and)e(exc)o(hange)22 b FB(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(57)248 1153 y(3.14.8)27 b(Deriv)o(ed)16 b(datat)o(yp)q(es)37 b FB(:)23 b(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FC(58)248 1211 y(3.14.9)27 b(Summary)33 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(58)75 1316 y FD(4)42 b(Collectiv)o(e)19 b(Comm)o(unication)1077 b(59)143 1374 y FC(4.1)46 b(In)o(tro)q(duction)15 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FC(59)143 1433 y(4.2)46 b(Comm)o(unication)15 b(F)l(unctions)24 b FB(:)f(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(60)143 1491 y(4.3)46 b(Barrier)15 b(sync)o(hronization)44 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(60)143 1549 y(4.4)46 b(Data)14 b(mo)o(v)o(e)g(functions)29 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(60)248 1608 y(4.4.1)50 b(Broadcast)24 b FB(:)e(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(62)248 1666 y(4.4.2)50 b(Gather)13 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(62)248 1724 y(4.4.3)50 b(Scatter)c FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b FC(63)248 1782 y(4.4.4)50 b(All-to-all)17 b(broadcast)d FB(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(63)248 1841 y(4.4.5)50 b(All-to-all)17 b(scatter-gather)36 b FB(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(64)143 1899 y(4.5)46 b(Global)16 b(Compute)e(Op)q(erations)20 b FB(:)i(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b FC(64)248 1957 y(4.5.1)50 b(Reduce)45 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FC(65)248 2016 y(4.5.2)50 b(Scan)25 b FB(:)d(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(69)143 2074 y(4.6)46 b(Correctness)32 b FB(:)22 b(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(70)75 2179 y FD(5)42 b(Pro)q(cess)17 b(T)l(op)q(ologies)1250 b(74)143 2237 y FC(5.1)46 b(In)o(tro)q(duction)15 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FC(74)143 2296 y(5.2)46 b(Virtual)16 b(T)l(op)q(ologies)37 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(75)143 2354 y(5.3)46 b(Em)o(b)q(edding)16 b(in)g(MPI)22 b FB(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(75)143 2412 y(5.4)46 b(Ov)o(erview)16 b(of)e(the)i(prop)q(osed)f (MPI)g(functions)20 b FB(:)j(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(76)248 2470 y(5.4.1)50 b(Lo)o(w{lev)o(el)16 b(top)q(ology)f(functions)37 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FC(77)248 2529 y(5.4.2)50 b(High{lev)o(el)17 b(top)q(ology)e(functions)25 b FB(:)d(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(78)248 2587 y(5.4.3)50 b(T)l(op)q(ology)15 b(inquiry)i(functions) 42 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(80)248 2645 y(5.4.4)50 b(P)o(artitioning)16 b(of)e(Cartesian)h(structures)45 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)69 b FC(81)143 2704 y(5.5)46 b(A)15 b(simple)i(example)f FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(83)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 4 3 3 bop 75 45 a FD(6)42 b(Language)19 b(Binding)1268 b(85)143 106 y FC(6.1)46 b(In)o(tro)q(duction)15 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FC(85)143 168 y(6.2)46 b(Data)14 b(T)o(yp)q(es)h(and)g(Naming)h (Con)o(v)o(en)o(tions)43 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(85)143 229 y(6.3)46 b(F)l(ortran)14 b(77)g(Binding)j(Issues)34 b FB(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(86)143 290 y(6.4)46 b(C)15 b(Binding)i(Issues)35 b FB(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(86)143 352 y(6.5)46 b(Sp)q(eci\014c)17 b(Bindings)32 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(86)248 413 y(6.5.1)50 b(C)15 b(bindings)i(for)e(P)o(oin)o(t-to-P)o (oin)o(t)f(Comm)o(unication)k FB(:)k(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b FC(87)248 474 y(6.5.2)50 b(C)15 b(Bindings)i(for)e (Collectiv)o(e)h(Comm)o(unication)g(Chapter)29 b FB(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(89)248 535 y(6.5.3)50 b(F)l(ortran)14 b(Bindings)j(for)e(P)o(oin)o(t-to-P)o(oin)o (t)f(Routines)34 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b FC(90)248 597 y(6.5.4)50 b(F)l(ortran)14 b(Bindings)j(for)e (Collectiv)o(e)h(Comm)o(unciation)g(Chapter)i FB(:)k(:)h(:)f(:)g(:)h(:) f(:)69 b FC(92)75 707 y FD(7)42 b(En)o(vironmen)o(tal)18 b(Managemen)o(t)f(and)h(Inquiry)743 b(94)143 768 y FC(7.1)46 b(In)o(tro)q(duction)15 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(94)248 829 y(7.1.1)50 b(Program)14 b(Startup)h(Issues)36 b FB(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FC(94)143 890 y(7.2)46 b(MPI)15 b(Program)f(Startup)g(Prop)q (osal{MPI)p 995 890 14 2 v 17 w(INIT)39 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(96)143 952 y(7.3)46 b(MPI-Sp)q(eci\014c)17 b(Enquiry)f(F)l(unctions)f FB(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(96)248 1013 y(7.3.1)50 b(P)o(arallel)16 b(programming)39 b FB(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(98)248 1074 y(7.3.2)50 b(non-MPI)d FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FC(99)75 1184 y FD(8)42 b(Pro\014ling)1458 b(100)143 1245 y FC(8.1)46 b(Requiremen)o(ts)28 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(100)143 1307 y(8.2)g(Discussion)21 b FB(:)i(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(100)143 1368 y(8.3)g(Logic)16 b(of)e(the)i(design)29 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(101)143 1429 y(8.4)g(Examples)36 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(101)248 1491 y(8.4.1)k(Pro\014ler)16 b(implemen)o(tation)35 b FB(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(101)248 1552 y(8.4.2)k(MPI)15 b(library)h(implemen)o(tation)h FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b FC(101)143 1613 y(8.5)g(Multiple)17 b(lev)o(els)f(of)f(in)o(terception)25 b FB(:)d(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FC(103)75 1723 y FD(9)c(Maps,)17 b(Groups)g(and)h(Con)o(texts) 1006 b(104)143 1784 y FC(9.1)46 b(In)o(tro)q(duction)15 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FC(104)143 1846 y(9.2)g(Maps)13 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FC(105)248 1907 y(9.2.1)k(Op)q(erations)16 b(on)f(maps)35 b FB(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(106)248 1968 y(9.2.2)k(Map)15 b(constructors)h FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(107)143 2030 y(9.3)g(Con)o(text)p 411 2030 V 15 w(id)15 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FC(108)248 2091 y(9.3.1)k(Op)q(erations)16 b(on)f(con)o(text)p 840 2091 V 16 w(id's)f FB(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(108)143 2152 y(9.4)g(Comm)o(unicators)20 b FB(:)j(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(109)248 2213 y(9.4.1)k(Op)q(erations)16 b(on)f(comm)o(unicators)h FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b FC(109)248 2275 y(9.4.2)k(Comm)o(unicator) 15 b(constructors)23 b FB(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(110)143 2336 y(9.5)g(W)l(orking)15 b(without)g(con)o(text)p 750 2336 V 16 w(id's)33 b FB(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(111)143 2397 y(9.6)g(W)l(orking)15 b(without)g(maps)g(and)g(con)o (text)p 957 2397 V 16 w(id's)38 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(112)143 2458 y(9.7)g(Examples)36 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(113)248 2520 y(9.7.1)k(Nonreen)o(tran)o(t)14 b(parallel)j(pro)q(cedures)29 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FC(113)248 2581 y(9.7.2)k(P)o(arallel)16 b(pro)q(cedures)g(that)f(are)g(nonreen)o(tran)o(t)f(within)j(eac)o(h)e (executing)h(group)d(114)248 2642 y(9.7.3)50 b(W)l(ell)17 b(nested)e(parallel)i(pro)q(cedures)28 b FB(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(114)143 2704 y(9.8)g(Left)41 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(115)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 5 4 4 bop 75 45 a FD(10)16 b(Initial)k(Implemen)o(tation)e(Subset)967 b(116)143 102 y FC(10.1)23 b(In)o(tro)q(duction)15 b FB(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FC(116)143 158 y(10.2)23 b(Criteria)15 b(and)h(Rational)c FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(116)143 214 y(10.3)23 b(Subset)15 b(F)l(unctionalit)o(y)31 b FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(118)248 271 y(10.3.1)27 b(Itemized)17 b(F)l(unctionalit)o(y)23 b FB(:)f(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(118)248 327 y(10.3.2)27 b(P)o(oin)o(t)15 b(to)g(P)o(oin)o(t)f(F)l(unctionalit)o(y)i FB(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(118)248 384 y(10.3.3)27 b(Collectiv)o(e)17 b(Comm)o(unication)e(F)l(unctionalit)o(y)24 b FB(:)e(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(119)75 486 y FD(MPI)17 b(F)l(unction)h(Index)1252 b(122)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 0 6 0 5 bop 875 722 a FD(Abstract)75 828 y FC(The)18 b(Message)f(P)o (assing)h(In)o(terface)f(F)l(orum)h(\(MPIF\),)e(with)i(participation)h (from)e(o)o(v)o(er)g(40)g(organiza-)75 885 y(tions,)f(has)h(b)q(een)g (meeting)g(since)h(Jan)o(uary)e(1993)f(to)h(discuss)h(and)g(de\014ne)g (a)f(set)h(of)f(library)h(in)o(terface)75 941 y(standards)f(for)h (message)f(passing.)25 b(MPIF)16 b(is)i(not)e(sanctioned)i(or)e(supp)q (orted)h(b)o(y)g(an)o(y)g(o\016cial)g(stan-)75 998 y(dards)e (organization.)166 1054 y(This)22 b(is)f(a)g(draft)f(of)h(what)g(will)h (b)q(ecome)g(the)f(Final)h(Rep)q(ort,)h(V)l(ersion)f(1.0,)f(of)f(the)i (Message)75 1111 y(P)o(assing)c(In)o(terface)h(F)l(orum.)29 b(This)19 b(do)q(cumen)o(t)g(con)o(tains)g(all)g(the)g(tec)o(hnical)h (features)e(prop)q(osed)h(for)75 1167 y(the)c(in)o(terface.)20 b(This)c(cop)o(y)f(of)g(the)g(draft)g(w)o(as)f(pro)q(cessed)i(b)o(y)f (L)1174 1161 y Fz(a)1195 1167 y FC(T)1220 1181 y(E)1246 1167 y(X)g(on)g(August)g(7,)g(1993.)166 1224 y(MPIF)j(in)o(vites)h (commen)o(ts)f(on)h(the)f(tec)o(hnical)i(con)o(ten)o(t)e(of)g(MPI,)g (as)g(w)o(ell)i(as)e(on)g(the)h(editorial)75 1280 y(presen)o(tation)14 b(in)g(the)g(do)q(cumen)o(t.)19 b(Commen)o(ts)13 b(receiv)o(ed)i(b)q (efore)e(July)i(1,)e(1993)g(will)i(b)q(e)f(considered)h(in)75 1336 y(pro)q(ducing)i(the)e(\014nal)h(draft)e(of)h(V)l(ersion)h(1.0)e (of)h(the)g(Message)g(P)o(assing)g(In)o(terface)g(Sp)q(eci\014cation.) 166 1393 y(The)i(goal)f(of)g(the)g(Message)g(P)o(assing)g(In)o (terface,)g(simply)i(stated,)e(is)g(to)g(dev)o(elop)h(a)g(widely)g (used)75 1449 y(standard)g(for)f(writing)i(message-passing)f(programs.) 24 b(As)17 b(suc)o(h)h(the)f(in)o(terface)g(should)h(establishing)75 1506 y(a)d(practical,)h(p)q(ortable,)f(e\016cien)o(t,)g(and)h (\015exible)h(standard)e(for)f(message)h(passing.)p eop %%Page: 1 7 1 6 bop 1852 -100 a FC(1)75 75 y FE(Ac)m(kno)m(wledgmen)m(ts)166 282 y FC(The)19 b(tec)o(hnical)i(dev)o(elopmen)o(t)f(w)o(as)e(carried)i (out)f(b)o(y)g(subgroups,)h(whose)f(w)o(ork)g(w)o(as)f(review)o(ed)75 338 y(b)o(y)d(the)h(full)h(committee.)k(During)15 b(the)h(p)q(erio)q(d) h(of)e(dev)o(elopmen)o(t)h(of)f(the)h(Message)f(P)o(assing)g(In)o (terface)75 395 y(\(MPI\),)f(man)o(y)h(p)q(eople)i(serv)o(ed)e(in)h(p)q (ositions)g(of)f(resp)q(onsibilit)o(y:)143 478 y Fy(\017)23 b FC(Jac)o(k)15 b(Dongarra,)e(Da)o(vid)i(W)l(alk)o(er,)g(Con)o(v)o (eners)g(and)g(Meeting)h(Chairs)143 566 y Fy(\017)23 b FC(Ewing)15 b(Lusk,)h(Bob)f(Knigh)o(ten,)h(Min)o(utes)143 653 y Fy(\017)23 b FC(Marc)14 b(Snir,)i(William)h(Gropp,)d(Ewing)i (Lusk,)f(P)o(oin)o(t)g(to)g(P)o(oin)o(t)f(Comm)o(unications)143 741 y Fy(\017)23 b FC(Al)16 b(Geist,)e(Marc)h(Snir,)h(Collectiv)o(e)g (Comm)o(unications)143 828 y Fy(\017)23 b FC(Rolf)15 b(Hemp)q(el,)i(Pro)q(cess)e(T)l(op)q(ologies)143 915 y Fy(\017)23 b FC(Scott)14 b(Berryman,)h(Ewing)g(Lusk,)h(Language)f (Binding)143 1003 y Fy(\017)23 b FC(Ewing)15 b(Lusk,)h(Correctness)143 1090 y Fy(\017)23 b FC(William)17 b(Gropp,)d(En)o(vironmen)o(tal)i (Managemen)o(t)e(and)h(Inquiry)143 1177 y Fy(\017)23 b FC(James)15 b(Co)o(wnie,)g(Pro\014ling)143 1265 y Fy(\017)23 b FC(T)l(on)o(y)16 b(Skjellum,)h(Lyndon)g(Clark)o(e,)f(Marc)g(Snir,)h (Ric)o(hard)g(Little\014eld,)i(Marc)d(Sears,)g(Comm)o(u-)189 1321 y(nication)g(Con)o(texts)143 1409 y Fy(\017)23 b FC(Stev)o(en)15 b(Huss-Lederman,)h(Initial)h(Implemen)o(tation)f (Subset)143 1496 y Fy(\017)23 b FC(Stev)o(e)15 b(Otto,)f(Editor.)166 1580 y(Do)h(w)o(e)f(w)o(an)o(t)h(an)g(organization/compan)o(y)f(list)i (here?)166 1636 y(The)22 b(follo)o(wing)h(list)f(includes)i(some)e(of)f (the)h(activ)o(e)g(participan)o(ts)h(in)f(the)g(MPI)g(pro)q(cess)g(not) 75 1693 y(men)o(tioned)16 b(ab)q(o)o(v)o(e:)100 1736 y(Ed)f(Anderson)245 b(Jo)q(e)16 b(Baron)206 b(Eric)15 b(Barszcz)100 1793 y(Rob)g(Bjornson)230 b(Anne)16 b(Elster)173 b(Jim)16 b(F)l(eeney)100 1849 y(Vince)g(F)l(ernando)192 b(Sam)15 b(Fineb)q(erg)135 b(Jon)15 b(Flo)o(w)o(er)100 1906 y(Daniel)h(F)l(ry)o(e)274 b(Ian)16 b(Glendinning)90 b(Adam)15 b(Green)o(b)q(erg)100 1962 y(Rob)q(ert)g(Harrison)180 b(Leslie)17 b(Hart)190 b(T)l(om)14 b(Haupt)100 2019 y(Don)h(Heller)290 b(T)l(om)15 b(Henderson)97 b(C.)14 b(T.)h(Ho)o(w)o(ard)f(Ho)100 2075 y(Stev)o(en)h(Huss-Lederman)51 b(John)16 b(Kap)q(enga)123 b(Bob)15 b(Leary)100 2131 y(Arth)o(ur)g(Maccab)q(e)178 b(P)o(eter)15 b(Madams)120 b(Alan)16 b(Main)o(w)o(aring)100 2188 y(Oliv)o(er)g(McBry)o(an)185 b(Phil)17 b(McKinley)123 b(Charles)15 b(Mosher)100 2244 y(Dan)g(Nessett)260 b(P)o(eter)15 b(P)o(ac)o(heco)129 b(Ho)o(w)o(ard)14 b(P)o(almer)100 2301 y(P)o(aul)h(Pierce)278 b(Sanja)o(y)15 b(Rank)m(a)137 b(P)o(eter)14 b(Rigsb)q(ee)100 2357 y(Arc)o(h)h(Robison)234 b(Eric)o(h)16 b(Sc)o(hikuta)118 b(Am)o(buj)15 b(Singh)100 2414 y(Alan)h(Sussman)222 b(Rob)q(ert)16 b(T)l(omlinson)51 b(Rob)q(ert)15 b(G.)g(V)l(oigt)100 2470 y(Dennis)h(W)l(eeks)228 b(Stephen)17 b(Wheat)102 b(Stev)o(e)15 b(Zenith)166 2525 y(The)d(follo)o(wing)h(organizations)f(made)g(the)h(draft)e(a)o(v)m (ailable)j(b)o(y)e(anon)o(ymous)g(FTP)f(access)i(and/or)75 2582 y(mail)h(serv)o(ers:)k(Oak)c(Ridge)g(National)g(Lab)q(oratory)l(,) e(These)i(outlets)f(w)o(ere)g(instrumen)o(tal)h(in)g(distribut-)75 2638 y(ing)i(the)f(do)q(cumen)o(t.)166 2695 y(T)l(alk)g(ab)q(out)g (\014nancial)i(supp)q(ort)f(here.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 8 2 7 bop 75 356 a Fx(Section)35 b(1)75 564 y FE(In)m(tro)s(duction)41 b(to)g(MPI)75 805 y Fw(1.1)59 b(Overview)19 b(and)g(Goals)75 906 y FC(Message)11 b(passing)i(is)f(a)g(paradigm)f(used)i(widely)g(on) f(certain)h(classes)f(of)f(parallel)j(mac)o(hines;)f(esp)q(ecially)75 963 y(those)j(with)i(distributed)g(memory)l(.)24 b(Although)17 b(there)g(are)f(man)o(y)h(v)m(ariations,)g(the)g(basic)g(concept)g(of) 75 1019 y(pro)q(cesses)12 b(comm)o(unicating)h(through)e(messages)g(is) i(w)o(ell)f(understo)q(o)q(d.)20 b(Ov)o(er)12 b(the)f(last)h(ten)g(y)o (ears,)f(sub-)75 1076 y(stan)o(tial)16 b(progress)g(has)g(b)q(een)h (made)f(in)h(casting)g(signi\014can)o(t)g(applications)h(in)f(this)f (paradigm.)23 b(Eac)o(h)75 1132 y(v)o(endor)c(has)h(implemen)o(ted)h (their)f(o)o(wn)f(v)m(arian)o(t.)33 b(More)19 b(recen)o(tly)l(,)j(sev)o (eral)d(systems)g(ha)o(v)o(e)h(demon-)75 1188 y(strated)12 b(that)h(a)g(message)f(passing)i(system)f(can)g(b)q(e)h(e\016cien)o (tly)h(and)e(b)q(e)h(p)q(ortably)f(implemen)o(ted.)21 b(It)13 b(is)75 1245 y(th)o(us)h(an)f(appropriate)h(time)g(to)f(try)g (to)g(de\014ne)i(b)q(oth)f(the)g(syn)o(tax)f(and)h(seman)o(tics)g(of)f (a)g(core)h(of)f(library)75 1301 y(routines)h(that)e(will)j(b)q(e)f (useful)h(to)e(a)g(wide)h(range)f(of)g(users)g(and)h(e\016cien)o(tly)g (implemen)o(table)i(on)d(a)g(wide)75 1358 y(range)i(of)g(computers.)166 1414 y(In)21 b(designing)i(MPI)e(w)o(e)f(ha)o(v)o(e)h(sough)o(t)f(to)g (mak)o(e)g(use)h(of)g(the)g(most)f(attractiv)o(e)f(features)i(of)f(a)75 1471 y(n)o(um)o(b)q(er)12 b(of)g(existing)h(message)e(passing)h (systems,)g(rather)f(than)h(selecting)h(one)f(of)g(them)g(and)g (adopting)75 1527 y(it)f(as)f(the)h(standard.)18 b(Th)o(us,)11 b(MPI)g(has)f(b)q(een)i(strongly)f(in\015uenced)i(b)o(y)d(w)o(ork)g(at) g(the)h(IBM)g(T.)f(J.)h(W)l(atson)75 1584 y(Researc)o(h)18 b(Cen)o(ter)g([1)o(,)g(2)o(],)g(In)o(tel's)g(NX/2)g([15)o(],)f(Express) h([14)o(],)g(nCUBE's)g(V)l(ertex)g([12)o(],)f(P4)h([3)o(],)g(and)75 1640 y(P)l(ARMA)o(CS)11 b([10)o(,)g(11)o(].)18 b(Other)11 b(imp)q(ortan)o(t)g(con)o(tributions)g(ha)o(v)o(e)g(come)g(from)f(Zip)q (co)q(de)i([16)o(,)f(17)o(],)g(Chimp)75 1697 y([6)o(,)k(7],)f(PVM)h([8) o(,)g(18],)f(and)h(PICL)h([9].)166 1753 y(One)j(of)f(the)g(ob)s(jectiv) o(es)g(of)g(this)g(pap)q(er)h(is)g(to)e(promote)h(a)f(discussion)j (within)g(the)e(concurren)o(t)75 1810 y(computing)d(researc)o(h)f(comm) o(unit)o(y)f(of)h(the)g(issues)h(that)f(m)o(ust)f(b)q(e)i(addressed)f (in)h(establishing)h(a)e(prac-)75 1866 y(tical,)j(p)q(ortable,)g(and)f (\015exible)j(standard)d(for)g(message)f(passing.)24 b(This)17 b(co)q(op)q(erativ)o(e)g(pro)q(cess)g(b)q(egan)75 1922 y(with)f(a)e(w)o(orkshop)h(on)g(standards)g(for)f(message)h (passing)h(held)g(in)g(April)h(1992)d([19)o(].)166 1979 y(The)k(main)h(adv)m(an)o(tages)e(of)h(establishing)i(a)e(message)g (passing)g(standard)g(are)g(p)q(ortabilit)o(y)h(and)75 2035 y(ease-of-use.)h(In)14 b(a)g(distributed)h(memory)e(comm)o (unication)i(en)o(vironmen)o(t)f(in)g(whic)o(h)h(the)f(higher)h(lev)o (el)75 2092 y(routines)c(and/or)g(abstractions)f(are)h(build)i(up)q(on) e(lo)o(w)o(er)g(lev)o(el)h(message)e(passing)i(routines)f(the)g(b)q (ene\014ts)75 2148 y(of)22 b(standardization)h(are)f(particularly)h (apparen)o(t.)41 b(F)l(urthermore,)24 b(the)e(de\014nition)i(of)e(a)g (message)75 2205 y(passing)16 b(standard,)e(suc)o(h)i(as)e(that)h(prop) q(osed)g(here,)h(pro)o(vides)f(v)o(endors)h(with)f(a)g(clearly)h (de\014ned)h(base)75 2261 y(set)10 b(of)g(routines)g(that)g(they)g(can) h(implemen)o(t)g(e\016cien)o(tly)l(,)i(or)c(in)i(some)f(cases)h(pro)o (vide)f(hardw)o(are)g(supp)q(ort)75 2318 y(for,)k(thereb)o(y)h (enhancing)i(scalabilit)o(y)l(.)166 2374 y(The)h(goal)g(of)g(the)g (Message)g(P)o(assing)g(In)o(terface)g(simply)i(stated)d(is)i(to)f(dev) o(elop)h(a)f(widely)h(used)75 2431 y(standard)e(for)f(writing)i (message-passing)f(programs.)24 b(As)17 b(suc)o(h)h(the)f(in)o(terface) g(should)h(establishing)75 2487 y(a)d(practical,)h(p)q(ortable,)f (e\016cien)o(t,)g(and)h(\015exible)h(standard)e(for)f(message)h (passing.)166 2543 y(A)g(complete)h(list)g(of)f(goals)g(follo)o(w.)143 2647 y Fy(\017)23 b FC(Design)11 b(an)f(application)i(programming)e(in) o(terface)h(\(not)f(necessarily)i(for)d(compilers)j(or)e(a)g(system)189 2704 y(implemen)o(tation)16 b(library\).)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 9 3 8 bop 75 -100 a Fv(1.2.)34 b(WHO)15 b(SHOULD)h(USE)g(THIS)g(ST)l(AND) o(ARD?)824 b FC(3)143 45 y Fy(\017)23 b FC(Allo)o(w)15 b(e\016cien)o(t)h(comm)o(unication:)21 b(Av)o(oid)16 b(memory)e(to)h(memory)g(cop)o(ying)g(and)h(allo)o(w)f(o)o(v)o(erlap) 189 102 y(of)e(computation)g(and)h(comm)o(unication)g(and)g(o\017oad)f (to)g(comm)o(unication)h(copro)q(cessor,)g(where)189 158 y(a)o(v)m(ailable.)143 255 y Fy(\017)23 b FC(Allo)o(w)15 b(\(but)g(no)h(mandate\))e(extensions)i(for)f(use)g(in)h(heterogeneous) f(en)o(vironmen)o(t.)143 352 y Fy(\017)23 b FC(Allo)o(w)15 b(con)o(v)o(enien)o(t)h(C,)f(F)l(ortran)f(77,)g(F)l(ortran)g(90,)g(and) i(C++)f(bindings)i(for)e(in)o(terface.)143 449 y Fy(\017)23 b FC(Assume)15 b(a)g(reliable)i(comm)o(unication)f(in)o(terface:)k (User)15 b(need)h(not)f(cop)q(e)h(with)f(comm)o(unication)189 506 y(failures.)21 b(Suc)o(h)15 b(failures)i(are)d(dealt)i(b)o(y)f(the) h(underlying)h(comm)o(unication)f(subsystem.)143 603 y Fy(\017)23 b FC(F)l(o)q(cus)15 b(on)g(a)g(prop)q(osal)h(that)e(can)h (b)q(e)h(agreed)f(up)q(on)h(in)g(6)f(mon)o(ths.)143 700 y Fy(\017)23 b FC(De\014ne)f(an)g(in)o(terface)g(that)f(is)h(not)f(to)q (o)g(di\013eren)o(t)h(from)f(curren)o(t)h(practice,)h(suc)o(h)f(as)g (PVM,)189 756 y(Express,)14 b(P4,)h(etc.)143 853 y Fy(\017)23 b FC(De\014ne)12 b(an)f(in)o(terface)h(that)f(can)h(b)q(e)g(quic)o(kly) h(implemen)o(ted)g(on)f(man)o(y)f(v)o(endor's)g(platforms,)h(with)189 909 y(no)j(signi\014can)o(t)h(c)o(hanges)f(in)h(the)f(underlying)j (comm)o(unication)d(and)h(system)f(soft)o(w)o(are.)143 1007 y Fy(\017)23 b FC(The)15 b(in)o(terface)h(should)g(not)f(con)o (tain)g(more)g(functions)h(than)f(are)g(really)h(necessary)l(.)143 1104 y Fy(\017)23 b FC(Seman)o(tics)15 b(of)g(the)g(in)o(terface)h (should)g(b)q(e)g(language)f(indep)q(enden)o(t.)75 1251 y Fw(1.2)59 b(Who)20 b(Should)g(Use)g(This)f(Standa)n(rd?)75 1354 y FC(This)d(standard)f(is)h(in)o(tended)h(for)e(use)g(b)o(y)h(all) g(those)f(who)g(w)o(an)o(t)g(to)f(write)i(p)q(ortable)g (message-passing)75 1411 y(programs)21 b(in)j(F)l(ortran)d(77,)i(C,)f (F)l(ortran)g(90,)h(or)f(C++.)43 b(This)23 b(includes)i(individual)g (application)75 1467 y(programmers,)14 b(dev)o(elop)q(ers)j(of)e(soft)o (w)o(are)f(designed)j(to)e(run)h(on)g(parallel)h(mac)o(hines,)f(and)g (creators)e(of)75 1524 y(en)o(vironmen)o(ts,)g(and)f(to)q(ols.)19 b(In)14 b(order)f(to)g(b)q(e)h(attractiv)o(e)e(to)h(this)h(wide)g (audience,)h(the)f(standard)f(m)o(ust)75 1580 y(pro)o(vide)j(a)f (simple,)i(easy-to-use)e(in)o(terface)h(for)e(the)i(basic)g(user)f (while)i(not)e(seman)o(tically)i(precluding)75 1637 y(the)e(high-p)q (erformance)i(message-passing)e(op)q(erations)g(a)o(v)m(ailable)i(on)e (adv)m(anced)i(mac)o(hines.)75 1784 y Fw(1.3)59 b(What)20 b(Platfo)n(rms)g(Are)g(T)-5 b(a)n(rgets)19 b(F)n(o)n(r)i(Implementat)o (ion?)75 1887 y FC(The)c(attractiv)o(eness)f(of)g(the)h (message-passing)g(paradigm)f(at)g(least)h(partially)h(stems)e(from)g (its)h(wide)75 1944 y(p)q(ortabilit)o(y)l(.)k(Programs)13 b(expressed)j(this)f(w)o(a)o(y)f(ma)o(y)g(run)h(on)g (distributed-memory)h(m)o(ultipro)q(cessors,)75 2000 y(net)o(w)o(orks)i(of)i(w)o(orkstations,)e(and)i(com)o(binations)g(of)g (all)g(of)f(these.)34 b(In)20 b(addition,)h(shared-memory)75 2057 y(implemen)o(tations)g(are)e(p)q(ossible.)34 b(The)20 b(paradigm)g(will)h(not)e(b)q(e)h(made)f(obsolete)h(b)o(y)g(arc)o (hitectures)75 2113 y(com)o(bining)15 b(the)e(shared-)h(and)g (distributed-memory)h(views,)f(or)f(b)o(y)g(increases)i(in)f(net)o(w)o (ork)f(sp)q(eeds.)20 b(It)75 2170 y(th)o(us)d(should)i(b)q(e)f(b)q(oth) g(p)q(ossible)h(and)f(useful)g(to)f(implemen)o(t)i(this)f(standard)f (on)h(a)f(great)g(v)m(ariet)o(y)g(of)75 2226 y(mac)o(hines,)i (including)h(those)e(\\mac)o(hines")g(consisting)g(of)g(collections)h (of)e(other)h(mac)o(hines,)g(parallel)75 2283 y(or)d(not,)f(connected)i (b)o(y)f(a)g(comm)o(unication)h(net)o(w)o(ork.)166 2340 y(The)21 b(in)o(terface)f(is)h(suitable)h(for)e(use)h(b)o(y)f(fully)i (general)f(MIMD)f(programs,)g(as)g(w)o(ell)h(as)f(those)75 2396 y(written)d(in)g(the)g(more)f(restricted)h(st)o(yle)g(of)f(SPMD.)g (Although)h(no)f(explicit)j(supp)q(ort)e(for)f(threads)g(is)75 2453 y(pro)o(vided,)g(the)f(in)o(terface)g(has)g(b)q(een)i(designed)f (so)f(as)g(not)g(to)f(prejudice)j(their)f(use.)75 2601 y Fw(1.4)59 b(What)20 b(Is)f(Included)g(In)g(The)g(Standa)n(rd?)75 2704 y FC(The)c(standard)g(includes)j(\(this)d(is)h(temp)q(orarily)f (as)g(inclusiv)o(e)j(as)d(p)q(ossible\):)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 10 4 9 bop 75 -100 a FC(4)925 b Fv(SECTION)16 b(1.)34 b(INTR)o(ODUCTION)17 b(TO)e(MPI)143 45 y Fy(\017)23 b FC(P)o(oin)o(t-to-p)q(oin)o(t)15 b(comm)o(unication)143 139 y Fy(\017)23 b FC(Collectiv)o(e)16 b(op)q(erations)143 233 y Fy(\017)23 b FC(Pro)q(cess)15 b(groups)143 327 y Fy(\017)23 b FC(Comm)o(unication)15 b(con)o(texts)143 420 y Fy(\017)23 b FC(A)15 b(simple)i(w)o(a)o(y)d(to) g(create)h(pro)q(cesses)h(for)e(the)i(SPMD)e(mo)q(del)143 514 y Fy(\017)23 b FC(Bindings)17 b(for)d(F)l(ortran)g(77,)g(F)l (ortran)h(90,)f(C)h(and)g(C++)143 608 y Fy(\017)23 b FC(A)15 b(mo)q(del)h(implemen)o(tation)143 702 y Fy(\017)23 b FC(A)15 b(formal)g(sp)q(eci\014cation.)143 796 y Fy(\017)23 b FC(Pro)q(cess)15 b(top)q(ology)143 889 y Fy(\017)23 b FC(A)15 b(v)m(alidation)i(suite)75 1033 y Fw(1.5)59 b(What)20 b(Is)f(Not)h(Included)e(In)i(The)f(Standa)n(rd?)75 1134 y FC(The)c(standard)g(do)q(es)h(not)f(sp)q(ecify:)143 1240 y Fy(\017)23 b FC(Explicit)17 b(shared-memory)e(op)q(erations)143 1334 y Fy(\017)23 b FC(Op)q(erations)c(that)e(require)i(more)f(op)q (erating)h(system)f(supp)q(ort)g(than)g(is)h(curren)o(tly)g(standard;) 189 1391 y(for)14 b(example,)i(in)o(terrupt-driv)o(en)g(receiv)o(es,)g (remote)f(execution,)h(or)e(activ)o(e)i(messages)143 1484 y Fy(\017)23 b FC(Program)13 b(construction)j(to)q(ols)143 1578 y Fy(\017)23 b FC(Debugging)15 b(facilities)143 1672 y Fy(\017)23 b FC(Auxiliary)17 b(functions)e(suc)o(h)h(as)f (timers)143 1766 y Fy(\017)23 b FC(Explicit)17 b(supp)q(ort)e(for)g (threads)166 1872 y(There)g(are)f(man)o(y)h(features)f(that)g(ha)o(v)o (e)g(b)q(een)i(considered)g(and)f(not)g(included)i(in)f(this)f (standard.)75 1929 y(This)j(happ)q(ened)h(for)e(a)g(n)o(um)o(b)q(er)h (of)f(reasons,)g(one)g(of)g(whic)o(h)i(is)f(the)f(time)h(constrain)o(t) f(that)g(w)o(as)f(self)75 1985 y(imp)q(osed)g(in)h(\014nishing)g(the)e (standard.)20 b(F)l(eatures)15 b(that)f(are)h(not)g(included)j(can)e (alw)o(a)o(ys)e(b)q(e)i(o\013ered)f(as)75 2042 y(extensions)h(b)o(y)f (sp)q(eci\014c)i(implemen)o(tations.)75 2185 y Fw(1.6)59 b(Organization)20 b(of)g(this)f(Do)r(cument)75 2286 y FC(Describ)q(e)d(where)g(ev)o(erything)f(is)h(lo)q(cated)g(in)g(the)f (MPIF)g(do)q(cumen)o(t.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 11 5 10 bop 75 359 a Fx(Section)35 b(2)75 569 y FE(MPI)40 b(T)-10 b(erms)39 b(and)g(Con)m(v)m(en)m(tions)75 794 y FC(This)12 b(Section)f(explains)i(notational)e(terms)f(and)h(con)o(v) o(en)o(tions)g(used)h(throughout)e(the)h(MPI)g(do)q(cumen)o(t,)75 851 y(some)k(of)g(the)g(c)o(hoices)h(that)e(ha)o(v)o(e)h(b)q(een)i (made,)d(and)i(the)f(rationale)h(b)q(ehind)h(those)e(c)o(hoices.)75 1009 y Fw(2.1)59 b(Do)r(cument)18 b(Conventions)75 1116 y FC(Discussions)f(that)f(are)g(not)f(part)h(of)g(the)g(standard)g(are) g(mark)o(ed)f(with)i FD(\\Discussion")g FC(and)g(app)q(ear)75 1172 y(in)f(small)g(t)o(yp)q(e:)166 1314 y Fu(Discussion:)34 b Ft(A)14 b(\\discussion")g(lo)q(oks)f(lik)o(e)g(this.)75 1456 y FC(Advice)k(to)d(implemen)o(tors)i(of)f(MPI)g(will)i(app)q(ear)e (in)h(a)f(note)g(lik)o(e)h(this:)166 1597 y Fu(Implemen)o(tati)o(on)c (note:)36 b Ft(Advice)15 b(on)e(implemen)o(tation)e(go)q(es)j(here.)75 1838 y Fw(2.2)59 b(Pro)r(cedure)19 b(Sp)r(eci\014cation)75 1945 y FC(MPI)f(pro)q(cedures)h(are)f(sp)q(eci\014ed)i(using)f(a)f (language)h(indep)q(enden)o(t)h(notation.)29 b(The)18 b(parameters)f(of)75 2001 y(pro)q(cedure)f(calls)g(are)f(mark)o(ed)g (as)30 b Ft(IN)p FC(,)g Ft(OUT)16 b FC(or)29 b Ft(INOUT)p FC(.)16 b(The)f(meanings)h(of)f(these)g(are:)143 2106 y Fy(\017)23 b FC(the)15 b(call)h(uses)g(but)f(do)q(es)h(not)e(up)q (date)i(a)f(parameter)f(mark)o(ed)30 b Ft(IN)p FC(,)143 2210 y Fy(\017)23 b FC(the)15 b(call)h(ma)o(y)f(up)q(date)h(a)e (parameter)h(mark)o(ed)30 b Ft(OUT)p FC(,)143 2314 y Fy(\017)23 b FC(the)15 b(call)h(b)q(oth)g(uses)f(and)g(up)q(dates)h(a)f (parameter)f(mark)o(ed)30 b Ft(INOUT)p FC(.)166 2419 y(If)22 b(a)f(parameter)f(is)i(a)f(handle)i(to)d(an)i(opaque)f(ob)s (ject,)h(and)f(the)h(ob)s(ject)f(is)g(up)q(dated)i(b)o(y)e(the)75 2475 y(pro)q(cedure)d(call,)g(then)f(the)g(parameter)f(is)h(mark)o(ed) 33 b Ft(OUT)p FC(.)17 b(This)g(is)h(done)f(ev)o(en)g(though)g(the)g (handle)75 2532 y(itself)d(is)g(not)f(mo)q(di\014ed)i({)e(w)o(e)g(use)g (the)27 b Ft(OUT)13 b FC(attribute)h(to)e(denote)i(that)e(what)h(the)g (handle)i Fs(r)n(efer)n(enc)n(es)75 2588 y FC(is)h(up)q(dated.)166 2647 y(The)f(de\014nition)j(of)c(MPI)i(tries)f(to)g(a)o(v)o(oid,)g(to)f (the)i(largest)f(p)q(ossible)i(exten)o(t,)d(the)i(use)f(of)30 b Ft(INOUT)75 2704 y FC(parameters,)14 b(b)q(ecause)i(suc)o(h)g(use)f (is)h(error-prone,)e(esp)q(ecially)k(for)d(scalar)g(parameters.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 12 6 11 bop 75 -100 a FC(6)737 b Fv(SECTION)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)166 45 y FC(A)j(common)f(o)q(ccurrence)i (for)e(MPI)g(functions)i(is)f(a)f(parameter)g(that)g(is)h(used)g(as)35 b Ft(IN)18 b FC(b)o(y)f(some)75 102 y(pro)q(cesses)j(and)40 b Ft(OUT)20 b FC(b)o(y)g(other)f(pro)q(cesses.)34 b(Suc)o(h)20 b(parameters)f(are,)h(syn)o(tactically)l(,)i(an)39 b Ft(INOUT)75 158 y FC(parameter,)14 b(although,)h(seman)o(tically)l(,)h (it)g(is)f(not)g(used)h(in)g(one)f(call)i(b)q(oth)e(for)g(input)h(and)f (for)g(output.)166 300 y Fu(Discussion:)34 b Ft(W)m(e)13 b(migh)o(t)f(w)o(an)o(t)i(to)f(distinguish)h(this)g(use;)g(e.g.)36 b Fr(IN/OUT)13 b Ft(vs)28 b Fr(INOUT)p Ft(.)166 441 y FC(Another)17 b(frequen)o(t)g(situation)h(arises)f(when)h(a)e (parameter)h(v)m(alue)h(is)g(needed)g(only)g(b)o(y)f(a)g(subset)75 498 y(of)e(the)h(pro)q(cesses.)22 b(When)17 b(a)e(parameter)g(is)i(not) e(signi\014can)o(t)i(at)e(a)h(pro)q(cess)g(then)g(an)g(arbitrary)f(v)m (alue)75 554 y(can)g(b)q(e)h(passed)g(as)e(argumen)o(t.)75 712 y Fw(2.3)59 b(Semantical)18 b(T)-5 b(erms)75 819 y FC(A)15 b(comm)o(unication)h(op)q(eration)g(is)f(termed)g(to)g(b)q (e:)75 938 y FD(non)o(blo)q(c)o(king)24 b FC(if)15 b(it)h(ma)o(y)e (return)h(b)q(efore)h(the)f(related)h(comm)o(unication)g(op)q(eration)f (completes;)75 1043 y FD(lo)q(cally-blo)r(c)o(king)26 b FC(if)18 b(return)g(from)f(the)h(pro)q(cedure)h(indicates)g(the)f (user)g(is)h(allo)o(w)o(ed)f(to)f(re-use)i(re-)189 1099 y(sources)c(\(suc)o(h)g(as)g(bu\013ers\))g(sp)q(eci\014ed)i(in)f(the)f (call;)75 1203 y FD(globally-blo)q(cking)25 b FC(if)c(return)f(from)g (the)g(pro)q(cedure)i(indicates)f(the)g(comm)o(unication)g(op)q (eration)189 1260 y(has)15 b(completed.)166 1379 y(An)20 b(op)q(eration)g(is)h FD(lo)q(cal)h FC(if)e(its)g(execution)h(do)q(es)f (not)g(en)o(tail)h(comm)o(unication,)g(and)f(do)q(es)g(not)75 1435 y(dep)q(end)c(on)f(the)g(state)f(of)g(other)g(pro)q(cesses;)h FD(global)p FC(,)h(if)g(its)e(completion)i(ma)o(y)e(dep)q(end)j(on)d (comm)o(uni-)75 1492 y(cation)h(o)q(ccurring;)h FD(collectiv)o(e)h FC(if)e(sev)o(eral)h(pro)q(cesses)f(need)h(to)f(participate)h(in)g(its) f(execution.)75 1650 y Fw(2.4)59 b(Data)19 b(T)-5 b(yp)r(es)75 1758 y Fq(2.4.1)49 b(Opaque)16 b(objects)75 1849 y FC(MPI)23 b(pro)q(cedures)g(use)g Fs(op)n(aque)h(obje)n(cts)p FC(.)42 b(These)23 b(hide)h(the)f(in)o(ternal)h(represen)o(tation)e(used,)j(th) o(us)75 1905 y(allo)o(wing:)c(similar)16 b(calls)h(in)f(C)f(and)g(F)l (ortran;)f(the)h(a)o(v)o(oidance)h(of)f(con\015icts)h(with)f(the)h(t)o (yping)f(rules)h(in)75 1962 y(these)f(languages;)g(future)g(extension)h (of)f(functionalit)o(y)l(.)21 b(The)16 b(mec)o(hanism)f(for)g(opaque)g (ob)s(jects)f(used)75 2018 y(here)i(lo)q(osely)g(follo)o(ws)f(the)g (POSIX)i(F)l(ortran)d(binding)j(standard.)166 2077 y(An)g(opaque)g(ob)s (ject)f(can)g(b)q(e)i(allo)q(cated)f(on)g(user)g(storage)e(or)h(on)h (system)f(\(i.e.)24 b(MPI)17 b(managed\))75 2134 y(storage.)23 b(W)l(e)17 b(call)h(the)f(former)f(a)g Fs(user)i(obje)n(ct)e FC(and)h(the)g(latter)g(a)f Fs(system)h(obje)n(ct)p FC(.)24 b(User)17 b(ob)s(jects)f(are)75 2190 y(allo)q(cated)e(and)f(deallo)q (cated)i(explicitly)g(b)o(y)e(the)g(user)g(as)g(a)g(regular)g (\(structured\))f(v)m(ariable.)21 b(The)13 b(t)o(yp)q(e)75 2247 y(de\014nitions)i(for)e(suc)o(h)h(ob)s(jects)f(are)g(pro)o(vided)i (b)o(y)e(MPI.)g(The)h(ob)s(jects)f(can)g(b)q(e)i(accessed)f(and)g(mo)q (di\014ed)75 2303 y(only)i(via)f(MPI)g(functions)h(that)f(tak)o(e)f (suc)o(h)i(ob)s(jects)e(as)h(parameters.)166 2362 y(System)d(ob)s (jects)g(are)g(accessed)i(only)f(via)f Fs(hand)r(les)p FC(.)19 b(In)13 b(C,)f(a)g(di\013eren)o(t)h(t)o(yp)q(e)g(of)f(handle)i (is)f(de\014ned)75 2419 y(b)o(y)g(MPI)g(for)g(eac)o(h)g(t)o(yp)q(e)h (of)f(opaque)g(system)g(ob)s(ject.)19 b(In)13 b(F)l(ortran,)g(all)h (handles)g(ha)o(v)o(e)f(t)o(yp)q(e)37 b Fp(INTEGER)p FC(.)75 2475 y(The)15 b(allo)q(cation)g(and)g(deallo)q(cation)h(of)e (suc)o(h)h(an)f(ob)s(ject,)g(and)g(an)o(y)h(access)f(or)g(up)q(date)h (to)f(it,)g(are)h(done)75 2532 y(only)h(via)f(MPI)g(calls)i(that)d (accept)i(as)e(parameters)h(handles)h(to)f(suc)o(h)g(ob)s(jects.)166 2591 y(An)k(opaque)g(ob)s(ject,)g(and)g(the)g(handle)h(to)e(suc)o(h)h (ob)s(ject)f(are)h(signi\014can)o(t)g(only)h(at)e(the)h(pro)q(cess)75 2647 y(where)g(the)g(ob)s(ject)f(w)o(as)g(created,)h(and)g(cannot)g(b)q (e)h(transferred)e(to)g(another)g(pro)q(cess)h(\(ob)s(jects)f(are)75 2704 y(lo)q(cal\).)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 13 7 12 bop 75 -100 a Fv(2.4.)34 b(D)o(A)l(T)l(A)15 b(TYPES)1375 b FC(7)166 45 y Fu(Discussion:)31 b Ft(This)11 b(ma)o(y)f(need)i(some)e (clari\014cation)g(if)h("\015atten/un\015atten")h(is)f(accepted)i(in)d (the)i(con)o(text)75 102 y(draft.)166 242 y FC(Opaque)k(system)e(ob)s (jects)h(are)g(allo)q(cated)h(and)f(deallo)q(cated)i(b)o(y)e(calls)h (that)f(are)f(sp)q(eci\014c)j(to)e(eac)o(h)75 299 y(ob)s(ject)k(t)o(yp) q(e.)32 b(These)20 b(are)f(listed)i(in)f(the)f(sections)h(where)g(the)f (ob)s(jects)g(are)g(describ)q(ed.)35 b(A)19 b(call)i(to)75 355 y(deallo)q(cate)15 b(an)f(opaque)h(system)e(ob)s(ject)h(do)q(es)g (not)g(necessarily)h(deallo)q(cate)h(the)e(ob)s(ject)f(immediately)l(,) 75 412 y(but)e(ma)o(y)f(only)h(mark)g(it)g(as)f(candidate)i(for)e (deallo)q(cation.)20 b(The)11 b(ob)s(ject)f(will)j(b)q(e)e(deallo)q (cated)h(as)f(as)f(so)q(on)75 468 y(as)17 b(there)g(there)g(is)h(no)f (p)q(ending)i(op)q(eration)e(that)g(is)g(using)h(this)g(ob)s(ject;)f (the)g(handle)i(then)e(b)q(ecomes)75 525 y(n)o(ull.)22 b(F)l(or)15 b(example,)h(if)f(a)g(comm)o(unication)i(ob)s(ject)d(is)i (freed)g(after)f(the)g(comm)o(unication)h(started)f(but)75 581 y(b)q(efore)h(it)h(completed,)g(then)f(the)h(p)q(ending)h(op)q (eration)e(is)h(not)e(a\013ected;)h(one)h(can)f(use)g(the)h(handle)g (in)75 637 y(calls)f(to)f Fo(MPI)p 317 637 14 2 v 16 w(W)l(AIT)g FC(or)g Fo(MPI)p 599 637 V 16 w(ST)l(A)l(TUS)p FC(.)h(In)g(fact,)f(one)g(m)o(ust)g(complete)h(the)f(comm)o(unication)i (suc)o(h)e(a)75 694 y(call.)25 b(Ho)o(w)o(ev)o(er,)15 b(once)i(the)f(op)q(eration)h(completes,)g(it)g(is)g(erroneous)f(to)g (re-use)h(the)f(freed)h(handle)h(for)75 750 y(a)d(new)g(comm)o (unication.)166 891 y Fu(Implemen)o(tati)o(on)j(note:)55 b Ft(Some)17 b(ob)r(jects,)k(suc)o(h)e(as)g(bu\013er)h(descriptor)g(ob) r(jects,)g(can)f(b)q(e)h(used)f(b)o(y)75 947 y(sev)o(eral,)c(distinct,) f(concurren)o(t)j(comm)o(unicatio)o(n)12 b(op)q(erations.)21 b(Th)o(us,)14 b(to)h(\014nd)f(out)h(whether)h(suc)o(h)f(ob)r(ject)h (can)75 1004 y(b)q(e)f(freed,)f(one)g(needs)h(a)f(reference)i(coun)o(t) e(of)f(p)q(ending)h(op)q(erations.)166 1227 y Fu(Discussion:)33 b Ft(Need)14 b(a)e(test)i(to)e(see)i(if)e(a)h(handle)f(is)h(n)o(ull.)k (In)12 b(C,)h(if)f(handles)h(are)g(p)q(oin)o(ters,)g(a)f(comparison)75 1284 y(to)g Fn(NULL)f Ft(can)h(b)q(e)h(used.)18 b(In)12 b(F)m(ortran,)g(w)o(e)g(ma)o(y)e(either)j(ha)o(v)o(e)f(a)g(function)f Fr(MPI)p 1300 1284 13 2 v 15 w(IS)p 1350 1284 V 14 w(NULL)i Ft(\(and)f(mak)o(e)f(it)g(a)o(v)n(ailable)75 1340 y(in)16 b(C,)g(to)q(o,)h(for)f(compatibilit)o(y)d(reasons\),)18 b(or)e(ha)o(v)o(e)h(a)f Fn(NULL)g Ft(v)n(alue)g(that)g(can)h(b)q(e)g (used)g(in)g(comparisons,)e(in)h(a)75 1397 y(manner)d(similar)e(to)j(a) g(C)g(n)o(ull)e(p)q(oin)o(ter.)166 1537 y FC(MPI)19 b(ma)o(y)g(pro)o (vide)h(prede\014ned)h(opaque)f(ob)s(jects)f(and)g(prede\014ned,)j (static)d(handles)i(to)e(these)75 1594 y(ob)s(jects.)g(Suc)o(h)d(ob)s (jects)f(ma)o(y)f(not)h(b)q(e)h(destro)o(y)o(ed.)75 1724 y Fq(2.4.2)49 b(Arra)o(y)16 b(of)h(handles)75 1813 y FC(An)i(MPI)g(call)h(ma)o(y)e(need)i(a)f(parameter)f(that)g(is)i(an)e Fs(arr)n(ay-of-hand)r(les)p FC(.)32 b(The)20 b(arra)o(y-of-handles)e (is)75 1870 y(a)h(regular)h(arra)o(y)e(with)i(en)o(tries)g(that)f(are)g (handles)i(to)e(ob)s(jects)g(of)g(the)h(same)f(t)o(yp)q(e)h(in)g (consecutiv)o(e)75 1926 y(lo)q(cations)j(in)h(the)e(arra)o(y)l(.)41 b(Whenev)o(er)23 b(suc)o(h)g(an)g(arra)o(y)e(is)i(used,)i(an)d (additional)47 b Fo(len)23 b FC(parameter)75 1983 y(is)g(required)h(to) d(indicate)j(the)f(n)o(um)o(b)q(er)g(of)f(v)m(alid)i(en)o(tries)f (\(unless)g(this)g(n)o(um)o(b)q(er)g(can)g(b)q(e)g(deriv)o(ed)75 2039 y(otherwise\).)75 2170 y Fq(2.4.3)49 b(State)75 2258 y FC(MPI)22 b(pro)q(cedures)h(use)g(at)e(v)m(arious)i(places)g (argumen)o(ts)e(with)h Fs(state)h FC(t)o(yp)q(es.)40 b(The)23 b(v)m(alues)g(of)f(suc)o(h)75 2315 y(data)16 b(t)o(yp)q(e)h(are)g(all)h(iden)o(ti\014ed)h(b)o(y)e(names,)g(and)g(no) g(op)q(eration)g(is)h(de\014ned)g(on)f(them.)25 b(F)l(or)16 b(example,)75 2371 y(the)42 b Fo(MPI)p 265 2371 14 2 v 16 w(COMM)p 423 2371 V 16 w(INIT)20 b FC(routine)h(has)g(a)g(state)f (t)o(yp)q(e)h(parameter)f(with)h(v)m(alues)41 b Fr(MPI)p 1632 2371 13 2 v 14 w(ST)m(AND)o(ARD)p FC(,)75 2428 y Fr(MPI)p 152 2428 V 14 w(READ)o(Y)p FC(,)14 b(etc.)75 2558 y Fq(2.4.4)49 b(Named)16 b(constants)75 2647 y FC(MPI)11 b(pro)q(cedures)h(sometimes)f(assign)g(a)g(sp)q(ecial)i(meaning)f(to)e (a)h(sp)q(ecial)i(v)m(alue)f(of)f(a)g(basic)g(t)o(yp)q(e)g(param-)75 2704 y(eter;)16 b(e.g.)44 b Fo(tag)15 b FC(is)i(an)f(in)o(teger)g(v)m (alued)h(parameter)e(of)h(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)f(op)q(erations,)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 8 14 8 13 bop 75 -100 a FC(8)737 b Fv(SECTION)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y FC(with)g(a)g(sp)q(ecial)i (wild-card)f(v)m(alue,)g Fr(MPI)p 770 45 13 2 v 14 w(ANY)p 869 45 V 15 w(T)m(A)o(G)p FC(.)d(Suc)o(h)j(parameters)e(will)j(ha)o(v)o (e)d(a)h(range)f(of)h(regular)75 102 y(v)m(alues,)j(whic)o(h)g(is)f(a)f (prop)q(er)h(subrange)g(of)f(the)h(range)g(of)f(v)m(alues)i(of)e(the)h (corresp)q(onding)h(basic)f(t)o(yp)q(e;)75 158 y(sp)q(ecial)g(v)m (alues)g(\(suc)o(h)e(as)29 b Fr(MPI)p 626 158 V 14 w(ANY)p 725 158 V 15 w(T)m(A)o(G)p FC(\))14 b(will)j(b)q(e)f(outside)g(the)f (regular)h(range.)k(The)15 b(range)g(of)g(reg-)75 214 y(ular)i(v)m(alues)h(can)f(b)q(e)h(queried,)g(and)f(sometimes)g(set,)g (using)h(en)o(vironmen)o(t)f(inquiry)h(or)f(en)o(vironmen)o(t)75 271 y(setting)e(functions)h(\(Chapter)f(7\).)166 411 y Fu(Implemen)o(tati)o(on)d(note:)35 b Ft(Implemen)o(tors)11 b(should)i(detect,)i(whenev)o(er)g(p)q(ossible,)e(illegal)e(uses)k(of)d (\\sp)q(e-)75 467 y(cial)h(v)n(alues".)19 b(Th)o(us,)14 b(the)h(use)g(of)f(the)29 b Fr(MPI)p 785 467 V 14 w(ANY)p 884 467 V 15 w(T)m(A)o(G)13 b Ft(v)n(alue)h(to)g(tag)g(a)g(message)g (sen)o(t)h(will)e(b)q(e)i(\015agged)f(as)g(an)75 524 y(error.)75 732 y Fq(2.4.5)49 b(Choice)75 820 y FC(MPI)22 b(functions)g(sometimes)f(use)h(parameters)f(with)h(a)f Fs(choic)n(e)g FC(\(or)g(union\))h(data)f(t)o(yp)q(e.)39 b(Distinct)75 876 y(calls)18 b(to)e(the)h(same)f(routine)h(ma)o(y)f (pass)h(b)o(y)f(reference)i(actual)e(parameters)g(of)g(di\013eren)o(t)h (t)o(yp)q(es.)25 b(The)75 933 y(mec)o(hanism)16 b(for)e(pro)o(viding)j (suc)o(h)e(parameters)f(will)j(di\013er)f(from)e(language)i(to)e (language.)166 1066 y Fu(Discussion:)47 b Ft(The)18 b(F)m(ortran)f(77)f (standard)i(sp)q(eci\014es)h(that)e(the)h(t)o(yp)q(e)g(of)f(actual)f (argumen)o(ts)h(need)h(to)75 1116 y(agree)12 b(with)e(the)i(t)o(yp)q(e) f(of)g(dumm)o(y)d(argumen)o(ts;)i(no)h(construct)i(equiv)n(alen)o(t)d (to)h(C)g(p)q(oin)o(ters)g(is)g(a)o(v)n(ailable.)k(Th)o(us,)c(it)75 1165 y(w)o(ould)e(seem)i(that)f(there)h(is)f(no)h(standard)f (conforming)e(mec)o(hanism)g(to)i(supp)q(ort)h(c)o(hoice)g(parameters.) 17 b(Ho)o(w)o(ev)o(er,)75 1215 y(most)10 b(F)m(ortran)i(compilers)e (either)j(don't)e(c)o(hec)o(k)h(t)o(yp)q(e)g(consistency)i(of)c(calls)i (to)f(external)h(routines,)g(or)g(supp)q(ort)g(a)75 1265 y(sp)q(ecial)g(mec)o(hanism)d(to)i(link)g(foreign)f(\(e.g.,)h(C\))h (routines.)18 b(W)m(e)11 b(accept)h(this)g(non-conformit)o(y)d(with)i (the)h(F)m(ortran)75 1315 y(77)g(standard.)18 b(I.e.,)12 b(w)o(e)i(accept)g(that)f(the)g(same)f(routine)h(ma)o(y)e(b)q(e)j (passed)f(an)g(actual)g(parameter)f(of)g(a)h(di\013eren)o(t)75 1365 y(t)o(yp)q(e)h(at)g(distinct)g(calls.)166 1415 y(Generic)j (routines)g(can)f(b)q(e)h(used)h(in)d(F)m(ortran)i(90)e(to)h(pro)o (vide)h(a)f(standard)g(conforming)f(solution.)24 b(This)75 1465 y(solution)13 b(will)f(b)q(e)j(consisten)o(t)g(with)f(our)f (nonstandard)i(conforming)c(F)m(ortran)j(77)f(solution.)75 1695 y Fw(2.5)59 b(Pro)r(cesses)75 1798 y FC(An)22 b(MPI)h(program)e (is)h(executed)h(b)o(y)g(sev)o(eral)f(autonomous)f(pro)q(cesses)i(that) e(eac)o(h)i(execute)f(their)75 1855 y(o)o(wn)d(co)q(de,)h(in)g(a)e (MIMD)h(st)o(yle.)31 b(The)19 b(co)q(des)h(executed)g(b)o(y)f(eac)o(h)g (pro)q(cess)g(need)h(not)f(b)q(e)g(iden)o(tical.)75 1911 y(The)h(pro)q(cesses)g(comm)o(unicate)g(via)g(calls)g(to)f(MPI)h(comm)o (unication)g(primitiv)o(es.)35 b(T)o(ypically)l(,)22 b(eac)o(h)75 1967 y(pro)q(cessor)17 b(executes)i(in)f(its)g(o)o(wn)f (address)g(space,)h(although)g(shared-memory)g(implemen)o(tations)g(of) 75 2024 y(MPI)i(are)f(p)q(ossible.)35 b(This)20 b(do)q(cumen)o(t)g(sp)q (eci\014es)i(the)d(b)q(eha)o(vior)h(of)g(a)f(parallel)i(program)e (assuming)75 2080 y(that)c(only)g(MPI)h(calls)g(are)f(used)h(for)e (comm)o(unication.)21 b(The)16 b(in)o(teraction)f(of)g(an)g(MPI)g (program)g(with)75 2137 y(other)g(p)q(ossible)i(means)e(of)g(comm)o (unication)h(\(e.g.,)d(shared)j(memory\))e(is)i(not)e(sp)q(eci\014ed.) 166 2194 y(MPI)d(do)q(es)g(not)g(sp)q(ecify)h(the)f(execution)h(mo)q (del)g(for)e(eac)o(h)h(pro)q(cess.)19 b(A)11 b(pro)q(cess)g(can)h(b)q (e)f(sequen)o(tial,)75 2250 y(or)f(can)h(b)q(e)h(m)o(ulti-threaded,)g (with)f(threads)g(p)q(ossibly)h(executing)g(concurren)o(tly)l(.)20 b(Care)10 b(has)g(b)q(een)i(tak)o(en)75 2307 y(to)j(mak)o(e)f(MPI)h (\\thread-safe",)f(b)o(y)i(a)o(v)o(oiding)f(the)g(use)h(of)f(implicit)i (global)f(states.)166 2364 y(MPI)i(do)q(es)g(not)f(sp)q(ecify)j(either) e(the)g(in)o(teraction)h(of)e(pro)q(cesses)h(with)h(signals,)g(in)f(a)g (UNIX)g(en-)75 2421 y(vironmen)o(t,)e(or)g(other)f(ev)o(en)o(ts)h(that) g(do)g(not)f(relate)i(to)e(MPI)h(comm)o(unication.)23 b(That)16 b(is,)g(signals)h(are)75 2477 y(not)c(signi\014can)o(t)h (from)e(the)h(view)h(p)q(oin)o(t)g(of)e(MPI,)h(and)g(implemen)o(tors)h (should)g(attempt)e(to)h(implemen)o(t)75 2533 y(MPI)h(so)g(that)f (signals)i(are)f(transparen)o(t:)19 b(an)14 b(MPI)g(call)h(susp)q (ended)h(b)o(y)e(a)g(signal)h(should)g(resume)g(and)75 2590 y(complete)h(after)e(the)i(signal)g(is)f(handled.)166 2647 y(MPI)j(do)q(es)g(not)f(pro)o(vide)h(mec)o(hanisms)g(to)f(sp)q (ecify)i(the)f(initial)i(allo)q(cation)f(of)e(pro)q(cesses)h(to)f(an)75 2704 y(MPI)d(computation)g(and)g(their)g(binding)i(to)d(ph)o(ysical)j (pro)q(cessors.)j(It)14 b(is)g(exp)q(ected)h(that)f(v)o(endors)f(will) 1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 9 15 9 14 bop 75 -100 a Fv(2.6.)34 b(ERR)o(OR)17 b(HANDLING)1239 b FC(9)75 45 y(pro)o(vide)20 b(mec)o(hanisms)h(to)e(do)g(so)h(either)g (at)f(load)h(time)g(or)g(at)f(run)h(time.)34 b(Suc)o(h)20 b(mec)o(hanisms)g(will)75 102 y(allo)o(w)c(the)f(sp)q(eci\014cation)j (of)d(the)h(initial)i(n)o(um)o(b)q(er)d(of)h(required)g(pro)q(cesses,)g (the)g(co)q(de)g(to)f(b)q(e)h(executed)75 158 y(b)o(y)k(eac)o(h)h (initial)i(pro)q(cess,)e(and)g(the)g(allo)q(cation)g(of)f(pro)q(cesses) h(to)f(pro)q(cessors.)36 b(Also,)21 b(the)g(curren)o(t)75 214 y(prop)q(osal)f(do)q(es)h(not)e(pro)o(vide)i(for)e(dynamic)i (creation)f(or)g(deletion)h(of)f(pro)q(cesses)g(during)h(program)75 271 y(execution)c(\(the)f(total)f(n)o(um)o(b)q(er)i(of)f(pro)q(cesses)g (is)h(\014xed\),)f(although)g(it)h(is)f(in)o(tended)i(to)d(b)q(e)i (consisten)o(t)75 327 y(with)h(suc)o(h)f(extensions.)27 b(Finally)l(,)19 b(the)e(curren)o(t)g(prop)q(osal)h(do)q(es)f(not)g(sp) q(ecify)h(a)f(naming)h(sc)o(heme)g(for)75 384 y(pro)q(cesses.)33 b(W)l(e)20 b(prop)q(ose)f(to)g(alw)o(a)o(ys)g(iden)o(tify)i(pro)q (cesses)f(according)g(to)f(their)h(relativ)o(e)g(rank)f(in)h(a)75 440 y(group,)15 b(so)f(that,)g(e\013ectiv)o(ely)l(,)i(pro)q(cesses)g (are)f(iden)o(ti\014ed)i(b)o(y)e(consecutiv)o(e)h(in)o(tegers.)75 592 y Fw(2.6)59 b(Erro)n(r)21 b(Handling)75 696 y FC(MPI)e(pro)o(vides) h(the)f(user)g(with)h(reliable)h(message)e(transmission.)32 b(A)19 b(message)g(sen)o(t)g(is)g(alw)o(a)o(ys)g(re-)75 752 y(ceiv)o(ed)e(correctly)l(,)e(and)h(the)g(user)g(do)q(es)f(not)h (need)g(to)f(c)o(hec)o(k)h(for)f(transmission)h(errors,)e(time-outs,)h (or)75 809 y(other)g(error)f(conditions.)22 b(In)15 b(other)g(w)o (ords,)f(MPI)h(do)q(es)h(not)f(pro)o(vide)h(mec)o(hanisms)f(for)g (dealing)i(with)75 865 y(failures)f(in)g(the)f(comm)o(unication)h (system.)j(If)c(the)g(MPI)g(implemen)o(tation)i(is)e(built)i(on)d(an)h (unreliable)75 922 y(underlying)k(mec)o(hanism,)e(then)g(it)g(is)h(the) e(job)h(of)f(the)h(implemen)o(ter)h(of)f(the)f(MPI)h(subsystem)g(to)f (in-)75 978 y(sulate)h(the)f(user)g(from)g(this)g(unreliabili)q(t)o(y)l (,)i(or)e(to)g(re\015ect)g(unreco)o(v)o(erable)h(errors)e(as)h(global)h (program)75 1035 y(failures.)k(Similarly)l(,)c(MPI)e(itself)h(pro)o (vides)g(no)f(mec)o(hanisms)h(for)e(handling)j(no)q(de)f(failures.)166 1092 y(Of)g(course,)h(MPI)f(programs)f(ma)o(y)h(still)i(b)q(e)f (erroneous.)23 b(A)16 b FD(program)i(error)d FC(can)i(o)q(ccur)f(when) 75 1149 y(an)f(MPI)g(call)g(is)h(called)g(with)f(an)g(incorrect)g (parameter)f(\(non-existing)i(destination)g(in)g(a)e(send)i(op)q(er-)75 1205 y(ation,)h(bu\013er)g(to)q(o)f(small)h(in)h(a)e(receiv)o(e)i(op)q (eration,)f(etc.\))25 b(This)17 b(t)o(yp)q(e)g(of)f(error)g(w)o(ould)h (o)q(ccur)h(in)f(an)o(y)75 1262 y(implemen)o(tation.)31 b(In)19 b(addition,)h(a)f FD(resource)h(error)d FC(ma)o(y)h(o)q(ccur)h (when)g(a)f(program)f(exceeds)j(the)75 1318 y(amoun)o(t)d(of)f(a)o(v)m (ailable)j(system)e(resources)h(\(n)o(um)o(b)q(er)f(of)g(p)q(ending)i (messages,)e(system)g(bu\013ers,)g(etc.\).)75 1375 y(The)h(o)q (ccurrence)h(of)f(this)g(t)o(yp)q(e)g(of)g(error)f(dep)q(ends)j(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(in)h(the)75 1431 y(system)13 b(and)h(the)g(resource)f(allo)q(cation)i(mec)o(hanism) f(used;)h(this)f(ma)o(y)f(di\013er)h(from)f(system)g(to)g(system.)75 1488 y(The)i(recommended)g(implemen)o(tation)g(will)h(pro)o(vide)f (generous)g(limits)g(on)g(some)f(of)g(the)g(resources)g(so)75 1544 y(as)i(to)g(alleviate)i(the)f(p)q(ortabilit)o(y)g(problem)h(this)f (represen)o(ts.)24 b(One)17 b(can)g(also)f(write)h FD(safe)g FC(programs,)75 1600 y(that)d(are)h(not)g(sub)s(ject)g(to)g(resource)g (errors.)166 1658 y(All)21 b(MPI)f(calls)h(return)f(an)f(error)h (parameter)f(that)g(indicates)i(successful)g(completion)g(of)f(the)75 1715 y(op)q(eration,)15 b(or)g(the)g(error)g(condition)h(that)f(o)q (ccurred,)g(otherwise.)21 b(Ho)o(w)o(ev)o(er,)13 b(MPI)j(do)q(es)f(not) g(require)75 1771 y(the)f(user)f(to)g(c)o(hec)o(k)h(error)e (parameters.)19 b(It)13 b(is)h(recommended)h(to)e(pro)o(vide)h(an)f (implemen)o(tation)i(where)75 1828 y(the)g(en)o(vironmen)o(t)g(for)f (an)h(MPI)g(execution)g(can)g(b)q(e)h(set)e(so)h(that)f(an)o(y)g(error) g(encoun)o(tered)i(in)f(an)g(MPI)75 1884 y(call)k(will)h(cause)e(a)f (program)g(to)g(fail)i(with)f(a)g(suitable)h(error)e(message)h (returned)g(to)g(the)g(user.)28 b(The)75 1941 y(en)o(vironmen)o(t)15 b(for)f(an)g(MPI)h(execution)h(can)e(also)h(b)q(e)g(set)g(so)f(that)g (\(non-fatal\))g(errors)g(do)g(not)g(cause)h(a)75 1997 y(program)f(to)h(fail,)g(and)h(the)f(user)g(can)h(pro)o(vide)f(his)h (or)f(her)g(o)o(wn)g(error)g(reco)o(v)o(ery)f(co)q(de.)166 2055 y(MPI)f(calls)h(ma)o(y)f(initiate)h(op)q(erations)g(that)e(con)o (tin)o(ue)i(async)o(hronously)g(after)e(the)i(call)g(returned.)75 2111 y(Th)o(us,)22 b(the)g(op)q(eration)f(ma)o(y)g(return)g(with)h(a)f (co)q(de)h(indicating)h(successful)g(completion,)g(y)o(et)e(later)75 2168 y(cause)15 b(an)g(error)g(exception)h(to)e(b)q(e)i(raised.)k(If)15 b(there)g(is)h(a)f(subsequen)o(t)g(call)h(that)f(relates)g(to)f(the)h (same)75 2224 y(op)q(eration)f(\(e.g.,)f(a)g(call)i(that)e(v)o (eri\014es)i(that)e(an)h(async)o(hronous)f(op)q(eration)h(has)g (completed\))h(then)f(the)75 2281 y(error)f(parameter)g(asso)q(ciated)h (with)g(this)g(call)h(will)g(b)q(e)g(used)f(to)f(indicate)i(the)f (nature)g(of)f(the)h(error.)k(In)75 2337 y(a)c(few)h(cases,)f(the)g (error)g(ma)o(y)g(o)q(ccur)h(after)f(all)h(calls)h(that)d(relate)i(to)f (the)h(op)q(eration)f(ha)o(v)o(e)g(completed,)75 2394 y(so)j(that)g(no)g(error)g(parameter)g(can)h(b)q(e)g(used)g(to)f (indicate)i(the)f(nature)f(of)g(the)h(error)f(\(e.g.,)f(an)i(error)75 2450 y(in)f(a)e(send)h(with)g(the)g(ready)g(mo)q(de\).)21 b(Suc)o(h)16 b(an)g(error)f(m)o(ust)g(b)q(e)h(treated)g(as)f(fatal,)g (since)i(information)75 2507 y(cannot)e(b)q(e)h(returned)f(for)g(the)g (user)h(to)e(reco)o(v)o(er)h(from)f(it.)166 2647 y Fu(Implemen)o(tati)o (on)h(note:)45 b Ft(A)16 b(p)q(ossible)h(mec)o(hanism)c(is)j(to)g(ha)o (v)o(e)g(MPI)h(calls)e(generate)j(a)e(signal)f(that)75 2704 y(can)d(b)q(e)h(ignored)f(when)h(an)f(error)h(is)f(encoun)o (tered.)20 b(A)12 b(more)f(mo)q(dular)g(mec)o(hanism)e(that)k(allo)o (ws)e(one)h(to)g(handle)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 10 16 10 15 bop 75 -100 a FC(10)714 b Fv(SECTION)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y Ft(an)f(exception)g (within)f(its)h(con)o(text)h(is)f(preferred.)166 184 y FC(The)21 b(b)q(eha)o(vior)g(of)g(erroneous)f(programs)g(is)h(not)g (de\014ned)h(b)o(y)e(this)i(do)q(cumen)o(t.)37 b(The)21 b(desired)75 241 y(b)q(eha)o(vior)d(is)f(that)f(a)h(relev)m(an)o(t)h (error)e(co)q(de)h(b)q(e)h(returned;)g(ho)o(w)o(ev)o(er,)e(e\016ciency) j(considerations)f(ma)o(y)75 297 y(dictate)e(that)e(the)h(system)g(do)q (es)h(not)e(detect)i(certain)f(t)o(yp)q(es)h(of)e(errors.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 11 17 11 16 bop 75 391 a Fx(Section)35 b(3)75 633 y FE(P)m(oin)m(t)41 b(to)f(P)m(oin)m(t)h(Comm)m(unication)75 908 y Fw(3.1)59 b(Intro)r(duction)75 1076 y FC(This)17 b(c)o(hapter)e(is)i(a)f(draft)f (of)h(the)g(curren)o(t)g(prop)q(osal)g(for)g(p)q(oin)o(t-to-p)q(oin)o (t)g(comm)o(unication.)23 b(Sending)75 1133 y(and)c(receiving)h(of)e (messages)g(b)o(y)g(pro)q(cesses)h(is)g(the)g(basic)g(MPI)g(comm)o (unication)g(mec)o(hanism.)31 b(All)75 1189 y(other)20 b(comm)o(unication)h(functions)f(can)h(b)q(e)g(implemen)o(ted)g(on)f (top)g(of)g(this)g(basic)h(comm)o(unication)75 1246 y(la)o(y)o(er)15 b(\(although)g(a)g(more)g(direct)h(implemen)o(tation)g(ma)o(y)f(lead)h (to)e(greater)g(e\016ciency\).)166 1336 y(The)19 b(basic)g(p)q(oin)o(t) g(to)f(p)q(oin)o(t)h(comm)o(unication)g(op)q(erations)g(are)f FD(send)g FC(and)h FD(receiv)o(e)p FC(.)29 b(A)18 b FD(send)75 1393 y FC(op)q(eration)13 b(creates)f(and)h(sends)g(a)g(message.)18 b(The)13 b(op)q(eration)g(sp)q(eci\014es)i(a)d FD(send)i(bu\013er)e FC(in)i(the)e(sender)75 1449 y(memory)f(from)g(whic)o(h)h(the)g (message)f(data)g(is)h(tak)o(en.)18 b(In)12 b(addition,)h(the)f(send)g (op)q(eration)g(asso)q(ciates)f(an)75 1506 y FD(en)o(v)o(elop)q(e)16 b FC(with)g(the)g(message.)22 b(This)16 b(en)o(v)o(elop)q(e)h(sp)q (eci\014es)h(the)e(message)g(destination)h(and)f(con)o(tains)75 1562 y(distinguishing)h(information)d(that)f(can)i(b)q(e)g(used)f(b)o (y)g(the)g FD(receiv)o(e)g FC(op)q(eration)g(to)g(select)h(a)e (particular)75 1619 y(message.)166 1710 y(A)22 b FD(receiv)o(e)f FC(op)q(eration)h(consumes)g(a)f(message.)39 b(The)22 b(message)f(to)g(b)q(e)i(receiv)o(ed)f(is)h(selected)75 1766 y(according)13 b(to)f(the)g(v)m(alue)i(on)e(its)g(en)o(v)o(elop)q (e,)i(and)f(the)f(message)g(data)g(is)h(put)f(in)o(to)g(the)h FD(receiv)o(e)g(bu\013er)p FC(.)166 1857 y(The)f(next)g(sections)h (describ)q(e)g(the)g(basic)f(\(blo)q(c)o(king\))h(send)g(and)f(receiv)o (e)h(op)q(erations.)19 b(W)l(e)12 b(discuss)75 1913 y(send,)21 b(receiv)o(e,)g(basic)f(comm)o(unication)g(seman)o(tics,)h(t)o(yp)q(e)e (matc)o(hing)h(requiremen)o(ts,)h(t)o(yp)q(e)e(con)o(v)o(er-)75 1970 y(sion)14 b(in)h(heterogeneous)f(en)o(vironmen)o(ts,)f(and)h(more) g(general)g(comm)o(unication)g(mo)q(des.)20 b(Non)o(blo)q(c)o(king)75 2026 y(comm)o(unication)c(is)h(addressed)f(next,)f(follo)o(w)o(ed)h(b)o (y)g(c)o(hannel-lik)o(e)i(constructs)e(and)f(send-receiv)o(e)j(op-)75 2083 y(erations.)i(W)l(e)15 b(then)g(consider)h(general)f(datat)o(yp)q (es)f(that)g(allo)o(w)h(one)g(to)g(transfer)f(heterogeneous)h(and)75 2139 y(noncon)o(tiguous)f(data,)f(and)h(conclude)h(with)f(a)g (description)h(of)e(an)h(implemen)o(tation)h(of)e(MPI)g(p)q(oin)o(t)i (to)75 2196 y(p)q(oin)o(t)h(comm)o(unication)g(using)g(a)e(small)i(n)o (um)o(b)q(er)g(of)f(primitiv)o(es.)75 2535 y Fw(3.2)59 b(Basic)19 b(send)h(op)r(eration)75 2704 y FC(The)15 b(syn)o(tax)g(of)g(the)g(simplest)h(send)g(op)q(eration)f(is)h(giv)o (en)g(b)q(elo)o(w.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 12 18 12 17 bop 75 -100 a FC(12)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(SEND)f(\(sta)o(rt,)g(count,)h(datat)o(yp)q(e,)h (dest,)f(tag,)f(comm\))117 136 y Ft(IN)155 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 238 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts) g(in)g(send)h(bu\013er)h(\(nonnegativ)o(e)e(in)o(te-)905 294 y(ger\))117 396 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t) 117 498 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 600 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 702 y(IN)155 b Fo(comm)470 b Ft(comm)o(unicator)11 b(\(handle\))75 968 y Fq(3.2.1)49 b(Message)16 b(data)75 1080 y FC(The)i(send)h(bu\013er)f(sp)q (eci\014ed)i(b)o(y)d(the)h Fo(MPI)p 828 1080 V 16 w(SEND)h FC(op)q(eration)f(consists)g(of)f Fo(count)i FC(successiv)o(e)h(en)o (tries)75 1136 y(of)e(the)g(t)o(yp)q(e)g(indicated)h(b)o(y)f Fo(datat)o(yp)q(e)p FC(,)i(starting)e(with)g(the)g(en)o(try)f(at)h (address)g Fo(sta)o(rt)p FC(.)28 b(Note)18 b(that)f(w)o(e)75 1193 y(sp)q(ecify)d(the)e(message)g(length)h(in)g(terms)e(of)h(n)o(um)o (b)q(er)h(of)e Fs(elements)p FC(,)h(not)f(n)o(um)o(b)q(er)i(of)f Fs(bytes)p FC(.)18 b(The)13 b(former)75 1249 y(is)j(mac)o(hine)g(indep) q(enden)o(t)h(and)f(closer)f(to)g(the)g(application)i(lev)o(el.)166 1319 y(The)g(data)f(part)g(of)g(the)g(message)h(consists)f(of)h(a)f (sequence)i(of)e Fo(count)i FC(v)m(alues,)f(eac)o(h)g(of)f(the)h(t)o (yp)q(e)75 1375 y(indicated)i(b)o(y)e Fo(datat)o(yp)q(e)p FC(.)52 b Fo(count)18 b FC(ma)o(y)e(b)q(e)i(zero,)e(in)i(whic)o(h)g (case)f(the)g(data)f(part)g(of)h(the)g(message)f(is)75 1432 y(empt)o(y)l(.)30 b(The)18 b(basic)h(datat)o(yp)q(es)f(that)g(can) h(b)q(e)g(sp)q(eci\014ed)h(for)e(message)g(data)g(v)m(alues)i(corresp)q (ond)f(to)75 1488 y(the)13 b(basic)h(datat)o(yp)q(es)f(of)g(the)g(host) g(language.)19 b(The)14 b(p)q(ossible`)g(v)m(alues)h(of)d(this)i (parameter)f(for)f(F)l(ortran)75 1545 y(and)j(the)h(corresp)q(onding)g (F)l(ortran)e(t)o(yp)q(es)h(are)g(listed)h(b)q(elo)o(w)p 473 1788 1004 2 v 472 1844 2 57 v 498 1827 a(MPI)f(datat)o(yp)q(e)p 1044 1844 V 292 w(F)l(ortran)f(datat)o(yp)q(e)p 1476 1844 V 473 1846 1004 2 v 472 1902 2 57 v 513 1885 a Fo(MPI)p 598 1885 14 2 v 16 w(INTEGER)p 1044 1902 2 57 v 270 w Fp(INTEGER)p 1476 1902 V 472 1959 V 513 1942 a Fo(MPI)p 598 1942 14 2 v 16 w(REAL)p 1044 1959 2 57 v 348 w Fp(REAL)p 1476 1959 V 472 2015 V 513 1998 a Fo(MPI)p 598 1998 14 2 v 16 w(DOUBLE)p 1044 2015 2 57 v 280 w Fp(DOUBLE)23 b(PRECISION)p 1476 2015 V 472 2071 V 513 2055 a Fo(MPI)p 598 2055 14 2 v 16 w(COMPLEX)p 1044 2071 2 57 v 246 w Fp(COMPLEX)p 1476 2071 V 472 2128 V 513 2111 a Fo(MPI)p 598 2111 14 2 v 16 w(DOUBLE)p 793 2111 V 17 w(COMPLEX)p 1044 2128 2 57 v 50 w Fp(DOUBLE)g(COMPLEX)p 1476 2128 V 472 2184 V 513 2167 a Fo(MPI)p 598 2167 14 2 v 16 w(LOGICAL)p 1044 2184 2 57 v 274 w Fp(LOGICAL)p 1476 2184 V 472 2241 V 513 2224 a Fo(MPI)p 598 2224 14 2 v 16 w(CHARA)o(CTER)p 1044 2241 2 57 v 194 w Fp(CHARACTER)p 1476 2241 V 472 2297 V 513 2280 a Fo(MPI)p 598 2280 14 2 v 16 w(BYTE)p 1044 2297 2 57 v 1476 2297 V 473 2299 1004 2 v 75 2549 a FC(The)13 b(p)q(ossible)i(v)m(alues)g(for)d(this)i(parameter)e(for)h (C)f(and)i(the)f(corresp)q(onding)h(C)f(t)o(yp)q(es)g(are)g(listed)h(b) q(elo)o(w)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 13 19 13 18 bop 75 -100 a Fv(3.2.)34 b(BASIC)16 b(SEND)f(OPERA)l(TION)1075 b FC(13)p 645 5 660 2 v 644 62 2 57 v 670 45 a(MPI)15 b(datat)o(yp)q(e)p 1031 62 V 107 w(C)g(datat)o(yp)q(e)p 1304 62 V 645 63 660 2 v 644 120 2 57 v 685 103 a Fo(MPI)p 770 103 14 2 v 16 w(SHORT)p 1031 120 2 57 v 124 w Fp(short)p 1304 120 V 644 176 V 685 159 a Fo(MPI)p 770 159 14 2 v 16 w(INT)p 1031 176 2 57 v 198 w Fp(int)p 1304 176 V 644 233 V 685 216 a Fo(MPI)p 770 216 14 2 v 16 w(LONG)p 1031 233 2 57 v 154 w Fp(long)p 1304 233 V 644 289 V 685 272 a Fo(MPI)p 770 272 14 2 v 16 w(UNSIGNED)p 1031 289 2 57 v 51 w Fp(unsigned)p 1304 289 V 644 346 V 685 329 a Fo(MPI)p 770 329 14 2 v 16 w(FLO)o(A)l(T)p 1031 346 2 57 v 134 w Fp(float)p 1304 346 V 644 402 V 685 385 a Fo(MPI)p 770 385 14 2 v 16 w(DOUBLE)p 1031 402 2 57 v 95 w Fp(double)p 1304 402 V 644 459 V 685 442 a Fo(MPI)p 770 442 14 2 v 16 w(CHAR)p 1031 459 2 57 v 154 w Fp(char)p 1304 459 V 644 515 V 685 498 a Fo(MPI)p 770 498 14 2 v 16 w(BYTE)p 1031 515 2 57 v 1304 515 V 645 517 660 2 v 166 695 a FC(The)g(datat)o(yp)q(e)28 b Fo(MPI)p 548 695 14 2 v 16 w(BYTE)15 b FC(do)q(es)g(not)f(corresp)q (ond)h(to)e(a)i(F)l(ortran)e(or)h(C)g(datat)o(yp)q(e.)19 b(A)c(v)m(alue)g(of)75 752 y(t)o(yp)q(e)33 b Fo(MPI)p 280 752 V 16 w(BYTE)17 b FC(consists)g(of)f(8)g(binary)i(digits.)24 b(A)17 b(b)o(yte)f(is)h(unin)o(terpreted)h(and)f(is)g(di\013eren)o(t)g (from)75 808 y(a)g(c)o(haracter.)25 b(Di\013eren)o(t)17 b(mac)o(hines)g(ma)o(y)g(ha)o(v)o(e)g(di\013eren)o(t)g(represen)o (tations)g(for)g(c)o(haracters,)f(or)h(ma)o(y)75 865 y(use)g(more)g(than)g(one)g(b)o(yte)g(to)g(represen)o(t)g(c)o (haracters.)25 b(On)18 b(the)f(other)g(hand,)g(a)g(b)o(yte)g(has)g(the) g(same)75 921 y(binary)f(v)m(alue)g(on)f(all)h(mac)o(hines.)166 1056 y Fu(Discussion:)166 1108 y Ft(Need)f(to)f(decide)g(whether)i(w)o (e)e(w)o(an)o(t)f(\(for)h(C\))166 1159 y Fn(unsigned)20 b(char)p Ft(,)13 b Fn(unsigned)20 b(short)p Ft(,)12 b Fn(unsigned)20 b(long)p Ft(,)13 b Fn(long)21 b(double)166 1211 y(DOUBLE)f(COMPLEX)13 b Ft(ma)o(y)f(not)i(b)q(e)g(standard)g(F)m (ortran)g(77)166 1263 y Fr(MPI)p 243 1263 13 2 v 14 w(DOUBLE)f Ft(is)h(no)o(w)f(used)i(b)q(oth)f(for)g(F)m(ortran)f(and)h(C.)f(Should) h(w)o(e)g(use)h(di\013eren)o(t)f(names?)75 1479 y Fq(3.2.2)49 b(Message)16 b(envelop)q(e)75 1568 y FC(In)g(addition)g(to)e(the)h (data)g(part,)f(messages)g(con)o(tain)h(information)h(that)e(can)h(b)q (e)h(used)f(to)g(distinguish)75 1625 y(messages)h(and)g(selectiv)o(ely) i(receiv)o(e)f(them.)23 b(This)17 b(information)f(is)h(con)o(tained)g (in)g(a)f(\014xed)h(n)o(um)o(b)q(er)f(of)75 1681 y(\014xed-format)f (\014elds,)h(whic)o(h)g(w)o(e)f(collectiv)o(ely)i(call)g(the)e FD(message)i(en)o(v)o(elop)q(e)p FC(.)i(These)d(\014elds)g(are)75 1783 y FD(source)75 1884 y(destination)75 1986 y(tag)75 2088 y(con)o(text)166 2190 y FC(The)c(message)g(source)g(is)h (implicitly)i(determined)f(b)o(y)e(the)g(iden)o(tit)o(y)h(of)f(the)g (message)g(sender.)20 b(The)75 2246 y(other)15 b(\014elds)h(are)f(sp)q (eci\014ed)i(b)o(y)f(parameters)e(in)i(the)f(send)h(op)q(eration.)166 2304 y(The)j(in)o(teger-v)m(alued)h(message)e(tag)g(is)h(sp)q (eci\014ed)i(b)o(y)d(the)37 b Fo(tag)19 b FC(parameter.)29 b(This)19 b(in)o(teger)g(can)75 2361 y(b)q(e)e(used)g(b)o(y)g(the)f (program)f(to)h(distinguish)j(di\013eren)o(t)d(t)o(yp)q(es)h(of)f (messages.)23 b(The)17 b(range)f(of)g(v)m(alid)i(tag)75 2417 y(v)m(alues)e(is)g(implemen)o(tation)g(dep)q(enden)o(t)g(and)f (can)g(b)q(e)h(queried)g(using)g(the)f Fo(T)l(agRange)g FC(en)o(vironmen)o(tal)75 2474 y(inquiry)i(function,)e(as)g(describ)q (ed)i(in)f(Chapter)f(7.)166 2532 y(The)k(con)o(text)g(of)g(the)g (message)g(sen)o(t)g(is)h(sp)q(eci\014ed)h(b)o(y)f(the)38 b Fo(comm)18 b FC(parameter.)31 b(the)19 b(message)75 2589 y(carries)c(the)h(con)o(text)e(asso)q(ciated)i(with)f(this)h(comm) o(unicator)f(\(see)g(Chapter)g(9\).)166 2647 y(The)d(message)f (destination)i(is)f(sp)q(eci\014ed)i(b)o(y)e(the)23 b Fo(dest)14 b FC(parameter)d(as)g(a)g(rank)h(within)h(the)f(pro)q(cess) 75 2704 y(group)18 b(asso)q(ciated)h(with)g(that)f(same)h(comm)o (unicator.)30 b(The)19 b(range)f(of)h(v)m(alid)h(v)m(alues)g(is)f Fp(0,)24 b(...)47 b(,)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 14 20 14 19 bop 75 -100 a FC(14)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fp(n-1)p FC(,)i(where)g Fp(n)g FC(is)h(the)f(n)o(um)o(b)q(er)g(of)g(pro)q (cesses)g(in)h(this)g(group.)28 b(Th)o(us,)19 b(p)q(oin)o(t-to-p)q(oin) o(t)f(comm)o(unica-)75 102 y(tions)e(do)g(not)f(use)i(absolute)f (addresses,)g(but)g(only)g(relativ)o(e)h(ranks)e(within)j(a)d(group.)22 b(This)16 b(pro)o(vides)75 158 y(imp)q(ortan)o(t)f(mo)q(dularit)o(y)l (.)166 215 y(The)j(message)f(en)o(v)o(elop)q(e)i(w)o(ould)f(normally)g (b)q(e)h(enco)q(ded)g(b)o(y)e(a)g(\014xed-length)j(message)d(header.)75 272 y(Ho)o(w)o(ev)o(er,)c(the)i(actual)f(mec)o(hanism)h(used)g(to)f (asso)q(ciate)g(an)g(en)o(v)o(elop)q(e)i(with)e(a)g(message)g(is)h (implemen-)75 328 y(tation)d(dep)q(enden)o(t:)20 b(some)12 b(of)g(the)g(information)g(\(e.g.,)g(source)g(or)g(destination\))g(ma)o (y)g(b)q(e)h(implicit,)i(and)75 385 y(need)f(not)g(b)q(e)g(explicitly)i (carried)e(b)o(y)g(messages;)f(pro)q(cesses)h(ma)o(y)f(b)q(e)h(iden)o (ti\014ed)i(b)o(y)d(relativ)o(e)h(ranks,)f(or)75 441 y(absolute)j(ids;)f(etc.)75 589 y Fw(3.3)59 b(Basic)19 b(receive)g(op)r(eration)75 692 y FC(The)c(syn)o(tax)g(of)g(the)g (simplest)h(receiv)o(e)g(op)q(eration)g(is)f(giv)o(en)h(b)q(elo)o(w.)75 796 y Fo(MPI)p 160 796 14 2 v 16 w(RECV)g(\(sta)o(rt,)f(count,)h(datat) o(yp)q(e,)g(source,)f(tag,)g(comm,)e(status\))117 874 y Ft(OUT)108 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f(receiv) o(e)h(bu\013er)g(\(c)o(hoice\))117 951 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1027 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)e(\(state\))117 1104 y(IN)155 b Fo(source)468 b Ft(rank)14 b(of)f(source)j(\(in)o(teger\))117 1181 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 1257 y(IN)155 b Fo(comm)470 b Ft(comm)o(unicator)11 b(\(handle\))117 1334 y(OUT)108 b Fo(status)476 b Ft(status)15 b(ob)r(ject)166 1459 y FC(The)f(receiv)o(e)h(bu\013er)f(consists)g(of)f(the)h(storage)f (con)o(taining)29 b Fo(count)15 b FC(consecutiv)o(e)g(elemen)o(ts)f(of) g(the)75 1516 y(t)o(yp)q(e)j(sp)q(eci\014ed)i(b)o(y)33 b Fo(datat)o(yp)q(e)p FC(,)18 b(starting)e(at)g(address)34 b Fo(sta)o(rt)p FC(.)24 b(The)17 b(length)h(of)e(the)h(receiv)o(ed)g (message)75 1572 y(m)o(ust)e(b)q(e)i(less)g(then)f(or)f(equal)i(to)e (the)h(length)h(of)f(the)g(receiv)o(e)g(bu\013er.)23 b(I.e.,)15 b(all)i(incoming)g(data)f(m)o(ust)75 1629 y(\014t,)23 b(without)e(truncation,)i(in)o(to)e(the)h(receiv)o(e)g (bu\013er.)39 b(The)22 b Fo(MPI)p 1262 1629 V 16 w(PROBE)g FC(function)h(describ)q(ed)g(in)75 1685 y(Section)16 b(3.9)e(can)i(b)q(e)g(used)f(to)g(receiv)o(e)h(messages)f(of)f(unkno)o (wn)i(length.)166 1742 y(The)j(selection)h(of)f(a)f(message)h(b)o(y)f (a)h(receiv)o(e)h(op)q(eration)f(is)g(done)g(uniquely)i(according)f(to) e(the)75 1799 y(v)m(alue)i(of)f(the)g(message)g(en)o(v)o(elop)q(e.)32 b(The)19 b(receiv)o(e)h(op)q(eration)g(sp)q(eci\014es)h(an)d FD(en)o(v)o(elop)q(e)k(pattern)p FC(;)f(a)75 1855 y(message)13 b(can)g(b)q(e)g(receiv)o(ed)h(b)o(y)f(that)g(receiv)o(e)h(op)q(eration) f(only)g(if)h(its)f(en)o(v)o(elop)q(e)h(matc)o(hes)f(that)f(pattern.)75 1912 y(A)k(pattern)g(sp)q(eci\014es)h(v)m(alues)h(for)d(the)32 b Fo(source)p FC(,)g Fo(tag)16 b FC(and)32 b Fo(context)18 b FC(\014elds)f(of)f(the)g(message)f(en)o(v)o(elop)q(e.)75 1968 y(The)g(receiv)o(er)h(ma)o(y)e(sp)q(ecify)i(a)e(wildcard)j Fr(MPI)p 882 1968 13 2 v 14 w(ANY)p 981 1968 V 15 w(SOURCE)c FC(v)m(alue)j(for)29 b Fo(source)p FC(,)15 b(and/or)f(a)h(wildcard)75 2025 y Fr(MPI)p 152 2025 V 14 w(ANY)p 251 2025 V 15 w(T)m(A)o(G)21 b FC(v)m(alue)j(for)44 b Fo(tag)p FC(,)24 b(indicating)g(that)e(an)o(y) g(source)h(and/or)f(tag)f(are)h(acceptable.)43 b(It)75 2081 y(cannot)15 b(sp)q(ecify)i(a)f(wildcard)h(v)m(alue)f(for)31 b Fo(context)p FC(.)23 b(Th)o(us,)15 b(a)g(message)g(can)h(b)q(e)g (receiv)o(ed)h(b)o(y)f(a)f(receiv)o(e)75 2138 y(op)q(eration)23 b(only)f(if)h(it)g(is)g(addressed)f(to)g(the)h(receiving)g(pro)q(cess,) h(has)f(a)f(matc)o(hing)g(con)o(text,)h(has)75 2194 y(matc)o(hing)18 b(source)g(unless)h(source=)p Fr(MPI)p 791 2194 V 15 w(ANY)p 891 2194 V 14 w(SOURCE)d FC(in)j(the)f(pattern,)g(and)g(has)f (a)h(matc)o(hing)g(tag)75 2250 y(unless)e(tag=)e Fr(MPI)p 399 2250 V 14 w(ANY)p 498 2250 V 15 w(T)m(A)o(G)g FC(in)i(the)f (pattern.)166 2308 y(The)22 b(message)f(tag)g(is)h(sp)q(eci\014ed)i(b)o (y)e(the)43 b Fo(tag)22 b FC(parameter)f(of)g(the)h(receiv)o(e)h(op)q (eration.)40 b(The)75 2364 y(message)26 b Fo(context)15 b FC(is)f(the)g(con)o(text)f(attac)o(hed)g(with)g(the)h(comm)o (unicator)f(sp)q(eci\014ed)j(b)o(y)d(the)h(parameter)75 2421 y Fo(comm)p FC(.)j(The)12 b(message)g(source,)g(if)h(di\013eren)o (t)f(from)22 b Fr(MPI)p 1040 2421 V 14 w(ANY)p 1139 2421 V 15 w(SOURCE)p FC(,)10 b(is)i(sp)q(eci\014ed)j(as)c(a)h(rank)g(within) 75 2477 y(the)h(pro)q(cess)h(group)f(asso)q(ciated)g(with)h(that)e (same)h(comm)o(unicator.)19 b(Th)o(us,)13 b(the)g(range)g(of)g(v)m (alid)i(v)m(alues)75 2533 y(for)20 b(the)43 b Fo(source)21 b FC(parameter)g(is)g Fy(f)j Fp(0,)g(...)47 b(,)24 b(n-1)f Fy(g)h([f)f Fp(MPI)p 1250 2533 15 2 v 17 w(ANY)p 1339 2533 V 17 w(SOURCE)g Fy(g)p FC(,)f(where)f Fp(n)g FC(is)h(the)75 2590 y(n)o(um)o(b)q(er)16 b(of)e(pro)q(cesses)i(in)g(this)g(group.)166 2647 y(Note)i(the)i(asymmetry)d(b)q(et)o(w)o(een)j(send)f(and)g(receiv) o(e)h(op)q(erations:)28 b(A)19 b(receiv)o(e)h(op)q(eration)f(ma)o(y)75 2704 y(accept)13 b(messages)f(from)g(an)h(arbitrary)f(sender;)i(on)f (the)g(other)f(hand,)i(a)e(send)h(op)q(eration)g(m)o(ust)g(sp)q(ecify) 1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 15 21 15 20 bop 75 -100 a Fv(3.3.)34 b(BASIC)16 b(RECEIVE)g(OPERA)l(TION)990 b FC(15)75 45 y(a)15 b(unique)j(receiv)o(er.)k(This)16 b(matc)o(hes)f(a)h(\\push")f(comm)o(unication)i(mec)o(hanism,)f(where)g (data)f(transfer)75 102 y(is)i(e\013ected)h(b)o(y)f(the)g(sender)g (\(rather)f(than)h(a)g(\\pull")h(mec)o(hanism,)g(where)f(data)f (transfer)g(is)i(e\013ected)75 158 y(b)o(y)d(the)g(receiv)o(er\).)166 214 y(Source)h(=)f(destination)h(is)g(allo)o(w)o(ed:)k(a)15 b(pro)q(cess)g(can)h(send)g(a)e(message)h(to)g(itself.)75 335 y Fq(3.3.1)49 b(Return)16 b(status)75 504 y Fu(Discussion:)f(This)g (is)g(a)h(c)o(hange)f(from)g(previous)f(draft)166 643 y FC(The)f(source)g(or)f(the)h(tag)f(of)h(a)f(receiv)o(ed)i(message)f (ma)o(y)f(not)g(b)q(e)i(kno)o(wn)e(if)i(wildcard)g(v)m(alues)g(where)75 699 y(used)j(in)g(the)g(receiv)o(e)g(op)q(eration.)24 b(Also,)16 b(the)h(actual)f(length)i(of)e(the)g(message)g(receiv)o(ed)i (ma)o(y)d(not)h(b)q(e)75 756 y(kno)o(wn.)22 b(Th)o(us,)15 b(this)i(information)f(needs)g(to)g(b)q(e)g(returned)g(b)o(y)g(the)g (receiv)o(e)h(op)q(eration.)22 b(F)l(or)15 b(reasons)75 812 y(that)i(will)j(b)q(ecome)f(clear)f(later,)g(it)g(is)h(preferable)g (to)e(return)h(this)g(information)h(in)f(a)g(separate)35 b Fo(out)75 869 y FC(v)m(ariable,)23 b(rather)d(then)i(using)f(the)42 b Fo(count)p FC(,)j Fo(source)21 b FC(and)42 b Fo(tag)21 b FC(parameters)f(b)q(oth)h(for)f(input)i(and)75 925 y(output.)j(The)17 b(information)g(is)h(returned)f(b)o(y)g(the)34 b Fo(status)19 b FC(parameter)d(of)h(the)g Fo(MPI)p 1554 925 14 2 v 16 w(RECV)g FC(function.)75 981 y(This)h(is)f(a)g(parameter) f(of)g(a)h(sp)q(ecial)h(MPI-de\014ned)h(t)o(yp)q(e.)25 b(The)17 b(status)f(v)m(ariable)j(can)e(b)q(e)g(\\deco)q(ded")75 1038 y(to)e(retriev)o(e)g(the)30 b Fo(count)p FC(,)i Fo(source)15 b FC(and)31 b Fo(tag)15 b FC(\014elds,)h(using)g(the)f (query)h(functions)g(listed)g(b)q(elo)o(w.)75 1142 y Fo(MPI)p 160 1142 V 16 w(GET)p 264 1142 V 17 w(SOURCE\(status,)i (source\))117 1219 y Ft(IN)155 b Fo(status)476 b Ft(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 1293 y(OUT)108 b Fo(source)468 b Ft(source)15 b(rank)f(\(in)o(teger\)) 166 1417 y FC(Returns)e(the)f(rank)h(of)f(the)g(message)g(source)h (\(in)g(the)f(group)g(asso)q(ciated)h(with)g(the)f(comm)o(unicator)75 1474 y(that)j(w)o(as)h(used)h(to)e(receiv)o(e\).)75 1577 y Fo(MPI)p 160 1577 V 16 w(GET)p 264 1577 V 17 w(T)l(A)o(G\(status,)j (tag\))117 1655 y Ft(IN)155 b Fo(status)476 b Ft(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 1729 y(OUT)108 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))166 1853 y FC(Returns)j(the)f(tag)f(of)h(receiv)o(ed)h(message.)75 1957 y Fo(MPI)p 160 1957 V 16 w(GET)p 264 1957 V 17 w(COUNT\(status,)h (count\))117 2034 y Ft(IN)155 b Fo(status)476 b Ft(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 2108 y(OUT)108 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(elemen)o(ts)f(\(in)o(teger\))166 2232 y FC(Returns)h(the)f(n)o(um)o(b)q(er)h(of)f(elemen)o(ts)h(receiv)o (ed.)21 b(\(Here,)14 b(again,)h(w)o(e)f(coun)o(t)g Fs(elements)p FC(,)f(not)h Fs(bytes)p FC(.\))166 2289 y(The)20 b(information)f (returned)h(b)o(y)g(these)f(query)h(functions)g(is)g(the)g(information) g(last)f(stored)g(in)75 2345 y(the)28 b Fo(status)16 b FC(v)m(ariable)f(b)o(y)f(a)g(receiv)o(e)h(function.)20 b(It)14 b(is)g(erroneous)g(to)f(call)i(these)g(query)f(functions)g(if)h (the)75 2402 y Fo(status)i FC(v)m(ariable)g(w)o(as)d(nev)o(er)h(set)g (b)o(y)h(a)e(receiv)o(e.)166 2458 y(Note)h(that)g(it)h(is)g(not)f (mandatory)g(to)g(query)g(the)h(return)g(status)e(after)h(a)g(receiv)o (e.)22 b(The)16 b(receiv)o(er)75 2515 y(will)h(use)e(these)h(calls,)g (or)e(some)h(of)g(them,)g(only)h(when)f(the)h(information)f(they)g (return)g(is)h(needed.)166 2571 y(information)f(returned)h(in)g(it)f(b) o(y)g(the)h(last)f(receiv)o(e)h(op)q(eration)f(that)g(up)q(dated)166 2704 y Fu(Discussion:)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 16 22 16 21 bop 75 -100 a FC(16)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Ft(The)g(use)g(of)e (a)h(separate)i(status)f(parameter)e(prev)o(en)o(ts)j(errors)f(that)g (are)f(often)h(attac)o(hed)f(with)30 b(INOUT)75 95 y(parameters)14 b(\(e.g.,)f(passing)g(the)28 b Fr(MPI)p 701 95 13 2 v 15 w(ANY)p 801 95 V 14 w(T)m(A)o(G)13 b Ft(constan)o(t)h(as)g(the)h (actual)e(argumen)o(t)g(for)27 b Fr(tag)p Ft(\).)18 b(The)c(use)h(of)75 145 y(an)h(explicit)f(user)i(ob)r(ject)g(b)q(ecomes)f(imp)q(ortan)o(t)f (for)g(non)o(blo)q(c)o(king)g(comm)o(unication)e(as)j(it)f(allo)o(ws)g (the)i(receiv)o(e)75 195 y(op)q(eration)c(to)g(b)q(e)h(stateless)h (and,)d(hence,)i(reen)o(tran)o(t.)20 b(This)13 b(prev)o(en)o(ts)h (confusions)f(in)g(the)h(case)g(where)g(m)o(ultiple)75 244 y(receiv)o(es)i(can)e(b)q(e)g(p)q(osted)h(b)o(y)f(a)f(pro)q(cess.) 166 461 y Fu(Implemen)o(tati)o(on)f(note:)166 518 y Ft(One)j(exp)q (ects)g(deco)q(de)h(functions)e(to)f(b)q(e)i(in-lined)e(in)g(man)o(y)f (implemen)o(tations.)166 658 y FC(All)20 b(send)g(and)f(receiv)o(e)h (op)q(erations)f(use)g(the)38 b Fo(sta)o(rt)p FC(,)i Fo(count)p FC(,)21 b Fo(datat)o(yp)q(e)p FC(,)41 b Fo(source)p FC(,)f Fo(dest)p FC(,)g Fo(tag)p FC(,)75 715 y Fo(comm)17 b FC(and)37 b Fo(status)21 b FC(parameters)c(in)j(the)f(same)f(w)o(a)o (y)f(as)i(the)f(basic)38 b Fo(MPI)p 1402 715 14 2 v 16 w(SEND)19 b FC(and)37 b Fo(MPI)p 1746 715 V 16 w(RECV)75 771 y FC(op)q(erations)15 b(describ)q(ed)i(in)g(this)e(section.)75 921 y Fw(3.4)59 b(Semantics)18 b(of)i(p)r(oint-to-p)r(oint)e (communication)75 1025 y FC(A)g(v)m(alid)h(MPI)e(implemen)o(tation)i (guaran)o(tees)e(certain)h(general)g(prop)q(erties)g(of)f(p)q(oin)o (t-to-p)q(oin)o(t)h(com-)75 1081 y(m)o(unication,)e(whic)o(h)g(are)f (describ)q(ed)i(in)f(this)f(section.)75 1208 y Fo(Order)46 b FC(Messages)14 b(are)h Fs(non-overtaking)p FC(,)g(within)h(eac)o(h)f (con)o(text:)20 b(if)c(t)o(w)o(o)d(messages)i(are)g(successiv)o(ely)75 1264 y(sen)o(t)10 b(from)g(the)g(same)g(source,)h(to)e(the)i(same)f (destination,)h(with)g(the)f(same)g(con)o(text,)h(then)f(the)g (messages)75 1320 y(are)17 b(receiv)o(ed)i(in)f(the)f(order)g(they)h(w) o(ere)f(sen)o(t.)26 b(I.e.,)17 b(if)h(the)f(receiv)o(er)i(p)q(osts)e(t) o(w)o(o)f(successiv)o(e)i(receiv)o(es)75 1377 y(that)g(b)q(oth)g(matc)o (h)g(either)i(message,)e(then)h(the)f(\014rst)h(receiv)o(e)g(will)h (receiv)o(e)g(the)e(\014rst)g(message,)h(and)75 1433 y(the)e(second)h(receiv)o(e)h(will)g(receiv)o(e)f(the)f(second)h (message.)26 b(This)18 b(requiremen)o(t)g(facilitates)g(matc)o(hing)75 1490 y(of)e(sends)h(to)f(receiv)o(es.)25 b(It)16 b(guaran)o(tees)g (that)g(message-passing)g(co)q(de)h(is)g(deterministic,)i(if)e(pro)q (cesses)75 1546 y(are)e(single-threaded)i(and)e(wildcard)31 b Fr(MPI)p 831 1546 13 2 v 14 w(ANY)p 930 1546 V 15 w(SOURCE)13 b FC(is)j(not)f(used)g(in)h(receiv)o(es.)166 1604 y(If)h(a)g(pro)q (cess)h(has)f(a)g(single)h(thread)f(of)g(execution,)i(then)e(an)o(y)g (t)o(w)o(o)f(comm)o(unications)i(executed)75 1660 y(b)o(y)e(this)h(pro) q(cess)g(are)f(ordered.)23 b(On)17 b(the)f(other)g(hand,)h(if)g(the)f (pro)q(cess)h(is)g(m)o(ulti-threaded,)g(then)g(the)75 1717 y(seman)o(tics)e(of)f(thread)h(execution)h(ma)o(y)e(not)g (de\014ne)i(a)e(relativ)o(e)i(order)e(b)q(et)o(w)o(een)h(t)o(w)o(o)e (send)j(op)q(erations)75 1773 y(executed)g(b)o(y)f(t)o(w)o(o)f (distinct)j(threads:)i(the)d(op)q(erations)f(are)g(logically)i (concurren)o(t,)e(ev)o(en)h(though)f(one)75 1830 y(ph)o(ysically)22 b(precedes)e(the)g(other.)33 b(In)20 b(suc)o(h)g(a)f(case,)i(the)e(t)o (w)o(o)g(messages)g(sen)o(t)g(can)h(b)q(e)g(receiv)o(ed)h(in)75 1886 y(an)o(y)f(order.)37 b(Similarly)l(,)24 b(if)d(t)o(w)o(o)e(receiv) o(e)j(op)q(erations)f(that)f(are)g(logically)j(concurren)o(t)e(receiv)o (e)g(t)o(w)o(o)75 1943 y(successiv)o(ely)i(sen)o(t)d(messages,)h(then)g (the)g(t)o(w)o(o)f(messages)g(can)h(matc)o(h)f(the)h(t)o(w)o(o)e (receiv)o(es)j(in)f(either)75 1999 y(order.)75 2125 y Fo(Progress)45 b FC(If)13 b(a)g(pair)g(of)f(matc)o(hing)h(send)h(and)f (receiv)o(es)g(ha)o(v)o(e)g(b)q(een)h(initiated)g(on)f(t)o(w)o(o)e(pro) q(cesses,)j(then)75 2182 y(at)k(least)h(one)h(of)e(these)h(t)o(w)o(o)f (op)q(erations)h(will)i(complete,)f(indep)q(enden)o(tly)i(of)c(other)h (actions)g(in)h(the)75 2238 y(system:)e(the)13 b(send)h(op)q(eration)f (will)h(complete,)g(unless)g(the)f(receiv)o(e)h(is)f(satis\014ed)h(b)o (y)e(another)h(message,)75 2295 y(and)h(completes;)g(the)g(receiv)o(e)h (op)q(eration)f(will)h(complete,)g(unless)g(the)f(message)f(sen)o(t)g (is)i(consumed)f(b)o(y)75 2351 y(another)h(matc)o(hing)g(receiv)o(e)h (that)f(w)o(as)f(p)q(osted)h(at)g(the)g(same)g(destination)h(pro)q (cess.)75 2478 y Fo(F)o(airness)46 b FC(MPI)15 b(mak)o(es)g(no)g (guaran)o(tee)g(of)g Fs(fairness)f FC(in)i(the)g(handling)h(of)e(comm)o (unication.)21 b(Supp)q(ose)75 2534 y(that)15 b(a)g(send)h(w)o(as)f(p)q (osted.)21 b(Then)15 b(it)h(is)g(p)q(ossible)h(that)e(the)h (destination)g(rep)q(eatedly)h(p)q(osts)e(a)g(receiv)o(e)75 2591 y(that)e(matc)o(hes)h(this)g(send,)h(y)o(et)e(the)h(message)g(is)g (nev)o(er)g(receiv)o(ed,)h(b)q(ecause)g(it)f(is)h(eac)o(h)f(time)g(o)o (v)o(ertak)o(en)75 2647 y(b)o(y)e(another)h(message,)f(sen)o(t)g(from)g (another)g(source.)19 b(Similarly)l(,)c(supp)q(ose)e(that)f(a)g(receiv) o(e)h(w)o(as)f(p)q(osted.)75 2704 y(Then)18 b(it)f(is)h(p)q(ossible)h (that)e(messages)g(that)f(matc)o(h)h(this)h(receiv)o(e)g(are)f(rep)q (eatedly)h(receiv)o(ed,)h(y)o(et)e(the)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 17 23 17 22 bop 75 -100 a Fv(3.4.)29 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)455 b FC(17)75 45 y(receiv)o(e)17 b(is)g(nev)o(er)g (satis\014ed,)f(b)q(ecause)i(it)e(is)h(o)o(v)o(ertak)o(en)e(b)o(y)h (other)g(receiv)o(es)h(p)q(osted)g(at)f(this)g(no)q(de)h(\(b)o(y)75 102 y(other)12 b(executing)h(threads\).)19 b(It)12 b(is)h(the)f (programmer's)e(resp)q(onsibili)q(t)o(y)k(to)e(prev)o(en)o(t)g(starv)m (ation)g(in)h(suc)o(h)75 158 y(situations.)166 291 y Fu(Discussion:)166 342 y(Need)i(to)h(b)q(e)f(discussed)f(b)o(y)h(MPIF) 166 392 y Ft(Alternativ)o(e)f(p)q(ositions:)166 443 y(1.)j(Sends)e(are) f(handled)f(fairly;)e(e.g.)18 b(a)13 b(p)q(osted)h(send)h(cannot)e(b)q (e)h(rep)q(eatedly)h(o)o(v)o(ertak)o(en)e(b)o(y)g(other)h(sends.)75 492 y(An)h(example)g(of)f(an)h(implemen)o(tation)d(that)k(violates)e (this:)21 b(Assume)16 b(that)f(arriving)f(messages)i(are)g(k)o(ept)f (in)g(a)75 542 y(hash)i(table)g(organized)f(b)o(y)h(source,)h(and)f (that)g(a)f(receiv)o(e)i(with)f(source=don)o(tcare)i(is)d(handled)h(b)o (y)f(searc)o(hing)75 592 y(this)d(table)g(sequen)o(tially)m(.)j(Then)e (a)e(message)h(p)q(osted)g(at)g(the)g(end)h(of)e(this)h(table)g(ma)o(y) d(b)q(e)k(rep)q(eatedly)g(o)o(v)o(ertak)o(en)75 642 y(b)o(y)g(messages) g(that)g(get)g(p)q(osted)h(at)e(the)i(head)f(of)f(the)i(table.)j (\(Easy)c(\014x:)k(rotate)c(starting)g(p)q(oin)o(t)f(of)h(searc)o(h.\)) 166 692 y(2.)22 b(Receiv)o(es)16 b(are)g(handled)f(fairly;)f(e.g.)22 b(a)15 b(p)q(osted)h(receiv)o(e)g(cannot)g(b)q(e)f(rep)q(eatedly)i(o)o (v)o(ertak)o(en)e(b)o(y)g(other)75 742 y(receiv)o(es.)24 b(An)15 b(example)f(of)h(an)g(implemen)o(tatio)o(n)e(that)i(violates)g (this:)21 b(assume)15 b(that)g(there)h(are)g(t)o(w)o(o)f(di\013eren)o (t)75 792 y(p)q(osting)k(tables)g(for)g(receiv)o(es)h(with)f(a)g(sp)q (eci\014c)h(source)g(and)f(for)g(receiv)o(es)h(with)f(source=don)o (tcare;)k(assume)75 842 y(further)c(that)g(these)g(t)o(w)o(o)f (structures)j(are)e(searc)o(hed)h(in)e(a)g(\014xed)h(order)g(when)g(a)f (message)g(arriv)o(es.)31 b(Then)19 b(a)75 892 y(receiv)o(e)d(with)f (source=don)o(tcare)i(ma)o(y)c(b)q(e)i(rep)q(eatedly)h(o)o(v)o(ertak)o (en)f(b)o(y)f(receiv)o(es)j(with)d(source)i(sp)q(eci\014ed.)23 b(\(Fix:)75 941 y(alternate)15 b(searc)o(h)g(order;)g(but)f(this)h(has) f(p)q(erformance)g(problems,)f(if)h(w)o(e)g(assume)g(that)h(one)f(t)o (yp)q(e)h(of)f(receiv)o(e)h(is)75 991 y(m)o(uc)o(h)e(more)g(frequen)o (t)h(than)g(the)h(other)f(t)o(yp)q(e.\))166 1042 y(3.)k(Both)c(send)h (and)e(receiv)o(es)j(are)e(handled)g(fairly)m(.)166 1099 y(Opinions?)75 1305 y Fo(Resource)g(limitations)45 b FC(The)13 b(curren)o(t)f(practice)h(for)f(man)o(y)g(commercial)i (message-passing)e(libraries)i(is)75 1362 y(that)f(\(short\))f (messages)h(are)g(bu\013ered)g(b)o(y)h(the)f(system,)g(th)o(us)g(allo)o (wing)h(blo)q(c)o(king)h(send)f(op)q(erations)g(to)75 1418 y(complete)d(ahead)f(of)g(the)h(matc)o(hing)f(receiv)o(es.)19 b(It)11 b(is)f(exp)q(ected)i(that)e(man)o(y)f(MPI)i(implemen)o(tations) g(will)75 1475 y(follo)o(w)j(this)g(practice,)g(and)f(pro)o(vide)h(the) g(same)f(lev)o(el)i(of)e(bu\013ering)h(that)f(is)h(a)o(v)m(ailable)h (on)f(the)f(libraries)75 1531 y(they)21 b(replace.)37 b(Ho)o(w)o(ev)o(er,)20 b(message)g(bu\013ering)h(is)h(not)e(a)g(univ)o (ersal)i(practice.)36 b(Ev)o(en)21 b(on)f(systems)75 1588 y(where)e(bu\013ering)g(o)q(ccur,)g(the)f(amoun)o(t)g(of)g (bu\013er)h(space)f(a)o(v)m(ailable)j(and)d(the)h(w)o(a)o(y)e(it)i(is)g (allo)q(cated)g(is)75 1644 y(b)q(ound)e(to)f(b)q(e)h(implemen)o(tation) g(dep)q(enden)o(t.)166 1701 y(Therefore,)24 b(message)e(bu\013ering)h (is)g(not)f(mandated)g(b)o(y)g(MPI)h(and)f(is)h(seen)g(as)f(a)g(qualit) o(y)h(of)75 1758 y(implemen)o(tation)c(issue.)28 b(A)18 b(v)m(alid)h(MPI)f(implemen)o(tation)h(of)35 b Fo(MPI)p 1291 1758 14 2 v 16 w(SEND)18 b FC(is)g(to)f(blo)q(c)o(k)i(the)e (sender)75 1814 y(un)o(til)23 b(a)f(matc)o(hing)h(receiv)o(e)g(has)f(b) q(een)h(initiated.)43 b(In)23 b(general,)h(the)e(programmer)f(can)h (mak)o(e)g(no)75 1871 y(assumptions)f(on)g(the)g(a)o(v)m(ailabilit)o(y) j(of)c(bu\013er)h(space,)i(and)e(ho)o(w)f(this)i(space)f(is)h(allo)q (cated.)38 b(Th)o(us,)75 1927 y(p)q(ortable)14 b(\(safe\))e(MPI)h(co)q (de)h(should)g(w)o(ork)f(under)h(the)f(assumption)g(that)g(an)g (arbitrary)g(subset)g(of)g(the)75 1983 y(send)18 b(op)q(erations)f(are) g(going)g(to)g(return)g(b)q(efore)h(a)f(matc)o(hing)g(receiv)o(e)h(is)g (p)q(osted,)g(and)f(the)g(rest)g(will)75 2040 y(blo)q(c)o(k)f(un)o(til) g(a)f(matc)o(hing)g(receiv)o(e)i(is)e(p)q(osted.)166 2097 y(MPI)g(implemen)o(tation)i(will)h(pro)o(vide)e(information)g(on)f (the)h(amoun)o(t)f(of)g(a)o(v)m(ailable)i(bu\013er)f(space)75 2153 y(and)f(on)h(the)f(bu\013ering)h(p)q(olicy)g(via)g(the)f(en)o (vironmen)o(tal)h(enquiries)h(describ)q(ed)g(in)f(Chapter)f(7.)166 2211 y(Examples)h(\(in)o(v)o(olving)g(t)o(w)o(o)e(pro)q(cessors)h(with) g(ranks)g(0)g(and)g(1\))166 2268 y(The)g(follo)o(wing)h(program)e(is)i (safe,)f(and)g(should)h(alw)o(a)o(ys)f(succeed.)75 2421 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2478 y(IF)h(\(rank.EQ.0\))123 2534 y(THEN)170 2591 y(CALL)g (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\)) 170 2647 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(ierr\))123 2704 y(ELSE)95 b(!)23 b(rank.EQ.1)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 18 24 18 23 bop 75 -100 a FC(18)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)170 45 y Fp(CALL)24 b(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\)) 170 102 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(context\))75 158 y(END)g(IF)166 252 y FC(The)15 b(follo)o(wing)h(program)e(is)i(erroneous,)f(and)g(should)h(alw)o(a)o (ys)f(fail.)75 402 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 459 y(IF)h(\(rank.EQ.0\))123 515 y(THEN)170 572 y(CALL)g(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(ierr\))170 628 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i (MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))123 685 y(ELSE)95 b(!)23 b(rank.EQ.1)170 741 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))170 798 y(CALL)h (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\)) 75 854 y(END)g(IF)166 948 y FC(The)15 b(receiv)o(e)g(op)q(eration)g(of) f(the)h(\014rst)f(pro)q(cess)h(m)o(ust)f(complete)i(b)q(efore)f(its)f (send,)h(and)g(can)g(com-)75 1004 y(plete)j(only)g(if)g(the)f(matc)o (hing)g(send)h(of)f(the)g(second)h(pro)q(cessor)f(is)h(executed;)g(the) g(receiv)o(e)g(op)q(eration)75 1061 y(of)c(the)h(second)g(pro)q(cess)g (m)o(ust)f(complete)h(b)q(efore)g(its)f(send)i(and)e(can)h(complete)g (only)g(if)g(the)g(matc)o(hing)75 1117 y(send)h(of)f(the)g(\014rst)g (pro)q(cess)g(is)h(executed.)21 b(This)15 b(program)f(will)j(deadlo)q (c)o(k.)166 1174 y(The)10 b(follo)o(wing)h(program)f(is)g(unsafe,)h (and)g(ma)o(y)e(succeed)j(or)e(fail,)h(dep)q(ending)i(on)d(implemen)o (tation.)75 1337 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\)) 75 1393 y(IF)h(\(rank.EQ.0\))123 1450 y(THEN)170 1506 y(CALL)g(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g (ierr\))170 1562 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,) h(tag,)f(comm,)g(ierr\))123 1619 y(ELSE)95 b(!)23 b(rank.EQ.1)170 1675 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))170 1732 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))75 1788 y(END)g(IF)166 1951 y FC(The)f(message)g(sen)o(t)f(b)o(y)h(eac)o(h)g(pro)q(cess)g(has) g(to)g(b)q(e)g(copied)h(out)f(b)q(efore)g(the)g(send)h(op)q(eration)75 2008 y(returns)15 b(and)h(the)f(receiv)o(e)h(op)q(eration)g(starts.)i (F)l(or)d(the)g(program)g(to)f(complete,)i(it)f(is)h(necessary)g(that) 75 2064 y(at)h(least)g(one)h(of)f(the)g(t)o(w)o(o)f(messages)h(sen)o(t) g(is)h(bu\013ered.)27 b(Th)o(us,)17 b(this)h(program)e(can)i(succeed)h (only)e(if)75 2120 y(the)h(comm)o(unication)g(system)g(has)f (su\016cien)o(t)i(bu\013er)e(space)h(to)g(bu\013er)35 b Fo(count)19 b FC(w)o(ords)e(of)g(data,)h(and)75 2177 y(the)d(MPI)g(implemen)o(tation)i(bu\013ers)e(messages.)75 2320 y Fw(3.5)59 b(Data)19 b(T)-5 b(yp)r(e)19 b(Matching)75 2422 y FC(One)d(can)f(think)h(of)f(message)g(transmission)g(as)g (consisting)h(of)f(three)g(phases:)131 2516 y(1.)22 b(Data)14 b(is)h(pulled)j(out)d(of)f(the)i(send)f(bu\013er)h(and)f(a)g(message)g (is)g(assem)o(bled)131 2610 y(2.)22 b(A)15 b(message)g(is)g (transferred)g(from)f(sender)i(to)f(receiv)o(er)131 2704 y(3.)22 b(Data)14 b(is)h(pulled)j(from)c(the)h(incoming)i(message)e (and)g(disassem)o(bled)i(in)o(to)e(the)g(receiv)o(e)h(bu\013er)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 19 25 19 24 bop 75 -100 a Fv(3.5.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) 1105 b FC(19)166 45 y(T)o(yp)q(e)20 b(matc)o(hing)f(has)h(to)e(b)q(e)j (observ)o(ed)e(at)g(eac)o(h)g(of)h(these)f(three)h(phases:)29 b(The)19 b(t)o(yp)q(e)h(of)f(eac)o(h)75 102 y(v)m(ariable)i(in)f(the)g (sender)g(bu\013er)g(has)g(to)f(matc)o(h)g(the)g(t)o(yp)q(e)h(sp)q (eci\014ed)i(for)d(that)g(en)o(try)g(b)o(y)h(the)f(send)75 158 y(op)q(eration;)d(the)f(t)o(yp)q(e)h(sp)q(eci\014ed)i(b)o(y)e(the)f (send)i(op)q(eration)e(has)h(to)f(matc)o(h)g(the)h(t)o(yp)q(e)f(sp)q (eci\014ed)j(b)o(y)e(the)75 214 y(receiv)o(e)g(op)q(eration;)e(and)h (the)g(t)o(yp)q(e)f(of)h(eac)o(h)f(v)m(ariable)i(in)g(the)e(receiv)o(e) i(bu\013er)f(has)f(to)g(matc)o(h)g(the)h(t)o(yp)q(e)75 271 y(sp)q(eci\014ed)f(for)e(that)g(en)o(try)g(b)o(y)g(the)g(receiv)o (e)h(op)q(eration.)20 b(A)12 b(program)f(that)h(fails)h(to)e(observ)o (e)h(these)h(three)75 327 y(rules)j(is)g(erroneous.)166 384 y(T)l(o)f(de\014ne)i(t)o(yp)q(e)f(matc)o(hing)g(more)f(precisely)l (,)j(w)o(e)d(need)i(to)e(deal)h(with)g(t)o(w)o(o)f(issues:)22 b(matc)o(hing)15 b(of)75 441 y(t)o(yp)q(es)e(of)g(the)h(host)f (language)g(with)h(t)o(yp)q(es)g(sp)q(eci\014ed)h(in)f(comm)o (unication)h(op)q(erations;)e(and)h(matc)o(hing)75 497 y(of)h(t)o(yp)q(es)g(at)g(sender)g(and)h(receiv)o(er.)166 554 y(A)h(t)o(yp)q(e)f(sp)q(eci\014ed)j(for)d(an)h(en)o(try)f(b)o(y)h (a)f(send)h(op)q(eration)g(matc)o(hes)f(the)h(t)o(yp)q(e)g(sp)q (eci\014ed)i(for)d(that)75 610 y(en)o(try)11 b(b)o(y)h(a)f(receiv)o(e)h (op)q(eration)g(if)g(b)q(oth)f(op)q(erations)h(use)g(iden)o(tical)h (names:)36 b Fo(MPI)p 1500 610 14 2 v 16 w(INTEGER)12 b FC(matc)o(hes)75 667 y Fo(MPI)p 160 667 V 16 w(INTEGER)p FC(,)30 b Fo(MPI)p 490 667 V 16 w(REAL)16 b FC(matc)o(hes)30 b Fo(MPI)p 906 667 V 16 w(REAL)p FC(,)15 b(and)g(so)g(on.)166 723 y(The)k(t)o(yp)q(e)g(of)f(a)g(v)m(ariable)i(in)g(a)e(host)h (program)e(matc)o(hes)h(the)h(t)o(yp)q(e)g(sp)q(eci\014ed)i(in)e(the)g (comm)o(u-)75 780 y(nication)g(op)q(eration)g(if)f(the)h(datat)o(yp)q (e)f(name)g(used)h(b)o(y)f(that)f(op)q(eration)i(corresp)q(onds)f(to)g (the)g(basic)75 836 y(t)o(yp)q(e)i(of)g(the)g(host)g(program)f(v)m (ariable:)31 b(an)20 b(en)o(try)f(with)i(t)o(yp)q(e)f(name)40 b Fo(MPI)p 1449 836 V 16 w(INTEGER)21 b FC(matc)o(hes)e(a)75 893 y(F)l(ortran)13 b(v)m(ariable)i(of)e(t)o(yp)q(e)38 b Fp(INTEGER)p FC(,)12 b(an)i(en)o(try)f(with)h(t)o(yp)q(e)g(name)27 b Fo(MPI)p 1376 893 V 16 w(REAL)14 b FC(matc)o(hes)g(a)f(F)l(ortran)75 949 y(v)m(ariable)18 b(of)d(t)o(yp)q(e)40 b Fp(REAL)p FC(,)15 b(and)i(so)f(on.)22 b(There)17 b(is)f(one)g(exception)i(to)d (this)i(last)f(rule:)22 b(An)17 b(en)o(try)f(with)75 1005 y(t)o(yp)q(e)i(name)37 b Fo(MPI)p 408 1005 V 15 w(BYTE)19 b FC(can)f(b)q(e)h(used)g(to)e(matc)o(h)h(an)o(y)g(b)o(yte)g (of)g(storage)e(\(on)i(a)g(b)o(yte-addressable)75 1062 y(mac)o(hine\),)d(irresp)q(ectiv)o(e)i(of)d(the)h(datat)o(yp)q(e)g(of)f (the)h(v)m(ariable)i(that)d(con)o(tains)h(this)g(b)o(yte.)20 b(The)15 b(v)m(alue)h(of)75 1118 y(the)f(message)g(en)o(try)g(will)i(b) q(e)f(the)f(binary)h(v)m(alue)g(of)f(the)g(corresp)q(onding)h(b)o(yte)f (in)h(memory)l(.)166 1175 y(W)l(e)f(th)o(us)g(ha)o(v)o(e)g(t)o(w)o(o)f (cases:)143 1270 y Fy(\017)23 b FC(Comm)o(unication)11 b(of)f(t)o(yp)q(ed)i(v)m(alues)g(\(e.g.,)e(with)h(datat)o(yp)q(e)g (di\013eren)o(t)g(from)21 b Fo(MPI)p 1586 1270 V 16 w(BYTE)p FC(\),)11 b(where)189 1326 y(the)k(datat)o(yp)q(es)f(of)g(the)h (corresp)q(onding)h(en)o(tries)f(in)h(the)f(sender)g(program,)e(in)j (the)f(send)g(call,)h(in)189 1383 y(the)f(receiv)o(e)h(call)g(and)g(in) g(the)f(receiv)o(er)h(program)e(should)i(all)g(matc)o(h.)143 1477 y Fy(\017)23 b FC(Comm)o(unication)15 b(of)f(un)o(t)o(yp)q(ed)h(v) m(alues)h(\(e.g.,)d(of)h(datat)o(yp)q(e)29 b Fo(MPI)p 1334 1477 V 16 w(BYTE)p FC(\),)15 b(where)g(b)q(oth)f(sender)189 1534 y(and)h(receiv)o(er)g(use)h(the)f(datat)o(yp)q(e)29 b Fo(MPI)p 890 1534 V 16 w(BYTE)p FC(.)15 b(In)g(this)h(case,)e(there)h (are)g(no)g(requiremen)o(ts)g(on)189 1590 y(the)f(t)o(yp)q(es)h(of)f (the)g(corresp)q(onding)h(en)o(tries)g(in)h(the)e(sender)h(and)g(the)f (receiv)o(er)h(programs,)e(nor)h(is)189 1647 y(it)h(required)h(that)f (they)g(b)q(e)h(the)f(same.)166 1741 y(Examples:)166 1798 y(1st)f(program:)75 1893 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f (rank,)h(ierr\))75 1949 y(IF\(rank.EQ.0\))123 2006 y(THEN)170 2062 y(CALL)h(MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f (ierr\))123 2119 y(ELSE)170 2175 y(CALL)h(MPI_RECV\(a\(1\),)e(15,)h (MPI_REAL,)g(0,)g(tag,)h(comm,)f(ierr\))75 2232 y(END)g(IF)166 2326 y FC(This)c(co)q(de)f(is)h(correct)e(if)i(b)q(oth)f(sender)h(and)f (receiv)o(er)h(programs)d(ha)o(v)o(e)i(allo)q(cated)h(consecutiv)o(e)75 2383 y(storage)14 b(for)h(ten)g(real)g(n)o(um)o(b)q(ers,)g(starting)g (from)f Fp(a\(1\))p FC(.)20 b(2nd)15 b(program:)75 2478 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2534 y(IF\(rank.EQ.0\))123 2591 y(THEN)170 2647 y(CALL)h (MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))123 2704 y(ELSE)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 20 26 20 25 bop 75 -100 a FC(20)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)170 45 y Fp(CALL)24 b(MPI_RECV\(a\(1\),)e(40,)h(MPI_BYTE,)g(0,)g(tag,)h(comm,)f(ierr\))75 102 y(END)g(IF)166 191 y FC(This)d(co)q(de)g(is)f(erroneous,)h(since)g (sender)g(and)g(receiv)o(er)g(do)f(not)f(pro)o(vide)i(matc)o(hing)f (datat)o(yp)q(e)75 247 y(parameters.)g(3rd)c(program:)75 337 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 393 y(IF\(rank.EQ.0\))123 450 y(THEN)170 506 y(CALL)h (MPI_SEND\(a\(1\),)e(40,)h(MPI_BYTE,)g(1,)g(tag,)h(comm,)f(ierr\))123 563 y(ELSE)170 619 y(CALL)h(MPI_RECV\(a\(1\),)e(60,)h(MPI_BYTE,)g(0,)g (tag,)h(comm,)f(ierr\))75 676 y(END)g(IF)166 765 y FC(This)16 b(co)q(de)f(is)h(correct,)f(irresp)q(ectiv)o(e)h(of)f(the)g(t)o(yp)q(e) h(of)e Fp(a\(1\))h FC(and)g(follo)o(wing)h(v)m(ariables)h(in)f(store.) 166 897 y Fu(Discussion:)166 947 y Ft(The)f(b)q(eha)o(vior)f(of)g (erroneous)i(co)q(de)g(is)e(implemen)o(tation-dep)q(enden)o(t.)k(It)d (is)g(desirable)g(but)g(not)f(required)75 997 y(that)i(errors)i(are)e (rep)q(orted)i(at)e(run-time.)24 b(An)16 b(MPI)g(implemen)o(tation)d (ma)o(y)h(decide)j(to)f(a)o(v)o(oid)f(t)o(yp)q(e-c)o(hec)o(king)75 1047 y(for)d(the)h(sak)o(e)f(of)g(e\016ciency)m(,)h(in)f(whic)o(h)g (case)h(none)g(of)e(the)i(three)h(programs)d(will)g(rep)q(ort)i(an)f (error.)19 b(On)12 b(a)g(system)75 1097 y(where)f(real)g(n)o(um)o(b)q (ers)f(o)q(ccup)o(y)h(four)f(b)o(ytes)h(and)f(b)q(oth)g(sender)i(and)e (receiv)o(er)i(execute)g(in)e(the)h(same)f(en)o(vironmen)o(t,)75 1146 y(it)k(is)f(lik)o(ely)g(that)h(all)e(three)k(co)q(des)f(will)d(ha) o(v)o(e)i(the)g(same)f(b)q(eha)o(vior.)75 1372 y Fw(3.6)59 b(Data)19 b(conversion)75 1473 y FC(One)d(of)f(the)g(goals)g(of)g(MPI)g (is)h(to)e(supp)q(ort)i(parallel)g(computations)f(across)g (heterogeneous)g(en)o(viron-)75 1529 y(men)o(ts.)20 b(Comm)o(unication) 15 b(in)h(a)f(heterogeneous)g(en)o(vironmen)o(t)h(ma)o(y)e(require)i (data)f(con)o(v)o(ersions.)166 1586 y(W)l(e)g(use)h(the)f(follo)o(wing) h(terminology:)75 1675 y FD(t)o(yp)q(e)h(con)o(v)o(ersion)22 b FC(c)o(hanges)14 b(the)h(datat)o(yp)q(e)f(of)f(a)i(v)m(alue,)g(e.g.)k (b)o(y)14 b(rounding)i(a)28 b Fo(REAL)15 b FC(to)e(an)29 b Fo(INTE-)189 1732 y(GER)p FC(.)75 1824 y FD(represen)o(tation)17 b(con)o(v)o(ersion)22 b FC(c)o(hanges)16 b(the)h(binary)g(represen)o (tation)f(of)g(a)g(v)m(alue,)h(e.g.)23 b(from)16 b(Hex)189 1880 y(\015oating)f(p)q(oin)o(t)g(to)g(IEEE)g(\015oating)h(p)q(oin)o (t.)166 1970 y(Represen)o(tation)d(con)o(v)o(ersion)g(is)h(needed)g (when)f(data)f(is)i(mo)o(v)o(ed)e(across)g(di\013eren)o(t)h(en)o (vironmen)o(ts,)75 2026 y(e.g.)19 b(di\013eren)o(t)d(mac)o(hines)g (that)e(use)i(di\013eren)o(t)f(binary)h(enco)q(dings)h(for)d(the)i (same)e(datat)o(yp)q(e,)h(or)f(co)q(des)75 2083 y(compiled)k(with)e (di\013eren)o(t)h(compiler)g(options)f(for)g(datat)o(yp)q(e)f(represen) o(tation.)23 b(Represen)o(tation)17 b(con-)75 2139 y(v)o(ersion)12 b(preserv)o(es,)g(to)f(the)h(greatest)f(p)q(ossible)i(exten)o(t,)f(the) g FD(v)m(alue)g FC(represen)o(ted.)19 b(Ho)o(w)o(ev)o(er,)11 b(rounding)75 2195 y(errors)k(and)g(o)o(v)o(er\015o)o(w)f(and)i (under\015o)o(w)g(exceptions)g(ma)o(y)f(o)q(ccur)h(during)g(\015oating) f(p)q(oin)o(t)h(con)o(v)o(ersions;)75 2252 y(con)o(v)o(ersion)k(of)f (in)o(tegers)g(ma)o(y)g(also)h(lead)g(to)f(exceptions)i(when)f(a)f(v)m (alue)i(that)d(can)i(represen)o(ted)g(in)75 2308 y(one)h(system)g (cannot)g(b)q(e)g(represen)o(ted)h(in)g(the)f(other)f(system.)37 b(MPI)21 b(do)q(es)h(not)e(sp)q(ecify)j(rules)f(for)75 2365 y(represen)o(tation)15 b(con)o(v)o(ersion.)166 2421 y(The)e(t)o(yp)q(e)g(matc)o(hing)g(rules)g(imply)h(that)e(MPI)h(comm)o (unication)h(nev)o(er)f(en)o(tails)g(t)o(yp)q(e)g(con)o(v)o(ersion.)75 2478 y(On)20 b(the)f(other)f(hand,)i(MPI)f(requires)h(that)e(a)h (represen)o(tation)g(con)o(v)o(ersion)g(b)q(e)h(p)q(erformed)f(when)g (a)75 2534 y(t)o(yp)q(ed)i(v)m(alue)h(is)f(transferred)f(across)g(en)o (vironmen)o(ts)h(that)f(use)h(di\013eren)o(t)f(represen)o(tations)h (for)f(the)75 2591 y(datat)o(yp)q(e)15 b(of)h(this)g(v)m(alue.)24 b(An)16 b(exception)h(o)q(ccurring)g(during)g(represen)o(tation)f(con)o (v)o(ersion)g(results)g(in)75 2647 y(a)f(failure)h(of)f(the)g(comm)o (unication;)g(an)h(error)e(o)q(ccurs)h(either)h(in)g(the)f(send)h(op)q (eration,)f(or)g(the)g(receiv)o(e)75 2704 y(op)q(eration,)g(or)g(b)q (oth.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 21 27 21 26 bop 75 -100 a Fv(3.6.)34 b(D)o(A)l(T)l(A)15 b(CONVERSION)1197 b FC(21)166 45 y(If)20 b(a)g(v)m(alue)i(sen)o(t)e(in)h(a)e(message)h (is)h(un)o(t)o(yp)q(ed)g(\(i.e.,)f(of)g(t)o(yp)q(e)41 b Fo(MPI)p 1374 45 14 2 v 15 w(BYTE)p FC(\),)21 b(then)f(the)g(binary) 75 102 y(represen)o(tation)15 b(of)f(the)h(b)o(yte)g(stored)f(at)g(the) h(receiv)o(er)h(is)f(iden)o(tical)i(to)d(the)h(binary)h(represen)o (tation)e(of)75 158 y(the)f(b)o(yte)g(loaded)h(at)e(the)h(sender.)20 b(This)14 b(holds)f(true,)g(whether)g(sender)h(and)f(receiv)o(er)h(run) f(in)h(the)f(same)75 214 y(or)i(in)h(distinct)g(en)o(vironmen)o(ts;)f (no)g(represen)o(tation)h(con)o(v)o(ersion)f(is)h(required.)166 288 y(Note)d(that)g(no)g(con)o(v)o(ersion)h(ev)o(er)f(o)q(ccurs)h(when) g(an)f(MPI)g(program)g(executes)h(in)g(a)f(homogeneous)75 344 y(system,)18 b(where)g(all)h(pro)q(cesses)g(run)f(in)h(the)f(same)g (en)o(vironmen)o(t.)29 b(Also)18 b(note)g(the)g(di\013eren)o(t)g(b)q (eha)o(v-)75 401 y(ior)g(of)37 b Fo(MPI)p 305 401 V 15 w(BYTE)19 b FC(and)g(of)36 b Fo(MPI)p 704 401 V 16 w(CHARA)o(CTER)p FC(.)19 b(A)g(bu\013er)f(descriptor)h(en)o(try)f(with)h(datat)o(yp)q(e) e(of)75 457 y Fo(MPI)p 160 457 V 16 w(CHARA)o(CTER)j FC(can)f(only)g(matc)o(h)f(a)g(F)l(ortran)g(v)m(ariable)i(of)e(t)o(yp)q (e)42 b Fp(CHARACTER)p FC(;)17 b(and)i(represen-)75 514 y(tation)c(con)o(v)o(ersion)h(ma)o(y)f(o)q(ccur)i(when)f(v)m(alues)h (of)e(t)o(yp)q(e)32 b Fo(MPI)p 1165 514 V 16 w(CHARA)o(CTER)17 b FC(are)e(transferred.,)g(e.g.,)75 570 y(from)f(an)i(EBCDIC)f(enco)q (ding)h(to)f(an)g(ASCI)q(I)h(enco)q(ding.)166 643 y(Consider)g(the)f (previous)h(three)f(examples.)166 717 y(The)c(\014rst)f(program)f(is)i (correct,)g(assuming)g(b)q(oth)f(sender)i(and)e(receiv)o(er)i(declared) f(ten)g(consecutiv)o(e)75 773 y(real)19 b(v)m(ariables)h(in)g(storage)e (starting)g(at)g Fp(a\(1\))p FC(.)30 b(If)19 b(the)g(sender)h(and)f (receiv)o(er)g(execute)h(in)g(di\013eren)o(t)75 830 y(en)o(vironmen)o (ts,)12 b(then)f(the)g(ten)f(real)i(v)m(alues)f(that)g(are)f(fetc)o (hed)h(from)f(the)h(send)h(bu\013er)e(will)j(b)q(e)e(con)o(v)o(erted)75 886 y(to)g(the)h(represen)o(tation)f(for)g(reals)h(on)f(the)h(receiv)o (er)g(site)g(b)q(efore)g(they)g(are)f(stored)g(in)h(the)g(receiv)o(e)g (bu\013er.)75 942 y(While)19 b(the)e(n)o(um)o(b)q(er)h(of)f(real)g (elemen)o(ts)h(fetc)o(hed)g(from)f(the)g(send)h(bu\013er)f(equal)i(the) e(n)o(um)o(b)q(er)h(of)f(real)75 999 y(elemen)o(ts)d(stored)e(in)i(the) f(receiv)o(e)h(bu\013er,)f(the)g(n)o(um)o(b)q(er)g(of)g(b)o(ytes)g (stored)f(need)i(not)f(equal)h(the)f(n)o(um)o(b)q(er)75 1055 y(of)f(b)o(ytes)g(loaded:)19 b(e.g.)g(the)12 b(sender)h(ma)o(y)e (use)i(a)f(four)g(b)o(yte)g(represen)o(tation)h(and)f(the)g(receiv)o (er)i(an)e(eigh)o(t)75 1112 y(b)o(yte)17 b(represen)o(tation)h(for)f (reals.)27 b(If)18 b(the)g(send)g(or)f(receiv)o(e)h(bu\013er)g(do)f (not)h(con)o(tain)f(ten)h(consecutiv)o(e)75 1168 y(real)e(v)m (ariables,)g(then)f(the)g(program)g(is)g(erroneous,)g(and)g(its)h(b)q (eha)o(vior)f(is)h(unde\014ned.)166 1242 y(The)f(second)h(program)e(is) i(erroneous,)f(and)g(its)g(b)q(eha)o(vior)h(is)g(unde\014ned.)166 1315 y(The)e(third)g(program)e(is)i(correct.)19 b(The)13 b(exact)g(same)g(sequence)i(of)e(fort)o(y)f(b)o(ytes)h(that)g(w)o(ere)g (loaded)75 1371 y(from)18 b(the)h(send)h(bu\013er)f(will)i(b)q(e)e (stored)g(in)h(the)f(receiv)o(e)h(bu\013er,)f(ev)o(en)h(if)f(sender)h (and)f(receiv)o(er)h(run)75 1428 y(in)e(a)g(di\013eren)o(t)f(en)o (vironmen)o(t.)28 b(The)18 b(message)f(sen)o(t)g(has)h(exactly)f(the)h (same)f(length)i(\(in)f(b)o(ytes\))f(and)75 1484 y(the)f(same)f(binary) h(represen)o(tation)f(as)g(the)h(message)f(receiv)o(ed.)22 b(If)16 b(the)f(v)m(ariables)i(in)f(the)g(send)g(bu\013er)75 1541 y(are)i(of)h(di\013eren)o(t)g(t)o(yp)q(es)g(as)f(the)h(v)m (ariables)h(in)g(the)f(receiv)o(e)g(bu\013er,)h(or)e(they)h(are)f(of)h (the)f(same)h(t)o(yp)q(e)75 1597 y(but)e(di\013eren)o(t)g(data)g (represen)o(tations)f(are)h(used,)h(then)f(the)g(bits)h(stored)e(in)i (the)f(receiv)o(e)h(bu\013er)f(ma)o(y)75 1654 y(enco)q(de)f(v)m(alues)h (that)d(are)h(di\013eren)o(t)g(from)g(the)g(v)m(alues)h(they)g(enco)q (ded)g(in)g(the)g(send)f(bu\013er.)166 1727 y(Data)f(represen)o(tation) i(con)o(v)o(ersion)f(also)h(applies)h(to)e(the)g(en)o(v)o(elop)q(e)i (of)e(a)g(message:)20 b(source,)15 b(des-)75 1783 y(tination)h(and)f (tag)f(are)h(all)h(in)o(tegers)g(that)e(ma)o(y)h(need)h(to)e(b)q(e)i (con)o(v)o(erted.)166 1857 y(The)k(curren)o(t)g(draft)f(do)q(es)h(not)g (pro)o(vide)g(a)g(mec)o(hanism)h(for)e(in)o(ter-language)i(comm)o (unication:)75 1913 y(messages)d(sen)o(t)h(b)o(y)f(F)l(ortran)g(calls)i (should)f(b)q(e)h(receiv)o(ed)g(b)o(y)e(F)l(ortran)g(calls)h(and)g (messages)g(sen)o(t)f(b)o(y)75 1970 y(C)e(calls)g(should)h(b)q(e)g (receiv)o(ed)g(b)o(y)e(C)h(calls)h(\(this)f(follo)o(ws)f(from)h(the)f (requiremen)o(ts)i(of)e(t)o(yp)q(e)h(matc)o(hing)75 2026 y(and)g(the)h(fact)f(that)f(F)l(ortran)g(and)i(C)f(datat)o(yp)q(es)g (are)g(distinct\).)24 b(If)16 b(in)o(ter-language)h(comm)o(unication)75 2082 y(is)g(needed)i(then)e(one)g(needs)h(to)e(in)o(v)o(ok)o(e)h(C)f (comm)o(unication)i(routines)f(from)f(a)h(F)l(ortran)f(program)f(or)75 2139 y(F)l(ortran)f(comm)o(unication)i(routines)g(from)e(a)h(C)g (program.)166 2288 y Fu(Implemen)o(tati)o(on)d(note:)166 2355 y Ft(The)k(curren)o(t)i(de\014nition)e(do)q(es)h(not)f(require)h (messages)f(to)g(carry)h(data)f(t)o(yp)q(e)g(information.)22 b(A)16 b(message)75 2405 y(can)c(b)q(e)h(comp)q(osed)f(and)g(sen)o(t)h (using)f(only)g(the)h(information)c(pro)o(vided)j(b)o(y)g(the)h(send)g (call,)e(and)h(can)h(b)q(e)g(receiv)o(ed)75 2455 y(and)k(stored)g (using)g(only)f(the)h(information)d(pro)o(vided)j(b)o(y)g(the)g(receiv) o(e)h(call.)26 b(If)17 b(messages)g(are)g(sen)o(t)h(b)q(et)o(w)o(een)75 2504 y(di\013eren)o(t)c(mac)o(hines)f(then)g(one)h(can)f(either)h(use)g (a)f(\\univ)o(ersal")g(data)g(enco)q(ding)g(for)g(messages,)g(use)h (kno)o(wledge)75 2554 y(of)g(the)i(receiv)o(er)h(en)o(vironmen)o(t)d (in)g(order)i(to)f(con)o(v)o(ert)h(data)e(at)h(the)h(sender,)h(or)e (use)g(kno)o(wledge)g(of)g(the)g(sender)75 2604 y(en)o(vironmen)o(t)j (in)g(order)i(to)f(con)o(v)o(ert)g(data)g(at)g(the)g(receiv)o(er.)35 b(In)19 b(either)g(case)h(the)g(datat)o(yp)q(e)f(parameter)f(in)75 2654 y(the)f(lo)q(cal)f(call)g(can)g(b)q(e)h(used)h(to)e(deriv)o(e)h (the)g(t)o(yp)q(es)h(of)e(the)h(v)n(alues)f(transferred.)28 b(Ho)o(w)o(ev)o(er,)17 b(additional)d(t)o(yp)q(e)75 2704 y(information)d(carried)k(b)o(y)e(messages)h(can)g(pro)o(vide)g(b)q (etter)h(error)g(detection.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 22 28 22 27 bop 75 -100 a FC(22)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fu(Discussion:)166 99 y Ft(MPI)e(do)q(es)h(not)f(handle)g(in)o(ter-language)g(comm)o (unicatio)o(n)e(b)q(ecause)k(there)f(seem)f(to)g(b)q(e)h(no)f(agreed)h (stan-)75 149 y(dards)20 b(for)f(in)o(ter-language)f(t)o(yp)q(e)i(con)o (v)o(ersions.)34 b(The)20 b(curren)o(t)h(\(non\)solution)d(means)g (that)i(in)o(ter-language)75 199 y(comm)o(unication)10 b(relies)15 b(on)f(the)g(v)o(endor)g(sp)q(eci\014c)i(rules)e(for)g(in)o (ter-language)f(calls)g(at)h(eac)o(h)g(site.)166 253 y(It)i(ma)o(y)e(b)q(e)i(desirable)h(to)e(pro)o(vide)h(information)d(on) j(the)g(corresp)q(ondence)j(b)q(et)o(w)o(een)e(MPI)g(C)e(datat)o(yp)q (es)75 302 y(\(suc)o(h)i(as)38 b Fn(MPI)p 331 302 14 2 v 15 w(FLOAT)p Ft(\))15 b(and)h(MPI)g(F)m(ortran)g(datat)o(yp)q(es)g (\(suc)o(h)h(as)38 b Fn(MPI)p 1265 302 V 15 w(REAL)p Ft(\))16 b(with)f(suitable)h(en)o(vironmen)o(t)75 352 y(query)e(functions.)75 602 y Fw(3.7)59 b(Communication)18 b(Mo)r(des)75 711 y FC(The)12 b(basic)h(send)g(op)q(eration)g(describ)q (ed)h(in)f(Section)g(3.2)e(used)i(the)f FD(standard)h FC(comm)o(unication)g(mo)q(de.)75 768 y(In)h(suc)o(h)g(a)f(comm)o (unication)h(mo)q(de,)g(a)f(send)h(op)q(eration)g(can)f(b)q(e)h (started)f(whether)h(or)f(not)g(a)g(matc)o(hing)75 824 y(receiv)o(e)g(w)o(as)e(p)q(osted.)20 b(The)12 b(completion)h(of)f(the) g(send)h(op)q(eration)f(indicates)i(that)d(the)i(message)e(and)i(its)75 881 y(en)o(v)o(elop)q(e)j(ha)o(v)o(e)e(b)q(een)i(safely)f(stored)f(a)o (w)o(a)o(y)g(and)g(that)h(the)f(sender)i(is)f(free)g(to)f(access)h(and) g(mo)q(dify)g(the)75 937 y(sender)g(bu\013er.)k(Th)o(us,)14 b(the)g(op)q(eration)h(is)f FD(blo)q(c)o(king)p FC(:)21 b(it)14 b(do)q(es)h(not)e(return)i(un)o(til)g(the)f(send)h(op)q (eration)75 993 y(has)g(completed)h FD(lo)q(cally)p FC(,)h(on)f(the)f (sender)h(side.)166 1054 y(The)f(completion)i(of)e(a)g(send)h(op)q (eration)f(giv)o(es)h(no)f(indication)i(that)e(the)g(message)g(w)o(as)g (receiv)o(ed)75 1110 y(on)g(the)h(receiv)o(er)g(side.)21 b(A)15 b(blo)q(c)o(king)i(send)f(ma)o(y)e(b)q(e)i(implemen)o(ted)h(so)e (that)g(a)g(blo)q(c)o(king)h(send)g(returns)75 1167 y(only)g(after)f(a) g(matc)o(hing)g(receiv)o(e)h(has)g(b)q(een)g(executed)g(on)g(the)f (receiv)o(er)h(side.)22 b(This)16 b(a)o(v)o(oids)f(the)g(need)75 1223 y(to)i(bu\013er)g(message)g(data)g(out)g(of)g(sender)h(or)f (receiv)o(er)h(memory)l(.)26 b(In)18 b(this)g(case)f(the)h(send)g(op)q (eration)75 1280 y(completes)j(only)f(after)f(the)h(matc)o(hing)g (receiv)o(e)h(has)f(started)f(executing.)35 b(On)20 b(the)g(other)g (hand,)h(it)75 1336 y(is)g(also)f(p)q(ossible)i(for)e(MPI)g(to)g (bu\013er)g(messages,)h(so)f(as)g(to)f(allo)o(w)i(the)f(sender)h(to)f (pro)q(ceed)h(ahead)75 1393 y(of)e(the)h(receiv)o(er.)35 b(In)20 b(suc)o(h)g(a)f(case)h(the)g(send)g(op)q(eration)g(ma)o(y)g (complete)g(successfully)i(b)q(efore)e(the)75 1449 y(message)c(w)o(as)g (receiv)o(ed.)26 b(Th)o(us,)16 b(the)h(basic)h(send)f(op)q(eration)g (describ)q(ed)h(in)g(the)f(previous)g(section)g(is)75 1506 y FD(async)o(hronous)p FC(,)12 b(since)i(its)f(return)g(do)q(es)h (not)f(imply)h(a)f(sync)o(hronization)h(with)f(the)g(\(remote\))f (receiv)o(e)75 1562 y(op)q(eration,)j(and)g(do)q(es)h(not)f(imply)h FD(global)h FC(termination)f(of)f(the)g(comm)o(unication.)166 1623 y(There)g(are)g(t)o(w)o(o)f(additional)j(comm)o(unication)f(mo)q (des:)166 1683 y(A)h(send)g(that)f(uses)i(the)f FD(ready)f FC(comm)o(unication)i(mo)q(de)f(ma)o(y)f(b)q(e)h(started)g(only)g(if)g (a)g(matc)o(hing)75 1740 y(receiv)o(e)d(is)f(already)g(p)q(osted;)g (otherwise)g(the)g(op)q(eration)g(is)g(erroneous)f(and)h(its)g(outcome) f(is)h(unde\014ned.)75 1796 y(In)19 b(some)f(systems,)h(this)f(allo)o (ws)h(the)f(remo)o(v)m(al)h(of)f(a)g(hand-shak)o(e)h(op)q(eration)g (that)e(is)i(otherwise)g(re-)75 1853 y(quired,)g(and)f(results)g(in)g (impro)o(v)o(ed)g(p)q(erformance.)27 b(The)18 b(completion)h(of)e(the)h (send)g(op)q(eration)g(do)q(es)75 1909 y(not)e(dep)q(end)i(on)e(the)g (status)f(of)h(a)g(matc)o(hing)g(receiv)o(e,)h(and)f(merely)h (indicates)h(that)d(the)h(send)h(bu\013er)75 1966 y(can)i(b)q(e)g (reused.)30 b(A)19 b(send)g(op)q(eration)g(that)f(uses)g(the)h(ready)f (mo)q(de)h(has)g(the)f(same)g(seman)o(tics)h(as)f(a)75 2022 y(standard)g(send)h(op)q(eration.)29 b(It)18 b(is)h(merely)g(that) f(the)g(sender)h(pro)o(vides)f(additional)i(information)f(to)75 2078 y(the)c(comm)o(unication)h(subsystem,)f(when)h(using)g(the)f (ready)g(mo)q(de.)166 2139 y(A)j(send)g(that)f(uses)g(the)h FD(sync)o(hronous)e FC(mo)q(de)i(can)g(b)q(e)g(started)f(whether)g(or)g (not)h(a)f(matc)o(hing)75 2195 y(receiv)o(e)12 b(w)o(as)e(p)q(osted.)19 b(Ho)o(w)o(ev)o(er,)10 b(the)h(send)h(will)g(complete)g(successfully)h (only)f(if)f(a)g(matc)o(hing)g(receiv)o(e)g(is)75 2252 y(p)q(osted,)h(and)g(the)g(receiv)o(e)g(op)q(eration)g(has)f(started)g (to)g(receiv)o(e)i(the)e(message)g(sen)o(t)h(b)o(y)f(the)h(sync)o (hronous)75 2308 y(send.)29 b(\(I.e.,)19 b(the)f(receiv)o(e)h(has)f(b)q (een)h(p)q(osted,)g(and)g(the)f(incoming)h(message)f(has)g(b)q(een)i (matc)o(hed)e(to)75 2365 y(this)g(p)q(osted)g(receiv)o(e.\))28 b(Th)o(us,)18 b(the)g(completion)g(of)g(a)f(sync)o(hronous)h(send)g (not)f(only)i(indicates)g(that)75 2421 y(the)g(send)h(bu\013er)f(can)g (b)q(e)h(reused,)g(but)f(also)g(indicates)i(that)d(the)i(receiv)o(er)f (has)g(reac)o(hed)h(a)f(certain)75 2478 y(p)q(oin)o(t)e(in)h(its)f (execution,)h(namely)g(that)e(it)h(has)g(started)f(executing)i(the)f (matc)o(hing)g(receiv)o(e.)26 b(If)17 b(b)q(oth)75 2534 y(sends)f(and)h(receiv)o(es)f(are)g(blo)q(c)o(king)h(op)q(erations)f (then)h(the)f(use)g(of)f(the)h(sync)o(hronous)g(mo)q(de)g(pro)o(vides) 75 2591 y(sync)o(hronous)f(comm)o(unication)h(seman)o(tics:)k(a)14 b(comm)o(unication)i(do)q(es)f(not)g(complete)g(at)g(either)g(ends)75 2647 y(b)q(efore)d(b)q(oth)g(pro)q(cesses)g(\\attend")f(to)g(the)h (comm)o(unication;)h(the)f(completion)h(of)e(a)h(sync)o(hronous)f(send) 75 2704 y(is)16 b(a)f FD(global)i FC(ev)o(en)o(t.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 23 29 23 28 bop 75 -100 a Fv(3.7.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b FC(23)166 45 y(Tw)o(o)22 b(additional)j(send)e (functions)h(are)f(pro)o(vided)h(for)f(the)g(t)o(w)o(o)f(additional)i (comm)o(unication)75 102 y(mo)q(des.)32 b(The)19 b(comm)o(unication)h (mo)q(de)f(is)h(indicated)g(b)o(y)f(a)g(one)g(letter)h(pre\014x:)28 b Fp(R)18 b FC(for)h(ready)g(and)g Fp(S)75 158 y FC(for)c(sync)o (hronous.)166 216 y(Send)h(in)g(ready)f(mo)q(de)75 320 y Fo(MPI)p 160 320 14 2 v 16 w(RSEND)h(\(sta)o(rt,)f(count,)h(datat)o (yp)q(e,)g(dest,)g(tag,)f(comm\))117 399 y Ft(IN)155 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 476 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 554 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)117 631 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 709 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 786 y(IN)155 b Fo(comm)470 b Ft(comm)o(unicator)11 b(\(handle\))166 912 y FC(Send)16 b(in)g(sync)o(hronous)f(mo)q(de)75 1017 y Fo(MPI)p 160 1017 V 16 w(SSEND)h(\(sta)o(rt,)e(count,)j(datat)o(yp)q(e,)f(dest,)g (tag,)f(comm\))117 1095 y Ft(IN)155 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1172 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1250 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)117 1327 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1405 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 1482 y(IN)155 b Fo(comm)470 b Ft(comm)o(unicator)11 b(\(handle\))166 1608 y FC(There)16 b(is)g(only)g(one)g(receiv)o(e)h(mo)q(de,)e(whic)o(h)i(can)f(matc)o(h)f (an)o(y)g(of)h(the)f(send)i(mo)q(des.)k(The)16 b(receiv)o(e)75 1664 y(op)q(eration)g(describ)q(ed)h(in)g(the)e(last)h(section)g(is)g FD(blo)q(c)o(king)p FC(:)22 b(it)16 b(returns)f(only)h(after)f(the)h (receiv)o(e)g(bu\013er)75 1721 y(con)o(tains)h(the)h(newly)g(receiv)o (ed)g(message.)26 b(It)17 b(is)h FD(async)o(hronous)p FC(:)23 b(the)17 b(completion)i(of)d(a)h(receiv)o(e)h(is)75 1777 y(a)e(lo)q(cal)i(op)q(eration,)f(and)f(a)g(receiv)o(e)i(can)f (complete)g(b)q(efore)g(the)f(matc)o(hing)h(send)g(has)f(completed)i (\(of)75 1834 y(course,)d(it)g(can)h(complete)g(only)f(after)g(the)g (matc)o(hing)g(send)h(has)f(started\).)166 1891 y(Comm)o(unication)e (imp)q(oses)g(an)f(order)h(on)f(the)h(ev)o(en)o(ts)f(o)q(ccurring)h(at) f(the)h(comm)o(unicating)g(no)q(des.)75 1948 y(It)i(is)g(alw)o(a)o(ys)e (the)i(case)g(that)e(the)i(completion)h(of)e(a)g(receiv)o(e)i(o)q (ccurs)e(after)g(the)h(start)e(of)h(the)h(matc)o(hing)75 2004 y(send.)29 b(If)19 b(the)f(sync)o(hronous)g(send)h(mo)q(de)f(is)h (used,)g(then)g(it)f(is)h(also)f(the)g(case)g(that)g(the)g(completion) 75 2061 y(of)i(a)h(send)g(o)q(ccurs)g(after)f(the)g(start)g(of)g(the)h (matc)o(hing)f(receiv)o(e.)38 b(Of)20 b(course,)i(on)f(eac)o(h)f(pro)q (cess,)i(a)75 2117 y(comm)o(unication)14 b(completes)h(after)d(it)i(is) g(started.)19 b(No)13 b(other)h(order)f(is)h(imp)q(osed)h(b)o(y)e(MPI.) g(E.g.,)g(if)h(the)75 2174 y(standard)h(send)g(mo)q(de)h(is)f(used,)h (the)f(send)h(op)q(eration)f(ma)o(y)f(complete)i(b)q(efore)g(the)f (matc)o(hing)g(receiv)o(e)75 2230 y(w)o(as)f(started.)166 2288 y(In)i(a)e(m)o(ulti-threaded)j(implemen)o(tation)f(of)f(MPI,)f(a)h (blo)q(c)o(king)h(send)g(or)e(receiv)o(e)i(op)q(eration)g(ma)o(y)75 2344 y(only)j(blo)q(c)o(k)g(the)g(executing)h(thread,)f(not)f(the)h(en) o(tire)g(pro)q(cess.)30 b(Th)o(us,)19 b(while)h(a)e(thread)h(is)g(blo)q (c)o(k)o(ed)75 2401 y(w)o(aiting)k(for)e(a)h(send)h(or)f(receiv)o(e)i (to)d(complete,)k(other)d(threads)g(ma)o(y)g(execute)h(within)h(the)e (same)75 2457 y(address)16 b(space.)22 b(In)17 b(suc)o(h)f(a)f(case)h (it)g(the)g(user)g(resp)q(onsibilit)o(y)i(not)e(to)f(mo)q(dify)i(the)f (send)g(bu\013er)g(un)o(til)75 2513 y(the)11 b(send)h(completes)g(and)g (not)e(to)h(access)g(or)g(mo)q(dify)h(the)f(receiv)o(e)i(bu\013er)e(un) o(til)h(the)f(receiv)o(e)i(completes;)75 2570 y(otherwise)i(the)h (outcome)f(of)f(the)i(computation)f(is)g(unde\014ned.)166 2704 y Fu(Discussion:)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 24 30 24 29 bop 75 -100 a FC(24)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Ft(Do)d(w)o(e)g(w)o (an)o(t)g(to)g(sa)o(y)g(that)g(a)g(blo)q(c)o(king)g(op)q(eration)g(MA)m (Y)g(or)g(MUST)g(only)g(blo)q(c)o(k)g(the)h(executing)g(thread?)166 263 y Fu(Implemen)o(tati)o(on)c(note:)34 b Ft(A)13 b(ready)f(send)h (can)f(b)q(e)h(implemen)o(ted)d(as)i(a)g(standard)h(send;)g(in)e(suc)o (h)i(a)f(case)75 313 y(there)j(will)e(b)q(e)h(no)g(p)q(erformance)g (adv)n(an)o(tage)f(\(or)h(disadv)n(an)o(tage\))f(for)g(the)i(use)f(of)g (ready)g(send.)166 366 y(A)f(standard)h(send)h(can)e(b)q(e)h(implemen)o (ted)e(as)h(a)g(sync)o(hronous)i(send.)j(In)c(suc)o(h)g(a)f(case,)h(no) f(data)h(bu\013ering)75 416 y(is)g(needed.)166 469 y(A)g(p)q(ossible)g (comm)o(unication)c(proto)q(col)k(for)g(the)g(v)n(arious)f(comm)o (unication)e(mo)q(des)i(is)h(outlined)f(b)q(elo)o(w:)166 521 y Fn(ready)21 b(send)p Ft(:)c(The)d(message)g(is)f(sen)o(t)i(as)f (so)q(on)g(as)g(p)q(ossible.)166 574 y Fn(synchronous)19 b(send:)29 b Ft(The)20 b(sender)g(sends)h(a)e(request-to-send)j (message.)34 b(The)19 b(receiv)o(er)i(stores)g(this)75 624 y(request.)f(When)15 b(a)f(matc)o(hing)e(receiv)o(e)j(is)f(p)q (osted,)h(the)g(receiv)o(er)h(sends)f(bac)o(k)f(a)g(p)q (ermission-to-send)g(message,)75 674 y(and)g(the)g(sender)i(no)o(w)d (sends)i(the)g(message.)166 727 y Fn(standard)20 b(send:)e Ft(First)d(proto)q(col)f(ma)o(y)e(b)q(e)j(used)h(for)d(short)i (messages,)f(and)g(second)i(proto)q(col)e(for)g(long)75 777 y(messages.)166 829 y(Additional)h(con)o(trol)h(messages)h(migh)o (t)e(b)q(e)i(needed)h(for)e(\015o)o(w)g(con)o(trol)h(and)f(error)i (reco)o(v)o(ery)m(.)27 b(Of)16 b(course,)75 879 y(there)f(are)f(man)o (y)f(other)h(p)q(ossible)g(c)o(hoices.)75 1122 y Fw(3.8)59 b(Nonblo)r(cking)20 b(communication)75 1230 y FC(One)e(can)g(impro)o(v) o(e)f(p)q(erformance)g(on)h(man)o(y)e(systems)h(b)o(y)h(o)o(v)o (erlapping)f(comm)o(unication)i(and)e(com-)75 1286 y(putation.)35 b(This)21 b(is)g(esp)q(ecially)i(true)d(on)g(systems)g(where)g(comm)o (unication)h(can)g(b)q(e)g(executed)g(au-)75 1343 y(tonomously)12 b(b)o(y)h(an)g(in)o(telligen)o(t)h(comm)o(unication)g(con)o(troller.)19 b(Ligh)o(t-w)o(eigh)o(t)13 b(threads)f(are)h(one)g(mec)o(h-)75 1399 y(anism)21 b(for)g(ac)o(hieving)h(suc)o(h)g(o)o(v)o(erlap.)37 b(An)21 b(alternativ)o(e)h(mec)o(hanism)g(that)e(often)h(leads)g(to)g (b)q(etter)75 1456 y(p)q(erformance)e(is)g(to)f(use)h FD(non)o(blo)q(c)o(king)j(comm)o(unication)p FC(.)32 b(A)19 b(non)o(blo)q(c)o(king)h(send)f(call)h(initiates)75 1512 y(the)15 b(send)h(op)q(eration,)f(but)h(do)q(es)f(not)g(complete)h (it.)k(The)15 b(send)h(call)h(will)f(return)g(b)q(efore)f(the)g (message)75 1568 y(w)o(as)d(copied)h(out)f(of)g(the)g(send)h(bu\013er.) 19 b(A)13 b(separate)e(call)j(is)f(needed)g(to)f(complete)h(the)g(comm) o(unication,)75 1625 y(i.e.)20 b(to)13 b(v)o(erify)i(that)e(the)h(data) g(has)g(b)q(een)h(copied)g(out)f(of)g(the)g(send)h(bu\013er.)k(With)c (suitable)g(hardw)o(are,)75 1681 y(the)k(transfer)f(of)h(data)f(out)h (of)g(the)g(sender)g(memory)g(ma)o(y)f(pro)q(ceed)i(concurren)o(tly)g (with)f(computa-)75 1738 y(tions)g(done)g(at)f(the)g(sender)h(after)f (the)h(send)g(w)o(as)f(initiated)i(and)e(b)q(efore)h(it)g(completed.)31 b(Similarly)l(,)75 1794 y(a)19 b(non)o(blo)q(c)o(king)h(receiv)o(e)g (initiates)h(the)e(receiv)o(e)h(op)q(eration,)g(but)f(do)q(es)g(not)g (complete)h(it.)32 b(The)19 b(call)75 1851 y(will)f(return)f(b)q(efore) g(a)f(message)g(is)i(stored)e(in)o(to)g(the)h(receiv)o(e)h(bu\013er.)24 b(A)17 b(separate)f(call)h(is)h(needed)g(to)75 1907 y(complete)f(the)f (receiv)o(e)i(op)q(eration)e(and)g(v)o(erify)h(that)e(the)i(data)e(has) i(b)q(een)g(receiv)o(ed)g(in)o(to)g(the)f(receiv)o(e)75 1964 y(bu\013er.)k(With)14 b(suitable)h(hardw)o(are,)f(the)g(transfer)f (of)h(data)g(in)o(to)g(the)g(receiv)o(er)h(memory)f(ma)o(y)f(pro)q (ceed)75 2020 y(concurren)o(tly)f(with)g(computations)f(done)g(after)g (the)g(receiv)o(e)h(w)o(as)f(initiated)i(and)e(b)q(efore)h(it)f (completed.)166 2080 y(Non)o(blo)q(c)o(king)16 b(sends)g(can)g(use)f (the)h(same)f(three)g(mo)q(des)h(as)f(blo)q(c)o(king)h(sends:)21 b Fp(standard)p FC(,)14 b Fp(ready)75 2136 y FC(and)21 b Fp(synchronous)p FC(.)35 b(These)21 b(carry)g(the)g(same)f(meaning.) 37 b(The)21 b(initiation)i(and)e(completion)h(of)e(a)75 2192 y(standard)e(send)i(do)f(not)f(dep)q(end)j(on)d(the)h(status)f(of) h(a)f(matc)o(hing)h(receiv)o(e.)32 b(A)19 b(ready)g(send)g(can)g(b)q(e) 75 2249 y(initiated)12 b(only)f(if)g(a)g(matc)o(hing)f(receiv)o(e)i (has)e(already)h(b)q(een)h(initiated,)g(otherwise)f(the)g(call)g(is)g (erroneous;)75 2305 y(its)18 b(completion)g(do)q(es)g(not)f(dep)q(end)i (on)f(the)f(status)g(of)g(a)g(matc)o(hing)h(receiv)o(e.)27 b(A)18 b(sync)o(hronous)f(send)75 2362 y(can)f(b)q(e)h(initiated)h(b)q (efore)f(a)f(matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)i(initiated,)g (but)e(will)i(complete)f(only)g(after)75 2418 y(a)f(matc)o(hing)f (receiv)o(e)i(has)f(b)q(een)h(initiated,)g(and)f(has)g(started)f (receiving)j(the)e(message)f(generated)h(b)o(y)75 2475 y(the)f(send)h(op)q(eration.)166 2534 y(In)f(all)h(cases)e(the)h(op)q (eration)g(that)f(initiates)i(the)f(comm)o(unication)g(\(send)g(or)f (receiv)o(e\))i(is)f(a)f(lo)q(cal,)75 2591 y(non)o(blo)q(c)o(king)k (call)f(that)f(returns)h(as)f(so)q(on)g(as)g(p)q(ossible.)26 b(A)16 b(non)o(blo)q(c)o(king)i(comm)o(unication)f(call)h(ma)o(y)75 2647 y(fail)e(b)q(ecause)h(b)q(ecause)g(the)e(system)g(has)h(exhausted) g(a)o(v)m(ailable)h(resources)f(\(e.g.,)e(exceeded)j(the)f(limit)75 2704 y(on)h(n)o(um)o(b)q(er)h(of)f(p)q(ending)i(comm)o(unications)f(p)q (er)f(no)q(de\).)27 b(Go)q(o)q(d)17 b(qualit)o(y)h(implemen)o(tations)g (of)f(MPI)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 25 31 25 30 bop 75 -100 a Fv(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FC(25)75 45 y(will)17 b(set)d(these)i(limits)g(high)g(enough)f(so)g (that)f(a)h(non)o(blo)q(c)o(king)h(comm)o(unication)g(call)g(will)h (fail)f(only)f(in)75 102 y(\\pathological")h(cases.)166 159 y(Non)o(blo)q(c)o(king)g(sends)g(can)f(b)q(e)h(matc)o(hed)f(with)h (blo)q(c)o(king)g(receiv)o(es,)g(and)f(vice-v)o(ersa.)75 287 y Fq(3.8.1)49 b(Communication)17 b(Objects)75 375 y FC(Non)o(blo)q(c)o(king)c(comm)o(unications)g(use)f(opaque)g(comm)o (unication)h(ob)s(jects)f(to)f(iden)o(tify)i(comm)o(unication)75 432 y(op)q(erations)18 b(and)h(matc)o(h)e(the)i(op)q(eration)f(that)f (initiates)j(the)e(comm)o(unication)h(with)f(the)h(op)q(eration)75 488 y(that)e(terminates)i(it.)29 b(These)18 b(are)g(system)g(ob)s (jects)g(that)f(are)h(accessed)h(via)f(a)g(handle.)30 b(An)19 b(opaque)75 545 y(comm)o(unication)f(ob)s(ject)e(iden)o (ti\014es)j(v)m(arious)f(prop)q(erties)f(of)g(a)g(comm)o(unication)h (op)q(eration,)f(suc)o(h)g(as)75 601 y(the)g(\(send)g(or)f(receiv)o (e\))h(bu\013er)g(that)f(is)h(asso)q(ciated)g(with)g(it,)g(its)g(con)o (text,)f(the)g(tag)g(and)h(destination)75 658 y(parameters)c(to)f(b)q (e)j(used)f(for)e(a)i(send,)g(or)f(the)g(tag)g(and)g(source)h (parameters)e(to)h(b)q(e)h(used)g(for)f(a)g(receiv)o(e.)75 714 y(In)j(addition,)g(this)g(ob)s(ject)f(stores)f(information)i(ab)q (out)f(the)g(status)g(of)g(the)g(p)q(ending)i(comm)o(unication)75 771 y(op)q(eration)e(that)g(is)h(p)q(erformed)f(with)h(this)f(ob)s (ject.)75 899 y Fq(3.8.2)49 b(Communication)17 b(initiation)75 987 y FC(W)l(e)h(use)g(the)g(same)g(naming)g(con)o(v)o(en)o(tions)g(as) f(for)g(blo)q(c)o(king)j(comm)o(unication:)26 b(a)17 b(pre\014x)h(of)g Fp(R)f FC(\()p Fp(S)p FC(\))g(is)75 1043 y(used)h(for)f Fp(READY)g FC(\()p Fp(SYNCHRONOUS)p FC(\))f(mo)q(de.)27 b(In)19 b(addition)f(a)g(pre\014x)g(of)f Fp(I)h FC(\(for)e Fp(IMMEDIATE)p FC(\))h(indicates)75 1100 y(that)d(the)i(call)g(is)g(non)o(blo)q(c)o(king.)166 1157 y(Initiate)g(a)f(standard)g(mo)q(de)g(non)o(blo)q(c)o(king)i(comm) o(unication.)75 1262 y Fo(MPI)p 160 1262 14 2 v 16 w(ISEND\(handle,)f (sta)o(rt,)f(count,)h(datat)o(yp)q(e,)g(dest,)h(tag,)d(comm\))117 1340 y Ft(OUT)108 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati) o(on)d(ob)r(ject)117 1418 y(IN)155 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1495 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1573 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)117 1650 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1727 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 1805 y(IN)155 b Fo(comm)470 b Ft(comm)o(unicator)11 b(\(handle\))166 1930 y FC(Initiate)16 b(a)f(ready)g(mo)q(de)h(non)o(blo)q(c)o(king)g(comm)o(unication.)75 2035 y Fo(MPI)p 160 2035 V 16 w(IRSEND\(handle,)g(sta)o(rt,)f(count,)h (datat)o(yp)q(e,)h(dest,)f(tag,)f(comm\))117 2114 y Ft(OUT)108 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject) 117 2191 y(IN)155 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 2268 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h (bu\013er)f(\(in)o(teger\))117 2346 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t) 117 2423 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in) o(teger\))117 2501 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 2578 y(IN)155 b Fo(comm)470 b Ft(comm)o(unicator)11 b(\(handle\))166 2704 y FC(Initiate)16 b(a)f(sync)o(hronous)g(mo)q(de)h(non)o(blo)q(c)o(king)g(comm)o (unication.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 26 32 26 31 bop 75 -100 a FC(26)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(ISSEND\(handle,)g(sta)o(rt,)f(count,)h(datat)o(yp)q (e,)h(dest,)f(tag,)f(comm\))117 126 y Ft(OUT)108 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject)117 208 y(IN)155 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 291 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 373 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t) 117 455 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 538 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 620 y(IN)155 b Fo(comm)470 b Ft(comm)o(unicator)11 b(\(handle\))166 748 y FC(Initiate)16 b(a)f(non)o(blo)q(c)o(king)i(receiv)o(e.)75 855 y Fo(MPI)p 160 855 V 16 w(IRECV)e(\(handle,)h(sta)o(rt,)f(count,)h(datat)o(yp)q (e,)h(source,)e(tag,)g(comm\))117 936 y Ft(OUT)108 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject)117 1018 y(OUT)108 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1101 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h (receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1183 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv) o(e)h(bu\013er)g(elemen)o(t)117 1265 y(IN)155 b Fo(source)468 b Ft(rank)14 b(of)f(source)j(\(in)o(teger\))117 1348 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 1430 y(IN)155 b Fo(comm)470 b Ft(comm)o(unicator)11 b(\(handle\))166 1558 y FC(The)22 b(call)g(allo)q(cates)g(a)f(comm)o(unication)h(ob)s (ject)f(and)h(asso)q(ciates)f(it)h(with)f(the)h(handle.)40 b(The)75 1615 y(handle)20 b(can)e(b)q(e)i(used)f(later)f(to)g(query)h (ab)q(out)f(the)h(status)f(of)g(the)g(comm)o(unication)i(or)e(w)o(ait)g (for)g(its)75 1671 y(completion.)166 1731 y(A)11 b(sender)h(should)h (not)e(up)q(date)g(an)o(y)g(part)g(of)g(a)g(send)h(bu\013er)f(after)g (a)g(non)o(blo)q(c)o(king)i(send)f(op)q(eration)75 1787 y(returns,)20 b(un)o(til)g(the)g(send)f(completes.)33 b(A)20 b(receiv)o(er)g(should)g(not)f(access)g(or)g(up)q(date)h(an)o(y) f(part)f(of)h(a)75 1844 y(receiv)o(e)d(bu\013er)f(after)g(a)g(non)o (blo)q(c)o(king)h(receiv)o(e)g(op)q(eration)g(returns,)e(un)o(til)j (the)e(receiv)o(e)h(completes.)75 1986 y Fq(3.8.3)49 b(Communication)17 b(Completion)75 2079 y FC(The)j(completion)h(of)f(a) f(send)i(op)q(eration)f(indicates)h(that)e(the)h(sender)h(is)f(no)o(w)f (free)h(to)f(up)q(date)i(the)75 2135 y(lo)q(cations)i(in)g(the)f(send)h (bu\013er,)h(or)e(an)o(y)g(other)g(lo)q(cation)h(that)e(can)i(b)q(e)g (referenced)g(b)o(y)f(the)g(send)75 2192 y(op)q(eration)d(\(the)g(send) g(op)q(eration)g(itself)h(lea)o(v)o(es)e(the)h(con)o(ten)o(t)f(of)h (the)g(send)g(bu\013er)g(unc)o(hanged\).)31 b(It)75 2248 y(do)q(es)19 b(not)f(indicate)j(that)d(the)h(message)f(has)g(b)q(een)i (receiv)o(ed;)i(rather,)c(it)h(ma)o(y)f(ha)o(v)o(e)h(b)q(een)h (bu\013ered)75 2305 y(b)o(y)f(the)g(comm)o(unication)h(subsystem.)32 b(Ho)o(w)o(ev)o(er,)19 b(if)h(a)f Fp(synchronous)f FC(mo)q(de)h(send)h (w)o(as)e(used,)j(the)75 2361 y(completion)c(of)e(the)g(send)h(op)q (eration)g(indicates)h(that)e(a)g(matc)o(hing)g(receiv)o(e)i(w)o(as)d (initiated,)j(and)f(that)75 2418 y(the)f(message)g(will)i(ev)o(en)o (tually)f(b)q(e)g(receiv)o(ed)g(b)o(y)f(this)h(matc)o(hing)f(receiv)o (e.)166 2478 y(The)20 b(completion)h(of)e(a)g(receiv)o(e)i(op)q (eration)f(indicates)h(that)e(the)h(receiv)o(e)g(bu\013er)g(con)o (tains)g(the)75 2534 y(receiv)o(ed)f(message,)f(and)g(that)f(the)h (status)f(ob)s(ject)h(is)g(set;)h(the)f(receiv)o(er)h(is)f(no)o(w)f (free)h(to)g(access)g(the)75 2591 y(receiv)o(e)c(bu\013er,)f(or)f(an)o (y)g(other)h(lo)q(cation)h(that)e(can)h(b)q(e)g(referenced)h(b)o(y)f (the)g(receiv)o(e)g(op)q(eration.)20 b(It)13 b(do)q(es)75 2647 y(not)g(indicate)i(that)d(the)i(matc)o(hing)f(send)h(op)q(eration) g(has)f(completed)h(\(but)g(indicates,)g(of)f(course,)h(that)75 2704 y(the)h(send)h(w)o(as)e(initiated\).)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 27 33 27 32 bop 75 -100 a Fv(3.8.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)868 b FC(27)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(W)l(AIT)15 b(\(handle,)h(status\))117 122 y Ft(IN)155 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject)117 197 y(OUT)108 b Fo(status)476 b Ft(status)15 b(ob)r(ject)166 322 y FC(A)e(call)h(to)26 b Fo(MPI)p 446 322 V 16 w(W)l(AIT)13 b FC(returns)g(when)g(the)g(send)h(op)q(eration)f(iden)o(ti\014ed)j(b)o (y)26 b Fo(handle)14 b FC(is)g(complete.)75 378 y(If)g(the)f(comm)o (unication)i(ob)s(ject)d(asso)q(ciated)i(with)g(this)g(handle)h(w)o(as) d(created)i(b)o(y)f(a)h(non)o(blo)q(c)o(king)g(send)75 434 y(or)19 b(receiv)o(e)i(call,)h(then)e(the)g(ob)s(ject)f(is)i (deallo)q(cated)g(b)o(y)f(the)g(call)h(to)39 b Fo(MPI)p 1422 434 V 16 w(W)l(AIT)20 b FC(and)g(the)g(handle)75 491 y(b)q(ecomes)c(n)o(ull.)166 547 y(The)g(call)g(returns)f(in)32 b Fo(status)17 b FC(information)f(on)f(the)g(completed)i(op)q(eration.) j(The)c(status)f(ob)s(ject)75 604 y(for)h(a)g(receiv)o(e)i(op)q (eration)f(can)g(b)q(e)g(queried)h(using)f(the)g(functions)g(describ)q (ed)i(in)f(Section)f(3.3.1.)23 b(The)75 660 y(status)15 b(parameter)g(is)i(not)e(used)i(or)e(up)q(dated)i(for)e(a)h(send)g(op)q (eration,)g(and)g(an)g(arbitrary)g(v)m(alue)h(ma)o(y)75 717 y(b)q(e)f(passed)f(for)g(this)h(parameter)e(in)i(the)f(call.)166 849 y Fu(Discussion:)166 899 y Ft(Do)e(w)o(e)h(agree)h(on)e(the)i(last) f(sen)o(tence?)20 b(\(this)14 b(implies)e(some)h(restriction)i(on)e (the)i(implemen)o(tatio)o(n\))75 1085 y Fo(MPI)p 160 1085 V 16 w(TEST)g(\(handle,)h(\015ag,)f(status\))117 1163 y Ft(IN)155 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o (on)d(ob)r(ject)117 1237 y(OUT)124 b Fo(\015ag)502 b Ft(logical)117 1312 y(OUT)108 b Fo(status)476 b Ft(status)15 b(ob)r(ject)166 1437 y FC(A)i(call)h(to)34 b Fo(MPI)p 462 1437 V 16 w(TEST)17 b FC(returns)34 b Fo(\015ag=true)18 b FC(if)g(the)f(op)q(eration)h(iden)o(ti\014ed)h(b)o(y)34 b Fo(handle)18 b FC(is)g(com-)75 1493 y(plete.)37 b(In)21 b(suc)o(h)g(a)f(case,)i(the)e(status)g(ob)s(ject)g(is)h(set)f(to)g(con) o(tain)h(information)g(on)f(the)h(completed)75 1550 y(op)q(eration;)15 b(if)h(the)f(comm)o(unication)h(ob)s(ject)e(w)o(as)h(created)g(b)o(y)g (a)g(non)o(blo)q(c)o(king)h(send)g(or)f(receiv)o(e,)h(then)75 1606 y(it)i(is)g(deallo)q(cated)h(and)e(the)h(handle)h(b)q(ecomes)f(n)o (ull.)28 b(The)18 b(call)g(returns)35 b Fo(\015ag=false)p FC(,)18 b(otherwise.)27 b(In)75 1662 y(suc)o(h)16 b(a)e(case,)h(the)h (v)m(alue)g(of)f(the)g(status)f(ob)s(ject)h(is)h(unde\014ned.)166 1795 y Fu(Discussion:)166 1851 y Ft(W)m(e)11 b(could)h(disp)q(ose)g(of) f(the)i(\015ag)e(parameter)g(and)h(add)f(an)h(additional)e(query)i (function)f(for)h(status)g(ob)r(jects.)166 2047 y Fo(MPI)p 251 2047 V 16 w(TEST)i FC(is)h(a)f(lo)q(cal,)h(non)o(blo)q(c)o(king)g (op)q(eration.)20 b(If)14 b(rep)q(eatedly)i(called)g(for)d(an)h(op)q (eration)h(that)75 2103 y(is)h(enabled,)g(it)f(m)o(ust)g(ev)o(en)o (tually)h(succeed.)166 2160 y(The)11 b(return)g(status)f(ob)s(ject)h (for)f(a)h(receiv)o(e)h(op)q(eration)f(carries)g(information)g(that)g (can)g(b)q(e)g(accessed)75 2216 y(using)k(the)f(functions)h(describ)q (ed)h(in)g(Section)f(3.3.1.)j(The)c(status)g(argumen)o(t)f(is)i(not)f (used)g(or)g(up)q(dated)75 2273 y(for)h(a)f(send)i(op)q(eration)g(and)f (an)g(arbitrary)g(v)m(alue)h(can)f(b)q(e)h(passed)g(for)e(this)i (parameter)e(in)i(the)g(call.)166 2329 y(In)i(m)o(ulti-threaded)h(en)o (vironmen)o(t,)f(the)g(use)g(of)f(a)h(blo)q(c)o(king)h(receiv)o(e)f(op) q(eration)g(\()g Fo(MPI)p 1732 2329 V 15 w(W)l(AIT)p FC(\))75 2386 y(ma)o(y)g(allo)o(w)i(the)f(op)q(erating)g(system)g(to)g (de-sc)o(hedule)i(the)e(blo)q(c)o(k)o(ed)h(thread)f(and)h(sc)o(hedule)g (another)75 2442 y(thread)h(for)f(execution,)j(if)e(suc)o(h)g(is)h(a)o (v)m(ailable.)38 b(The)21 b(use)g(of)g(a)f(non)o(blo)q(c)o(king)j (receiv)o(e)e(op)q(eration)g(\()75 2499 y Fo(MPI)p 160 2499 V 16 w(TEST)p FC(\))15 b(allo)o(ws)h(the)g(user)g(to)f(sc)o (hedule)i(alternativ)o(e)f(activities)h(within)g(a)f(single)h(thread)e (of)g(exe-)75 2555 y(cution.)166 2612 y(Example:)75 2704 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 28 34 28 33 bop 75 -100 a FC(28)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fp(IF\(rank.EQ.0\)) 123 102 y(THEN)170 158 y(CALL)24 b(MPI_ISEND\(handle,)d(a\(1\),)i(10,)h (MPI_REAL,)f(1,)g(tag,)g(comm,)h(ierr\))170 214 y(****)g(do)f(some)h (computation)e(to)i(mask)f(latency)g(****)170 271 y(CALL)h (MPI_WAIT\(handle,)d(status,)i(ierr\))123 327 y(ELSE)170 384 y(CALL)h(MPI_IRECV\(handle,)d(a\(1\),)i(15,)h(MPI_REAL,)f(0,)g (tag,)g(comm,)h(ierr\))170 440 y(****)g(do)f(some)h(computation)e(to)i (mask)f(latency)g(****)170 497 y(CALL)h(MPI_WAIT\(handle,)d(status,)i (ierr\))75 553 y(END)g(IF)166 676 y FC(The)c(functions)38 b Fo(MPI)p 566 676 14 2 v 16 w(W)l(AIT)18 b FC(and)38 b Fo(MPI)p 906 676 V 16 w(TEST)19 b FC(can)g(b)q(e)g(used)g(to)f (complete)i(b)q(oth)f(sends)g(and)75 733 y(receiv)o(es;)26 b(they)c(will)i(b)q(e)e(also)g(used)h(to)f(complete)g(an)o(y)g(other)g (non)o(blo)q(c)o(king)h(comm)o(unication)g(call)75 789 y(pro)o(vided)16 b(b)o(y)f(MPI.)166 853 y(Non)o(blo)q(c)o(king)g(comm)o (unications)g(cannot)f(b)q(e)g(replaced)i(b)o(y)e(blo)q(c)o(king)h (comm)o(unications,)g(ev)o(en)f(in)75 910 y(the)h Fp(synchronous)f FC(comm)o(unication)i(mo)q(de.)k(Consider)c(the)f(follo)o(wing)h (example:)75 1040 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\)) 75 1097 y(IF\(rank.EQ.0\))123 1153 y(THEN)170 1210 y(CALL)h (MPI_ISEND\(handle1,)d(a\(1\),)i(count,)g(MPI_REAL,)g(1,)h(tag1,)f (comm,)g(ierr\))170 1266 y(CALL)h(MPI_ISEND\(handle2,)d(a\(2\),)i (count,)g(MPI_REAL,)g(1,)h(tag2,)f(comm,)g(ierr\))170 1322 y(CALL)h(MPI_WAIT\(handle1,)d(status,)i(ierr\))170 1379 y(CALL)h(MPI_WAIT\(handle2,)d(status,)i(ierr\))123 1435 y(ELSE)170 1492 y(CALL)h(MPI_IRECV\(handle2,)d(a\(2\),)i(count,)g (MPI_REAL,)g(0,)h(tag2,)f(comm,)g(ierr\))170 1548 y(CALL)h (MPI_IRECV\(handle1,)d(a\(1\),)i(count,)g(MPI_REAL,)g(0,)h(tag1,)f (comm,)g(ierr\))170 1605 y(CALL)h(MPI_WAIT\(handle2,)d(status,)i (ierr\))170 1661 y(CALL)h(MPI_WAIT\(handle1,)d(status,)i(ierr\))75 1718 y(END)g(IF)166 1841 y FC(The)16 b(co)q(de)g(is)g(guaran)o(teed)f (to)g(execute)h(correctly)l(,)g(ev)o(en)g(in)g(an)g(implemen)o(tation)h (that)d(do)q(es)i(not)75 1897 y(bu\013er)g(messages)f(\(the)h(only)g (resource)g(requiremen)o(t)h(is)f(the)g(abilit)o(y)h(to)e(ha)o(v)o(e)h (t)o(w)o(o)f(p)q(ending)i(comm)o(u-)75 1954 y(nications\).)30 b(On)19 b(the)g(other)f(hand,)h(the)f(co)q(de)i(ma)o(y)d(deadlo)q(c)o (k)j(if)e(the)h(non)o(blo)q(c)o(king)h(op)q(erations)e(are)75 2010 y(replaced)e(b)o(y)g(blo)q(c)o(king)g(ones.)75 2174 y Fq(3.8.4)49 b(Multiple)17 b(Completions)75 2274 y FC(It)23 b(is)h(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)g(to)e(w)o(ait)h(for)g (the)g(completion)i(of)d(an)o(y)h(or)g(all)h(the)g(op)q(erations)f(in) 75 2331 y(a)e(set,)i(rather)e(than)g(ha)o(ving)h(to)f(w)o(ait)h(for)f (a)g(sp)q(eci\014c)i(message.)39 b(A)22 b(call)h(to)42 b Fo(MPI)p 1598 2331 V 16 w(W)l(AIT)l(ANY)22 b FC(or)75 2387 y Fo(MPI)p 160 2387 V 16 w(TEST)l(ANY)c FC(can)f(b)q(e)g(used)h (to)e(w)o(ait)g(for)h(the)g(completion)h(of)e(one)h(out)f(of)h(sev)o (eral)g(op)q(erations;)g(a)75 2443 y(call)f(to)30 b Fo(MPI)p 314 2443 V 16 w(W)l(AIT)l(ALL)15 b FC(or)30 b Fo(MPI)p 687 2443 V 16 w(TEST)l(ALL)15 b FC(can)h(b)q(e)f(used)h(to)f(w)o(ait)g (for)f(all)j(p)q(ending)g(op)q(erations)e(in)75 2500 y(a)g(list.)166 2640 y Fu(Discussion:)166 2704 y Fr(MPI)p 243 2704 13 2 v 14 w(TEST)m(ALL)i Fu(is)e(new.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 29 35 29 34 bop 75 -100 a Fv(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FC(29)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(W)l(AIT)l(ANY)16 b(\(count,)g(a)o(rra)o(y)p 640 45 V 14 w(of)p 691 45 V 16 w(handles,)h(index,)f(status\))117 135 y Ft(IN)155 b Fo(count)482 b Ft(list)14 b(length)f(\(in)o(teger\))117 237 y(IN)155 b Fo(a)o(rra)o(y)p 416 237 V 15 w(of)p 468 237 V 16 w(handles)285 b Ft(arra)o(y)14 b(of)f(handles)h(to)g(comm)o (unication)c(ob)r(jects)117 338 y(OUT)108 b Fo(index)489 b Ft(index)12 b(of)f(handle)g(for)g(op)q(eration)h(that)f(completed)g (\(in)o(teger\))117 439 y(OUT)108 b Fo(status)476 b Ft(return)15 b(status)g(ob)r(ject)166 577 y FC(Blo)q(c)o(ks)j(un)o(til)g(one)g(of)f (the)g(op)q(erations)h(asso)q(ciated)g(with)f(the)h(comm)o(unication)g (handles)h(in)f(the)75 633 y(arra)o(y)13 b(has)i(completed.)21 b(Returns)15 b(the)g(index)h(of)e(that)g(handle)i(in)f(the)g(arra)o(y)l (,)f(and)g(returns)h(the)g(status)75 690 y(of)g(that)f(op)q(eration)i (in)g(the)f(ob)s(ject)g(asso)q(ciated)g(with)h(the)f(status.)166 759 y(The)i(successful)i(execution)f(of)33 b Fo(MPI)p 830 759 V 16 w(W)l(AIT)l(ANY\(count,)19 b(a)o(rra)o(y)p 1297 759 V 15 w(of)p 1349 759 V 16 w(handles,)f(index,)g(status\))h FC(has)75 815 y(the)d(same)f(e\013ect)h(as)f(the)h(successful)h (execution)f(of)31 b Fo(MPI)p 1087 815 V 16 w(W)l(AIT\(a)o(rra)o(y)p 1329 815 V 14 w(of)p 1380 815 V 16 w(handles[i],)18 b(status\))p FC(,)f(where)75 872 y Fo(i)j FC(is)g(the)h(v)m(alue)g(returned)f(b)o(y) 40 b Fo(index)p FC(.)c(In)20 b(particular,)i(the)e(asso)q(ciated)g (comm)o(unication)h(ob)s(ject)e(is)75 928 y(deallo)q(cated,)26 b(and)d(the)f(handle)i(to)e(it)h(in)47 b Fo(a)o(rra)o(y)p 969 928 V 14 w(of)p 1020 928 V 16 w(handles)25 b FC(is)e(set)f(to)h(n)o (ull.)43 b(If)23 b(more)g(then)g(one)75 985 y(op)q(eration)16 b(is)g(enabled)h(and)e(can)h(terminate,)f(one)g(is)h(arbitrarily)g(c)o (hosen.)21 b(There)16 b(is)g(no)f(requiremen)o(t)75 1041 y(that)f(the)i(c)o(hoice)g(satisfy)f(an)o(y)g(fairness)g(criterion.)166 1187 y Fu(Discussion:)166 1250 y Ft(If)e(statemen)o(t)h(of)f(fairness)i (is)f(c)o(hanged)g(in)f(Section)i(3.4)d(then)j(will)d(need)j(to)f(c)o (hange)g(here,)h(to)q(o.)75 1449 y Fo(MPI)p 160 1449 V 16 w(TEST)l(ANY\()h(count,)g(a)o(rra)o(y)p 641 1449 V 15 w(of)p 693 1449 V 16 w(handles,)g(index,)g(status\))117 1539 y Ft(IN)155 b Fo(count)482 b Ft(list)14 b(length)f(\(in)o(teger\)) 117 1641 y(IN)155 b Fo(a)o(rra)o(y)p 416 1641 V 15 w(of)p 468 1641 V 16 w(handles)285 b Ft(arra)o(y)14 b(of)f(handles)h(to)g (comm)o(unication)c(ob)r(jects)117 1742 y(OUT)108 b Fo(index)489 b Ft(index)15 b(of)e(handle)h(for)g(op)q(eration)h(that)f(completed,)g (or)g(-1)g(if)905 1798 y(none)g(completed)g(\(in)o(teger\))117 1900 y(OUT)108 b Fo(status)476 b Ft(return)15 b(status)g(ob)r(ject)166 2037 y FC(Causes)d(either)h(one)f(or)g(none)h(of)f(the)g(op)q(erations) g(asso)q(ciated)h(with)f(the)h(comm)o(unication)g(handles)75 2094 y(to)d(return.)19 b(In)12 b(the)f(former)f(case,)i(it)f(returns)g (the)g(index)i(of)d(that)h(handle)h(in)g(the)f(arra)o(y)l(,)g(and)g (returns)g(the)75 2150 y(status)g(of)h(that)f(op)q(eration)h(in)h(the)f (ob)s(ject)f(asso)q(ciated)h(with)h(the)f(status.)18 b(The)12 b(comm)o(unication)h(ob)s(ject)75 2206 y(is)18 b(deallo)q(cated)g(and)f(the)g(handle)i(to)d(it)h(in)35 b Fo(a)o(rra)o(y)p 956 2206 V 15 w(of)p 1008 2206 V 16 w(handles)19 b FC(is)e(set)g(to)f(n)o(ull.)27 b(In)18 b(the)f(latter)g(case,)g(it)75 2263 y(returns)12 b(a)g(v)m(alue)i(of)e (-1)g(in)25 b Fo(index)13 b FC(and)25 b Fo(status)14 b FC(is)f(unde\014ned.)21 b(Lik)o(e)13 b Fo(MPI)p 1322 2263 V 16 w(TEST)p FC(,)f(this)h(is)g(a)f(non)o(blo)q(c)o(king)75 2319 y(op)q(eration,)j(that)f(returns)h(immediately;)i(and)e(if)g(one)h (busy-w)o(aits)f(with)g Fo(MPI)p 1444 2319 V 16 w(TEST)l(ANY)p FC(,)h(and)f(some)75 2376 y(op)q(eration)g(on)h(the)f(list)h(is)g (enabled,)g(then)f Fo(MPI)p 909 2376 V 16 w(TEST)l(ANY)i FC(will)f(ev)o(en)o(tually)h(return)30 b Fo(\015ag)15 b(=)h(true)p FC(.)166 2521 y Fu(Discussion:)166 2584 y Ft(Could)g(ha)o(v)o(e)h(a)33 b Fr(\015ag)17 b Ft(parameter,)g(rather) h(than)f(using)f(index)h(=)g(-1)g(for)f(unsuccessful)j(return.)28 b(This)17 b(is)75 2634 y(more)c(consisten)o(t)i(and)f(more)f(elegan)o (t,)g(but)h(requires)h(an)f(additional)e(parameter.)166 2704 y(If)h(w)o(e)i(stic)o(k)f(with)f(curren)o(t)j(v)o(ersion,)d(ma)o (y)f(replace)j(-1)e(b)o(y)h(an)g(MPI)g(constan)o(t.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 30 36 30 35 bop 75 -100 a FC(30)635 b Fv(SECTION)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(W)l(AIT)l(ALL\()e(count,)i(a)o(rra)o(y)p 626 45 V 15 w(of)p 678 45 V 16 w(handles,)h(a)o(rra)o(y)p 958 45 V 14 w(of)p 1009 45 V 16 w(status\))117 122 y Ft(IN)155 b Fo(count)482 b Ft(lists)14 b(length)g(\(in)o(teger\))117 196 y(IN)155 b Fo(a)o(rra)o(y)p 416 196 V 15 w(of)p 468 196 V 16 w(handles)285 b Ft(arra)o(y)14 b(of)f(handles)h(to)g(comm)o (unication)c(ob)r(jects)117 270 y(OUT)108 b Fo(a)o(rra)o(y)p 416 270 V 15 w(of)p 468 270 V 16 w(status)313 b Ft(arra)o(y)14 b(of)f(status)i(ob)r(jects)166 395 y FC(Blo)q(c)o(ks)e(un)o(til)h(all)g (comm)o(unication)g(op)q(erations)f(asso)q(ciated)g(with)g(handles)h (in)g(the)f(list)h(complete,)75 451 y(and)j(return)g(the)g(status)f(of) h(all)h(these)f(op)q(erations.)26 b(Both)16 b(arra)o(ys)g(ha)o(v)o(e)h (the)g(same)f(n)o(um)o(b)q(er)i(of)e(v)m(alid)75 508 y(en)o(tries.)22 b(The)32 b Fo(i)p FC(-th)16 b(en)o(try)f(in)32 b Fo(a)o(rra)o(y)p 721 508 V 15 w(of)p 773 508 V 16 w(status)18 b FC(is)e(set)g(to)f(the)h(return)f(status)g(of)h(the)31 b Fo(i)p FC(-th)16 b(op)q(eration.)75 564 y(All)g(comm)o(unication)g (ob)s(jects)f(are)g(deallo)q(cated)h(and)g(all)g(handles)g(are)f(set)g (to)g(n)o(ull.)75 668 y Fo(MPI)p 160 668 V 16 w(TEST)l(ALL\(count,)h(a) o(rra)o(y)p 613 668 V 14 w(of)p 664 668 V 16 w(handles,)h(\015ag,)d(a)o (rra)o(y)p 1040 668 V 15 w(of)p 1092 668 V 16 w(status\))117 745 y Ft(IN)155 b Fo(count)482 b Ft(lists)14 b(length)g(\(in)o(teger\)) 117 819 y(IN)155 b Fo(a)o(rra)o(y)p 416 819 V 15 w(of)p 468 819 V 16 w(handles)285 b Ft(arra)o(y)14 b(of)f(handles)h(to)g(comm) o(unication)c(ob)r(jects)117 893 y(OUT)108 b Fo(\015ag)518 b Ft(\(logical\))117 967 y(OUT)108 b Fo(a)o(rra)o(y)p 416 967 V 15 w(of)p 468 967 V 16 w(status)313 b Ft(arra)o(y)14 b(of)f(status)i(ob)r(jects)166 1091 y FC(Causes)f(either)g(all)h(or)f (none)g(of)f(the)i(op)q(erations)f(asso)q(ciated)g(with)g(the)g(comm)o (unication)h(handles)75 1148 y(to)i(complete.)29 b(It)18 b(returns)36 b Fo(\015ag)18 b(=)h(true)f FC(if)h(all)g(comm)o (unications)f(asso)q(ciated)g(with)h(handles)g(in)g(the)75 1204 y(arra)o(y)10 b(ha)o(v)o(e)h(completed.)19 b(In)12 b(this)g(case,)f(eac)o(h)h(status)e(en)o(try)h(is)h(set)f(to)f(the)i (status)e(of)h(the)g(corresp)q(onding)75 1261 y(comm)o(unication.)20 b(All)15 b(comm)o(unication)g(ob)s(jects)e(are)h(deallo)q(cated,)h(and) f(all)h(handles)g(are)f(set)g(to)f(n)o(ull.)75 1317 y(Otherwise,)30 b Fo(\015ag)14 b(=)h(false)g FC(is)g(returned)g(and)g(the)g(v)m(alues)h (of)e(the)h(status)e(en)o(tries)i(are)g(unde\014ned.)21 b(This)75 1374 y(is)14 b(a)f(non)o(blo)q(c)o(king)i(op)q(eration)e (that)g(returns)g(immediately;)j(if)e(one)f(busy)h(w)o(aits)f(with)h Fo(MPI)p 1660 1374 V 15 w(TEST)l(ALL)p FC(,)75 1430 y(and)19 b(all)h(op)q(erations)f(in)g(the)g(list)h(b)q(ecome)f(enabled,)i(then)e Fo(MPI)p 1219 1430 V 16 w(TEST)l(ALL)g FC(will)h(ev)o(en)o(tually)g (return)75 1487 y Fo(\015ag)15 b(=)h(true)p FC(.)166 1543 y(Example:)75 1687 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 1744 y(IF\(rank)g(<)h(2\))123 1800 y(THEN)170 1857 y(CALL)g(MPI_ISEND\(handle,)d(a,)j(n,)f(MPI_REAL,)g(2,)h(tag,)f (comm,)g(ierr\))170 1913 y(****)h(do)f(some)h(computation)e(to)i(mask)f (latency)g(****)170 1970 y(CALL)h(MPI_WAIT\(handle,)d(status,)i(ierr\)) 123 2026 y(ELSE)95 b(!)23 b(rank.EQ.2)170 2083 y(CALL)h (MPI_IRECV\(handle_list\(0\))o(,)d(a,)j(n,)f(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))170 2139 y(CALL)h(MPI_IRECV\(handle_list\(1\))o(,)d(b,) j(n,)f(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))170 2195 y(****)h(do)f(some)h(computation)e(to)i(mask)f(latency)g(****)170 2252 y(CALL)h(MPI_WAITANY\(2,)e(handle_list,)g(index,)h(status,)g (ierr\))170 2308 y(IF\(index.EQ.0\))218 2365 y(THEN)266 2421 y(****)g(handle)g(message)g(from)g(process)g(0)h(****)266 2478 y(CALL)f(MPI_WAIT\(handle_list\(1\),)e(status,)i(ierr\))266 2534 y(****)g(handle)g(message)g(from)g(process)g(1)h(****)218 2591 y(ELSE)266 2647 y(****)f(handle)g(message)g(from)g(process)g(1)h (****)266 2704 y(CALL)f(MPI_WAIT\(handle_list\(0\),)e(status,)i(ierr\)) 1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 31 37 31 36 bop 75 -100 a Fv(3.9.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1161 b FC(31)266 45 y Fp(****)23 b(handle)g(message)g(from)g(process)g(0)h (****)170 102 y(END)g(IF)75 158 y(END)f(IF)166 371 y FC(The)16 b(calls)h(in)o(tro)q(duced)g(in)g(this)g(subsection)f(can)h (b)q(e)f(used)h(to)e(w)o(ait)h(or)f(test)g(for)h(the)g(completion)75 428 y(of)d(an)h(arbitrary)f(mix)h(of)f(non)o(blo)q(c)o(king)i(comm)o (unication)g(calls,)f(including)j(a)c(mix)h(sends)g(and)g(receiv)o(es) 75 484 y(and)h(of)g(an)o(y)g(additional)i(non)o(blo)q(c)o(king)f(comm)o (unication)g(calls)g(pro)o(vided)g(b)o(y)f(MPI.)166 571 y(The)40 b Fo(status)22 b FC(argumen)o(t)e(in)g Fo(MPI)p 768 571 14 2 v 16 w(W)l(AIT)l(ANY)h FC(and)f Fo(MPI)p 1181 571 V 16 w(W)l(AIT)l(ALL)g FC(and)g(the)40 b Fo(a)o(rra)o(y)p 1697 571 V 15 w(of)p 1749 571 V 16 w(status)75 627 y FC(argumen)o(t)20 b(in)h Fo(MPI)p 427 627 V 16 w(W)l(AIT)l(ALL)f FC(and)h Fo(MPI)p 828 627 V 16 w(TEST)l(ALL)f FC(are)h(not)f(used)h(or) f(up)q(dated)i(if)f(the)f(complet-)75 684 y(ing)f(comm)o(unication)h (is)g(a)e(send.)32 b(If)19 b(all)h(op)q(erations)f(in)g(the)g(list)h (pro)o(vided)g(b)o(y)f(the)g(argumen)o(t)37 b Fo(a)o(r-)75 740 y(ra)o(y)p 136 740 V 15 w(of)p 188 740 V 16 w(handles)14 b FC(are)d(send)h(op)q(erations,)h(then)f(an)f(arbitrary)g(v)m(alue)i (can)f(b)q(e)g(passed)g(to)f(the)24 b Fo(status)14 b FC(\(resp.)75 796 y Fo(a)o(rra)o(y)p 173 796 V 15 w(of)p 225 796 V 16 w(status)p FC(\))i(argumen)o(t.)166 959 y Fu(Discussion:)166 1039 y Ft(Is)e(the)h(last)e(paragraph)h(agreed?) 166 1118 y(W)m(e)d(ma)o(y)e(w)o(an)o(t)i(to)g(allo)o(w)e(n)o(ull)h(p)q (oin)o(ters)i(in)f(an)g(arra)o(y)p 1007 1118 13 2 v 15 w(of)p 1056 1118 V 14 w(handles)g(This)h(will)d(allo)o(w)h(to)h(lo)q (op)f(with)h(the)h(same)75 1168 y(arra)o(y)p 172 1168 V 15 w(of)p 221 1168 V 14 w(handles,)i(un)o(til)g(all)f(comm)o(uni)o (cations)f(ha)o(v)o(e)i(completed.)k(On)d(the)f(other)h(hand,)f(this)g (will)f(encourage)75 1218 y(p)q(oten)o(tially)g(ine\016cien)o(t)h(co)q (de.)166 1304 y(I)j(added)h(the)h(requiremen)o(t)e(that)h(the)g(handed) g(to)g(a)f(comm)o(unicatio)o(n)e(that)j(completes)f(is)g(set)i(to)e(n)o (ull.)75 1361 y(This)d(prev)o(en)o(ts)h(the)g(o)q(ccurrence)h(of)d (dangling)g(references,)j(but)e(add)g(some)f(o)o(v)o(erhead.)75 1757 y Fw(3.9)59 b(Prob)r(e)20 b(and)f(Cancel)75 1917 y FC(The)27 b Fo(MPI)p 265 1917 14 2 v 15 w(PROBE)15 b FC(and)26 b Fo(MPI)p 624 1917 V 16 w(IPROBE)14 b FC(op)q(erations)f (allo)o(w)h(incoming)g(messages)f(to)f(b)q(e)i(c)o(hec)o(k)o(ed)g(for,) 75 1973 y(without)i(actually)h(receiving)h(them.)k(The)17 b(user)f(can)g(then)h(decide)g(ho)o(w)f(to)g(receiv)o(e)h(them,)f (based)g(on)75 2030 y(the)g(information)h(returned)f(b)o(y)h(the)f (prob)q(e)h(\(basically)l(,)h(the)e(information)g(returned)h(b)o(y)33 b Fo(status)p FC(\).)24 b(In)75 2086 y(particular,)16 b(the)g(user)g(ma)o(y)f(allo)q(cate)h(memory)f(for)g(the)h(receiv)o(e)h (bu\013er,)e(according)h(to)f(the)h(length)g(of)75 2143 y(the)f(prob)q(ed)h(message.)166 2229 y(The)31 b Fo(MPI)p 360 2229 V 16 w(CANCEL)16 b FC(op)q(eration)g(allo)o(ws)f(p)q(ending)j (comm)o(unications)e(to)f(b)q(e)h(cancelled.)23 b(This)16 b(is)75 2286 y(required)k(for)f(clean)o(up.)33 b(P)o(osting)19 b(a)g(send)h(or)f(a)g(receiv)o(e)h(ties)f(up)h(user)f(resources)h (\(send)f(or)g(receiv)o(e)75 2342 y(bu\013ers\),)14 b(and)i(a)f(cancel) h(ma)o(y)f(b)q(e)g(needed)i(to)d(free)i(these)f(resources)g(gracefully) l(.)166 2504 y Fu(Discussion:)166 2591 y Ft(Made)f(one)h(c)o(hange)f (from)e(appro)o(v)o(ed)i(draft)g(on)g(prob)q(e:)19 b(remo)o(v)o(ed)13 b(the)i(datat)o(yp)q(e)f(argumen)o(t)f(from)g(prob)q(e.)75 2647 y(If)18 b(generalized)h(datat)o(yp)q(e)f(are)g(adopted,)h(then)g (it)f(seems)g(more)f(natural)h(to)g(pro)o(vide)g(the)h(argumen)o(t)e (in)g(the)75 2704 y(status)e(deco)q(de)g(function.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 32 38 32 37 bop 75 -100 a FC(32)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(IPROBE\(source,)g(tag,)f(comm,)d(\015ag,)j(status\)) 117 125 y Ft(IN)155 b Fo(source)468 b Ft(source)15 b(rank,)f(or)27 b Fr(MPI)p 1281 125 13 2 v 15 w(ANY)p 1381 125 V 14 w(SOURCE)13 b Ft(\(in)o(teger\))117 207 y(IN)155 b Fo(tag)526 b Ft(tag)14 b(v)n(alue)f(or)28 b Fr(MPI)p 1226 207 V 14 w(ANY)p 1325 207 V 15 w(T)m(A)o(G)12 b Ft(\(in)o(teger\))117 288 y(IN)155 b Fo(comm)470 b Ft(handle)14 b(to)g(comm)o(unicator)117 369 y(OUT)108 b Fo(\015ag)518 b Ft(\(logical\))117 451 y(OUT)108 b Fo(status)476 b Ft(status)15 b(ob)r(ject)166 578 y Fo(MPI)p 251 578 14 2 v 16 w(IPROBE)j FC(returns)34 b Fo(\015ag)17 b(=)g(true)h FC(if)f(there)h(is)f(a)g(message)g(that)f (can)h(b)q(e)h(receiv)o(ed)g(and)g(that)75 634 y(matc)o(hes)e(the)g (pattern)f(sp)q(eci\014ed)j(b)o(y)e(the)g(parameters)31 b Fo(source)p FC(,)i Fo(tag)p FC(,)16 b(and)32 b Fo(comm)p FC(.)20 b(It)c(returns)32 b Fo(\015ag)75 691 y(=)19 b(false)p FC(,)g(otherwise.)31 b(If)38 b Fo(MPI)p 629 691 V 16 w(IPROBE)19 b FC(returns)38 b Fo(\015ag)18 b(=)h(true)p FC(,)h(then)f(the)g(source)g(and)g(tag)f(of)g(the)75 747 y(message)d(matc)o(hed)h(are)g(returned)g(in)g(the)g(status)f(ob)s (ject.)21 b(These)c(are)e(the)h(same)g(v)m(alues)g(that)g(w)o(ould)75 804 y(ha)o(v)o(e)f(b)q(een)h(returned)g(b)o(y)f(a)g(call)h(to)30 b Fo(MPI)p 812 804 V 16 w(RECV)16 b FC(executed)g(at)e(the)i(same)e(p)q (oin)o(t)i(in)g(the)f(program.)166 863 y(These)22 b(v)m(alues)g(can)g (b)q(e)g(deco)q(ded)h(from)e(the)g(return)h(status)e(ob)s(ject)h(using) i(the)e(status)g(query)75 920 y(functions)16 b(describ)q(ed)h(in)f (Section)g(3.3.1.)i(The)e(return)f(status)f(ob)s(ject)h(is)h (unde\014ned)h(if)31 b Fo(\015ag=false)p FC(.)166 979 y(The)16 b(call)h(to)31 b Fo(MPI)p 501 979 V 16 w(PROBE)17 b FC(do)q(es)f(not)g(carry)f(a)h(datat)o(yp)q(e)f(argumen)o(t:)20 b(the)c(message)g(t)o(yp)q(e)g(ma)o(y)75 1036 y(not)k(b)q(e)g(kno)o(wn) g(when)h(the)f(prob)q(e)g(is)h(executed.)35 b(Accordingly)l(,)23 b(the)d(prob)q(e)h(call)g(do)q(es)f(not)g(return)75 1092 y(in)g(the)f(status)f(ob)s(ject)h(the)g(length)g(of)g(the)g(prob)q(ed)h (message,)f(and)g Fo(MPI)p 1382 1092 V 16 w(GET)p 1486 1092 V 17 w(COUNT)g FC(cannot)g(b)q(e)75 1149 y(used)c(after)e(a)g (call)i(to)f Fo(MPI)p 542 1149 V 15 w(PROBE)i FC(to)d(query)h(on)g(the) g(length)g(of)g(the)g(prob)q(ed)g(message.)19 b(\(Remem)o(b)q(er)75 1205 y(that)f(the)g(length)i(of)e(a)g(message,)g(in)i(elemen)o(ts,)f (dep)q(ends)h(on)f(the)f(t)o(yp)q(es)h(of)f(these)h(elemen)o(ts;)h (some)75 1262 y(implemen)o(tations)e(ma)o(y)f(not)g(carry)f(in)j(the)e (transmitted)g(message)f(t)o(yp)q(e)i(information.\))26 b(Instead,)17 b(a)75 1318 y(di\013eren)o(t)g(status)g(deco)q(ding)h (function)g(that)e(pro)o(vides)i(the)f(datat)o(yp)q(e)g(as)f(argumen)o (t)h(can)g(b)q(e)g(used)h(in)75 1375 y(this)e(case.)75 1481 y Fo(MPI)p 160 1481 V 16 w(PROBE)p 324 1481 V 17 w(COUNT\(status,)i(datat)o(yp)q(e,)e(count\))117 1562 y Ft(IN)155 b Fo(status)476 b Ft(status)15 b(ob)r(ject)117 1643 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(message)14 b(datat)o(yp)q(e)117 1724 y(OUT)108 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(message)h(\(in)o (teger\))166 1852 y FC(Returns)k(the)g(length)h(of)e(the)h(message)g (that)f(w)o(as)g(last)h(prob)q(ed)h(with)f Fo(status)i FC(as)d(an)h(argumen)o(t.)75 1908 y(The)d(call)i(is)e(erroneous)h(if)30 b Fo(datat)o(yp)q(e)17 b FC(do)q(es)f(not)e(matc)o(h)h(the)g(t)o(yp)q (e)h(of)e(the)i(message.)166 2044 y Fu(Discussion:)166 2097 y Ft(Message)d(passing)f(in)g(MPI)g(can)h(b)q(e)g(implem)o(en)o (ted)d(without)i(app)q(ending)g(t)o(yp)q(e)h(information)c(to)j (messages.)75 2146 y(A)h(message)h(is)f(merely)f(a)h(string)h(of)f(b)o (ytes)h(and)f(the)h(in)o(terpretation)g(of)e(these)j(b)o(ytes)f(in)o (to)f(a)g(sequence)j(of)c(t)o(yp)q(ed)75 2196 y(elemen)o(ts)18 b(is)f(done)h(using)g(the)g(datat)o(yp)q(e)g(information)d(pro)o(vided) j(b)o(y)g(the)g(lo)q(cal)f(comm)o(unication)d(call.)30 b(The)75 2246 y(abilit)o(y)17 b(to)i(use)h(suc)o(h)g(an)f(implemen)o (tation)d(strategy)k(is)f(deemed)g(to)g(b)q(e)g(an)g(imp)q(ortan)o(t)f (goal.)32 b(In)19 b(suc)o(h)h(an)75 2296 y(implemen)o(tatio)o(n,)14 b(when)j(a)f(message)g(arriv)o(es,)h(it)f(is)g(not)g(kno)o(wn)g(ho)o(w) g(man)o(y)e(elemen)o(ts)j(it)f(con)o(tains,)g(or)g(ev)o(en)75 2346 y(ho)o(w)j(m)o(uc)o(h)f(storage)h(is)g(needed)i(to)e(receiv)o(e)h (that)f(message)g(\(b)q(ecause)i(of)e(p)q(ossible)g(represen)o(tation)i (con)o(v)o(er-)75 2395 y(sion)c(in)g(a)g(heterogeneous)j(en)o(vironmen) o(t\).)27 b(The)18 b(length)g(of)e(the)i(incoming)d(message)j(can)f(b)q (e)h(computed)f(b)o(y)75 2445 y Fr(MPI)p 152 2445 13 2 v 14 w(PROBE)p 304 2445 V 14 w(LEN)d Ft(from)e(the)j(b)o(yte)f (length)g(of)f(the)i(message)e(and)h(from)e(the)j(datat)o(yp)q(e)f (parameter.)166 2498 y(The)c(curren)o(t)i(solution)d(sa)o(v)o(es)i(us)f (the)h(need)g(for)f(one)g(additional)e(w)o(ord)i(p)q(er)h(message)f (that)g(w)o(ould)f(otherwise)75 2548 y(b)q(e)15 b(needed)g(to)f (transfer)g(the)h(message)f(length)f(\(in)h(elemen)o(ts\))g(with)f(the) i(message.)166 2601 y Fu(Need)g(b)q(e)h(discussed)d(b)o(y)j(MPIF)166 2654 y Ft(W)m(e)g(ma)o(y)e(w)o(an)o(t)i(a)g(call)g(to)33 b Fr(MPI)p 699 2654 V 14 w(PROBE)p 851 2654 V 13 w(COUNT)16 b Ft(to)g(b)q(e)h(signi\014can)o(t)f(not)g(just)h(after)g(prob)q(e,)g (but)g(also)75 2704 y(after)d(receiv)o(e,)h(for)e(consistency)j (reasons.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 33 39 33 38 bop 75 -100 a Fv(3.9.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1161 b FC(33)166 45 y Ft(It's)11 b(tempting)f(to)h(merge)22 b Fr(MPI)p 667 45 13 2 v 15 w(GET)p 763 45 V 14 w(COUNT)11 b Ft(and)22 b Fr(MPI)p 1096 45 V 14 w(PROBE)p 1248 45 V 14 w(COUNT)10 b Ft(in)o(to)h(one)g(call.)17 b(E.g.,)10 b(alw)o(a)o(ys)75 95 y(pro)o(vide)15 b(a)g(datat)o(yp)q(e)g(argumen)o (t,)f(ev)o(en)h(though)g(it)g(is)g(redundan)o(t)h(after)f(a)g(receiv)o (e.)23 b(But)15 b(then)h(w)o(e)f(imp)q(ose)f(an)75 145 y(additional)e(burden)j(on)e(regular)h(receiv)o(es.)166 195 y(W)m(e)h(ma)o(y)f(w)o(an)o(t)i(to)g(relax)g(t)o(yp)q(e)g (restrictions)h(so)f(that)32 b Fr(datat)o(yp)q(e)17 b Ft(=)32 b Fr(MPI)p 1373 195 V 15 w(BYTE)15 b Ft(could)h(b)q(e)g(used)h (as)f(an)75 245 y(argumen)o(t)e(to)30 b Fr(MPI)p 406 245 V 14 w(PROBE)p 558 245 V 14 w(COUNT)p Ft(,)13 b(ev)o(en)o(t)j(if)e (the)i(message)f(w)o(as)g(sen)o(t)h(with)e(another)i(datat)o(yp)q(e;)f (this)g(will)75 295 y(allo)o(w)d(to)i(coun)o(t)g(the)h(n)o(um)o(b)q(er) e(of)g(b)o(ytes)i(receiv)o(ed,)f(and)g(use)h(this)f(coun)o(t)g(in)f(a)h (mallo)q(c.)166 435 y FC(A)f(subsequen)o(t)i(receiv)o(e)f(executed)g (with)g(the)g(same)f(con)o(text,)g(and)g(the)h(source)f(and)h(tag)f (returned)75 491 y(b)o(y)i(the)h(call)g(to)30 b Fo(MPI)p 456 491 14 2 v 16 w(IPROBE)16 b FC(will)h(receiv)o(e)f(the)f(message)g (that)g(w)o(as)f(matc)o(hed)h(b)o(y)h(the)f(prob)q(e,)g(if)h(no)75 548 y(other)11 b(in)o(terv)o(ening)h(receiv)o(e)g(o)q(ccurred)g(after)e (the)h(prob)q(e.)19 b(If)11 b(the)g(receiving)i(pro)q(cess)e(is)h(m)o (ulti-threaded,)75 604 y(it)j(is)h(the)f(user)h(resp)q(onsibilit)o(y)h (to)e(ensure)h(that)e(the)i(last)f(condition)h(holds.)166 737 y Fu(Discussion:)166 788 y Ft(MPI)f(guaran)o(tees)g(that)g (successiv)o(e)i(messages)e(sen)o(t)h(from)d(a)h(source)i(to)f(a)f (destination)h(within)e(the)j(same)75 838 y(con)o(text)21 b(are)g(receiv)o(ed)h(in)e(the)h(order)g(they)g(are)g(sen)o(t.)39 b(Th)o(us,)22 b(MPI)e(m)o(ust)g(supp)q(ort,)i(either)g(explicitly)d(or) 75 888 y(implicitly)l(,)13 b(a)i(FIF)o(O)g(structure)j(to)d(manage)e (messages)j(b)q(et)o(w)o(een)h(eac)o(h)e(pair)g(of)g(pro)q(cesses,)j (for)d(eac)o(h)h(con)o(text.)75 937 y Fr(MPI)p 152 937 13 2 v 14 w(PROBE)e Ft(returns)j(information)c(on)i(the)h(\014rst)h (matc)o(hing)c(message)j(in)f(this)g(FIF)o(O;)h(this)f(will)f(also)h(b) q(e)h(the)75 987 y(message)10 b(receiv)o(ed)h(b)o(y)f(the)h(\014rst)g (subsequen)o(t)h(receiv)o(e)f(with)f(the)h(same)e(source,)j(tag)d(and)h (con)o(text)h(as)g(the)f(message)75 1037 y(matc)o(hed)j(b)o(y)28 b Fr(MPI)p 391 1037 V 14 w(PROBE)p Ft(.)75 1224 y Fo(MPI)p 160 1224 14 2 v 16 w(PROBE\()16 b(source,)f(tag,)g(comm,)e(datat)o(yp)q (e,)j(status\))117 1302 y Ft(IN)155 b Fo(source)468 b Ft(source)15 b(rank,)f(or)27 b Fr(MPI)p 1281 1302 13 2 v 15 w(ANY)p 1381 1302 V 14 w(SOURCE)13 b Ft(\(in)o(teger\))117 1379 y(IN)155 b Fo(tag)526 b Ft(tag)14 b(v)n(alue,)e(or)28 b Fr(MPI)p 1237 1379 V 14 w(ANY)p 1336 1379 V 15 w(T)m(A)o(G)13 b Ft(\(in)o(teger\))117 1455 y(IN)155 b Fo(comm)470 b Ft(handle)14 b(to)g(comm)o(unicator)117 1532 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(assumed)14 b(t)o(yp)q(e)g(of)f(data)h(in)f (message)h(\(status\))117 1608 y(OUT)108 b Fo(status)476 b Ft(status)15 b(ob)r(ject)166 1734 y Fo(MPI)p 251 1734 14 2 v 16 w(PROBE)h FC(b)q(eha)o(v)o(es)f(lik)o(e)31 b Fo(MPI)p 780 1734 V 16 w(IPROBE)16 b FC(except)g(that)e(it)h(is)h(a)f (blo)q(c)o(king)h(call)g(whic)o(h)g(returns)75 1790 y(only)g(after)e(a) h(matc)o(hing)g(message)g(has)g(b)q(een)h(found.)166 1923 y Fu(Discussion:)166 1974 y(CANCEL)h(has)f(b)q(een)f(c)o(hanged)f (since)h(last)g(draft)75 2161 y Fo(MPI)p 160 2161 V 16 w(CANCEL\(handle\))117 2239 y Ft(IN)155 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject)166 2364 y FC(A)k(call)i(to)29 b Fo(MPI)p 454 2364 V 16 w(CANCEL)16 b FC(marks)e(for)h(cancellation)i(a)e(p)q(ending)i(non)o(blo)q(c)o (king)f(comm)o(unication)75 2421 y(op)q(eration)g(\(send)g(or)g(receiv) o(e\).)23 b(The)16 b(cancel)h(call)g(is)g(non-blo)q(c)o(king,)g(and)f (lo)q(cal.)24 b(It)16 b(returns)f(immedi-)75 2477 y(ately)l(,)g(p)q (ossibly)i(b)q(efore)e(the)h(comm)o(unication)g(is)f(actually)h (cancelled.)166 2534 y(It)c(is)g(still)h(necessary)f(to)g(complete)g(a) g(comm)o(unication)g(that)f(has)h(b)q(een)h(mark)o(ed)e(for)h (cancellation,)75 2591 y(using)h(a)e(call)j(to)23 b Fo(MPI)p 455 2591 V 16 w(W)l(AIT)12 b FC(or)23 b Fo(MPI)p 742 2591 V 16 w(TEST)12 b FC(\(or)f(an)o(y)h(of)g(the)g(deriv)o(ed)h(op)q (erations\).)18 b(If)13 b(the)f(op)q(eration)75 2647 y(has)20 b(b)q(een)h(cancelled,)i(then)d(information)g(ab)q(out)g(to)f (that)h(e\013ect)f(will)j(b)q(e)f(returned)f(in)h(the)f(status)75 2704 y(argumen)o(t)10 b(of)g(the)h(op)q(eration)g(that)f(completes)h (the)g(comm)o(unication.)19 b(If)11 b(a)g(comm)o(unication)g(is)g(mark) o(ed)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 34 40 34 39 bop 75 -100 a FC(34)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FC(for)i (cancellation,)j(then)e(a)37 b Fo(MPI)p 667 45 14 2 v 16 w(W)l(AIT)18 b FC(call)i(for)e(that)g(comm)o(unication)h(is)g (guaran)o(teed)f(to)g(return,)75 102 y(irresp)q(ectiv)o(e)d(of)e(the)g (status)f(of)h(other)g(pro)q(cesses;)h(similarly)h(if)27 b Fo(MPI)p 1270 102 V 16 w(TEST)13 b FC(is)h(rep)q(eatedly)g(called)h (in)f(a)75 158 y(busy)e(w)o(ait)g(lo)q(op)g(for)g(a)f(cancelled)k(comm) o(unication,)d(then)h Fo(MPI)p 1177 158 V 16 w(TEST)f FC(will)h(ev)o(en)o(tually)g(b)q(e)g(successful.)166 231 y(Either)19 b(the)f(cancellation)i(succeeds,)f(or)f(the)g(comm)o (unication)h(succeeds,)h(but)e(not)g(b)q(oth.)29 b(If)18 b(a)75 288 y(send)h(is)h(mark)o(ed)e(for)g(cancellation,)j(then)e(it)g (m)o(ust)f(b)q(e)i(the)f(case)g(that)f(either)h(the)g(send)g(completes) 75 344 y(normally)l(,)f(in)h(whic)o(h)f(case)f(the)h(message)f(sen)o(t) g(w)o(as)g(receiv)o(ed)h(at)f(the)h(destination)g(pro)q(cess,)g(or)f (that)75 400 y(the)e(send)h(is)f(successfully)i(cancelled,)g(in)e(whic) o(h)h(case)f(no)g(part)f(of)h(the)g(message)f(w)o(as)g(receiv)o(ed)j (at)d(the)75 457 y(destination.)21 b(Then,)15 b(an)o(y)f(matc)o(hing)h (receiv)o(e)h(has)e(to)h(b)q(e)g(satis\014ed)h(b)o(y)e(another)h(send.) 20 b(If)15 b(a)g(receiv)o(e)g(is)75 513 y(mark)o(ed)d(for)f (cancellation,)k(then)d(it)h(m)o(ust)e(b)q(e)i(the)g(case)f(that)f (either)i(the)g(receiv)o(e)g(completes)g(normally)l(,)75 570 y(or)k(that)g(the)h(receiv)o(e)h(is)f(successfully)i(cancelled,)g (in)f(whic)o(h)f(case)g(no)g(part)f(of)g(the)h(receiv)o(e)h(bu\013er)e (is)75 626 y(altered.)j(Then,)c(an)o(y)e(matc)o(hing)i(send)g(has)f(to) f(b)q(e)i(satis\014ed)g(b)o(y)f(another)g(receiv)o(e.)75 747 y Fo(MPI)p 160 747 V 16 w(TEST)p 290 747 V 16 w(CANCELLED\(status,) i(\015ag\))117 840 y Ft(IN)155 b Fo(status)476 b Ft(return)15 b(status)g(ob)r(ject)117 949 y(IN)155 b Fo(\015ag)518 b Ft(\(logical\))166 1090 y FC(Returns)28 b Fo(\015ag)13 b(=)h(true)g FC(if)g(the)g(comm)o(unication)g(asso)q(ciated)g(with)g (the)g(return)f(status)g(ob)s(ject)g(w)o(as)75 1146 y(cancelled)21 b(successfully)l(.)34 b(In)19 b(suc)o(h)g(a)g(case,)h(all)f(other)g (\014elds)h(of)38 b Fo(status)20 b FC(\(suc)o(h)f(as)38 b Fo(length)20 b FC(or)37 b Fo(tag)p FC(\))75 1203 y(are)17 b(unde\014ned.)28 b(Returns)35 b Fo(\015ag)17 b(=)h(false)p FC(,)f(otherwise.)27 b(If)17 b(a)g(receiv)o(e)h(op)q(eration)g(migh)o (t)f(b)q(e)h(cancelled)75 1259 y(then)i(one)g(should)h(call)41 b Fo(MPI)p 613 1259 V 16 w(TEST)p 743 1259 V 16 w(CANCELLED)20 b FC(\014rst,)g(to)g(c)o(hec)o(k)g(whether)g(the)g(op)q(eration)g(w)o (as)75 1316 y(cancelled,)d(b)q(efore)f(c)o(hec)o(king)g(on)f(the)g (other)g(v)m(alues)h(of)f(the)g(return)g(status.)75 1554 y Fw(3.10)59 b(P)n(ersistent)19 b(communication)f(objects)75 1764 y Fu(Discussion:)166 1837 y(there)c(are)i(some)f(c)o(hanges)g (here)g(not)g(y)o(et)g(appro)o(v)o(ed)e(b)o(y)j(MPIF)166 1993 y FC(Often)e(a)e(comm)o(unication)i(with)g(the)f(same)g(parameter) g(list)h(is)g(rep)q(eatedly)g(executed)g(within)h(the)75 2049 y(inner)k(lo)q(op)g(of)e(a)h(parallel)i(computation.)28 b(In)18 b(suc)o(h)h(a)f(situation,)g(it)h(ma)o(y)e(b)q(e)i(p)q(ossible) g(to)f(optimize)75 2106 y(the)c(comm)o(unication)g(b)o(y)g(binding)i (the)d(list)i(of)e(comm)o(unication)i(parameters)d(to)h(the)h(comm)o (unication)75 2162 y(ob)s(ject)f(once)h(and,)g(then,)g(rep)q(eatedly)g (using)h(the)e(comm)o(unication)i(handle)g(to)d(initiate)j(and)f (complete)75 2219 y(messages.)22 b(The)16 b(comm)o(unication)h(handle)g (th)o(us)f(created)g(can)g(b)q(e)g(though)o(t)f(of)h(as)g(a)f(comm)o (unication)75 2275 y(p)q(ort)c(or)f(a)h(\\half-c)o(hannel")h(.)18 b(It)12 b(do)q(es)f(not)f(pro)o(vide)i(the)f(full)h(functionalit)o(y)h (of)d(a)h(con)o(v)o(en)o(tional)g(c)o(hannel,)75 2332 y(since)i(there)g(is)g(no)f(binding)i(of)e(the)g(send)h(p)q(ort)f(to)f (the)i(receiv)o(e)g(p)q(ort:)18 b(this)13 b(construct)f(allo)o(ws)g (reduction)75 2388 y(of)18 b(the)g(o)o(v)o(erhead)f(for)h(comm)o (unication)g(b)q(et)o(w)o(een)h(pro)q(cessor)e(and)h(comm)o(unication)h (con)o(troller,)g(but)75 2444 y(not)c(the)g(o)o(v)o(erhead)g(for)f (comm)o(unication)i(b)q(et)o(w)o(een)g(one)f(comm)o(unication)h(con)o (troller)g(and)f(another.)166 2518 y(A)j(comm)o(unication)h(ob)s(ject)f (is)g(created)h(using)f(one)h(of)e(the)i(four)f(follo)o(wing)g(calls.) 30 b(These)19 b(calls)75 2574 y(in)o(v)o(olv)o(e)d(no)f(comm)o (unication.)166 2647 y(The)e(function)g Fo(MPI)p 517 2647 V 16 w(CREA)l(TE)p 702 2647 V 17 w(SEND)g FC(creates)f(a)g(comm)o (unication)h(ob)s(ject)f(for)g(a)g(standard)g(mo)q(de)75 2704 y(send)k(op)q(eration,)f(and)g(binds)i(to)d(it)i(all)g(the)f (parameters)f(of)h(a)g(send)h(op)q(eration.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 35 41 35 40 bop 75 -100 a Fv(3.10.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (OBJECTS)673 b FC(35)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(CREA)l(TE)p 345 45 V 17 w(SEND\(handle,)17 b(sta)o(rt,)e(count,)h (datat)o(yp)q(e,)g(dest,)h(tag,)d(comm\))117 137 y Ft(OUT)108 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject) 117 242 y(IN)155 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 348 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\)) 117 453 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)117 558 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o(teger\))117 663 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 768 y(IN)155 b Fo(comm)470 b Ft(handle)14 b(to)g(comm)o(unicator)d(ob)r (ject)166 908 y FC(The)i(call)i(allo)q(cates)f(a)f(new)h(comm)o (unication)g(ob)s(ject)e(and)i(asso)q(ciates)f(the)h(comm)o(unication)g (han-)75 964 y(dle)i(with)g(it.)166 1112 y Fu(Discussion:)166 1176 y Ft(Tw)o(o)d(c)o(hanges:)166 1241 y(1.)42 b Fr(bu\013er)p 341 1241 13 2 v 17 w(handle)16 b Ft(is)f(replaced)h(with)29 b Fr(sta)o(rt,)15 b(count,)g(datat)o(yp)q(e)p Ft(,)h(to)f(b)q(e)g(in)g (sync)g(with)g(the)g(new)h(datat)o(yp)q(e)75 1291 y(prop)q(osal.)166 1356 y(2.)g(T)m(o)q(ok)10 b(out)20 b Fr(p)q(ersistence)p Ft(.)g(Do)q(es)10 b(not)g(seem)h(v)o(ery)f(useful)h(here.)18 b(Instead,)11 b(I)f(in)o(tro)q(duce)h(in)f(the)h(last)f(section,)75 1406 y(a)k(\\univ)o(ersal")f(comm)o(unicati)o(on)e(function,)i(whic)o (h)h(w)o(ould)f(not)h(usually)f(used)i(directly)f(b)o(y)f(programmers,) f(but)75 1455 y(can)i(b)q(e)h(used)f(to)g(de\014ne)h(the)g(other)f (comm)o(unication)c(functions.)166 1610 y FC(The)16 b(function)h Fo(MPI)p 524 1610 14 2 v 16 w(CREA)l(TE)p 709 1610 V 17 w(RSEND)g FC(creates)e(a)h(comm)o(unication)h(ob)s(ject)e(for)g(a)h (ready)g(mo)q(de)75 1666 y(send)g(op)q(eration.)75 1785 y Fo(MPI)p 160 1785 V 16 w(CREA)l(TE)p 345 1785 V 17 w(RSEND\(handle,)h(sta)o(rt,)e(count,)h(datat)o(yp)q(e,)h(dest,)f(tag,) f(comm\))117 1877 y Ft(OUT)108 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject)117 1982 y(IN)155 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)117 2087 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 2192 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)117 2297 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o(teger\))117 2403 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 2508 y(IN)155 b Fo(comm)470 b Ft(handle)14 b(to)g(comm)o(unicator)d(ob) r(ject)166 2647 y FC(The)k(function)i Fo(MPI)p 523 2647 V 15 w(CREA)l(TE)p 707 2647 V 18 w(SSEND)f FC(creates)f(a)g(comm)o (unication)h(ob)s(ject)e(for)h(a)g(sync)o(hronous)75 2704 y(mo)q(de)g(send)h(op)q(eration.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 36 42 36 41 bop 75 -100 a FC(36)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(CREA)l(TE)p 345 45 V 17 w(SSEND\(handle,)h(sta)o (rt,)e(count,)h(datat)o(yp)q(e,)g(dest,)h(tag,)e(comm\))117 123 y Ft(OUT)108 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o (on)d(ob)r(ject)117 201 y(IN)155 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)117 278 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f (sen)o(t)i(\(in)o(teger\))117 356 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)117 433 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 511 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 588 y(IN)155 b Fo(comm)470 b Ft(handle)14 b(to)g(comm)o(unicator)d(ob)r(ject)166 714 y FC(The)k(function)h Fo(MPI)p 522 714 V 15 w(CREA)l(TE)p 706 714 V 18 w(RECV)g FC(creates)e(a)h(comm)o(unication)g(ob)s(ject)f (for)h(a)f(receiv)o(e)i(op)q(era-)75 771 y(tion.)75 875 y Fo(MPI)p 160 875 V 16 w(CREA)l(TE)p 345 875 V 17 w(RECV\(handle,)h (sta)o(rt,)e(count,)h(datat)o(yp)q(e,)h(source,)e(tag,)g(comm\))117 954 y Ft(OUT)108 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o (on)d(ob)r(ject)117 1031 y(OUT)108 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)117 1109 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(receiv)o(ed)j(\(in)o(teger\))117 1186 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)117 1264 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(source)j(or)d(MPI)p 1309 1264 13 2 v 16 w(ANY)p 1418 1264 V 15 w(SOUR)o(CE)h(\(in)o(teger\))117 1341 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(or)h(MPI)p 1270 1341 V 15 w(ANY)p 1378 1341 V 16 w(T)m(A)o(G)f(\(in)o(teger\))117 1419 y(IN)155 b Fo(comm)470 b Ft(handle)14 b(to)g(comm)o(unicator)d(ob) r(ject)166 1544 y FC(A)19 b(comm)o(unication)g(\(send)g(or)f(receiv)o (e\))i(that)e(uses)h(a)f(prede\014ned)j(handle)f(is)f(initiated)h(b)o (y)f(the)75 1601 y(function)d Fo(MPI)p 338 1601 14 2 v 16 w(ST)l(ART)p FC(.)75 1706 y Fo(MPI)p 160 1706 V 16 w(ST)l(ART\(handle\))117 1784 y Ft(INOUT)62 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject)166 1910 y FC(The)17 b(comm)o(unication)h(ob)s(ject)e(asso)q(ciated)h(with) 34 b Fo(handle)18 b FC(should)g(b)q(e)f(an)g(ob)s(ject)f(that)h(w)o(as) f(cre-)75 1966 y(ated)h(b)o(y)g(one)h(of)f(the)g(previous)h(four)f (functions,)h(so)f(that)f(all)j(the)e(comm)o(unication)h(parameters)e (are)75 2023 y(already)k(de\014ned.)34 b(A)19 b(send)h(can)g(b)q(e)g (started)f(pro)o(vided)h(that)f(the)g(previous)h(send)h(using)f(the)f (same)75 2079 y(ob)s(ject)d(has)h(completed,)h(or)f(as)f(so)q(on)h(as)g (the)g(ob)s(ject)f(is)h(created,)g(if)h(it)f(has)g(not)f(y)o(et)h(b)q (een)h(used)f(in)h(a)75 2136 y(comm)o(unication.)j(In)16 b(addition,)h(if)f(the)f(comm)o(unication)h(mo)q(de)g(is)g Fp(ready)f FC(then)g(a)h(matc)o(hing)f(receiv)o(e)75 2192 y(should)k(b)q(e)f(p)q(osted.)28 b(The)18 b(send)g(bu\013er)g (should)h(not)e(b)q(e)i(up)q(dated)f(after)f(the)h(send)g(is)h (started,)e(un)o(til)75 2248 y(the)e(op)q(eration)h(completes.)166 2306 y(A)e(receiv)o(e)g(can)g(b)q(e)g(started)f(pro)o(vided)i(that)e (the)h(preceding)h(receiv)o(e)f(using)h(the)f(same)f(ob)s(ject)g(has)75 2363 y(completed,)f(or)f(as)f(so)q(on)h(as)f(the)h(ob)s(ject)f(is)i (created,)f(if)g(it)g(has)g(not)g(y)o(et)f(b)q(een)i(used)f(in)h(a)f (comm)o(unication.)75 2419 y(The)18 b(receiv)o(e)h(bu\013er)f(should)h (not)e(b)q(e)i(accessed)g(after)e(the)h(receiv)o(e)h(is)f(started,)g (un)o(til)h(the)f(op)q(eration)75 2475 y(completes.)166 2533 y(The)23 b(call)g(is)g(non)o(blo)q(c)o(king,)i(with)e(similar)g (seman)o(tics)g(as)f(the)h(non)o(blo)q(c)o(king)g(comm)o(unication)75 2590 y(op)q(erations)15 b(describ)q(ed)i(in)g(Section)f(3.8.)166 2647 y(A)10 b(comm)o(unication)h(started)e(with)i(a)f(call)h(to)e Fo(MPI)p 1016 2647 V 16 w(ST)l(ART)i FC(is)g(completed)g(b)o(y)f(a)g (call)h(to)f Fo(MPI)p 1740 2647 V 16 w(W)l(AIT)p FC(,)75 2704 y Fo(MPI)p 160 2704 V 16 w(TEST)p FC(,)i(or)f(one)h(of)g(the)g (deriv)o(ed)h(functions)f(describ)q(ed)i(in)f(Section)g(3.8.4.)k(These) 12 b(comm)o(unication)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 37 43 37 42 bop 75 -100 a Fv(3.11.)34 b(SEND-RECEIVE)1277 b FC(37)75 45 y(completion)18 b(functions)g(do)e(not)h(deallo)q(cate)h (the)f(comm)o(unication)g(ob)s(ject,)g(and)g(this)g(can)g(b)q(e)g (reused)75 102 y(anew)k(b)o(y)f(a)h Fo(MPI)p 394 102 14 2 v 15 w(ST)l(ART)h FC(call.)37 b(The)21 b(ob)s(ject)f(needs)i(to)e (b)q(e)h(explicitly)j(deallo)q(cated)e(b)o(y)e(a)h(call)h(to)75 158 y(the)15 b(function)g Fo(MPI)p 415 158 V 16 w(COMM)p 573 158 V 16 w(FREE)p FC(,)g(b)q(elo)o(w.)20 b(W)l(e)15 b(th)o(us)f(ha)o(v)o(e)g(t)o(w)o(o)f(t)o(yp)q(es)i(of)f(comm)o (unication)h(ob)s(jects:)75 214 y FD(p)q(ersisten)o(t)j FC(ob)s(jects,)f(whic)o(h)i(are)e(allo)q(cated)i(b)o(y)f(a)f(call)i(to) e Fo(MPI)p 1230 214 V 16 w(CREA)l(TE)p 1415 214 V 18 w(xxx)p FC(,)h(and)g(are)f(explicitly)75 271 y(deallo)q(cated)h(b)o(y)f Fo(MPI)p 465 271 V 16 w(COMM)p 623 271 V 17 w(FREE)p FC(,)g(and)g FD(ephemeral)g FC(ob)s(jects)f(that)g(p)q(ersist)i(for)e (one)h(comm)o(uni-)75 327 y(cation)e(only:)21 b(they)16 b(are)f(created)g(b)o(y)g(a)g(non)o(blo)q(c)o(king)i(comm)o(unication)f (initiation)h(function,)f(and)f(are)75 384 y(freed)h(b)o(y)f(the)g (comm)o(unication)h(completion)g(call.)75 490 y Fo(MPI)p 160 490 V 16 w(COMM)p 318 490 V 16 w(FREE\(handle\))117 570 y Ft(INOUT)62 b Fo(handle)465 b Ft(handle)14 b(to)g(comm)o(unicati) o(on)d(ob)r(ject)166 697 y FC(Marks)h(the)h(comm)o(unication)g(ob)s (ject)f(for)g(deallo)q(cation.)21 b(The)13 b(ob)s(ject)f(will)j(b)q(e)e (deallo)q(cated)h(when)75 754 y(there)23 b(are)f(no)g(p)q(ending)j (comm)o(unications)e(in)o(v)o(olving)g(this)g(ob)s(ject,)h(at)e(whic)o (h)h(p)q(oin)o(t)g(the)g(handle)75 810 y(b)q(ecomes)16 b(n)o(ull.)166 869 y(The)g(call)g(is)g(non)o(blo)q(c)o(king,)g(and)g (the)f(ob)s(ject)g(ma)o(y)g(not)g(b)q(e)h(deallo)q(cated)h(when)e(the)h (call)g(returns.)75 926 y(It)f(is)g(p)q(ermissible)i(to)d(call)i Fo(MPI)p 628 926 V 16 w(COMM)p 786 926 V 16 w(FREE\(handle\))g FC(after)e(a)h(comm)o(unication)g(that)f(uses)30 b Fo(handle)75 982 y FC(has)18 b(b)q(een)h(initiated,)h(but)f(b)q(efore)f(it)h(has)f (completed.)29 b(The)19 b(ob)s(ject)e(will)j(b)q(e)f(deallo)q(cated)g (after)f(the)75 1039 y(comm)o(unication)23 b(completes.)41 b(It)22 b(is)h(erroneous)f(to)f(try)h(to)f(reuse)i(the)f(handle)h(for)f (a)f(subsequen)o(t)75 1095 y(comm)o(unication.)166 1154 y(A)15 b(correct)f(in)o(v)o(o)q(cation)i(of)e(the)h(functions)h (describ)q(ed)g(in)g(this)f(section)h(will)g(o)q(ccur)g(in)f(a)g (sequence)75 1211 y(of)g(the)g(form)613 1275 y FD(Create)i FC(\()p FD(Start)h(Complete)p FC(\))1172 1257 y Fm(\003)1209 1275 y FD(F)l(ree)e FB(;)75 1368 y FC(where)j Fy(\003)f FC(indicates)i(zero)e(or)g(more)g(rep)q(etitions.)31 b(If)18 b(the)h(same)f(comm)o(unication)h(ob)s(ject)f(is)h(used)g(in)75 1425 y(sev)o(eral)12 b(concurren)o(t)g(threads,)g(it)g(is)h(the)f(user) g(resp)q(onsibilit)o(y)i(to)d(co)q(ordinate)i(calls)g(so)e(that)g(the)h (correct)75 1481 y(sequence)k(is)g(ob)q(ey)o(ed.)166 1540 y(A)11 b(send)g(op)q(eration)g(initiated)h(with)22 b Fo(MPI)p 881 1540 V 15 w(ST)l(ART)12 b FC(can)f(b)q(e)g(matc)o(hed)g (with)g(an)o(y)f(receiv)o(e)i(op)q(eration)75 1597 y(and,)g(lik)o (ewise,)h(a)e(receiv)o(e)i(op)q(eration)e(initiated)i(with)23 b Fo(MPI)p 1102 1597 V 16 w(ST)l(ART)12 b FC(can)g(receiv)o(e)g (messages)f(generated)75 1653 y(b)o(y)k(an)o(y)g(send)h(op)q(eration.) 75 1812 y Fw(3.11)59 b(Send-receive)75 1995 y Fu(Discussion:)166 2054 y(This)15 b(section)f(has)h(not)g(y)o(et)g(b)q(een)g(appro)o(v)o (ed)f(b)o(y)h(MPIF.)166 2195 y FC(The)20 b FD(send-receiv)o(e)e FC(op)q(erations)i(com)o(bines)g(in)g(one)g(call)h(the)e(sending)i(of)e (a)g(message)g(to)g(one)75 2252 y(destination)h(and)f(the)g(receiving)h (of)e(another)h(message,)g(from)f(another)g(destination,)i(p)q(ossibly) h(the)75 2308 y(same.)f(The)c FD(exc)o(hange)f FC(op)q(erations)h(are)f (the)h(same)f(as)g(send-receiv)o(e,)i(except)f(that)e(the)i(send)g (bu\013er)75 2365 y(and)i(the)g(receiv)o(e)g(bu\013er)g(are)g(iden)o (tical.)29 b(A)18 b(send-receiv)o(e)h(op)q(eration)f(is)g(v)o(ery)g (useful)h(for)e(executing)75 2421 y(a)h(shift)i(op)q(eration)f(across)f (a)g(c)o(hain)i(of)e(pro)q(cesses.)31 b(If)19 b(blo)q(c)o(king)h(sends) f(and)g(receiv)o(es)h(are)e(used)i(for)75 2478 y(suc)o(h)d(shift,)g (then)f(one)h(needs)g(to)f(order)g(correctly)h(the)f(sends)h(and)g (receiv)o(es)g(\(e.g.)23 b(ev)o(en)17 b(sends,)g(next)75 2534 y(receiv)o(es,)j(o)q(dd)f(receiv)o(es)h(\014rst,)f(next)g(sends\)) f(so)h(as)f(to)g(prev)o(en)o(t)h(cyclic)h(dep)q(endencies)i(that)c (lead)i(to)75 2591 y(deadlo)q(c)o(k.)f(When)12 b(a)e(send-receiv)o(e)j (or)d(exc)o(hange)h(op)q(eration)g(is)h(used,)f(the)g(comm)o(unication) h(subsystem)75 2647 y(tak)o(es)i(care)h(of)g(these)g(issues.)21 b(Also,)15 b(a)g(send-receiv)o(e)i(op)q(eration)e(is)h(useful)g(for)e (implemen)o(ting)k(remote)75 2704 y(pro)q(cedure)e(calls.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 38 44 38 43 bop 75 -100 a FC(38)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FC(A)h(message)f (sen)o(t)h(b)o(y)g(a)f(send-receiv)o(e)j(or)d(exc)o(hange)h(op)q (eration)g(can)g(b)q(e)h(receiv)o(ed)g(b)o(y)e(a)h(regular)75 102 y(receiv)o(e)f(op)q(eration,)f(and)h(vice)g(v)o(ersa.)75 206 y Fo(MPI)p 160 206 14 2 v 16 w(SENDRECV\(send)p 509 206 V 19 w(sta)o(rt,)26 b(send)p 736 206 V 18 w(count,)g(send)p 981 206 V 18 w(t)o(yp)q(e,)h(dest,)g(recv)p 1312 206 V 16 w(sta)o(rt,)f(recv)p 1530 206 V 16 w(count,)h(recv)p 1768 206 V 16 w(t)o(yp)q(e,)75 263 y(source,)15 b(tag,)g(comm,)e (status\))117 341 y Ft(IN)155 b Fo(send)p 404 341 V 18 w(sta)o(rt)400 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 418 y(IN)155 b Fo(send)p 404 418 V 18 w(count)381 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 495 y(IN)155 b Fo(send)p 404 495 V 18 w(t)o(yp)q(e)406 b Ft(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f (send)i(bu\013er)117 572 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o(teger\))117 649 y(OUT)108 b Fo(recv)p 398 649 V 17 w(sta)o(rt)407 b Ft(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 726 y(IN)155 b Fo(recv)p 398 726 V 17 w(count)388 b Ft(n)o(um)o(b)q(er) 13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 803 y(IN)155 b Fo(recv)p 398 803 V 17 w(t)o(yp)q(e)413 b Ft(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(receiv)o(e)i(bu\013er)117 880 y(IN)155 b Fo(source)468 b Ft(rank)14 b(of)f(source)j(\(in)o (teger\))117 957 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 1034 y(IN)155 b Fo(comm)470 b Ft(handle)14 b(to)g(comm)o(unicator)117 1111 y(OUT)108 b Fo(status)476 b Ft(status)15 b(ob)r(ject)166 1237 y FC(Execute)e(a)f(blo)q(c)o(king)i(send)g(and)f(receiv)o(e)g(op)q (eration.)19 b(Both)13 b(send)g(and)g(receiv)o(e)h(use)f(the)f(same)h (tag)75 1293 y(v)m(alue)h(and)g(the)f(same)g(comm)o(unicator.)19 b(Ho)o(w)o(ev)o(er,)12 b(the)i(send)g(bu\013er)f(and)g(receiv)o(e)h (bu\013er)g(are)f(disjoin)o(t,)75 1350 y(and)i(ma)o(y)g(ha)o(v)o(e)g (di\013eren)o(t)g(length)h(and)f(di\013eren)o(t)h(datat)o(yp)q(es.)166 1483 y Fu(Discussion:)166 1534 y Ft(F)m(or)d(a)h(shift)f(it's)h(more)e (natural)i(to)f(ha)o(v)o(e)h(same)f(t)o(yp)q(e)h(and)g(same)e(length)i (b)q(oth)g(for)f(send)i(and)f(receiv)o(e;)g(for)75 1584 y(a)g(remote)f(pro)q(cedure)j(call,)d(the)h(additional)e(freedom)h(mak) o(es)g(sense.)75 1771 y Fo(MPI)p 160 1771 V 16 w(EX)o(CHANGE\(sta)o (rt,)j(count,)g(datat)o(yp)q(e,)h(dest,)f(source,)f(tag,)g(comm,)e (status\))117 1849 y Ft(IN)155 b Fo(sta)o(rt)501 b Ft(initial)12 b(address)j(of)f(send)g(and)g(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1926 y(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)9 b(of)g(elemen)o(ts) h(in)f(send)i(and)e(receiv)o(e)i(bu\013er)g(\(in)o(teger\))117 2003 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(and)f(receiv)o(e)h(bu\013er)117 2080 y(IN)155 b Fo(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 2157 y(IN)155 b Fo(source)468 b Ft(rank)14 b(of)f(source)j(\(in)o(teger\))117 2234 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 2311 y(IN)155 b Fo(comm)470 b Ft(handle)14 b(to)g(comm)o(unicator)117 2388 y(OUT)108 b Fo(status)476 b Ft(status)15 b(ob)r(ject)166 2514 y FC(Execute)g(a)f(blo)q(c)o(king)i(send)f(and)g(receiv)o(e;)g (the)g(same)f(bu\013er)g(is)h(used)h(b)q(oth)e(for)g(the)h(send)g(and)g (for)75 2570 y(the)g(receiv)o(e.)166 2704 y Fu(Discussion:)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 39 45 39 44 bop 75 -100 a Fv(3.12.)34 b(NULL)16 b(PR)o(OCESSES)1218 b FC(39)166 45 y Ft(I)17 b(omitted)e(the)j(ready)f(and)f(sync)o (hronous)i(mo)q(des)e(for)h(send/receiv)o(e)i(and)d(exc)o(hange.)28 b(Also,)17 b(I)f(omitted)75 95 y(non)o(blo)q(c)o(king)d(send/receiv)o (e)i(and)f(exc)o(hange.)19 b(Do)13 b(w)o(e)h(w)o(an)o(t)g(an)o(y)f(of)h (those?)166 234 y FC(The)20 b(seman)o(tics)h(of)f(a)g(send-receiv)o(e)i (op)q(eration)e(is)h(what)f(w)o(ould)g(obtain)h(if)g(the)f(caller)h (fork)o(ed)75 291 y(t)o(w)o(o)16 b(concurren)o(t)h(threads,)g(one)g(to) f(execute)i(the)f(send,)g(and)h(one)f(to)f(execute)i(the)f(receiv)o(e,) h(follo)o(w)o(ed)75 347 y(b)o(y)f(a)f(join)h(of)f(these)h(t)o(w)o(o)e (threads.)24 b(A)16 b(send-receiv)o(e)j(cannot)d(b)q(e)h(implemen)o (ted)i(b)o(y)d(a)g(blo)q(c)o(king)i(send)75 404 y(follo)o(w)o(ed)e(b)o (y)f(a)g(blo)q(c)o(king)i(receiv)o(e)g(or)e(a)g(blo)q(c)o(king)i (receiv)o(e,)f(follo)o(w)o(ed)g(b)o(y)f(a)g(blo)q(c)o(king)i(send.)k (Consider)75 460 y(the)15 b(follo)o(wing)h(co)q(de,)g(where)f(t)o(w)o (o)f(pro)q(cesses)i(exc)o(hange)f(messages:)75 567 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 624 y(IF)h(\(rank.EQ.0\))123 680 y(THEN)170 737 y(CALL)g (MPI_SENDRECV\(send_buff,)c(count,)k(MPI_REAL,)e(0,)i(recv_buff,)e (count,)h(MPI_REAL,)290 793 y(1,)g(tag,)h(comm,)f(status,)g(ierr\))123 850 y(ELSE)170 906 y(CALL)h(MPI_SENDRECV\(send_buff,)c(count,)k (MPI_REAL,)e(1,)i(recv_buff,)e(count,)h(MPI_REAL,)290 962 y(0,)g(tag,)h(comm,)f(status,)g(ierr\))75 1019 y(END)g(IF)166 1126 y FC(If)12 b(the)f(send)h(receiv)o(es)g(are)g(replaced)g(either)g (b)o(y)g(blo)q(c)o(king)g(send,)h(follo)o(w)o(ed)f(b)o(y)f(blo)q(c)o (king)i(receiv)o(e,)f(or)75 1182 y(blo)q(c)o(king)g(receiv)o(e,)h (follo)o(w)o(ed)e(b)o(y)g(blo)q(c)o(king)i(send,)f(then)g(the)f(co)q (de)g(ma)o(y)g(deadlo)q(c)o(k.)19 b(On)12 b(the)f(other)g(hand,)75 1239 y(send-receiv)o(e)21 b(can)f(b)q(e)h(implemen)o(ted)g(using)g(non) o(blo)q(c)o(king)g(sends)f(and)g(receiv)o(es.)35 b(Note)19 b(that)g(some)75 1295 y(system)e(bu\013ering)i(is)f(required)h(for)e(a) h(correct)f(implemen)o(tation)i(of)35 b Fo(MPI)p 1402 1295 14 2 v 16 w(EX)o(CHANGE)p FC(.)19 b(\(Consider)75 1352 y(the)c(last)g(example,)h(with)g(send-receiv)o(e)g(replaced)h(b)o (y)e(exc)o(hange.\))75 1496 y Fw(3.12)59 b(Null)21 b(p)n(ro)r(cesses)75 1680 y Fu(Discussion:)34 b Ft(This)14 b(section)g(has)g(not)g(b)q(een)h (review)o(ed)g(b)o(y)f(MPIF.)166 1820 y FC(In)21 b(man)o(y)f (instances,)h(it)g(is)g(con)o(v)o(enien)o(t)g(to)e(sp)q(ecify)j(a)e (\\dumm)o(y")f(source)i(or)e(destination)j(for)75 1876 y(comm)o(unication.)27 b(This)18 b(simpli\014es)i(the)d(co)q(de)h(that) f(is)h(needed)g(for)f(dealing)i(with)e(b)q(oundaries,)i(e.g.,)75 1933 y(in)d(the)f(case)h(of)e(a)h(non-circular)i(shift)e(done)h(with)g (calls)g(to)e(send-receiv)o(e.)166 1989 y(The)f(sp)q(ecial)j(v)m(alue) 26 b Fr(MPI)p 608 1989 13 2 v 15 w(PROCNULL)12 b FC(can)i(b)q(e)g(used) g(instead)g(of)f(a)g(rank)g(wherev)o(er)g(a)g(source)g(or)g(a)75 2046 y(destination)i(parameter)e(is)i(required)g(in)g(a)e(call.)21 b(A)14 b(comm)o(unication)h(with)f(pro)q(cess)27 b Fr(MPI)p 1649 2046 V 15 w(PROCNULL)75 2102 y FC(has)13 b(no)h(e\013ect:)k(a)13 b(send)h(to)25 b Fr(MPI)p 633 2102 V 14 w(PROCNULL)13 b FC(succeeds)i(and)e(returns)g(as)g(so)q(on)h(as)f(p)q(ossible.)21 b(A)13 b(receiv)o(e)75 2159 y(from)26 b Fr(MPI)p 271 2159 V 14 w(PROCNULL)13 b FC(succeeds)i(and)f(returns)g(as)f(so)q(on)h (as)f(p)q(ossible)j(with)e(no)g(mo)q(di\014cations)h(to)e(the)75 2215 y(receiv)o(e)k(bu\013er.)22 b(When)16 b(a)g(receiv)o(e)h(with)32 b Fo(source)17 b FC(=)30 b Fr(MPI)p 1095 2215 V 15 w(PROCNULL)15 b FC(is)i(executed)f(then)h(the)f(status)75 2272 y(ob)s(ject)f(returns) 30 b Fo(source)16 b FC(=)29 b Fr(MPI)p 661 2272 V 14 w(PROCNULL)p FC(,)g Fo(tag)16 b FC(=)29 b Fr(MPI)p 1147 2272 V 14 w(ANY)p 1246 2272 V 15 w(T)m(A)o(G)14 b FC(and)31 b Fo(count=0)p FC(.)166 2404 y Fu(Discussion:)166 2454 y Ft(It)18 b(w)o(ould)f(b)q(e)i(nice)f(to)g(ha)o(v)o(e)36 b Fr(MPI)p 740 2454 V 14 w(PROCNULL)17 b Ft(\\equiv)n(alenced")h(to)g (-1)f(and)h(to)g(group)p 1624 2454 V 15 w(size,)h(so)f(that,)75 2504 y(when)f(a)f(shift)h(is)f(executed)i(in)e(a)h(group)f(the)h (\014rst)g(and)g(last)f(pro)q(cessors)j(automatically)13 b(comm)o(unicate)h(with)75 2554 y(the)g(n)o(ull)f(pro)q(cess.)166 2604 y(A)e(c)o(hoice)g(is)f(to)h(sa)o(y)f(that)h(a)g (source/destination)g(with)g(rank)f(-1)g(or)h(group)p 1329 2604 V 15 w(size)g(is)g(alw)o(a)o(ys)f(the)h(n)o(ull)f(pro)q (cess.)75 2654 y(W)m(e)j(then)i(lose)f(some)f(error)i(catc)o(hing)f (capabilit)o(y)m(.)166 2704 y(Another)h(c)o(hoice)f(is)g(to)f(sa)o(y)h (that)g(only)f(for)h(send-receiv)o(e)i(and)d(exc)o(hange.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 40 46 40 45 bop 75 -100 a FC(40)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Ft(A)d(third)g(c)o (hoice)g(is)g(to)f(liv)o(e)h(without)f(this)h(nicet)o(y)m(.)k(It)c (means)f(that)h(one)g(still)f(ha)o(v)o(e)h(to)g(execute)h(conditional) 75 95 y(co)q(de:)k(\(if)10 b(m)o(yrank=0)g(then)i(m)o(yneigh)o(b)q (or=mpi)p 850 95 13 2 v 12 w(pro)q(cn)o(ull\).)17 b(But)12 b(this)g(co)q(de)g(need)g(b)q(e)g(executed)i(only)c(when)i(the)75 145 y(comm)o(unication)e(pattern)15 b(is)f(set,)g(and)g(the)g (conditional)f(b)q(oundary)h(co)q(de)h(is)e(not)h(needed)h(in)f(the)g (inner)h(lo)q(op.)75 374 y Fw(3.13)59 b(Derived)19 b(datat)n(yp)r(es)75 553 y Fu(Discussion:)166 604 y(This)c(is)g(a)h(new)g(section,)e(and)h (the)g(material)f(has)h(not)g(y)o(et)g(b)q(een)g(discussed)f(b)o(y)h (MPIF.)166 743 y FC(Up)10 b(to)g(no)o(w,)g(all)h(p)q(oin)o(t)g(to)f(p)q (oin)o(t)g(comm)o(unication)h(in)o(v)o(olv)o(ed)g(only)g(con)o(tiguous) f(bu\013ers)g(con)o(taining)75 800 y(a)18 b(sequence)h(of)f(elemen)o (ts)g(of)g(the)g(same)g(t)o(yp)q(e.)29 b(This)18 b(is)h(to)q(o)e (constraining)i(on)f(t)o(w)o(o)f(accoun)o(ts:)25 b(One)75 856 y(often)16 b(w)o(an)o(ts)f(to)g(pass)h(messages)f(that)h(con)o (tain)g(v)m(alues)h(with)f(di\013eren)o(t)h(datat)o(yp)q(es)e(\(e.g.,)g (an)h(in)o(teger)75 913 y(coun)o(t,)i(follo)o(w)o(ed)g(b)o(y)f(coun)o (t)h(real)g(n)o(um)o(b)q(ers\);)g(and)g(one)g(often)f(w)o(an)o(ts)f(to) h(send)i(noncon)o(tiguous)f(data)75 969 y(\(e.g.)g(a)12 b(sub-blo)q(c)o(k)i(of)e(a)g(matrix\).)18 b(One)13 b(solution)g(is)g (to)f(pro)o(vide)h(functions)g(that)e(pac)o(k)i(noncon)o(tiguous)75 1026 y(data)f(in)o(to)h(a)g(con)o(tiguous)g(bu\013er)g(at)f(the)h (sender)h(site)f(and)g(unpac)o(k)h(it)f(bac)o(k)g(at)f(the)h(receiv)o (er)h(site.)19 b(This)75 1082 y(has)e(the)h(disadv)m(an)o(tage)g(of)f (requiring)i(additional)g(memory)e(to)g(memory)g(cop)o(y)h(at)f(b)q (oth)g(sites,)i(ev)o(en)75 1139 y(when)c(the)g(comm)o(unication)h (subsystem)f(has)g(scatter-gather)e(capabilities.)23 b(Instead,)15 b(MPI)f(pro)o(vides)75 1195 y(mec)o(hanisms)j(to)f(sp)q (ecify)j(more)d(general,)h(mixed)h(and)f(noncon)o(tiguous)g(comm)o (unication)g(bu\013ers.)25 b(It)75 1251 y(is)16 b(up)g(to)f(the)h (implemen)o(tation)h(to)e(decide)i(whether)f(data)f(should)h(b)q(e)g (\014rst)g(pac)o(k)o(ed)f(in)i(a)e(con)o(tiguous)75 1308 y(bu\013er)e(b)q(efore)f(b)q(een)i(transmitted,)e(or)g(whether)h(it)g (can)g(b)q(e)g(collected)h(directly)g(from)e(where)h(it)f(resides.)166 1365 y(The)h(general)g(mec)o(hanisms)g(pro)o(vided)g(here)g(allo)o(ws)g (one)f(to)g(transfer)g(directly)l(,)i(without)f(cop)o(ying,)75 1421 y(ob)s(jects)k(of)g(v)m(arious)h(shap)q(e)h(and)f(size.)28 b(It)17 b(is)i(not)e(assumed)h(that)f(the)g(MPI)h(library)g(is)h (cognizan)o(t)e(of)75 1478 y(the)f(ob)s(jects)g(declared)h(in)g(the)f (host)g(language;)g(th)o(us,)g(if)g(one)g(w)o(an)o(ts)f(to)h(transfer)f (a)h(structure,)f(or)h(an)75 1534 y(arra)o(y)d(section,)i(it)f(will)i (b)q(e)f(necessary)f(to)g(pro)o(vide)h(in)g(MPI)f(a)g(de\014nition)i (of)e(a)g(comm)o(unication)h(bu\013er)75 1591 y(that)e(mimics)j(the)e (de\014nition)i(of)e(the)g(structure)g(or)f(arra)o(y)g(section)i(in)g (question.)20 b(These)15 b(facilities)h(can)75 1647 y(b)q(e)j(used)g(b) o(y)f(library)h(designers)h(to)d(de\014ne)j(comm)o(unication)f (functions)g(that)f(can)g(transfer)g(ob)s(jects)75 1704 y(de\014ned)g(in)f(the)f(host)g(language)g({)g(b)o(y)h(deco)q(ding)g (their)g(de\014nitions)h(as)e(a)o(v)m(ailable)i(in)f(a)f(sym)o(b)q(ol)h (table)75 1760 y(or)e(a)g(dop)q(e)g(v)o(ector.)20 b(Suc)o(h)15 b(higher-lev)o(el)j(comm)o(unication)e(functions)g(are)f(not)g(part)f (of)h(MPI.)166 1817 y(More)d(general)g(comm)o(unication)i(bu\013ers)e (are)g(sp)q(eci\014ed)i(b)o(y)f(replacing)g(the)g(basic)g(datat)o(yp)q (es)f(that)75 1874 y(ha)o(v)o(e)18 b(used)h(so)f(far)g(with)g(deriv)o (ed)i(datat)o(yp)q(es)e(that)f(are)h(constructed)h(from)e(basic)i (datat)o(yp)q(es)f(using)75 1930 y(the)13 b(constructors)e(describ)q (ed)j(in)g(this)f(section.)19 b(These)13 b(metho)q(ds)f(of)g (constructing)h(deriv)o(ed)h(datat)o(yp)q(es)75 1987 y(can)h(b)q(e)h(applied)h(recursiv)o(ely)l(.)166 2044 y(A)e FD(general)j(datat)o(yp)q(e)e FC(is)g(an)f(opaque)h(ob)s(ject)e (that)h(sp)q(eci\014es)i(t)o(w)o(o)d(things:)143 2140 y Fy(\017)23 b FC(A)15 b(sequence)h(of)f(basic)h(t)o(yp)q(es)143 2237 y Fy(\017)23 b FC(A)15 b(sequence)h(of)f(in)o(teger)g(\(b)o(yte\)) g(displacemen)o(ts)166 2333 y(The)22 b(displacemen)o(ts)g(are)g(not)f (required)h(to)f(b)q(e)h(p)q(ositiv)o(e,)i(distinct,)g(or)d(in)h (increasing)h(order;)75 2389 y(therefore)17 b(the)g(order)g(of)f(items) i(need)g(not)f(coincide)i(with)e(their)h(order)f(in)g(store,)g(and)g (an)g(item)h(ma)o(y)75 2446 y(app)q(ear)d(more)g(than)g(once.)166 2503 y(Let)471 2561 y FB(D)q(ataty)r(pe)e FC(=)g Fy(f)p FC(\()p FB(ty)r(pe)844 2568 y Fl(0)864 2561 y FB(;)8 b(disp)969 2568 y Fl(0)987 2561 y FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)1188 2568 y Fk(n)p Fm(\000)p Fl(1)1254 2561 y FB(;)g(disp)1359 2568 y Fk(n)p Fm(\000)p Fl(1)1426 2561 y FC(\))p Fy(g)p FB(;)75 2647 y FC(b)q(e)16 b(suc)o(h)f(a)g (general)g(datat)o(yp)q(e,)f(where)h FB(ty)r(pe)851 2654 y Fk(i)881 2647 y FC(are)f(basic)i(t)o(yp)q(es,)f(and)g FB(disp)1375 2654 y Fk(i)1403 2647 y FC(are)g(displacemen)o(ts.)21 b(This)75 2704 y(general)g(datat)o(yp)q(e,)g(together)f(with)h(a)f (base)h(address)41 b Fo(sta)o(rt)p FC(,)22 b(sp)q(eci\014es)h(a)d(comm) o(unication)h(bu\013er:)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 41 47 41 46 bop 75 -100 a Fv(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(41)75 45 y(the)22 b(comm)o(unication)g(bu\013er)f(that)g (consists)h(of)f FB(n)h FC(en)o(tries,)h(where)f(the)g FB(i)p FC(-th)f(en)o(try)g(is)h(at)f(address)75 102 y Fo(sta)o(rt)14 b FC(+)g FB(disp)308 109 y Fk(i)342 102 y FC(and)21 b(has)f(t)o(yp)q(e)h FB(ty)r(pe)714 109 y Fk(i)728 102 y FC(.)36 b(A)21 b(message)f(assem)o(bled)h(from)f(this)h (comm)o(unication)g(bu\013er)75 158 y(will)g(con)o(tain)e(a)f(sequence) j(of)d FB(n)h FC(v)m(alues,)i(where)e(v)m(alue)i FB(i)d FC(has)h(t)o(yp)q(e)g FB(ty)r(pe)1383 165 y Fk(i)1398 158 y FC(.)31 b(The)19 b(general)g(datat)o(yp)q(e)75 214 y(determines)g(the)e FD(t)o(yp)q(e)j(signature)e FC(of)f(the)h(message)f(\(i.e.)27 b(the)18 b(n)o(um)o(b)q(er)f(of)h(v)m (alues)g(carried)g(b)o(y)g(the)75 271 y(message)d(and)g(the)g(basic)h (t)o(yp)q(e)g(of)e(eac)o(h)i(v)m(alue\).)166 328 y(W)l(e)j(can)f(use)h (a)g(handle)h(to)e(a)g(general)h(datat)o(yp)q(e)f(as)h(an)f(argumen)o (t)g(in)i(a)e(send)h(or)f(receiv)o(e)i(op-)75 385 y(eration,)c(in)h (replacemen)o(t)g(of)f(a)g(basic)h(datat)o(yp)q(e)f(argumen)o(t.)23 b(The)16 b(op)q(eration)33 b Fo(MPI)p 1576 385 14 2 v 16 w(SEND\(sta)o(rt,)17 b(1,)75 441 y(datat)o(yp)q(e,...\))35 b FC(will)22 b(use)e(the)g(send)h(bu\013er)f(de\014ned)i(b)o(y)e(the)g (base)g(address)40 b Fo(sta)o(rt)21 b FC(and)f(the)g(general)75 498 y(datat)o(yp)q(e)c(asso)q(ciated)g(with)33 b Fo(datat)o(yp)q(e)p FC(;)17 b(it)g(will)h(generate)e(a)f(message)h(with)h(the)f(t)o(yp)q(e) g(signature)g(de-)75 554 y(termined)g(b)o(y)g(the)31 b Fo(datat)o(yp)q(e)17 b FC(argumen)o(t.)40 b Fo(MPI)p 932 554 V 16 w(RECV\(sta)o(rt,)16 b(1,)f(datat)o(yp)q(e,...\))21 b FC(will)c(use)f(the)f(receiv)o(e)75 611 y(bu\013er)e(de\014ned)h(b)o (y)f(the)g(base)g(address)26 b Fo(sta)o(rt)13 b FC(and)g(the)g(general) g(datat)o(yp)q(e)g(asso)q(ciated)g(with)26 b Fo(datat)o(yp)q(e)p FC(.)166 668 y(General)16 b(datat)o(yp)q(es)e(can)i(b)q(e)g(used)f(in)h (all)h(send)e(and)h(receiv)o(e)g(op)q(erations.)k(W)l(e)c(address)f (later)g(in)75 724 y(Section)h(3.13.3)d(the)j(case)f(where)g(the)h (second)f(argumen)o(t)30 b Fo(count)16 b FC(has)f(v)m(alue)i FB(>)c FC(1.)166 782 y(The)22 b(prede\014ned)h(basic)f(datat)o(yp)q(es) f(presen)o(ted)g(in)i(Section)f(3.2.1)e(are)h(particular)h(cases)f(of)g (a)75 838 y(general)c(datat)o(yp)q(e.)22 b(Th)o(us,)33 b Fo(MPI)p 675 838 V 16 w(INT)16 b FC(is)h(a)f(prede\014ned)i(handle)f (to)f(the)g(datat)o(yp)q(e)g Fy(f)p FC(\()p Fp(int)o FB(;)8 b FC(0\))p Fy(g)p FC(,)15 b(with)75 895 y(one)g(en)o(try)g(of)g (t)o(yp)q(e)g Fp(int)g FC(and)g(displacemen)o(t)i(zero.)j(And)c (similarly)h(for)d(all)i(other)f(basic)h(datat)o(yp)q(es.)166 952 y(The)f FD(exten)o(t)f FC(of)g(a)g(datat)o(yp)q(e)g(is)h(de\014ned) g(to)f(b)q(e)h(the)g(span)f(from)g(the)g(\014rst)g(b)o(yte)h(to)e(the)i (last)f(b)o(yte)75 1008 y(o)q(ccupied)j(b)o(y)e(en)o(tries)h(in)g(this) g(datat)o(yp)q(e,)e(rounded)i(up)f(to)g(satisfy)g(alignmen)o(t)h (requiremen)o(ts.)k(I.e.,)15 b(if)471 1113 y FB(D)q(ataty)r(pe)e FC(=)g Fy(f)p FC(\()p FB(ty)r(pe)844 1120 y Fl(0)864 1113 y FB(;)8 b(disp)969 1120 y Fl(0)987 1113 y FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)1188 1120 y Fk(n)p Fm(\000)p Fl(1)1254 1113 y FB(;)g(disp)1359 1120 y Fk(n)p Fm(\000)p Fl(1)1426 1113 y FC(\))p Fy(g)p FB(;)75 1218 y(disp)159 1225 y Fk(r)190 1218 y FC(=)13 b(min)314 1225 y Fk(j)340 1218 y FB(disp)424 1225 y Fk(j)457 1218 y FC(and)i FB(disp)629 1225 y Fk(s)659 1218 y FC(=)e(max)792 1225 y Fk(j)818 1218 y FB(disp)902 1225 y Fk(j)935 1218 y FC(then)471 1335 y FB(extent)p FC(\()p FB(D)q(ataty)r(pe)p FC(\))h(=)f FB(disp)967 1342 y Fk(s)995 1335 y FC(+)d FB(siz)r(eof)5 b FC(\()p FB(ty)r(pe)1272 1342 y Fk(s)1291 1335 y FC(\))10 b Fy(\000)g FB(disp)1448 1342 y Fk(r)1467 1335 y FC(;)302 b(\(3)p FB(:)p FC(1\))75 1421 y(if)18 b(furthermore,)e FB(ty)r(pe)469 1428 y Fk(i)501 1421 y FC(requires)h(alignmen)o(t)h(to)e (a)h(b)o(yte)g(address)g(that)f(is)i(is)g(a)e(m)o(ultiple)j(of)e FB(k)1743 1428 y Fk(i)1757 1421 y FC(,)g(then)75 1478 y FB(extent)p FC(\()p FB(D)q(ataty)r(pe)p FC(\))g(is)e(rounded)h(up)g (to)e(the)i(next)f(m)o(ultiple)i(of)e(max)1280 1485 y Fk(i)1302 1478 y FB(k)1326 1485 y Fk(i)1340 1478 y FC(.)166 1535 y(Example:)20 b(Assume)c(that)652 1640 y FB(T)6 b(y)r(pe)p FC(1)13 b(=)g Fy(f)p FC(\()p Fp(double)n FB(;)8 b FC(0\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(8\))p Fy(g)75 1745 y FC(\(a)17 b Fp(double)f FC(at)h(displacemen)o(t)i(zero,)f(follo) o(w)o(ed)f(b)o(y)h(a)f Fp(char)g FC(at)g(displacemen)o(t)i(eigh)o(t\).) 26 b(Assume,)18 b(fur-)75 1801 y(thermore,)f(that)f(doubles)i(ha)o(v)o (e)f(to)f(b)q(e)i(strictly)f(aligned)h(at)f(addresses)g(that)f(are)h(m) o(ultiple)i(of)d(eigh)o(t.)75 1858 y(Then,)g(the)h(exten)o(t)e(of)h (this)h(datat)o(yp)q(e)e(is)i(16)e(\(9)h(rounded)h(to)e(the)h(next)h(m) o(ultiple)h(of)d(8\).)22 b(A)16 b(datat)o(yp)q(e)75 1914 y(that)d(consists)g(of)g(a)g(c)o(haracter)g(immediately)i(follo)o(w)o (ed)e(b)o(y)h(a)f(double)h(will)h(also)f(ha)o(v)o(e)f(an)g(exten)o(t)g (of)g(16.)75 2041 y Fq(3.13.1)49 b(Datat)o(yp)q(e)17 b(constructo)o(rs)75 2129 y FC(The)d(simplest)h(datat)o(yp)q(e)f (constructor)f(is)h Fo(MPI)p 901 2129 V 16 w(TYPE)p 1034 2129 V 17 w(CONTIGUOUS)i FC(whic)o(h)f(allo)o(ws)f(replication)h(of)75 2185 y(a)g(datat)o(yp)q(e)g(in)o(to)g(con)o(tiguous)g(lo)q(cations.)75 2290 y Fo(MPI)p 160 2290 V 16 w(TYPE)p 293 2290 V 17 w(CONTIGUOUS\(count,)i(oldt)o(yp)q(e,)f(newt)o(yp)q(e\))117 2368 y Ft(IN)155 b Fo(count)482 b Ft(replication)14 b(coun)o(t)g(\(in)o (teger\))117 2445 y(IN)155 b Fo(oldt)o(yp)q(e)450 b Ft(handle)14 b(to)g(input)f(datat)o(yp)q(e)i(ob)r(ject)117 2522 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Ft(handle)14 b(to)g(output)g(datat)o(yp)q(e)g (ob)r(ject)166 2647 y Fo(newt)o(yp)q(e)k FC(is)e(the)g(datat)o(yp)q(e)f (obtained)i(b)o(y)f(concatenating)32 b Fo(count)17 b FC(copies)g(of)31 b Fo(oldt)o(yp)q(e)p FC(.)23 b(P)o(adding)75 2704 y(ma)o(y)14 b(b)q(e)i(added)g(to)f(satisfy)g(alignmen)o(t)h (requiremen)o(ts)f(of)g(the)g(underlying)i(arc)o(hitecture.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 42 48 42 47 bop 75 -100 a FC(42)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FC(Example:)k(let) 31 b Fo(oldt)o(yp)q(e)16 b FC(b)q(e)g(a)f(handle)i(to)d(the)h(datat)o (yp)q(e)738 144 y Fy(f)p FC(\()p Fp(double)o FB(;)8 b FC(0\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(8\))p Fy(g)p FB(;)75 243 y FC(with)16 b(exten)o(t)e(16,)h(and)g(let)h Fo(count)e FC(=)f(3.)19 b(The)d(resulting)g(datat)o(yp)q(e)f(returned)g (b)o(y)30 b Fo(newt)o(yp)q(e)17 b FC(is)257 342 y Fy(f)p FC(\()p Fp(double)o FB(;)8 b FC(0\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(8\))p FB(;)g FC(\()p Fp(double)l FB(;)g FC(16\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(24\))p FB(;)g FC(\()p Fp(double)l FB(;)g FC(32\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(40\))p Fy(g)p FC(;)75 442 y(i.e.,)15 b(alternating)g Fp(double)g FC(and)g Fp(char)g FC(elemen)o(ts,)g(with)h (displacemen)o(ts)g(0)p FB(;)8 b FC(8)p FB(;)g FC(16)p FB(;)g FC(24)o FB(;)f FC(32)p FB(;)g FC(40.)166 498 y(In)16 b(general,)f(assume)g(that)30 b Fo(oldt)o(yp)q(e)16 b FC(is)g(a)f(handle)h(to)f(the)g(datat)o(yp)q(e)595 597 y Fy(f)p FC(\()p FB(ty)r(pe)720 604 y Fl(0)740 597 y FB(;)8 b(disp)845 604 y Fl(0)863 597 y FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)1064 604 y Fk(n)p Fm(\000)p Fl(1)1130 597 y FB(;)g(disp)1235 604 y Fk(n)p Fm(\000)p Fl(1)1302 597 y FC(\))p Fy(g)p FB(;)75 696 y FC(with)13 b(exten)o(t)g FB(extent)p FC(.)20 b(Then)26 b Fo(newt)o(yp)q(e)14 b FC(is)g(a)e(handle)i(to)e(the)h(datat)o(yp)q(e)f(with)h Fp(count)5 b Fy(\001)g Fp(n)12 b FC(en)o(tries)i(de\014ned)75 753 y(b)o(y:)75 861 y Fy(f)p FC(\()p FB(ty)r(pe)200 868 y Fl(0)219 861 y FB(;)8 b(disp)324 868 y Fl(0)343 861 y FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)544 868 y Fk(n)p Fm(\000)p Fl(1)609 861 y FB(;)g(disp)714 868 y Fk(n)p Fm(\000)p Fl(1)782 861 y FC(\))p FB(;)g FC(\()p FB(ty)r(pe)923 868 y Fl(0)941 861 y FB(;)g(disp)1046 868 y Fl(0)1065 861 y FC(+)p FB(extent)p FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)1428 868 y Fk(n)p Fm(\000)p Fl(1)1495 861 y FB(;)g(disp)1600 868 y Fk(n)p Fm(\000)p Fl(1)1667 861 y FC(+)p FB(extent)p FC(\))p FB(;)g(:::;)157 960 y FC(\()p FB(ty)r(pe)259 967 y Fl(0)278 960 y FB(;)g(disp)383 967 y Fl(0)412 960 y FC(+)i FB(extent)i Fy(\001)e FC(\()p Fp(count)f Fy(\000)h FC(1\)\))p FB(;)e(:::;)g FC(\()o FB(ty)r(pe)1052 967 y Fk(n)p Fm(\000)p Fl(1)1118 960 y FB(;)g(disp)1223 967 y Fk(n)p Fm(\000)p Fl(1)1301 960 y FC(+)i FB(extent)i Fy(\001)d FC(\()p Fp(count)g Fy(\000)i FC(1\)\))p Fy(g)p FB(:)166 1042 y Fo(MPI)p 251 1042 14 2 v 16 w(TYPE)p 384 1042 V 17 w(VECTOR)i FC(is)f(a)g(more)f(general)h (constructor)g(that)f(allo)o(ws)h(replication)h(of)f(a)f(datat)o(yp)q (e)75 1098 y(in)o(to)18 b(lo)q(cations)i(that)d(consist)i(of)f(equally) i(spaced)f(con)o(tiguous)f(blo)q(c)o(ks)h(of)f(equal)h(size)h({)e(blo)q (c)o(k)h(sizes)75 1155 y(and)c(blo)q(c)o(k)h(displacemen)o(ts)h(are)e (all)h(m)o(ultiples)h(of)e(the)g(old)h(t)o(yp)q(e)f(exten)o(t.)75 1258 y Fo(MPI)p 160 1258 V 16 w(TYPE)p 293 1258 V 17 w(VECTOR\()h(count,)g(oldt)o(yp)q(e,)g(stride,)f(blo)q(cklen,)h(newt)o (yp)q(e\))117 1336 y Ft(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))117 1410 y(IN)155 b Fo(oldt)o(yp)q(e)450 b Ft(handle)14 b(to)g(input)f(datat)o(yp)q(e)i (ob)r(ject)117 1485 y(IN)155 b Fo(stride)484 b Ft(n)o(um)o(b)q(er)13 b(of)f(elemen)o(ts)h(b)q(et)o(w)o(een)h(start)g(of)f(eac)o(h)g(blo)q(c) o(k)g(\(in)o(te-)905 1541 y(ger\))117 1616 y(IN)155 b Fo(blo)q(cklen)433 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h (eac)o(h)g(blo)q(c)o(k)g(\(p)q(ositiv)o(e)g(in)o(teger\))117 1691 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Ft(handle)14 b(to)g(output)g(datat)o(yp)q(e)g(ob)r(ject)166 1815 y FC(Example:)20 b(Assume,)15 b(again,)g(that)30 b Fo(oldt)o(yp)q(e)16 b FC(p)q(oin)o(ts)g(to)e(the)i(t)o(yp)q(e)738 1914 y Fy(f)p FC(\()p Fp(double)o FB(;)8 b FC(0\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(8\))p Fy(g)p FB(;)75 2013 y FC(with)19 b(exten)o(t)f(16.)30 b(A)18 b(call)i(to)e Fo(MPI)p 696 2013 V 16 w(TYPE)p 829 2013 V 16 w(VECTOR\()i(2,)f(oldt)o(yp)q(e,)h(4,) f(3,)f(newt)o(yp)q(e\))i FC(will)h(create)d(the)75 2070 y(datat)o(yp)q(e)269 2169 y Fy(f)p FC(\()p Fp(double)n FB(;)8 b FC(0\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(8\))p FB(;)g FC(\()p Fp(double)m FB(;)g FC(16\))p FB(;)g FC(\()p Fp(char)l FB(;)g FC(24\))p FB(;)g FC(\()p Fp(double)m FB(;)g FC(32\))p FB(;)g FC(\()p Fp(char)l FB(;)g FC(40\))p FB(;)228 2268 y FC(\()p Fp(double)o FB(;)g FC(64\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(72\))p FB(;)g FC(\()p Fp(dou)o(ble)m FB(;)g FC(80\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(88\))p FB(;)g FC(\()p Fp(doub)o(le)m FB(;)g FC(96\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(104\))p Fy(g)i FC(:)75 2350 y(t)o(w)o(o)k(blo)q(c)o(ks)i(with)f(three)h(copies) g(eac)o(h)f(of)g(the)g(old)h(t)o(yp)q(e,)f(starting)f(4*16)g(apart.)166 2406 y(A)h(call)h(to)f Fo(MPI)p 439 2406 V 16 w(TYPE)p 572 2406 V 17 w(VECTOR\(oldt)o(yp)q(e,)h(3,)f(-2,)g(1,)f(newt)o(yp)q (e\))j FC(will)g(create)e(the)g(datat)o(yp)q(e)186 2505 y Fy(f)p FC(\()p Fp(double)o FB(;)8 b FC(0\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(8\))p FB(;)g FC(\()p Fp(double)m FB(;)g Fy(\000)p FC(32\))p FB(;)g FC(\()p Fp(char)m FB(;)g Fy(\000)p FC(24\))p FB(;)g FC(\()p Fp(double)l FB(;)g Fy(\000)p FC(64\))p FB(;)g FC(\()p Fp(char)m FB(;)g Fy(\000)p FC(56\))p Fy(g)p FB(:)166 2604 y FC(In)16 b(general,)f(assume)g(that)30 b Fo(oldt)o(yp)q(e)16 b FC(is)g(a)f(handle)h(to)f(the)g(datat)o(yp)q(e) 595 2704 y Fy(f)p FC(\()p FB(ty)r(pe)720 2711 y Fl(0)740 2704 y FB(;)8 b(disp)845 2711 y Fl(0)863 2704 y FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)1064 2711 y Fk(n)p Fm(\000)p Fl(1)1130 2704 y FB(;)g(disp)1235 2711 y Fk(n)p Fm(\000)p Fl(1)1302 2704 y FC(\))p Fy(g)p FB(;)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 43 49 43 48 bop 75 -100 a Fv(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(43)75 45 y(with)13 b(exten)o(t)g FB(extent)p FC(.)21 b(The)13 b(newly)h(created)f(datat)o(yp)q(e)f(is)h(a)g (sequence)h(of)f(length)h Fp(count)5 b Fy(\001)h Fp(blocklen)s Fy(\001)g FB(n)75 102 y FC(with)16 b(en)o(tries:)606 164 y Fy(f)p FC(\()p FB(ty)r(pe)731 171 y Fl(0)751 164 y FB(;)8 b(disp)856 171 y Fl(0)874 164 y FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)1075 171 y Fk(n)p Fm(\000)p Fl(1)1141 164 y FB(;)g(disp)1246 171 y Fk(n)p Fm(\000)p Fl(1)1313 164 y FC(\))p FB(;)405 259 y FC(\()p FB(ty)r(pe)507 266 y Fl(0)527 259 y FB(;)g(disp)632 266 y Fl(0)660 259 y FC(+)j FB(extent)p FC(\))p FB(;)d(:::;)g FC(\()p FB(ty)r(pe)1034 266 y Fk(n)p Fm(\000)p Fl(1)1101 259 y FB(;)g(disp)1206 266 y Fk(n)p Fm(\000)p Fl(1)1283 259 y FC(+)i FB(extent)p FC(\))p FB(;)e(:::;)96 355 y FC(\()p FB(ty)r(pe)198 362 y Fl(0)218 355 y FB(;)g(disp)323 362 y Fl(0)352 355 y FC(+)i(\()p Fp(blocklen)f Fy(\000)h FC(1\))g Fy(\001)g FB(extent)p FC(\))p FB(;)e(:::;)g FC(\()p FB(ty)r(pe)1063 362 y Fk(n)p Fm(\000)p Fl(1)1130 355 y FB(;)g(disp)1235 362 y Fk(n)p Fm(\000)p Fl(1)1312 355 y FC(+)i(\()p Fp(blocklen)f Fy(\000)i FC(1\))e Fy(\001)h FB(extent)p FC(\))p FB(;)229 450 y FC(\()p FB(ty)r(pe)331 457 y Fl(0)351 450 y FB(;)e(disp)456 457 y Fl(0)484 450 y FC(+)j Fp(stride)e Fy(\001)h FB(extent)p FC(\))p FB(;)e(:::;)g FC(\()p FB(ty)r(pe)1034 457 y Fk(n)p Fm(\000)p Fl(1)1101 450 y FB(;)g(disp)1206 457 y Fk(n)p Fm(\000)p Fl(1)1283 450 y FC(+)i Fp(stride)g Fy(\001)f FB(extent)p FC(\))p FB(;)f(:::;)448 546 y FC(\()p FB(ty)r(pe)550 553 y Fl(0)570 546 y FB(;)g(disp)675 553 y Fl(0)703 546 y FC(+)j(\()p Fp(stride)e FC(+)h Fp(blocklen)f Fy(\000)i FC(1\))e Fy(\001)h FB(extent)p FC(\))p FB(;)e(:::;)393 641 y FC(\()p FB(ty)r(pe)495 648 y Fk(n)p Fm(\000)p Fl(1)564 641 y FB(;)g(disp)669 648 y Fk(n)p Fm(\000)p Fl(1)746 641 y FC(+)i(\()p Fp(stride)f FC(+)i Fp(blocklen)e Fy(\000)h FC(1\))g Fy(\001)g FB(extent)p FC(\))p FB(;)e(::::;)495 737 y FC(\()p FB(ty)r(pe)597 744 y Fl(0)617 737 y FB(;)g(disp)722 744 y Fl(0)751 737 y FC(+)i Fp(stride)f Fy(\001)h FC(\()p Fp(count)f Fy(\000)i FC(1\))e Fy(\001)h FB(extent)p FC(\))p FB(;)e(:::;)446 833 y FC(\()p FB(ty)r(pe)548 840 y Fk(n)p Fm(\000)p Fl(1)617 833 y FB(;)g(disp)722 840 y Fk(n)p Fm(\000)p Fl(1)799 833 y FC(+)j Fp(stride)e Fy(\001)h FC(\()p Fp(count)f Fy(\000)h FC(1\))g Fy(\001)g FB(extent)p FC(\))p FB(;)e(:::;)315 928 y FC(\()p FB(ty)r(pe)417 935 y Fl(0)437 928 y FB(;)g(disp)542 935 y Fl(0)571 928 y FC(+)i(\()p Fp(stride)f Fy(\001)h FC(\()p Fp(count)f Fy(\000)h FC(1\))g(+)g Fp(blocklen)f Fy(\000)i FC(1\))e Fy(\001)h FB(extent)p FC(\))p FB(;)e(:::;)290 1024 y FC(\()p FB(ty)r(pe)392 1031 y Fk(n)p Fm(\000)p Fl(1)461 1024 y FB(;)g(disp)566 1031 y Fk(n)p Fm(\000)p Fl(1)643 1024 y FC(+)j(\()p Fp(stride)e Fy(\001)h FC(\()p Fp(count)f Fy(\000)h FC(1\))g(+)g Fp(blocklen)f Fy(\000)i FC(1\))e Fy(\001)h FB(extent)p FC(\))p Fy(g)166 1191 y Fu(Discussion:)166 1243 y Ft(I)k(c)o(hanged)28 b Fr(count)15 b Ft(from)d(b)q(eing)i(the)g (n)o(um)o(b)q(er)f(of)h(en)o(tries)h(\(or)e(replicates\),)i(as)f(in)f (the)i(old)e(draft,)g(to)h(b)q(eing)75 1293 y(the)i(n)o(um)o(b)q(er)e (of)h(blo)q(c)o(ks.)22 b(The)16 b(reason)g(is)f(that)h(in)e(the)i(next) g(calls)f(I)g(allo)o(w)f(blo)q(c)o(k)h(lengths)g(to)g(v)n(ary;)g(it)g (is)h(easy)75 1343 y(to)f(sa)o(y)f(what)h(it)f(means)g(to)g(ha)o(v)o(e) h Fj(n)f Ft(blo)q(c)o(ks)h(of)f(sizes)i Fj(b)950 1349 y Fi(0)969 1343 y Fj(;)7 b(:::;)g(b)1061 1349 y Fh(n)o Fg(\000)p Fi(1)1123 1343 y Ft(;)14 b(it)h(is)f(harder)i(to)e(explain)g (what)h(it)f(means)g(to)75 1393 y(ha)o(v)o(e)g Fj(n)f Ft(elemen)o(ts)h(\014lling)f(up)g(successiv)o(e)k(p)q(ositions)c(in)h (blo)q(c)o(ks)f(of)h(sizes)h Fj(b)1256 1399 y Fi(0)1274 1393 y Fj(;)7 b(b)1311 1399 y Fi(1)1329 1393 y Fj(;)g(::::)p Ft(.)166 1444 y(W)m(e)19 b(lo)q(ose)h(the)h(abilit)o(y)d(to)h(use)i Fr(MPI)p 776 1444 13 2 v 14 w(TYPE)p 898 1444 V 15 w(VECTOR)d Ft(to)i(create)h(a)e(partially)g(\014lled)g(last)h(blo)q(c)o(k)f(\(the) 75 1494 y(more)11 b(general)i Fr(MPI)p 396 1494 V 14 w(TYPE)p 518 1494 V 14 w(INDEXED)f Ft(function)f(is)i(needed\);)g(w)o (e)g(gain)e(consistency)j(with)e(the)h(next)f(calls,)g(and)75 1544 y(con)o(v)o(enience)j(in)f(these)h(calls.)166 1596 y(Opinions?)166 1737 y FC(A)h(call)i(to)e Fo(MPI)p 443 1737 14 2 v 16 w(TYPE)p 576 1737 V 16 w(CONTIGUOUS\(oldt)o(yp)q(e,)j (count,)f(newt)o(yp)q(e\))g FC(is)f(equiv)m(alen)o(t)h(to)e(a)g(call)i (to)75 1793 y Fo(MPI)p 160 1793 V 16 w(TYPE)p 293 1793 V 17 w(VECTOR\(oldt)o(yp)q(e,)e(count,)h(1,)d(1,)h(newt)o(yp)q(e\))p FC(.)166 1851 y(The)k(function)g Fo(MPI)p 529 1851 V 16 w(TYPE)p 662 1851 V 17 w(HVECTOR)h FC(is)g(iden)o(tical)g(to)e Fo(MPI)p 1287 1851 V 16 w(TYPE)p 1420 1851 V 17 w(VECTOR)p FC(,)h(except)h(that)75 1908 y Fo(stride)12 b FC(is)f(giv)o(en)h(in)f (b)o(ytes,)g(rather)g(then)g(in)h(elemen)o(ts.)19 b(The)11 b(use)g(for)f(b)q(oth)h(t)o(yp)q(es)g(of)g(v)o(ector)f(constructors)75 1964 y(is)16 b(illustrated)g(in)g(Section)g(3.13.4.)i(\()p Fp(H)d FC(is)h(used)g(for)e(\\heterogeneous"\).)75 2070 y Fo(MPI)p 160 2070 V 16 w(TYPE)p 293 2070 V 17 w(HVECTOR\()i(count,)g (oldt)o(yp)q(e,)g(stride,)g(blo)q(cklen,)g(newt)o(yp)q(e\))117 2149 y Ft(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))117 2228 y(IN)155 b Fo(oldt)o(yp)q(e)450 b Ft(handle)14 b(to)g(input)f(datat)o(yp)q(e)i (ob)r(ject)117 2307 y(IN)155 b Fo(stride)484 b Ft(n)o(um)o(b)q(er)13 b(of)g(b)o(ytes)h(b)q(et)o(w)o(een)h(start)f(of)f(eac)o(h)h(blo)q(c)o (k)f(\(in)o(teger\))117 2385 y(IN)155 b Fo(blo)q(cklen)433 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o (k)g(\(p)q(ositiv)o(e)g(in)o(teger\))117 2464 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Ft(handle)14 b(to)g(output)g(datat)o(yp)q(e)g (ob)r(ject)166 2591 y Fo(MPI)p 251 2591 V 16 w(TYPE)p 384 2591 V 17 w(INDEXED)24 b FC(is)h(a)f(more)g(general)h(function)g (that)f(allo)o(ws)g(replication)i(of)e(an)g(old)75 2647 y(datat)o(yp)q(e)12 b(in)o(to)h(a)f(sequence)i(of)f(con)o(tiguous)g (blo)q(c)o(ks,)g(where)g(blo)q(c)o(k)h(lengths)f(and)g(blo)q(c)o(k)h (displacemen)o(ts)75 2704 y(can)h(all)h(di\013er)g({)f(but)g(are)g(all) h(m)o(ultiples)h(of)e(the)g(old)h(t)o(yp)q(e)f(exten)o(t.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 44 50 44 49 bop 75 -100 a FC(44)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(INDEXED\()e(count,)j(oldt)o(yp) q(e,)e(a)o(rra)o(y)p 932 45 V 15 w(of)p 984 45 V 16 w(indices,)h(a)o (rra)o(y)p 1249 45 V 15 w(of)p 1301 45 V 16 w(blo)q(cklen,)g(newt)o(yp) q(e\))117 124 y Ft(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))117 202 y(IN)155 b Fo(oldt)o(yp)q(e)450 b Ft(handle)14 b(to)g(input)f(datat)o(yp)q(e)i (ob)r(ject)117 281 y(IN)155 b Fo(a)o(rra)o(y)p 416 281 V 15 w(of)p 468 281 V 16 w(indices)299 b Ft(displacemen)o(t)17 b(for)g(eac)o(h)h(blo)q(c)o(k,)f(in)g(m)o(ultiples)f(of)34 b Fr(oldt)o(yp)q(e)905 337 y Ft(exten)o(t)15 b(\(arra)o(y)f(of)f(in)o (teger\))117 416 y(IN)155 b Fo(a)o(rra)o(y)p 416 416 V 15 w(of)p 468 416 V 16 w(blo)q(cklen)270 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(p)q(er)i(blo)q(c)o(k)f(\(arra)o(y)f(of)h(in)o (teger\))117 494 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Ft(handle)14 b(to)g(output)g(datat)o(yp)q(e)g(ob)r(ject)166 620 y FC(Example:)20 b(let)31 b Fo(oldt)o(yp)q(e)16 b FC(b)q(e)g(a)f(handle)i (to)d(the)h(t)o(yp)q(e)738 727 y Fy(f)p FC(\()p Fp(double)o FB(;)8 b FC(0\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(8\))p Fy(g)p FB(;)75 834 y FC(with)17 b(exten)o(t)g(16.)25 b(Let)17 b Fp(I)24 b(=)f(\(64,)h(0\))16 b FC(and)i(let)f Fp(B)24 b(=)f(\(3,)h(1\))p FC(.)h(A)17 b(call)h(to)e Fo(MPI)p 1480 834 V 16 w(TYPE)p 1613 834 V 17 w(INDEXED\(2,)75 890 y(oldt)o(yp)q(e,)g(I,)e(B,)h(newt)o(yp)q(e\))i FC(returns)e(in)h Fp(newtype)e FC(a)h(handle)i(to)d(the)i(t)o(yp)q(e)241 997 y Fy(f)p FC(\()p Fp(double)n FB(;)8 b FC(64\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(72\))p FB(;)g FC(\()p Fp(double)l FB(;)g FC(80\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(88\))p FB(;)g FC(\()p Fp(double)l FB(;)g FC(96\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(104\))743 1109 y(\()p Fp(double)o FB(;)g FC(0\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(8\))p Fy(g)j FC(:)75 1199 y(three)k(copies)h(of)e(the)i(old)f(t)o(yp)q(e)g (starting)f(at)h(displacemen)o(t)h(64,)e(and)h(one)h(cop)o(y)e (starting)h(at)f(displace-)75 1255 y(men)o(t)h(0.)166 1313 y(In)h(general,)f(assume)g(that)30 b Fo(oldt)o(yp)q(e)16 b FC(is)g(a)f(handle)h(to)f(the)g(datat)o(yp)q(e)595 1420 y Fy(f)p FC(\()p FB(ty)r(pe)720 1427 y Fl(0)740 1420 y FB(;)8 b(disp)845 1427 y Fl(0)863 1420 y FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)1064 1427 y Fk(n)p Fm(\000)p Fl(1)1130 1420 y FB(;)g(disp)1235 1427 y Fk(n)p Fm(\000)p Fl(1)1302 1420 y FC(\))p Fy(g)p FB(;)75 1527 y FC(with)14 b(exten)o(t)g Fs(extent)p FC(.)19 b(Let)29 b Fo(B)14 b FC(b)q(e)h(the)28 b Fo(a)o(rra)o(y)p 855 1527 V 15 w(of)p 907 1527 V 16 w(blo)q(cklen)15 b FC(parameter)f(and)28 b Fo(I)14 b FC(b)q(e)h(the)28 b Fo(a)o(rra)o(y)p 1684 1527 V 15 w(of)p 1736 1527 V 15 w(indices)75 1584 y FC(parameter.)18 b(The)13 b(newly)h(created)e(t)o(yp)q(e)h(returned)g(b)o(y)25 b Fo(newt)o(yp)q(e)15 b FC(is)e(the)g(sequence)h(of)e FB(n)5 b Fy(\001)1612 1551 y Ff(P)1656 1564 y Fe(count)n Fm(\000)p Fl(1)1656 1596 y Fk(i)p Fl(=0)1809 1584 y Fo(B)p FC([)p Fo(i)p FC(])75 1640 y(en)o(tries)300 1701 y Fy(f)p FC(\()p FB(ty)r(pe)425 1708 y Fl(0)445 1701 y FB(;)j(disp)550 1708 y Fl(0)578 1701 y FC(+)i Fo(I)p FC([)p Fo(0)p FC(])f Fy(\001)h FB(extent)p FC(\))p FB(;)e(:::;)g FC(\()p FB(ty)r(pe)1045 1708 y Fk(n)p Fm(\000)p Fl(1)1112 1701 y FB(;)g(disp)1217 1708 y Fk(n)p Fm(\000)p Fl(1)1294 1701 y FC(+)j Fo(I)p FC([)p Fo(0)p FC(])d Fy(\001)i FB(extent)p FC(\))p FB(;)e(:::;)75 1795 y FC(\()p FB(ty)r(pe)177 1802 y Fl(0)197 1795 y FB(;)g(disp)302 1802 y Fl(0)329 1795 y FC(+)h(\()p Fo(I)p FC([)p Fo(0)p FC(])e(+)i Fo(B)p FC([)p Fo(0)p FC(])f Fy(\000)h FC(1\))f Fy(\001)h FB(extent)p FC(\))p FB(;)f(:::;)g FC(\()p FB(ty)r(pe)1034 1802 y Fk(n)p Fm(\000)p Fl(1)1101 1795 y FB(;)g(disp)1206 1802 y Fk(n)p Fm(\000)p Fl(1)1282 1795 y FC(+)h(\()p Fo(I)p FC([)p Fo(0)p FC(])e(+)i Fo(B)p FC([)p Fo(0)p FC(])f Fy(\000)h FC(1\))f Fy(\001)g FB(extent)p FC(\))p FB(;)g(:::;)150 1889 y FC(\()p FB(ty)r(pe)252 1896 y Fl(0)271 1889 y FB(;)g(disp)376 1896 y Fl(0)405 1889 y FC(+)i Fo(I)p FC([)p Fo(count)h Fy(\000)f Fo(1)p FC(])g Fy(\001)g FB(extent)p FC(\))p FB(;)e(:::;)g FC(\()p FB(ty)r(pe)1034 1896 y Fk(n)p Fm(\000)p Fl(1)1101 1889 y FB(;)g(disp)1206 1896 y Fk(n)p Fm(\000)p Fl(1)1283 1889 y FC(+)i Fo(I)p FC([)p Fo(count)h Fy(\000)f Fo(1)p FC(])g Fy(\001)g FB(extent)p FC(\))p FB(;)e(:::;)384 1983 y FC(\()p FB(ty)r(pe)486 1990 y Fl(0)506 1983 y FB(;)g(disp)611 1990 y Fl(0)639 1983 y FC(+)j(\()p Fo(I)p FC([)p Fo(count)f Fy(\000)g Fo(1)p FC(])g(+)g Fo(B)p FC([)p Fo(count)i Fy(\000)e Fo(1)p FC(])g Fy(\000)g FC(1\))g Fy(\001)f FB(extent)p FC(\))p FB(;)f(:::;)353 2077 y FC(\()p FB(ty)r(pe)455 2084 y Fk(n)p Fm(\000)p Fl(1)523 2077 y FB(;)g(disp)628 2084 y Fk(n)p Fm(\000)p Fl(1)706 2077 y FC(+)i(\()p Fo(I)p FC([)p Fo(count)h Fy(\000)f Fo(1)p FC(])f(+)i Fo(B)p FC([)p Fo(count)g Fy(\000)g Fo(1)p FC(])e Fy(\000)i FC(1\))e Fy(\001)h FB(extent)p FC(\))p Fy(g)p FB(:)166 2168 y FC(A)k(call)g(to)f Fo(MPI)p 434 2168 V 16 w(TYPE)p 567 2168 V 17 w(VECTOR\(count,)j(oldt)o(yp)q(e,) f(stride,)f(blo)q(cklen,)h(newt)o(yp)q(e\))g FC(is)f(equiv)m(alen)o(t)i (to)75 2225 y(a)f(call)h(to)f Fo(MPI)p 337 2225 V 16 w(TYPE)p 470 2225 V 16 w(INDEX\(count,)h(oldt)o(yp)q(e,)g(I,)f(B,)g (newt)o(yp)q(e\))h FC(where)626 2332 y Fo(I)p FC([)p Fo(j)p FC(])c(=)h FB(j)f Fy(\001)e Fo(stride)16 b FB(;)8 b(j)14 b FC(=)f(0)p FB(;)8 b(:::;)g Fp(count)f Fy(\000)j FC(1)p FB(;)75 2439 y FC(and)619 2500 y Fo(B)p FC([)p Fo(j)p FC(])i(=)h Fo(blo)q(cklen)k FB(;)8 b(j)14 b FC(=)f(0)p FB(;)8 b(:::;)g Fp(cou)o(nt)f Fy(\000)k FC(1)p FB(:)166 2591 y FC(The)k(function)h Fo(MPI)p 522 2591 V 16 w(TYPE)p 655 2591 V 17 w(HINDEXED)f FC(is)h(iden)o(tical)g(to)f Fo(MPI)p 1280 2591 V 16 w(TYPE)p 1413 2591 V 16 w(INDEXED)p FC(,)g(except)h(that)75 2647 y(blo)q(c)o(k)j(displacemen)o(ts)i(in)38 b Fo(a)o(rra)o(y)p 665 2647 V 14 w(of)p 716 2647 V 16 w(indices)20 b FC(are)f(sp)q(eci\014ed)h(in)g(b)o(ytes,)f(rather)f (than)g(in)i(m)o(ultiples)g(of)75 2704 y(the)15 b(old)h(t)o(yp)q(e)f (exten)o(t.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 45 51 45 50 bop 75 -100 a Fv(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(45)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(HINDEXED\()15 b(count,)h(oldt)o(yp)q(e,)g(a)o(rra)o(y)p 965 45 V 14 w(of)p 1016 45 V 16 w(indices,)h(a)o(rra)o(y)p 1282 45 V 14 w(of)p 1333 45 V 16 w(blo)q(cklen,)f(newt)o(yp)q(e\))117 123 y Ft(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))117 198 y(IN)155 b Fo(oldt)o(yp)q(e)450 b Ft(handle)14 b(to)g(input)f(datat)o(yp)q(e)i (ob)r(ject)117 274 y(IN)155 b Fo(a)o(rra)o(y)p 416 274 V 15 w(of)p 468 274 V 16 w(indices)299 b Ft(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o(h)h(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o (teger\))117 350 y(IN)155 b Fo(a)o(rra)o(y)p 416 350 V 15 w(of)p 468 350 V 16 w(blo)q(cklen)270 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in) o(teger\))117 425 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Ft(handle)14 b(to)g(output)g(datat)o(yp)q(e)g(ob)r(ject)166 550 y Fp(MPI)p 241 550 15 2 v 17 w(TYPE)p 354 550 V 16 w(STRUCT)j FC(is)h(the)g(most)e(general)i(constructor:)24 b(It)17 b(further)h(generalizes)h(the)e(previous)75 607 y(one)e(in)h(that)f(it)g(allo)o(ws)h(eac)o(h)f(blo)q(c)o(k)h(to)e (consist)i(of)f(replications)i(of)d(a)h(di\013eren)o(t)h(datat)o(yp)q (e.)75 711 y Fo(MPI)p 160 711 14 2 v 16 w(TYPE)p 293 711 V 17 w(STRUCT\(count,)g(a)o(rra)o(y)p 733 711 V 15 w(of)p 785 711 V 16 w(t)o(yp)q(es,)g(a)o(rra)o(y)p 1022 711 V 14 w(of)p 1073 711 V 16 w(indices,)h(a)o(rra)o(y)p 1339 711 V 14 w(of)p 1390 711 V 16 w(blo)q(cklen,)f(newt)o(yp)q(e\))117 788 y Ft(IN)155 b Fo(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))117 864 y(IN)155 b Fo(a)o(rra)o(y)p 416 864 V 15 w(of)p 468 864 V 16 w(t)o(yp)q(es)327 b Ft(t)o(yp)q(e)15 b(of)f(elemen)o(ts)g(in)g(eac)o(h)h(blo)q(c)o(k)f (\(arra)o(y)g(of)g(datat)o(yp)q(e)g(ob-)905 920 y(jects\))117 996 y(IN)155 b Fo(a)o(rra)o(y)p 416 996 V 15 w(of)p 468 996 V 16 w(indices)299 b Ft(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o (h)h(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1072 y(IN)155 b Fo(a)o(rra)o(y)p 416 1072 V 15 w(of)p 468 1072 V 16 w(blo)q(cklen)270 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in) o(teger\))117 1147 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Ft(handle)14 b(to)g(output)g(datat)o(yp)q(e)g(ob)r(ject)166 1272 y FC(Example:)20 b(Let)31 b Fo(t)o(yp)q(e1)16 b FC(b)q(e)g(a)f(handle)h(to)f(the)g(t)o(yp)q(e)738 1375 y Fy(f)p FC(\()p Fp(double)o FB(;)8 b FC(0\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(8\))p Fy(g)p FB(;)75 1478 y FC(with)24 b(exten)o(t)g(16.)45 b(Let)24 b Fp(I)g(=)g(\(0,)f(16,)g (26\))p FC(,)j Fp(B)d(=)h(\(2,)g(1,)f(3\))h FC(and)g Fp(T)f(=)h(\(MPI)p 1551 1478 15 2 v 17 w(FLOAT,)f(type1,)75 1535 y(MPI)p 150 1535 V 17 w(BYTE\))p FC(.)18 b(Then)i(a)f(call)h(to)f Fo(MPI)p 712 1535 14 2 v 16 w(TYPE)p 845 1535 V 16 w(STRUC\(3,)h(T,)f (I,)g(B,)g(newt)o(yp)q(e\))i FC(returns)e(in)39 b Fo(newt)o(yp)q(e)21 b FC(a)75 1591 y(handle)16 b(to)f(the)g(datat)o(yp)q(e)166 1694 y Fy(f)p FC(\()p Fp(float)o FB(;)8 b FC(0\))p FB(;)g FC(\()p Fp(float)m FB(;)g FC(4\))p FB(;)g FC(\()p Fp(double)m FB(;)g FC(16\))p FB(;)g FC(\()p Fp(cha)o(r)m FB(;)g FC(24\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(26\))p FB(;)g FC(\()p Fp(char)m FB(;)g FC(27\))p FB(;)g FC(\()p Fp(cha)o(r)m FB(;)g FC(28\))p Fy(g)j FC(:)75 1797 y(t)o(w)o(o)19 b(copies)i(of)40 b Fo(MPI)p 467 1797 V 16 w(FLO)o(A)l(T)20 b FC(starting)g(at)g(0,)g (follo)o(w)o(ed)h(b)o(y)f(one)h(cop)o(y)f(of)40 b Fo(t)o(yp)q(e1)21 b FC(starting)e(at)h(16,)75 1853 y(follo)o(w)o(ed)d(b)o(y)f(three)h (copies)g(of)33 b Fo(MPI)p 724 1853 V 16 w(CHAR)p FC(,)17 b(starting)e(at)h(26.)23 b(\(W)l(e)17 b(assume)f(that)g(a)33 b Fo(\015oat)16 b FC(o)q(ccupies)75 1910 y(four)f(b)o(ytes.\))166 1967 y(In)h(general,)f(let)31 b Fo(T)15 b FC(b)q(e)h(the)30 b Fo(a)o(rra)o(y)p 772 1967 V 15 w(of)p 824 1967 V 16 w(t)o(yp)q(es)16 b FC(parameter,)e(where)31 b Fo(T[i])15 b FC(is)h(a)f(handle)h(to)547 2070 y FB(ty)r(pe)631 2077 y Fk(i)659 2070 y FC(=)d Fy(f)p FC(\()p FB(ty)r(pe)832 2051 y Fk(i)832 2081 y Fl(0)851 2070 y FB(;)8 b(disp)956 2051 y Fk(i)956 2081 y Fl(0)974 2070 y FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)1175 2051 y Fk(i)1175 2081 y(n)1196 2086 y Fd(i)1209 2070 y FB(;)g(disp)1314 2051 y Fk(i)1314 2081 y(n)1335 2086 y Fd(i)1349 2070 y FC(\))p Fy(g)p FB(;)75 2173 y FC(with)13 b(exten)o(t)f FB(extent)439 2180 y Fk(i)455 2173 y FC(.)19 b(Let)25 b Fo(B)13 b FC(b)q(e)h(the)25 b Fo(a)o(rra)o(y)p 867 2173 V 15 w(of)p 919 2173 V 16 w(blo)q(cklen)14 b FC(parameter)d(and)26 b Fo(I)12 b FC(b)q(e)i(the)25 b Fo(a)o(rra)o(y)p 1684 2173 V 15 w(of)p 1736 2173 V 15 w(indices)75 2229 y FC(parameter.)18 b(Then)12 b(the)f(newly)i(created)e(datat)o(yp)q(e)g(is)h(the)g(sequence)g(of)f (length)1452 2197 y Ff(P)1495 2210 y Fe(count)n Fm(\000)p Fl(1)1495 2242 y Fk(i)p Fl(=0)1648 2229 y Fo(B)p FC([)p Fo(i)p FC(])r Fy(\001)s FB(extent)1860 2236 y Fk(i)75 2286 y FC(with)16 b(en)o(tries)489 2343 y Fy(f)p FC(\()p FB(ty)r(pe)614 2324 y Fl(0)614 2354 y(0)634 2343 y FB(;)8 b(disp)739 2324 y Fl(0)739 2354 y(0)767 2343 y FC(+)j Fo(I)p FC([)p Fo(0)p FC(])n(\))p FB(;)d(:::;)g FC(\()p FB(ty)r(pe)1074 2324 y Fl(0)1074 2354 y Fk(n)1095 2359 y FA(0)1112 2343 y FB(;)g(disp)1217 2324 y Fl(0)1217 2354 y Fk(n)1238 2359 y FA(0)1266 2343 y FC(+)j Fo(I)p FC([)p Fo(0)p FC(])o(\))p FB(;)d(:::;)75 2428 y FC(\()p FB(ty)r(pe)177 2409 y Fl(0)177 2439 y(0)197 2428 y FB(;)g(disp)302 2409 y Fl(0)302 2439 y(0)330 2428 y FC(+)i Fo(I)p FC([)p Fo(0)p FC(])e(+)i(\()p Fo(B)p FC([)p Fo(0)p FC(])f Fy(\000)h FC(1\))f Fy(\001)g FB(extent)840 2435 y Fl(0)861 2428 y FC(\))p FB(;)f(:::;)g FC(\()p FB(ty)r(pe)1062 2409 y Fl(0)1062 2439 y Fk(n)1083 2444 y FA(0)1100 2428 y FB(;)g(disp)1205 2409 y Fl(0)1205 2439 y Fk(n)1226 2444 y FA(0)1254 2428 y FC(+)i Fo(I)p FC([)p Fo(0)p FC(])f(+)h(\()p Fo(B)p FC([)p Fo(0)p FC(])f Fy(\000)h Fo(1)p FC(\))f Fy(\001)g FB(extent)1765 2435 y Fl(0)1787 2428 y FC(\))p FB(;)f(:::;)91 2513 y FC(\()p FB(ty)r(pe)193 2494 y Fe(count)n Fm(\000)p Fl(1)193 2526 y(0)338 2513 y FB(;)g(disp)443 2494 y Fe(count)m Fm(\000)p Fl(1)443 2526 y(0)597 2513 y FC(+)j Fo(I)p FC([)p Fo(count)f Fy(\000)h Fo(1)p FC(])o(\))p FB(;)d(:::;)g FC(\()p FB(ty)r(pe)1066 2495 y Fe(co)o(unt)l Fm(\000)p Fl(1)1066 2525 y Fk(n)1087 2534 y Fe(cou)o(nt)m Fc(\000)p FA(1)1226 2513 y FB(;)g(disp)1331 2495 y Fe(count)m Fm(\000)p Fl(1)1331 2525 y Fk(n)1352 2534 y Fe(count)m Fc(\000)p FA(1)1503 2513 y FC(+)i Fo(I)p FC([)p Fo(count)h Fy(\000)f Fo(1)p FC(]\))p FB(;)e(:::;)186 2611 y FC(\()p FB(ty)r(pe)288 2592 y Fe(count)n Fm(\000)p Fl(1)288 2623 y(0)433 2611 y FB(;)g(disp)538 2592 y Fe(count)m Fm(\000)p Fl(1)538 2623 y(0)692 2611 y FC(+)j Fo(I)p FC([)p Fo(count)f Fy(\000)h Fo(1)p FC(])e(+)i(\()p Fo(B)p FC([)p Fo(count)g Fy(\000)f Fo(1)p FC(])f Fy(\000)i FC(1\))e Fy(\001)h FB(extent)1529 2618 y Fe(count)p Fm(\000)p Fl(1)1676 2611 y FC(\))p FB(;)e(:::;)192 2696 y FC(\()p FB(ty)r(pe)294 2677 y Fe(count)n Fm(\000)p Fl(1)294 2707 y Fk(n)315 2717 y Fe(count)o Fc(\000)p FA(1)457 2696 y FB(;)g(disp)562 2677 y Fe(count)m Fm(\000)p Fl(1)562 2707 y Fk(n)583 2717 y Fe(count)m Fc(\000)p FA(1)734 2696 y FC(+)i Fo(I)p FC([)p Fo(count)h Fy(\000)f Fo(1)p FC(])g(+)g(\()p Fo(B)p FC([)p Fo(count)h Fy(\000)g Fo(1)p FC(])e Fy(\000)i Fo(1)p FC(\))e Fy(\001)h FB(extent)1571 2703 y Fe(count)o Fm(\000)p Fl(1)1717 2696 y FC(\))p Fy(g)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 46 52 46 51 bop 75 -100 a FC(46)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FC(A)h(call)g(to)f Fo(MPI)p 443 45 14 2 v 16 w(TYPE)p 576 45 V 17 w(HINDEXED\()g(count,)i (oldt)o(yp)q(e,)g(I,)e(B,)g(newt)o(yp)q(e\))i FC(is)f(equiv)m(alen)o(t) i(to)d(a)g(call)75 102 y(to)f Fo(MPI)p 216 102 V 15 w(TYPE)p 348 102 V 17 w(STRUC\()h(count,)g(T,)f(I,)f(B,)h(newt)o(yp)q(e\))p FC(,)h(where)g(eac)o(h)f(en)o(try)g(of)g Fo(T)g FC(is)g(equal)h(to)30 b Fo(oldt)o(yp)q(e)p FC(.)166 237 y Fu(Discussion:)166 290 y Ft(I)14 b(am)e(op)q(en)i(to)g(suggestions)g(for)g(b)q(etter)h (names)f(to)f(these)j(functions.)75 512 y Fq(3.13.2)49 b(Additional)19 b(functions)75 604 y FC(The)i(displacemen)o(ts)h(in)g (a)f(general)g(datat)o(yp)q(e)f(are)h(relativ)o(e)g(to)g(some)f(start)g (address.)37 b FD(Absolute)75 660 y(addresses)13 b FC(can)h(b)q(e)h (substituted)g(for)e(these)i(displacemen)o(ts:)21 b(w)o(e)13 b(treat)h(them)g(as)g(displacemen)o(ts)h(rel-)75 717 y(ativ)o(e)f(to)e(\\address)i(zero",)f(the)h(start)e(of)h(the)h (address)f(space.)20 b(This)14 b(initial)i(address)e(zero)f(is)h (indicated)75 773 y(b)o(y)k(the)g(constan)o(t)33 b Fr(MPI)p 501 773 13 2 v 14 w(BOTTOM)p FC(.)17 b(Th)o(us,)h(a)g(datat)o(yp)q(e)f (can)h(sp)q(ecify)h(the)f(absolute)h(address)e(of)h(the)75 830 y(en)o(tries)e(in)g(the)f(comm)o(unication)h(bu\013er,)f(in)h(whic) o(h)g(case)f(the)31 b Fo(sta)o(rt)16 b FC(parameter)e(is)i(passed)f (the)h(v)m(alue)75 886 y Fr(MPI)p 152 886 V 14 w(BOTTOM)p FC(.)166 946 y(The)10 b(address)g(of)g(a)g(lo)q(cation)h(in)g(memory)f (can)g(b)q(e)h(found)f(b)o(y)g(in)o(v)o(oking)h(the)f(function)h Fo(MPI)p 1701 946 14 2 v 16 w(ADDRESS)p FC(.)75 1053 y Fo(MPI)p 160 1053 V 16 w(ADDRESS\(lo)q(cation,)16 b(address\))117 1133 y Ft(IN)155 b Fo(lo)q(cation)437 b Ft(lo)q(cation)13 b(in)h(caller)f(memory)f(\(c)o(hoice\))117 1214 y(OUT)108 b Fo(address)449 b Ft(address)15 b(of)f(lo)q(cation)f(\(in)o(teger\)) 166 1342 y FC(Returns)j(the)f(\(b)o(yte\))f(address)i(of)29 b Fo(lo)q(cation)p FC(.)166 1401 y(Another)11 b(useful)g(auxiliary)h (function)g(is)f Fo(MPI)p 950 1401 V 16 w(EXTENT)p FC(,)f(that)g (returns)h(the)g(exten)o(t)f(of)g(a)g(datat)o(yp)q(e)75 1458 y({)15 b(where)g(exten)o(t)g(is)h(as)f(de\014ned)h(in)g(Eq.)f (3.1.)75 1565 y Fo(MPI)p 160 1565 V 16 w(TYPE)p 293 1565 V 17 w(EXTENT\(datat)o(yp)q(e,)h(extent\))117 1645 y Ft(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(handle)14 b(to)g(datat)o(yp)q(e) g(ob)r(ject)117 1726 y(OUT)108 b Fo(extent)471 b Ft(datat)o(yp)q(e)14 b(exten)o(t)h(\(in)o(teger\))166 1854 y FC(As)i(de\014ned)i(no)o(w,)e (the)h(exten)o(t)f(of)g(a)g(datat)o(yp)q(e)g(go)q(es)g(from)g(its)h (\014rst)f(o)q(ccupied)i(b)o(yte)f(to)e(its)i(last)75 1910 y(o)q(ccupied)23 b(b)o(yte.)37 b(It)21 b(is)g(often)g(con)o(v)o (enien)o(t)g(to)g(de\014ne)h(a)e(datat)o(yp)q(e)h(that)f(has)h(a)g (\\hole")g(at)f(its)h(b)q(e-)75 1967 y(ginning)e(and/or)e(its)h(end;)h (or)f(a)f(datat)o(yp)q(e)g(with)h(en)o(tries)g(that)f(extend)i(b)q(ey)o (ond)f(the)g(datat)o(yp)q(e)f(\\ex-)75 2023 y(ten)o(t".)22 b(This)17 b(allo)o(ws)g(to)f(use)g(the)h(functions)33 b Fo(MPI)p 969 2023 V 16 w(TYPE)p 1102 2023 V 17 w(CONTIGUOUS)p FC(,)h Fo(MPI)p 1536 2023 V 15 w(TYPE)p 1668 2023 V 17 w(VECTOR)p FC(,)75 2079 y(or)24 b Fo(MPI)p 225 2079 V 16 w(TYPE)p 358 2079 V 16 w(INDEX)13 b FC(to)e(replicate)j(a)d(more)h (general)h(pattern;)f(examples)h(of)f(suc)o(h)g(a)g(usage)g(are)g(pro-) 75 2136 y(vided)g(in)g(Section)g(3.13.4.)k(T)l(o)11 b(ac)o(hiev)o(e)h (this,)f(w)o(e)g(add)g(t)o(w)o(o)f(additional)i(\\pseudo-datat)o(yp)q (es")22 b Fo(MPI)p 1795 2136 V 15 w(LB)p FC(,)75 2192 y(and)j Fo(MPI)p 258 2192 V 16 w(UB)13 b FC(that)e(can)i(b)q(e)g(used,) g(resp)q(ectiv)o(ely)l(,)i(to)d(mark)f(the)i(lo)o(w)o(er)f(b)q(ound)h (or)f(the)h(upp)q(er)g(b)q(ound)g(of)75 2249 y(a)i(datat)o(yp)q(e.)k (these)c(pseudo-datat)o(yp)q(es)g(o)q(ccup)o(y)h(no)f(space)g(\()p FB(extent)p FC(\()p Fo(MP)p 1369 2249 V 17 w(LB)p FC(\))e(=)g FB(extent)p FC(\()p Fo(MP)p 1734 2249 V 17 w(UB)p FC(\))g(=)75 2305 y(0\).)30 b(They)19 b(do)f(not)g(a\013ect)g(the)h(signature)g(of)f (a)h(message)f(created)g(with)h(this)h(datat)o(yp)q(e.)29 b(Ho)o(w)o(ev)o(er,)75 2362 y(they)16 b(do)g(a\013ect)f(the)h (de\014nition)i(of)d(the)h(exten)o(t)g(of)f(a)h(datat)o(yp)q(e)f(and,)h (therefore,)f(a\013ect)g(the)h(outcome)75 2418 y(of)f(a)g(replication)i (of)d(this)i(datat)o(yp)q(e)f(b)o(y)g(a)g(datat)o(yp)q(e)f (constructor.)166 2478 y(Example:)30 b(Let)21 b Fp(I)j(=)f(\(-3,)h(0,)f (6\))p FC(;)f Fp(T)i(=)g(\(MPI)p 1033 2478 15 2 v 16 w(LB,)g(MPI)p 1217 2478 V 17 w(INT,)f(MPI)p 1425 2478 V 17 w(UB\))p FC(,)c(and)h Fp(B)k(=)g(\(1,)f(1,)75 2534 y(1\))p FC(.)c(Then)c(a)f(call)h(to)e Fo(MPI)p 531 2534 14 2 v 16 w(TYPE)p 664 2534 V 17 w(STRUC\(3,)h(T,)g(I,)f(B,)h(t)o(yp)q (e1\))g FC(creates)g(a)g(new)g(datat)o(yp)q(e)g(that)f(has)h(an)75 2591 y(exten)o(t)h(of)g(9)g(\(from)f(-3)i(to)e(6\),)h(and)g(con)o (tains)h(an)f(in)o(teger)h(at)e(displacemen)o(t)j(0.)j(This)c(is)g(the) f(datat)o(yp)q(e)75 2647 y(de\014ned)f(b)o(y)e(the)g(sequence)i Fy(f)p Fp(\(lb,)23 b(-3\),)g(\(int,)g(0\),)h(\(ub,)f(6\))p Fy(g)g FC(.)c(If)13 b(this)g(t)o(yp)q(e)f(is)h(replicated)h(t)o(wice)75 2704 y(b)o(y)h(a)g(call)h(to)f Fo(MPI)p 400 2704 V 15 w(TYPE)p 532 2704 V 17 w(CONTIGUOUS\(2,)h(t)o(yp)q(e1,)g(t)o(yp)q(e2\)) f FC(then)h(the)f(newly)h(created)f(t)o(yp)q(e)g(can)g(b)q(e)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 47 53 47 52 bop 75 -100 a Fv(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(47)75 45 y(describ)q(ed)13 b(b)o(y)d(the)h(sequence)h Fy(f)p Fp(\(lb,)23 b(-3\),)g(\(int,)g(0\),)h(\(int,9\),)e(\(ub,)i(15\)) p Fy(g)f FC(.)18 b(\(En)o(tries)11 b(of)f(t)o(yp)q(e)75 102 y Fp(lb)15 b FC(or)g Fp(ub)f FC(can)i(b)q(e)g(deleted)g(if)g(they)f (are)g(not)g(at)f(the)i(endp)q(oin)o(ts)g(of)f(the)g(datat)o(yp)q(e.\)) 166 158 y(In)h(general,)f(if)471 253 y FB(D)q(ataty)r(pe)e FC(=)g Fy(f)p FC(\()p FB(ty)r(pe)844 260 y Fl(0)864 253 y FB(;)8 b(disp)969 260 y Fl(0)987 253 y FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)1188 260 y Fk(n)p Fm(\000)p Fl(1)1254 253 y FB(;)g(disp)1359 260 y Fk(n)p Fm(\000)p Fl(1)1426 253 y FC(\))p Fy(g)p FB(;)75 348 y FC(then)16 b(the)f FD(lo)o(w)o(er)h(b)q(ound)g FC(of)f FB(D)q(ataty)r(pe)h FC(is)f(de\014ned)i(to)e(b)q(e)131 474 y FB(l)q(b)p FC(\()p FB(D)q(ataty)r(pe)p FC(\))d(=)449 402 y Ff(\()503 445 y FC(min)579 452 y Fk(j)604 445 y FB(disp)688 452 y Fk(j)1062 445 y FC(if)j(no)g(en)o(try)g(has)g(basic)h(t)o(yp)q(e)f FB(ty)r(pe)1666 452 y Fk(j)1698 445 y FC(=)e Fp(lb)503 502 y FC(min)q Fy(f)p FB(disp)686 509 y Fk(j)731 502 y FC(:)27 b FB(ty)r(pe)855 509 y Fk(j)886 502 y FC(=)13 b Fp(lb)p Fy(g)41 b FC(otherwise)166 599 y(Similarly)l(,)17 b(the)e FD(upp)q(er)j(b)q(ound)e FC(of)e FB(D)q(ataty)r(pe)i FC(is)g(de\014ned)h(to)d(b)q(e)108 724 y FB(ub)p FC(\()p FB(D)q(ataty)r(pe)p FC(\))e(=)437 653 y Ff(\()492 696 y FC(max)576 703 y Fk(j)602 696 y FB(disp)686 703 y Fk(j)714 696 y FC(+)e FB(siz)r(eof)5 b FC(\()p FB(ty)r(pe)991 703 y Fk(j)1010 696 y FC(\))56 b(if)16 b(no)f(en)o(try)g(has)g(basic)h (t)o(yp)q(e)f FB(ty)r(pe)1689 703 y Fk(j)1720 696 y FC(=)e Fp(ub)492 753 y FC(max)o Fy(f)p FB(disp)683 760 y Fk(j)728 753 y FC(:)28 b FB(ty)r(pe)853 760 y Fk(j)884 753 y FC(=)13 b Fp(ub)p Fy(g)66 b FC(otherwise)166 850 y(Then)472 906 y FB(extent)p FC(\()p FB(D)q(ataty)r(pe)p FC(\))14 b(=)f FB(ub)p FC(\()p FB(D)q(ataty)r(pe)p FC(\))d Fy(\000)g FB(l)q(b)p FC(\()p FB(D)q(ataty)r(pe)p FC(\);)166 986 y(F)l(urthermore,)18 b(if)h FB(ty)r(pe)575 993 y Fk(i)608 986 y FC(requires)h(alignmen)o(t)f(to)f(an)g(address)h(that)f(is)h(a)f (m)o(ultiple)j(of)d FB(k)1741 993 y Fk(i)1755 986 y FC(,)h(then)75 1043 y FB(extent)p FC(\()p FB(D)q(ataty)r(pe)p FC(\))14 b(is)g(further)f(rounded)g(up)h(to)e(the)h(nearest)g(m)o(ultiple)i(of)d (max)1466 1050 y Fk(i)1488 1043 y FB(k)1512 1050 y Fk(i)1525 1043 y FC(.)19 b(The)14 b(formal)e(de\014-)75 1099 y(nitions)j(giv)o (en)g(for)e(the)h(v)m(arious)h(datat)o(yp)q(e)e(constructors)h(apply)g (no)o(w,)g(with)g(the)g(amended)h(de\014nition)75 1156 y(of)g FD(exten)o(t)p FC(.)166 1212 y(A)j(datat)o(yp)q(e)f(ob)s(ject)g (has)h(to)f(b)q(e)i FD(committed)f FC(b)q(efore)g(it)g(can)g(b)q(e)h (used)f(in)h(a)f(comm)o(unication.)75 1269 y(A)d(committed)h(datat)o (yp)q(e)e(ob)s(ject)h(cannot)g(b)q(e)h(used)g(to)e(build)k(new)d(deriv) o(ed)h(datat)o(yp)q(es.)k(The)15 b(system)75 1325 y(ma)o(y)e(use)g(a)g (di\013eren)o(t)h(in)o(ternal)g(represen)o(tation)f(for)g(committed)g (datat)o(yp)q(e)g(ob)s(ject)g(so)f(as)h(to)g(facilitate)75 1381 y(comm)o(unication,)h(e.g.)k(c)o(hange)c(from)e(a)g(compacted)i (represen)o(tation)f(to)f(a)h(\015at)f(represen)o(tation)h(of)g(the)75 1438 y(datat)o(yp)q(e,)h(and)i(select)g(the)f(most)f(con)o(v)o(enien)o (t)i(transfer)e(mec)o(hanism.)75 1542 y Fo(MPI)p 160 1542 14 2 v 16 w(TYPE)p 293 1542 V 17 w(COMMIT\(int)o(yp)q(e,)h(outt)o (yp)q(e\))117 1619 y Ft(IN)155 b Fo(int)o(yp)q(e)473 b Ft(handle)14 b(to)g(datat)o(yp)q(e)g(ob)r(ject)h(to)e(b)q(e)i (committed)117 1693 y(OUT)108 b Fo(outt)o(yp)q(e)445 b Ft(handle)14 b(to)g(committed)d(datat)o(yp)q(e)j(ob)r(ject)166 1817 y FC(There)i(is)g(no)f(need)h(to)f(commit)h(basic)g(datat)o(yp)q (es;)f(they)g(are)g(\\pre-committed".)21 b(They)16 b(are)f(the)75 1874 y(exception)h(to)e(the)h(rule)g(that)g(committed)f(datat)o(yp)q (es)h(cannot)f(b)q(e)i(used)f(to)f(form)g(deriv)o(ed)i(datat)o(yp)q (es.)166 2006 y Fu(Discussion:)166 2056 y Ft(1.)i(Do)13 b(w)o(e)h(really)f(need)i(a)f(commit)d(here?)166 2106 y(2.)18 b(Do)13 b(w)o(e)h(w)o(an)o(t)g(one)g(INOUT)m(argumen)o(t,)e(or) i(t)o(w)o(o)f(distinct)h(argumen)o(ts?)166 2156 y(3.)19 b(Are)c(committed)d(and)i(uncommitted)e(ob)r(jects)k(of)e(the)h(same)e (t)o(yp)q(e?)20 b(\(The)15 b(answ)o(er)g(m)o(ust)e(b)q(e)i(y)o(es,)g (if)e(it)75 2205 y(is)h(an)f(INOUT)m(argumen)o(t.\))166 2255 y(4.)k(If)c(w)o(e)h(ha)o(v)o(e)f(only)f(one)h(INOUT)m(argumen)o (t,)f(do)q(es)i(a)f(commit)d(c)o(hange)k(the)f(datat)o(yp)q(e)h(ob)r (ject,)f(or)g(do)q(es)h(it)75 2305 y(create)i(a)f(new)g(ob)r(ject)h (and)f(return)h(a)f(p)q(oin)o(ter)g(to)g(it)f(in)29 b Fr(datat)o(yp)q(e)p Ft(?)23 b(I.e.,)14 b(if)g(w)o(e)h(ha)o(v)o(e)g (another)g(p)q(oin)o(ter)h(to)e(the)75 2355 y(same)f(uncommitted)f(ob)r (ject,)i(can)g(w)o(e)g(con)o(tin)o(ue)g(to)g(use)h(it?)166 2405 y(W)m(e)g(prohibit)h(the)g(use)h(of)f(committed)d(datat)o(yp)q(e)j (in)g(the)h(construction)g(of)e(new)h(datat)o(yp)q(es)h(in)e(order)i (to)75 2455 y(allo)o(w)10 b(the)j(commit)c(function)j(to)f(c)o(hange)i (a)e(datat)o(yp)q(e)h(represen)o(tation)i(in)o(to)d(one)h(suitable)g (for)g(comm)o(unicatio)o(n,)75 2504 y(and)17 b(p)q(erhaps)i(unsuitable) e(for)g(recursiv)o(e)i(datat)o(yp)q(e)e(construction.)30 b(This)17 b(restriction)h(could)g(b)q(e)f(relaxed)h(b)o(y)75 2554 y(assuming)13 b(that)i(the)g(\\uncommitted")d(datat)o(yp)q(e)j (represen)o(tation)h(is)f(k)o(ept)g(alongside)f(the)h(\\committed")d (one.)75 2604 y(With)k(suc)o(h)h(an)f(implem)o(en)o(tation)30 b Fr(MPI)p 733 2604 13 2 v 14 w(TYPE)p 855 2604 V 14 w(COMMIT)16 b Ft(could)g(ha)o(v)o(e)g(only)f(one)i(argumen)o(t,)e(and)h (only)g(one)75 2654 y(t)o(yp)q(e)f(of)e(datat)o(yp)q(e)i(ob)r(ject)g (is)f(needed.)20 b(Suc)o(h)15 b(represen)o(tation)g(could)f(also)g (simplify)d(the)k(implemen)o(tatio)o(n)d(of)h(a)75 2704 y(datat)o(yp)q(e)h(\\\015atten")g(function)g(\(e.g.,)e(for)i (debugging\).)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 48 54 48 53 bop 75 -100 a FC(48)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(FREE\(datat)o(yp)q(e\))117 124 y Ft(INOUT)62 b Fo(datat)o(yp)q(e)424 b Ft(handle)14 b(to)g(datat)o(yp)q(e)g(ob)r(ject)166 251 y FC(Marks)h(the)g(datat)o (yp)q(e)g(ob)s(ject)g(asso)q(ciated)h(with)32 b Fo(datat)o(yp)q(e)17 b FC(for)e(deallo)q(cation.)22 b(The)16 b(ob)s(ject)f(will)75 307 y(b)q(e)h(deallo)q(cated)h(after)d(an)o(y)i(p)q(ending)h(comm)o (unication)f(that)e(uses)i(this)g(ob)s(ject)f(completes,)g(at)g(whic)o (h)75 364 y(p)q(oin)o(t)31 b Fo(datat)o(yp)q(e)17 b FC(b)q(ecomes)e(n)o (ull.)166 498 y Fu(Discussion:)166 550 y Ft(If)f(w)o(e)h(ha)o(v)o(e)f (di\013eren)o(t)i(datat)o(yp)q(es)f(for)f(committed)e(and)i (uncommitted,)e(then)j(w)o(e)g(ha)o(v)o(e)f(2)h(free)g(functions)75 600 y(\(or)f(nev)o(er)h(free)f(uncommitted)e(datat)o(yp)q(e\))75 816 y Fq(3.13.3)49 b(Use)16 b(of)h(general)f(datat)o(yp)q(es)g(in)h (communication)75 906 y FC(Handles)g(to)f(deriv)o(ed)h(datat)o(yp)q(es) f(can)h(b)q(e)f(passed)h(to)f(a)g(comm)o(unication)h(call)g(wherev)o (er)f(a)g(datat)o(yp)q(e)75 963 y(argumen)o(t)22 b(is)h(required.)43 b(A)23 b(call)h(of)e(the)h(form)f Fo(MPI)p 1050 963 V 15 w(SEND\(sta)o(rt,)j(count,)g(datat)o(yp)q(e,)h(...\))p FC(,)d(where)75 1019 y Fp(count)12 b FC(=)h Fp(1)f FC(uses)h(the)g (send)g(bu\013er)g(de\014ned)h(b)o(y)25 b Fo(sta)o(rt)13 b FC(and)26 b Fo(datat)o(yp)q(e)p FC(;)15 b(i.e.)k(if)26 b Fo(datat)o(yp)q(e)14 b FC(is)f(a)g(handle)h(to)75 1076 y(the)k(datat)o(yp)q(e)g(de\014ned)i(b)o(y)e(the)g(sequence)h Fy(f)p FC(\()p FB(ty)r(pe)973 1083 y Fl(0)993 1076 y FB(;)8 b(disp)1098 1083 y Fl(0)1116 1076 y FC(\))p FB(;)g(:::;)g FC(\()p FB(ty)r(pe)1317 1083 y Fk(n)p Fm(\000)p Fl(1)1383 1076 y FB(;)g(disp)1488 1083 y Fk(n)p Fm(\000)p Fl(1)1555 1076 y FC(\))p Fy(g)p FC(,)18 b(\(with)g(empt)o(y)75 1132 y(en)o(tries)j(deleted\))h(then)g(the)f(send)g(bu\013er)g (consists)g(of)g(the)g(en)o(tries)g(of)g(t)o(yp)q(es)g FB(ty)r(pe)1579 1139 y Fk(i)1593 1132 y FC(,)h(at)f(addresses)75 1188 y Fp(start)10 b FC(+)i FB(disp)336 1195 y Fk(i)349 1188 y FC(,)17 b FB(i)d FC(=)h(0)p FB(;)8 b(:::;)g(n)h Fy(\000)i FC(1.)24 b(The)17 b(message)f(generated)g(consists)h(of)f(a)g (sequence)i(of)e FB(n)h FC(en)o(tries,)75 1245 y(of)g(t)o(yp)q(es)h FB(ty)r(pe)336 1252 y Fl(0)356 1245 y FB(;)8 b(:::;)g(ty)r(pe)521 1252 y Fk(n)p Fm(\000)p Fl(1)587 1245 y FC(.)28 b(The)18 b(receiv)o(e)h(bu\013er)e(for)g(a)h(call)h Fo(MPI)p 1292 1245 V 16 w(RECV\()f(sta)o(rt,)g(count,)h(datat)o(yp)q(e)75 1301 y(,)h(...\))p FC(,)e(is)i(similarly)h(de\014ned,)g(for)e Fp(count)k(=1)p FC(.)32 b(The)19 b(same)g(applies)i(to)e(all)h(other)f (send)h(and)f(receiv)o(e)75 1358 y(op)q(erations)c(that)g(ha)o(v)o(e)g (a)30 b Fo(datat)o(yp)q(e)16 b FC(parameter.)166 1416 y(A)d(datat)o(yp)q(e)f(used)i(in)f(a)g(receiv)o(e)h(op)q(eration)f(ma)o (y)f(not)g(ha)o(v)o(e)h(o)o(v)o(erlapping)g(en)o(tries;)h(otherwise)f (the)75 1473 y(outcome)i(of)g(the)g(receiv)o(e)h(op)q(eration)f(is)h (unde\014ned.)166 1531 y(If)g(a)g(send)h(op)q(eration)f(using)h(a)f (general)g(datat)o(yp)q(e)32 b Fo(t)o(yp)q(e1)16 b FC(is)h(matc)o(hed)f (b)o(y)g(a)g(receiv)o(e)h(op)q(eration)75 1588 y(using)e(a)f(general)h (datat)o(yp)q(e)28 b Fo(t)o(yp)q(e2)16 b FC(then)e(the)h(sequence)g(of) f(\(nonempt)o(y\))g(basic)h(t)o(yp)q(e)g(en)o(tries)g(de\014ned)75 1644 y(b)o(y)f(eac)o(h)g(datat)o(yp)q(e)f(should)i(b)q(e)g(iden)o (tical.)21 b(F)l(urthermore,)13 b(eac)o(h)h(en)o(try)g(in)h(the)f(send) g(bu\013er)g(should)h(b)q(e)75 1701 y(of)i(t)o(yp)q(e)g(that)g(matc)o (hes)g(the)h(t)o(yp)q(e)f(of)g(the)g(corresp)q(onding)i(en)o(try)e(in) 35 b Fo(t)o(yp)q(e1)19 b FC(and)e(eac)o(h)h(en)o(try)f(in)h(the)75 1757 y(receiv)o(e)f(bu\013er)f(should)i(of)d(a)h(t)o(yp)q(e)h(that)e (matc)o(hes)h(the)g(t)o(yp)q(e)h(of)f(the)g(corresp)q(onding)h(en)o (try)f(in)33 b Fo(t)o(yp)q(e2)75 1814 y FC(\(with)13 b(t)o(yp)q(e)f(matc)o(hing)h(de\014ned)h(as)e(in)h(Section)g(3.5\).)18 b(Th)o(us,)13 b(t)o(yp)q(e)f(matc)o(hing)h(is)g(de\014ned)h(b)o(y)e (the)h(\\\015at")75 1870 y(structure)i(of)g(a)g(datat)o(yp)q(e,)f(and)h (do)q(es)h(not)f(dep)q(end)h(on)g(its)f(de\014ning)i(expression.)166 1929 y(Example:)75 2033 y Fp(...)75 2089 y(MPI_TYPE_CONTIGUOUS\()k (MPI_REAL,)i(2,)g(type2\))75 2146 y(MPI_TYPE_CONTIGUOUS\()e(MPI_REAL,)i (4,)g(type4\))75 2202 y(MPI_TYPE_CONTIGUOUS\()e(type2,)i(2,)h(type22\)) 166 2304 y FC(A)15 b(message)g(sen)o(t)g(with)g(datat)o(yp)q(e)30 b Fo(t)o(yp)q(e4)16 b FC(can)f(b)q(e)h(receiv)o(ed)h(with)e(datat)o(yp) q(e)30 b Fo(t)o(yp)q(e22)p FC(.)166 2363 y(T)o(yp)q(e)22 b(matc)o(hing)g(requiremen)o(ts)h(imply)h(that)d(\(nonempt)o(y\))g(en)o (tries)i(of)f(di\013eren)o(t)g(t)o(yp)q(es)g(in)h(a)75 2419 y(datat)o(yp)q(e)13 b(cannot)g(o)o(v)o(erlap,)g(unless)i(they)e (coincide)i(and)f(ha)o(v)o(e)f(the)g(same)g(basic)h(t)o(yp)q(e,)g(or)f (one)g(of)g(them)75 2476 y(ha)o(v)o(e)i(basic)h(t)o(yp)q(e)30 b Fo(b)o(yte)p FC(.)166 2534 y(The)12 b(length)g(of)f(a)h(message)f (receiv)o(ed)i(\(i.e.,)f(the)f(v)m(alue)i(returned)f(when)g(the)g (receiv)o(e)h(status)e(ob)s(ject)75 2591 y(is)k(deco)q(ded)h(b)o(y)f (the)f(function)i Fo(MPI)p 698 2591 V 16 w(GET)p 802 2591 V 16 w(LEN)p FC(\))f(is)g(equal)g(to)f(the)h(n)o(um)o(b)q(er)g(of) f(basic)h(en)o(tries)g(receiv)o(ed.)75 2647 y(Consider)h(again)f(the)g (datat)o(yp)q(es)g(de\014ned)i(in)f(the)f(last)g(example.)21 b(If)16 b(a)f(message)f(is)i(sen)o(t)f(using)h(either)75 2704 y(datat)o(yp)q(e)38 b Fo(t)o(yp)q(e22)19 b FC(or)38 b Fo(t)o(yp)q(e4)20 b FC(and)f(is)h(receiv)o(ed)g(using)g(either)g(of)f (these)g(t)o(w)o(o)f(datat)o(yp)q(es,)h(then)h(the)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 49 55 49 54 bop 75 -100 a Fv(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FC(49)75 45 y(length)17 b(of)f(the)h(message)f(receiv)o(ed)i (will)g(b)q(e)f(four.)24 b(In)17 b(the)g(general)g(case)f(the)h(basic)g (en)o(tries)g(need)g(not)75 102 y(b)q(e)f(all)g(of)f(the)g(same)g(t)o (yp)q(e.)166 236 y Fu(Discussion:)166 287 y Ft(An)f(alternativ)o(e)f (de\014nition)h(that)f(\014nds)i(fa)o(v)o(or)d(with)i(man)o(y)e(mem)o (b)q(ers)g(of)h(the)i(sub)q(committee)d(is)i(that)f(the)75 337 y(length)h(of)g(the)h(message)f(receiv)o(ed)h(is)f(measured)h(in)e (terms)i(of)e(subunits)i(of)f(the)h(receiving)f(datat)o(yp)q(e.)20 b(Th)o(us,)14 b(if)75 387 y(four)h(reals)g(are)h(receiv)o(ed)g(using)30 b Fr(t)o(yp)q(e4)p Ft(,)16 b(the)g(length)f(of)f(the)i(receiv)o(ed)g (message)f(is)g(four;)g(if)f(they)i(are)g(receiv)o(ed)75 437 y(using)29 b Fr(t)o(yp)q(e22)p Ft(,)14 b(the)h(length)f(of)g(the)h (receiv)o(ed)h(message)e(is)g(t)o(w)o(o.)20 b(With)13 b(suc)o(h)i(a)g(de\014nition,)e(it)h(is)h(required)g(that)75 486 y(the)f(arriving)f(message)h(\014lls)f(an)h(in)o(teger)g(n)o(um)o (b)q(er)f(of)h(subunits.)166 538 y(The)g(rationale)e(is)h(that)g(the)h (these)h(are)e(the)h(units)g(that)f(are)g(meaningful)e(to)i(the)h (programmer.)i(Consider)75 588 y(the)e(case)h(of)e(an)g(arra)o(y)g(of)g (structures)j(that)e(is)f(passed)i(in)e(a)g(message)h(\(example)e(4.1)g (b)q(elo)o(w\).)18 b(Then)c(one)g(migh)o(t)75 638 y(w)o(an)o(t)f(to)g (enforce)i(the)f(restriction)g(that)f(only)g(complete)g(structures)i (are)f(passed;)g(and)g(is)f(natural)g(to)g(return)h(in)75 687 y(the)h(receiv)o(e)g(status)g(the)f(n)o(um)o(b)q(er)f(of)h (structures)i(receiv)o(ed,)f(rather)g(than)f(this)g(n)o(um)o(b)q(er)f (m)o(ultiplied)f(b)o(y)h(14,)g(the)75 737 y(n)o(um)o(b)q(er)g(of)g(en)o (tries)i(p)q(er)g(structure.)166 789 y(Consider,)c(on)g(the)g(other)g (hand,)g(the)g(case)h(of)e(a)g(sequence)j(of)d(reals)h(that)g(is)f (receiv)o(ed)i(in)e(a)h(3D)f(arra)o(y)g(section)75 839 y(\(de\014ned)16 b(as)e(in)h(the)g(\014rst)g(example)e(b)q(elo)o(w\).) 20 b(W)m(e)15 b(ma)o(y)d(w)o(an)o(t)j(to)f(receiv)o(e)i(an)e(arbitrary) h(n)o(um)o(b)q(er)f(of)g(reals,)g(and)75 888 y(the)g(1D)g(and)g(2D)f (subsections)i(ma)o(y)d(not)i(carry)g(an)o(y)g(signi\014cance)g(for)g (the)g(purp)q(ose)h(of)f(this)f(comm)o(unication.)166 940 y(Opinions?)166 1081 y FC(A)18 b(call)h(of)f(the)g(form)f Fo(MPI)p 635 1081 14 2 v 16 w(SEND\(sta)o(rt,)i(count,)h(datat)o(yp)q (e)f(,)g(...\))p FC(,)e(where)h Fp(count)f FB(>)h Fp(1)p FC(,)g(is)h(in)o(ter-)75 1137 y(preted)c(as)g(if)g(the)g(call)h(w)o(as) e(passed)h(a)f(new)h(datat)o(yp)q(e)g(whic)o(h)g(is)h(the)f (concatenation)g(of)29 b Fo(count)16 b FC(copies)75 1194 y(of)35 b Fo(datat)o(yp)q(e)p FC(,)19 b(suc)o(h)f(as)g(w)o(ould)g(b)q (e)g(pro)q(duced)h(b)o(y)e(a)h(call)g(to)f Fo(MPI)p 1240 1194 V 16 w(CONTIGUOUS\(count,)k(datat)o(yp)q(e,)75 1250 y(newt)o(yp)q(e\))p FC(,)16 b(and)e(a)29 b Fo(count)15 b FC(argumen)o(t)f(of)g(one.)20 b(The)14 b(same)g(holds)i(true)e(for)g (the)g(other)g(comm)o(unication)75 1307 y(functions)i(that)e(ha)o(v)o (e)h(a)g(datat)o(yp)q(e)g(argumen)o(t.)75 1438 y Fq(3.13.4)49 b(Examples)75 1527 y FC(The)15 b(follo)o(wing)h(examples)g(illustrate)h (the)e(use)g(of)g(deriv)o(ed)h(datat)o(yp)q(es.)166 1585 y(First)f(example:)20 b(extract)15 b(a)g(section)h(of)e(a)h(3D)g (matrix.)218 1687 y Fp(REAL)23 b(a\(100,100,100\),)f(e\(9,9,9\))218 1744 y(INTEGER)h(oneslice,)g(twoslice,)f(threeslice,)h(finaltype,)f (sizeofreal)75 1857 y(C)143 b(extract)23 b(the)g(section)g(a\(1:17:2,)g (3:11,)g(2:10\))75 1913 y(C)143 b(and)23 b(store)h(it)f(in)h (e\(*,*,*\).)218 2026 y(CALL)f(MPI_TYPE_EXTENT\()f(MPI_REAL,)h (sizeofreal,)f(ierr\))75 2139 y(C)119 b(create)23 b(datatype)g(for)g(a) h(1D)g(section)218 2195 y(CALL)f(MPI_TYPE_VECTOR\()f(9,)i(MPI_REAL,)46 b(2,)24 b(1,)f(oneslice,)g(ierr\))75 2308 y(C)119 b(create)23 b(datatype)g(for)g(a)h(2D)g(section)218 2365 y(CALL)f (MPI_TYPE_HVECTOR\()f(9,)i(oneslice,)46 b(100*sizeofreal,)22 b(1,)i(twoslice,)e(ierr\))75 2478 y(C)119 b(create)23 b(datatype)g(for)g(the)h(entire)f(section)218 2534 y(CALL)g (MPI_TYPE_HVECTOR\()f(9,)i(twoslice,)46 b(100*100*sizeofreal,)21 b(1,)505 2591 y(threeslice,)h(ierr\))218 2704 y(CALL)h (MPI_TYPE_COMMIT\()f(threeslice,)h(finaltype,)f(ierr\))-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 50 56 50 55 bop 75 -100 a FC(50)635 b Fv(SECTION)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)218 45 y Fp(CALL)23 b(MPI_SENDRECV\(a\(1,3,2\),)e(1,)j(finaltype,)e(MPI_ME,)h(e,)h(9*9*9,) 505 102 y(MPI_REAL,)e(MPI_ME,)h(0,)h(MPI_ALL,)e(status,)h(ierr\))166 196 y FC(Second)16 b(example:)21 b(Cop)o(y)14 b(the)i(\(strictly\))f (lo)o(w)o(er)f(triangular)i(part)e(of)h(a)g(matrix.)218 291 y Fp(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 348 y(INTEGER)47 b(index\(100\),)22 b(blocklen\(100\),)g(ltype,)h (finaltype)75 461 y(C)119 b(copy)23 b(lower)h(triangular)e(part)h(of)h (array)f(a)75 517 y(C)119 b(onto)23 b(lower)h(triangular)e(part)h(of)h (array)f(b)75 630 y(C)119 b(compute)23 b(start)g(and)h(size)f(of)h (each)f(column)218 687 y(DO)h(i=1,)f(100)266 743 y(index\(i\))g(=)g (100*\(i-1\))g(+)h(i)266 800 y(block\(i\))f(=)g(100-i)218 856 y(END)h(DO)75 969 y(C)119 b(create)23 b(datatype)g(for)g(lower)h (triangular)e(part)218 1025 y(CALL)h(MPI_TYPE_INDEX\()f(MPI_REAL,)h (100,)g(index,)g(block,)g(ltype,)g(ierr\))218 1138 y(CALL)g (MPI_TYPE_COMMIT\(ltype,)e(finaltype,)i(ierr\))218 1195 y(CALL)g(MPI_SENDRECV\()g(a,)g(1,)h(finaltype,)e(MPI_ME,)h(b,)h(1,)457 1251 y(finaltype,)e(MPI_ME,)h(0,)h(MPI_ALL,)f(status,)f(ierr\))166 1346 y FC(Third)16 b(example:)21 b(T)l(ransp)q(ose)15 b(a)g(matrix)218 1441 y Fp(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 1498 y(INTEGER)g(row,)g(xpose,)g(finaltype,)g(sizeofreal)75 1610 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 1780 y(CALL)g(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 1893 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 1949 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(MPI_REAL,)g(100,)g(1,)h(row,)f (ierr\))75 2062 y(C)119 b(create)23 b(datatype)g(for)g(matrix)g(in)h (row-major)f(order)218 2119 y(CALL)g(MPI_TYPE_HVECTOR\()f(100,)h(row,)h (sizeofreal,)e(1,)i(xpose,)f(ierr\))218 2231 y(CALL)g (MPI_TYPE_COMMIT\()f(xpose,)h(finaltype,)g(ierr\))75 2344 y(C)119 b(send)23 b(matrix)h(in)f(row-major)g(order)g(and)g (receive)g(in)h(column)f(major)g(order)218 2401 y(CALL)g (MPI_SENDRECV\()g(a,)g(1,)h(finaltype,)e(MPI_ME,)h(b,)h(100*100,)457 2457 y(MPI_REAL,)e(MPI_ME,)h(0,)h(MPI_ALL,)f(status,)g(ierr\))166 2552 y FC(Another)15 b(approac)o(h)g(to)f(the)i(transp)q(ose)f (problem:)218 2647 y Fp(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 2704 y(INTEGER)47 b(index\(2\),)23 b(blen\(2\),)f(type\(2\),)h(row,)g (row1,)h(finaltype,)e(sizeofreal)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 51 57 51 56 bop 75 -100 a Fv(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FC(51)75 158 y Fp(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b) 218 327 y(CALL)g(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\)) 75 440 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 497 y(CALL)f(MPI_TYPE_VECTOR\()f(MPI_REAL,)h(100,)g(100,)g(1,)h(row,)f (ierr\))75 610 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row,)f (with)h(the)f(extent)g(of)h(one)f(real)g(number)218 666 y(index\(1\))g(=)h(0)218 723 y(index\(2\))f(=)h(sizeofreal)218 779 y(type\(1\))47 b(=)24 b(row)218 835 y(type\(2\))47 b(=)24 b(MPI_UB)218 892 y(blen\(1\))47 b(=)24 b(1)218 948 y(blen\(2\))47 b(=)24 b(1)218 1005 y(CALL)f(MPI_TYPE_STRUC\()f(2,)i (type,)f(index,)g(blen,)g(row1\))218 1118 y(CALL)g(MPI_TYPE_COMMIT\()f (row1,)h(finaltype,)g(ierr\))75 1231 y(C)119 b(send)23 b(100)h(rows)f(and)h(receive)f(in)g(column)g(major)g(order)218 1287 y(CALL)g(MPI_SENDRECV\()g(a,)g(100,)g(finaltype,)g(MPI_ME,)g(b,)g (100*100,)457 1344 y(MPI_REAL,)f(MPI_ME,)h(0,)h(MPI_ALL,)f(status,)g (ierr\))166 1431 y FC(F)l(ourth)15 b(example:)20 b(manipulate)d(an)e (arra)o(y)f(of)h(structures.)75 1518 y Fp(struct)23 b(Partstruct)147 1574 y({)147 1631 y(int)g(index;)95 b(/*)23 b(particle)g(type)g(*/)147 1687 y(double)g(d[6])47 b(/*)23 b(particle)g(coordinates)g(*/)147 1744 y(char)g(b[7])95 b(/*)23 b(some)h(additional)e(information)h(*/) 147 1800 y(};)75 1913 y(struct)g(Partstruct)g(particle[1000];)337 2026 y(/*)h(build)f(datatype)g(describing)g(first)g(array)g(entry)g(*/) 75 2139 y(MPI_datatype)f(Particletype;)75 2195 y(MPI_datatype)g (stype[3])h(=)h({MPI_int,)e(MPI_double,)h(MPI_char};)75 2252 y(int)238 b(sblock[3])23 b(=)h({1,)f(6,)h(7};)75 2308 y(int)238 b(sindex[3];)75 2421 y(MPI_address\()22 b(\(void)h(*\)particle,)g(sindex\);)75 2478 y(MPI_address\()f(\(void)h (*\)particle[0].d,)f(sindex+1\);)75 2534 y(MPI_address\()g(\(void)h (*\)particle[0].b,)f(sindex+2\);)75 2591 y(MPI_type_struct\()g(3,)h (stype,)g(sindex,)g(sblock,)g(Particletype\);)75 2647 y(/*)h(Particletype)e(describes)g(first)i(array)f(entry)g(--)h(using)f (absolute)147 2704 y(addresses)f(*/)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 52 58 52 57 bop 75 -100 a FC(52)635 b Fv(SECTION)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)266 158 y Fp(/*)23 b(4.1:)h(send)f(the)h(entire)f(array)g(*/)75 271 y(MPI_datatype)f(Type1;)75 327 y(MPI_commit\()g(Particletype,)g (Type1\);)75 384 y(MPI_send\()h(MPI_bottom,)f(1000,)h(Type1,)g(dest,)g (tag,)h(comm\);)266 553 y(/*)f(4.2:)h(send)f(the)h(entries)e(with)i (index)f(value)g(zero,)314 610 y(preceded)f(by)i(the)f(number)g(of)h (such)f(entries)g(*/)75 723 y(MPI_datatype)f(Zparticles;)70 b(/*)24 b(datatype)f(describing)f(all)i(particles)934 779 y(with)f(index)h(zero)f(\(needs)g(to)h(be)f(recomputed)934 835 y(if)h(indices)f(change\))f(*/)75 892 y(MPI_datatype)g(Ztype,)h (Type2;)75 1005 y(int)g(zindex[1000],)g(zblock[1000],)f(i,)h(j,)h(k;)75 1061 y(int)f(zzblock[2])g(=)h({1,1};)75 1118 y(int)f(zztype[2],)g (zzindex[2];)75 1287 y(/*)h(compute)e(indices)h(of)h(particles)f(with)g (index)g(zero)g(*/)75 1344 y(j)h(=)f(0;)75 1400 y(for\(i=0;)g(i)g(<)h (1000;)f(i++\))123 1456 y(if)g(\(particle[i].index==0\))e(then)194 1513 y({)194 1569 y(zindex[j])i(=)h(i;)194 1626 y(zblock[i])f(=)h(1;) 194 1682 y(j++;)194 1739 y(})75 1852 y(/*)g(create)f(datatype)f(for)i (particles)e(with)i(index)f(zero)g(*/)75 1908 y(MPI_type_index\()f(j,)h (Particletype,)46 b(zindex,)23 b(zblock,)g(Zparticles\);)75 2021 y(/*)h(prepend)e(particle)h(count)g(*/)75 2077 y (MPI_address\(\(void)f(*\)&j,)h(zzindex\);)75 2134 y(zzindex[1])f(=)i (MPI_bottom;)75 2190 y(zztype[0])f(=)g(MPI_int;)75 2247 y(zztype[1])g(=)g(Zparticles;)75 2303 y(MPI_type_struc\(2,)f(zztype,)g (zzindex,)h(zzblock,)g(Ztype\);)75 2416 y(MPI_type_commit\()f(Ztype,)h (Type2\);)75 2473 y(MPI_send\()g(MPI_bottom,)f(1,)i(Type2,)f(dest,)g (tag,)g(comm\);)266 2642 y(/*)g(A)h(possibly)f(more)g(efficient)g(way)g (of)h(defining)f(Zparticles)f(*/)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 53 59 53 58 bop 75 -100 a Fv(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(53)75 45 y Fp(/*)24 b(consecutive)e(particles)h(with)g(index) g(zero)g(are)h(handled)f(as)g(one)h(block)f(*/)75 102 y(j=0;)75 158 y(for)g(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 214 y(if)g(\(particle[i].index==0\))e(then)170 271 y({)170 327 y(for)j(\(k=i+1;)f(\(k)g(<)h(1000\)&&\(particle[k].index)d(==)i (0\))h(;)f(k++\);)170 384 y(zindex[j])g(=)h(i;)170 440 y(zblock[j])f(=)h(k-i;)170 497 y(j++;)170 553 y(i)g(=)g(k;)170 610 y(})75 666 y(MPI_type_index\()e(j,)h(Particletype,)f(zindex,)h (zblock,)g(Zparticles\);)266 835 y(/*)g(4.3:)h(send)f(the)h(first)f (two)g(coordinates)g(of)g(all)h(entries)f(*/)75 948 y(MPI_datatype)f (Allpairs;)142 b(/*)24 b(datatype)f(for)g(all)g(pairs)h(of)f (coordinates)g(*/)75 1005 y(MPI_datatype)f(Type3;)75 1118 y(int)238 b(sizeofentry;)75 1231 y(MPI_extent\()22 b(Particletype,)g(sizeofentry\);)194 1344 y(/*)i(sizeofentry)e(can)i (also)f(be)h(computed)e(by)i(subtracting)e(the)i(address)266 1400 y(of)f(particle[0])g(from)g(the)h(address)f(of)g(particle[1])f(*/) 75 1513 y(MPI_type_hvector\()g(1000,)h(MPI_real,)f(sizeofentry,)h(2,)g (Allpairs\);)75 1569 y(MPI_type_commit\()f(Allpairs,)g(Type3\);)75 1626 y(MPI_send\()h(particle.d,)f(1,)i(Type3,)f(dest,)g(tag,)g(comm\);) 218 1739 y(/*)h(an)f(alternative)g(solution)f(to)i(4.3)g(*/)75 1852 y(MPI_datatype)e(Onepair;)71 b(/*)23 b(datatype)g(for)h(one)f (pair)g(of)h(coordinates,)e(with)695 1908 y(the)i(extent)f(of)h(one)f (particle)g(entry)g(*/)75 1965 y(MPI_datatype)f(Type4;)75 2077 y(int)h(indexp[3];)75 2134 y(int)g(typep[3])g(=)h({MPI_lb,)f (MPI_double,)f(MPI_ub};)75 2190 y(int)h(blockp[3])g(=)h({1,)f(2,)h(1};) 75 2303 y(MPI_address\()e(\(void)h(*\)particle,)g(indexp\);)75 2360 y(MPI_address\()f(\(void)h(*\)particle[0].d,)f(indexp+1\);)75 2416 y(MPI_address\()g(\(void)h(*\)\(particle+1\),)f(indexp+2\);)75 2529 y(MPI_type_struc\()g(3,)h(typep,)g(indexp,)g(blockp,)g(Onepair\);) 75 2586 y(MPI_commit\()f(Onepair,)h(Type4\);)75 2642 y(MPI_send\()g(MPI_bottom,)f(1000,)h(Type4,)g(dest,)g(tag,)h(comm\);) -32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 54 60 54 59 bop 75 -100 a FC(54)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fu(Discussion:)166 95 y Ft(The)e(examples)e(carry)i(t)o(w)o(o)f(implicit)e(assumptions)h (on)h(the)h(w)o(a)o(y)f(padding)f(is)h(done)h(in)f(an)g(arra)o(y)g(of)g (struc-)75 145 y(tures.)166 195 y(First,)19 b(that)g(padding)e(is)i (done)f(in)g(the)h(same)f(w)o(a)o(y)g(in)g(the)h(successiv)o(e)h(arra)o (y)e(en)o(tries,)j(so)d(that)g(all)g(en-)75 244 y(tries)d(ha)o(v)o(e)g (an)f(iden)o(tical)g(memory)f(la)o(y)o(out.)19 b(E.g.,)13 b(w)o(e)i(assume)f(that)h(displacemen)o(t)f(from)f Fn(particle[i].b)f Ft(to)75 294 y Fn(particle[j].b)f Ft(is)j(the)g(same)f(as)h(the)h (displacemen)o(t)e(from)f Fn(particle[i].d)f Ft(to)j Fn(particle[j].d)p Ft(.)166 344 y(Second,)e(that)f(the)h(amoun)o(t)d (of)i(padding)f(added)i(is,)f(in)g(some)f(sense,)j(minim)o(al)o(.)i (I.e.,)10 b(if)h(the)g(most)f(stringen)o(t)75 394 y(alignmen)o(t)k (requiremen)o(t)j(for)g(a)f(basic)h(comp)q(onen)o(t)f(of)g(a)h (structure)h(is)f(alignmen)o(t)e(to)h(m)o(ultiples)f(of)h(2)1745 379 y Fh(k)1765 394 y Ft(,)h(then)75 444 y(no)h(con)o(tiguous)g (padding)f(space)i(has)f(size)h(greater)g(or)f(equal)g(to)g(2)1177 429 y Fh(k)1197 444 y Ft(.)31 b(This)18 b(assumption)e(is)i(implicit)e (in)h(the)75 493 y(de\014nition)c(of)h Fu(exten)o(t)p Ft(.)166 543 y(I)e(don't)g(think)h(either)g(assumptions)e(are)i (mandated)f(b)o(y)g(C)g(of)g(F)m(ortran)h(standards,)g(although)e(I)i (w)o(ould)e(b)q(e)75 593 y(v)o(ery)k(surprised)i(to)d(see)j(an)d (implemen)o(tation)e(that)j(do)q(es)g(not)g(ful\014ll)e(these)k (conditions.)k(If)14 b(I)h(am)e(wrong)i(\(e.g.,)75 643 y(if)g(some)h(compilers)f(alw)o(a)o(ys)g(align)g(structures)j(to)e (double)g(w)o(ord)g(b)q(oundaries,)h(ev)o(en)g(if)e(they)i(ha)o(v)o(e)f (no)g(double)75 693 y(w)o(ord)e(comp)q(onen)o(ts\),)f(then)i(the)f (de\014nition)f(of)28 b Fr(extent)15 b Ft(should)f(b)q(e)g(revisited.) 75 897 y Fq(3.13.5)49 b(Co)o(rrect)15 b(use)h(of)g(addresses)75 983 y FC(Successiv)o(ely)23 b(declared)f(v)m(ariables)h(in)e(C)g(or)g (F)l(ortran)e(are)i(not)g(necessarily)h(stored)f(at)f(con)o(tiguous)75 1039 y(lo)q(cations.)i(Th)o(us,)16 b(care)g(m)o(ust)f(b)q(e)h (exercised)i(that)d(displacemen)o(ts)i(do)f(not)f(cross)h(from)f(one)h (v)m(ariable)75 1095 y(to)e(another.)20 b(Also,)15 b(in)g(mac)o(hines)h (with)f(segmen)o(ted)g(address)g(space,)g(addresses)g(are)g(not)f (unique)j(and)75 1152 y(address)11 b(arithmetic)h(has)g(some)f(p)q (eculiar)i(prop)q(erties.)19 b(Th)o(us,)12 b(use)g(of)f FD(addresses)p FC(,)f(i.e.)20 b(displacemen)o(ts)75 1208 y(relativ)o(e)c(to)e(the)i(start)e(address)29 b Fr(MPI)p 733 1208 13 2 v 14 w(BOTTOM)p FC(,)14 b(has)h(to)g(b)q(e)h(restricted.) 166 1265 y(V)l(ariables)k(b)q(elong)f(to)f(the)h(same)f FD(sequen)o(tial)k(storage)d FC(if)g(they)g(b)q(elong)g(to)f(the)h (same)f(arra)o(y)l(,)75 1321 y(to)g(the)g(same)g Fp(COMMON)g FC(blo)q(c)o(k)h(in)g(F)l(ortran,)f(or)g(to)f(the)i(same)f(structure)g (in)h(C.)f(V)l(alid)i(addresses)e(are)75 1378 y(de\014ned)f(recursiv)o (ely)f(as)f(follo)o(ws:)131 1482 y(1.)22 b(The)13 b(function)h Fo(MPI)p 541 1482 14 2 v 16 w(ADDRESS)g FC(returns)f(a)g(v)m(alid)h (address,)g(when)f(passed)h(as)e(argumen)o(t)h(a)f(host)189 1538 y(program)i(v)m(ariable.)131 1632 y(2.)22 b(The)30 b Fo(sta)o(rt)15 b FC(parameter)f(of)h(a)f(comm)o(unication)i(function) g(ev)m(aluates)f(to)g(a)f(v)m(alid)j(address,)d(when)189 1688 y(passed)h(as)g(argumen)o(t)f(a)h(host)g(program)f(v)m(ariable.) 131 1781 y(3.)22 b(If)33 b Fo(v)16 b FC(is)h(a)f(v)m(alid)i(address,)e (and)33 b Fo(i)17 b FC(is)g(an)f(in)o(teger,)g(then)33 b Fo(v+i)17 b FC(is)g(a)f(v)m(alid)i(address,)e(pro)o(vided)34 b Fo(v)189 1838 y FC(and)c Fo(v+i)15 b FC(are)g(in)h(the)g(same)f (sequen)o(tial)h(storage.)131 1931 y(4.)22 b(If)30 b Fo(v)15 b FC(is)h(a)f(v)m(alid)i(address)e(then)29 b Fr(MPI)p 835 1931 13 2 v 15 w(BOTTOM)13 b(+)h(v)i FC(is)f(a)g(v)m(alid) i(address.)166 2035 y(A)22 b(correct)g(program)g(uses)g(only)h(v)m (alid)i(addresses)d(to)g(iden)o(tify)i(the)e(lo)q(cations)h(of)f(en)o (tries)h(in)75 2091 y(comm)o(unication)14 b(bu\013ers.)19 b(F)l(urthermore,)12 b(if)27 b Fo(u)13 b FC(and)26 b Fo(v)13 b FC(are)g(t)o(w)o(o)e(v)m(alid)k(addresses,)e(then)g(the)g (\(in)o(teger\))75 2148 y(di\013erence)32 b Fo(u)16 b(-)f(v)h FC(can)f(b)q(e)h(computed)g(only)g(if)g(b)q(oth)31 b Fo(u)15 b FC(and)h Fp(v)f FC(are)g(in)h(the)g(same)f(sequen)o(tial)i (storage;)75 2204 y(no)e(other)g(arithmetic)h(op)q(erations)f(can)g(b)q (e)h(meaningfully)i(executed)e(on)f(addresses.)166 2261 y(W)l(e)f(sa)o(y)g(that)g(a)g(datat)o(yp)q(e)g(is)h FD(absolute)g FC(if)g(all)h(displacemen)o(ts)g(within)f(this)g(datat)o(yp)q(e)f(are)g (v)m(alid)75 2317 y(\(absolute\))h(addresses;)g(it)g(is)h FD(relativ)o(e)g FC(otherwise.)166 2374 y(A)f(correct)g(program)f(ob)q (eys)i(the)f(follo)o(wing)h(constrain)o(ts)e(in)i(the)g(use)f(of)g (datat)o(yp)q(es:)143 2478 y Fy(\017)23 b FC(If)d(the)41 b Fo(oldt)o(yp)q(e)21 b FC(argumen)o(t)e(used)i(in)41 b Fo(MPI)p 981 2478 14 2 v 16 w(TYPE)p 1114 2478 V 17 w(CONTIGUOUS,)22 b(MPI)p 1536 2478 V 15 w(TYPE)p 1668 2478 V 17 w(VECTOR,)189 2534 y(MPI)p 274 2534 V 15 w(TYPE)p 406 2534 V 17 w(HVECTOR,)d(MPI)p 748 2534 V 16 w(TYPE)p 881 2534 V 17 w(INDEX)f FC(or)36 b Fo(MPI)p 1210 2534 V 16 w(TYPE)p 1343 2534 V 17 w(HINDEX)18 b FC(is)h(absolute,)g(then)189 2591 y(all)c(addresses)g(within)g(it)g(m)o(ust)f(b)q(e)h(to)f(v)m (ariables)i(con)o(tained)f(within)g(the)g(same)f(arra)o(y)f(or)h (struc-)189 2647 y(ture;)j(the)g(result)h(datat)o(yp)q(e)e(is)i(also)f (absolute,)h(and)f(all)h(computed)f(addresses)h(m)o(ust)e(also)h(fall) 189 2704 y(within)f(the)f(same)g(arra)o(y)f(or)h(structure.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 55 61 55 60 bop 75 -100 a Fv(3.14.)34 b(UNIVERSAL)17 b(COMMUNICA)l(TION)f (FUNCTIONS)633 b FC(55)143 45 y Fy(\017)23 b FC(If)c(all)h(en)o(tries)f (of)f(the)38 b Fo(a)o(rra)o(y)p 708 45 14 2 v 15 w(of)p 760 45 V 16 w(t)o(yp)q(es)20 b FC(argumen)o(t)e(of)h Fo(MPI)p 1238 45 V 16 w(TYPE)p 1371 45 V 16 w(STRUC)h FC(are)f(absolute)g(ad-)189 102 y(dresses)f(\(computed)h(b)o(y)g Fo(MPI)p 727 102 V 15 w(ADDR)p FC(\),)g(then)f(the)h(result)g(datat)o (yp)q(e)f(is)h(also)g(absolute.)30 b(Eac)o(h)189 158 y(new)14 b(address)h(computed)g(from)f(an)g(old)h(address)g(m)o(ust)f (fall)h(within)h(the)f(same)f(arra)o(y)f(or)h(record)189 214 y(as)h(the)i(old)f(address.)23 b(\(Addresses)16 b(in)h(the)f(old)h (t)o(yp)q(e)f(ma)o(y)f(not)h(b)q(e)h(all)g(within)g(the)f(same)g(arra)o (y)189 271 y(or)e(record.\))143 374 y Fy(\017)23 b FC(If)17 b(a)f(comm)o(unication)i(call)g(is)f(in)o(v)o(ok)o(ed)g(with)g (parameters)33 b Fo(sta)o(rt)17 b FC(and)34 b Fo(datat)o(yp)q(e)p FC(,)18 b(then)f(either)189 430 y Fo(sta)o(rt)f FC(=)31 b Fr(MPI)p 434 430 13 2 v 15 w(BOTTOM)15 b FC(and)33 b Fo(datat)o(yp)q(e)18 b FC(is)e(a)g(handle)i(to)d(an)h(absolute)h (datat)o(yp)q(e,)e(or)32 b Fo(sta)o(rt)16 b FC(is)189 487 y(set)h(to)f(the)h(address)h(of)f(a)f(program)g(v)m(ariable,)j(and) f(all)g(displacemen)o(ts)g(in)36 b Fo(datat)o(yp)q(e)18 b FC(relativ)o(e)189 543 y(to)c(this)h(base)g(yield)i(addresses)e(that) f(are)h(within)h(the)f(same)g(arra)o(y)f(or)g(record)h(as)g(this)g(v)m (ariable.)166 661 y(In)j(summary)l(,)g(the)f(t)o(yp)q(e)h(constructors) 34 b Fo(MPI)p 984 661 14 2 v 16 w(TYPE)p 1117 661 V 17 w(CONTIGUOUS,)19 b(MPI)p 1536 661 V 15 w(TYPE)p 1668 661 V 17 w(VECTOR,)75 718 y(MPI)p 160 718 V 16 w(TYPE)p 293 718 V 17 w(HVECTOR,)c(MPI)p 631 718 V 16 w(TYPE)p 764 718 V 17 w(INDEX)f FC(and)29 b Fo(MPI)p 1114 718 V 16 w(TYPE)p 1247 718 V 17 w(HINDEX)14 b FC(can)h(b)q(e)g(recursiv)o (ely)h(ap-)75 774 y(plied)22 b(to)e(build)j(datat)o(yp)q(es)d(that)f (will)k(com)o(bine)e(v)m(ariables)h(that)e(b)q(elong)h(to)f(the)h(same) f(sequen)o(tial)75 831 y(storage;)d(v)m(ariables)i(that)e(do)g(not)g(b) q(elong)i(to)d(the)i(same)f(sequen)o(tial)i(storage)d(can)i(b)q(e)g (com)o(bined)g(to-)75 887 y(gether)d(using)h(one)f(application)i(of)e Fo(MPI)p 788 887 V 16 w(TYPE)p 921 887 V 16 w(STRUC)p FC(.)166 1022 y Fu(Discussion:)166 1074 y Ft(It)f(is)g(not)g(exp)q (ected)j(that)d(MPI)g(implemen)o(tations)d(will)i(b)q(e)i(able)f(to)g (detect)i(erroneous,)f(\\out)f(of)f(b)q(ound")75 1124 y(displacemen)o(ts)h({)f(unless)i(those)g(o)o(v)o(er\015o)o(w)f(the)g (user)h(address)h(space)f({)f(since)g(the)h(MPI)f(call)g(ma)o(y)e(not)i (no)o(w)f(the)75 1174 y(exten)o(t)i(of)e(the)i(arra)o(ys)f(and)f (records)j(in)d(the)i(host)f(program.)75 1413 y Fw(3.14)59 b(Universal)20 b(communication)d(functions)75 1519 y FC(Although)g(the)f(n)o(um)o(b)q(er)g(of)g(comm)o(unication)g (functions)h(de\014ned)h(in)e(this)h(c)o(hapter)f(is)g(fairly)h(large,) f(it)75 1575 y(is)e(p)q(ossible)g(to)f(construct)g(all)h(of)e(them)h (from)g(a)f(m)o(uc)o(h)h(smaller)h(n)o(um)o(b)q(er)g(of)e(primitiv)o(e) j(functions.)20 b(The)75 1632 y(construction)d(is)h(outlined)g(in)g (this)f(section.)26 b(The)17 b(construction)g(can)g(b)q(e)h(used)g(to)e (build)j(a)d(p)q(ortable)75 1688 y(implemen)o(tation)22 b(of)e(MPI,)h(although)g(it)g(is)g(exp)q(ected)h(that)e(more)h (e\016cien)o(t)g(implemen)o(tation)h(will)75 1744 y(implemen)o(t)h (directly)f(most)f(or)g(all)h(of)f(the)h(functions)g(in)g(this)g (section.)39 b(Also,)23 b(this)f(construction)75 1801 y(can)15 b(b)q(e)g(used)g(in)g(order)g(to)e(deriv)o(e)j(the)e(b)q(eha)o (vior)h(of)f(MPI)h(p)q(oin)o(t)g(to)f(p)q(oin)o(t)h(comm)o(unication)g (from)f(the)75 1857 y(b)q(eha)o(vior)e(of)f(a)g(small)h(subset)g(of)f (functions.)19 b(W)l(e)12 b(assume)f(the)g(a)o(v)m(ailabilit)o(y)j(of)d (the)g(primitiv)o(e)i(functions)75 1914 y(listed)j(at)f(the)g(end)h(of) f(the)g(section,)h(and)f(the)g(abilit)o(y)i(to)d(allo)q(cate)i(memory)f (for)f(new)i(v)m(ariables.)166 1973 y(W)l(e)g(assume)g(that)f(the)i (function)g Fo(MPI)p 846 1973 V 15 w(TYPE)p 978 1973 V 17 w(COMMIT)f FC(is)g(not)g(c)o(hanging)h(the)f(represen)o(tation)75 2029 y(of)f(a)g(datat)o(yp)q(e,)f(but)h(merely)h(creates)f(a)g(new)g (cop)o(y)g(of)g(it.)166 2088 y(Comm)o(unication)g(with)g(a)30 b Fo(count)16 b FC(argumen)o(t)e(that)g(is)i FB(>)d FC(1)h(can)h(b)q(e) h(reduced)g(to)e(comm)o(unication)75 2144 y(with)33 b Fo(count)17 b(=1)p FC(,)f(b)o(y)g(replicating)i(the)32 b Fo(datat)o(yp)q(e)18 b FC(argumen)o(t)32 b Fo(count)17 b FC(times.)23 b(Comm)o(unication)17 b(with)75 2201 y(a)29 b Fo(a)14 b(sta)o(rt)h FC(argumen)o(t)e(that)h(is)h(not)28 b Fr(MPI)p 780 2201 13 2 v 14 w(BOTTOM)14 b FC(can)h(b)q(e)g(reduced)g (to)f(comm)o(unication)h(with)30 b Fo(sta)o(rt)75 2257 y FC(=)f Fr(MPI)p 216 2257 V 15 w(BOTTOM)14 b FC(b)o(y)i(creating)f(a)g (suitable)h(displaced)h(new)f(datat)o(yp)q(e.)j(Th)o(us:)166 2316 y Fo(MPI)p 251 2316 14 2 v 16 w(SEND\()c(sta)o(rt,)g(count,)h (datat)o(yp)q(e,)h(dest,)f(tag,)f(comm\))e FC(is)75 2421 y Fp(Type[1])23 b(=)h(datatype)75 2478 y(MPI_ADDRESS\()e(start,)h (Index[1]\);)75 2534 y(Block[1])g(=)g(count;)75 2591 y(MPI_TYPE_STRUCT\()f(1,)h(Type,)g(Index,)g(Block,)h(Newtype\))75 2647 y(MPI_TYPE_COMMIT\()e(Newtype,)g(datatype1\))75 2704 y(MPI_SEND\()h(MPI_BOTTOM,)f(1,)i(datatype1,)e(dest,)h(tag,)h (comm\))-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 56 62 56 61 bop 75 -100 a FC(56)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FC(The)i(same)g (construction)h(applies)g(to)f(all)h(other)f(comm)o(unication)g (functions)h(with)g(argumen)o(ts)75 102 y Fo(count)d FC(and)29 b Fo(datat)o(yp)q(e)p FC(.)21 b(W)l(e)15 b(shall)h (henceforth)f(restrict)f(ourselv)o(es)h(to)f(comm)o(unication)i(that)d (in)o(v)o(olv)o(es)75 158 y(only)j(one)f(elemen)o(t)h(with)g(an)f (absolute)g(address)g(\()g Fo(count)i(=1)p FC(,)e(and)30 b Fo(sta)o(rt)16 b FC(=)29 b Fr(MPI)p 1521 158 13 2 v 14 w(BOTTOM)p FC(\).)166 216 y(\(If)11 b(the)g(op)q(eration)g Fo(MPI)p 583 216 14 2 v 16 w(TYPE)p 716 216 V 17 w(COMMIT\))f FC(is)h(not)g(assumed)g(to)g(b)q(e)g(trivial,)i(then)e(the)g(ab)q(o)o (v)o(e)g(con-)75 273 y(struction)f(cannot)g(b)q(e)h(used;)h(this)f (adds)f(one)g(more)g(primitiv)o(e)h(function,)h(namely)f Fo(MPI)p 1562 273 V 16 w(TYPE)p 1695 273 V 16 w(COMMIT)p FC(,)75 329 y(to)k(our)f(list,)i(and)g(in)o(tro)q(duces)g(small)g(c)o (hanges)f(in)h(the)f(construction.\))166 387 y(It)10 b(is)h(con)o(v)o(enien)o(t)g(to)e(to)h(mak)o(e)f(explicit)k(the)d FD(p)q(ersistence)g FC(attribute)g(of)g(comm)o(unication)h(ob)s(jects.) 75 444 y(A)30 b Fo(p)q(ersistent)18 b FC(comm)o(unication)d(ob)s(ject)g (need)h(b)q(e)f(explicitly)j(deallo)q(cated)f(b)o(y)e(a)29 b Fo(MPI)p 1594 444 V 16 w(COMM)p 1752 444 V 16 w(FREE)75 500 y FC(op)q(eration.)j(On)20 b(the)f(other)g(hand,)h(an)38 b Fo(ephemeral)19 b FC(comm)o(unication)h(ob)s(ject)f(is)g(go)q(o)q(d)g (for)g(a)g(single)75 557 y(comm)o(unication.)h(It)14 b(is)g(deallo)q(cated)i(b)o(y)e(the)g(system)f(when)h(the)g(\014rst)g (comm)o(unication)h(it)f(is)g(used)h(for)75 613 y(completes.)166 672 y(The)c(function)g Fo(MPI)p 513 672 V 16 w(COMM)p 671 672 V 16 w(INIT)f FC(is)h(a)f(univ)o(ersal)i(function)f(for)f(the)h (creation)f(of)g(comm)o(unication)75 728 y(ob)s(jects:)75 834 y Fo(MPI)p 160 834 V 16 w(COMM)p 318 834 V 16 w(INIT\(handle,)15 b(datat)o(yp)q(e,)i(source-dest,)g(tag,)e(comm,)e(op-mo)q(de,)h(p)q (ersistence\))117 913 y Ft(OUT)108 b Fo(handle)465 b Ft(handle)14 b(to)g(newly)f(created)j(comm)o(unicatio)o(n)11 b(ob)r(ject)117 991 y(IN)155 b Fo(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(elemen)o(t)f(sen)o(t)i(or)f(receiv)o(ed) 117 1070 y(IN)155 b Fo(source-dest)377 b Ft(rank)14 b(of)f(destination) h(or)g(source)h(\(in)o(teger\))117 1149 y(IN)155 b Fo(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 1228 y(IN)155 b Fo(comm)470 b Ft(comm)o(unication)11 b(ob)r(ject)117 1306 y(IN)155 b Fo(op-mo)q(de)422 b Ft(one)10 b(of)18 b Fr(MPI)p 1106 1306 13 2 v 14 w(ST)m(AND)o(ARD)p Ft(,)f Fr(MPI)p 1443 1306 V 14 w(READ)o(Y)p Ft(,)g Fr(MPI)p 1698 1306 V 15 w(SYNCHRONOUS)905 1363 y Ft(or)28 b Fr(MPI)p 1047 1363 V 14 w(RECV)117 1442 y Ft(IN)155 b Fo(p)q(ersistence)383 b Ft(one)14 b(of)27 b Fr(MPI)p 1119 1442 V 15 w(PERSISTENT)13 b Ft(or)28 b Fr(MPI)p 1534 1442 V 14 w(EPHEMERAL)166 1568 y FC(The)15 b(comm)o(unication)h(in)o(v)o(olv)o(es)g(only)g(one)f (elemen)o(t,)h(with)f(an)g(absolute)h(address.)75 1700 y Fq(3.14.1)49 b(P)o(ersistent)14 b(communication)j(objects)75 1789 y Fo(MPI)p 160 1789 14 2 v 16 w(CREA)l(TE)p 345 1789 V 17 w(SEND\(handle,)g(MPI)p 731 1789 V 15 w(BOTTOM,)f(1,)e(datat) o(yp)q(e,)j(dest,)f(tag,)f(comm\))e FC(is)75 1907 y Fp (MPI_COMM_INIT\(handle,)21 b(datatype,)i(dest,)g(tag,)g(comm,)g (MPI_STANDARD,)f(MPI_PERSISTENT\))166 2022 y FC(The)f(functions)g Fo(MPI)p 551 2022 V 16 w(CREA)l(TE)p 736 2022 V 17 w(RSEND)p FC(,)g Fo(MPI)p 1015 2022 V 16 w(CREA)l(TE)p 1200 2022 V 17 w(RSEND)g FC(and)g Fo(MPI)p 1560 2022 V 16 w(CREA)l(TE)p 1745 2022 V 17 w(RECV)75 2078 y FC(are)15 b(dealt)h(in)g(a)e(similar)j (manner.)166 2137 y(The)e(functions)h Fo(MPI)p 540 2137 V 16 w(ST)l(ART)g FC(and)g Fo(MPI)p 885 2137 V 15 w(COMM)p 1042 2137 V 17 w(FREE)g FC(are)e(primitiv)o(e.)75 2269 y Fq(3.14.2)49 b(Nonblo)q(cking)19 b(communication)d(initiation)75 2358 y Fo(MPI)p 160 2358 V 16 w(ISEND\(handle,)g(MPI)p 557 2358 V 15 w(BOTTOM,)g(1,)f(datat)o(yp)q(e,)h(dest,)g(tag,)f(comm\)) e FC(is)75 2475 y Fp(MPI_COMM_INIT\(handle,)21 b(datatype,)i(dest,)g (tag,)g(comm,)g(MPI_STANDARD,)f(MPI_EPHEMERAL\))75 2532 y(MPI_START\(handle\))166 2647 y FC(The)g(functions)g Fo(MPI)p 553 2647 V 16 w(IRSEND)p FC(,)f Fo(MPI)p 844 2647 V 16 w(ISSEND)h FC(and)g Fo(MPI)p 1214 2647 V 16 w(IRECV)g FC(are)f(handled)i(in)g(a)e(similar)75 2704 y(manner.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 57 63 57 62 bop 75 -100 a Fv(3.14.)34 b(UNIVERSAL)17 b(COMMUNICA)l(TION)f (FUNCTIONS)633 b FC(57)75 45 y Fq(3.14.3)49 b(Communication)17 b(completion)75 131 y FC(The)e(t)o(w)o(o)f(primitiv)o(e)j(completion)f (op)q(erations)g(are)e Fo(MPI)p 1063 131 14 2 v 16 w(W)l(AIT)l(ANY)i FC(and)g Fo(MPI)p 1467 131 V 16 w(TEST)l(ALL)p FC(.)166 187 y(The)c(function)h Fo(MPI)p 516 187 V 15 w(W)l(AIT)f FC(can)g(b)q(e)h(implemen)o(ted)g(b)o(y)f(a)f(call)i(to)e Fo(MPI)p 1368 187 V 16 w(W)l(AIT)l(ANY)i FC(with)f(an)24 b Fo(a)o(rra)o(y-)75 244 y(of-handles)17 b FC(argumen)o(t)d(of)h (length)h(one.)166 300 y(The)h(function)h Fo(MPI)p 526 300 V 16 w(W)l(AIT)l(ALL)f FC(can)g(b)q(e)h(implemen)o(ted)h(as)d(a)h (lo)q(op)h(where)f(an)34 b Fo(MPI)p 1664 300 V 16 w(W)l(AIT)17 b FC(call)75 357 y(is)f(executed)g(for)e(eac)o(h)i(successiv)o(e)g (handle)h(in)f(the)30 b Fo(a)o(rra)o(y-of-handles)p FC(.)166 413 y(The)13 b(function)g Fo(MPI)p 517 413 V 16 w(TEST)g FC(can)f(b)q(e)i(implemen)o(ted)g(b)o(y)e(a)h(call)g(to)f Fo(MPI)p 1377 413 V 16 w(TEST)l(ALL)h FC(with)g(an)25 b Fo(a)o(rra)o(y-)75 470 y(ofhandles)17 b FC(of)d(length)i(one.)166 526 y(The)h(function)h Fo(MPI)p 526 526 V 16 w(TEST)l(ALL)e FC(can)h(b)q(e)h(implemen)o(ted)h(as)d(a)h(lo)q(op)g(where)g(an)34 b Fo(MPI)p 1663 526 V 16 w(TEST)17 b FC(call)75 583 y(is)f(executed)g (for)e(eac)o(h)i(successiv)o(e)g(handle)h(in)f(the)30 b Fo(a)o(rra)o(y-of-handle)p FC(.)75 704 y Fq(3.14.4)49 b(Blo)q(cking)18 b(communication)75 789 y Fo(MPI)p 160 789 V 16 w(SEND\(MPI)p 393 789 V 16 w(BOTTOM,)d(1,)g(datat)o(yp)q(e,)h (dest,)h(tag,)d(comm\))f FC(is)75 890 y Fp(MPI_COMM_INIT\(handle,)21 b(datatype,)i(dest,)g(tag,)g(comm,)g(MPI_STANDARD,)f(MPI_EPHEMERAL\))75 946 y(MPI_START\(handle\))75 1003 y(MPI_WAIT\(handle,)g (dontcarestatus\))166 1104 y FC(The)15 b(functions)h Fo(MPI)p 540 1104 V 16 w(RSEND)g FC(and)f Fo(MPI)p 888 1104 V 16 w(SSEND)h FC(are)f(handled)i(in)f(a)f(similar)h(manner.)166 1160 y Fo(MPI)p 251 1160 V 16 w(RECV\(MPI)p 482 1160 V 16 w(BOTTOM,)g(1,)e(datat)o(yp)q(e,)j(source,)e(tag,)g(comm\))e FC(is)75 1261 y Fp(MPI_COMM_INIT\(handle,)21 b(datatype,)i(dest,)g (tag,)g(comm,)g(MPI_RECV,)g(MPI_EPHEMERAL\))75 1317 y (MPI_START\(handle\))75 1374 y(MPI_WAIT\(handle,)f(status\))75 1495 y Fq(3.14.5)49 b(Prob)q(e)17 b(and)f(cancel)75 1581 y FC(The)f(functions)h Fo(MPI)p 449 1581 V 16 w(PROBE)h FC(and)e Fo(MPI)p 800 1581 V 16 w(CANCEL)g FC(are)g(primitiv)o(e.)75 1701 y Fq(3.14.6)49 b(Return)16 b(status)75 1787 y FC(The)k(functions)g Fo(MPI)p 458 1787 V 16 w(GET)p 562 1787 V 16 w(SOURCE)p FC(,)h Fo(MPI)p 868 1787 V 16 w(GET)p 972 1787 V 16 w(T)l(A)o(G)p FC(,)f Fo(MPI)p 1189 1787 V 16 w(GET)p 1293 1787 V 16 w(LEN)p FC(,)f Fo(MPI)p 1507 1787 V 16 w(PROBE)p 1671 1787 V 18 w(LEN)p FC(,)g(and)75 1844 y Fo(MPI)p 160 1844 V 16 w(IS)p 214 1844 V 16 w(CANCELLED)c FC(are)g(primitiv)o(e.)166 1900 y(These)g(functions)h(are)f(simple)i(macros)d(that)h(access)g (records)g(in)h(a)f(structure.)75 2021 y Fq(3.14.7)49 b(send-receive)15 b(and)i(exchange)75 2107 y Fo(MPI)p 160 2107 V 16 w(SENDRECV\()22 b(send)p 531 2107 V 18 w(sta)o(rt,)g(send)p 753 2107 V 18 w(count,)h(send)p 995 2107 V 18 w(t)o(yp)q(e,)g(dest,)h(recv)p 1319 2107 V 16 w(sta)o(rt,)e(recv)p 1533 2107 V 17 w(count,)h(recv)p 1768 2107 V 16 w(t)o(yp)q(e,)75 2163 y(source,)15 b(tag,)g(comm,)e (status\))k FC(is)75 2264 y Fp(MPI_ISEND\(handle[0],)k(send_start,)h (send_count,)h(send_type,)f(dest,)i(tag,)f(comm\))75 2321 y(MPI_IRECV\(handle[1],)e(recv_start,)h(recv_count,)h(recv_type,)f (source,)h(tag,)h(comm\))75 2377 y(MPI_WAITALL\(2,)e(handle,)h (status_array\))75 2434 y(status)g(=)h(status_array[1])166 2534 y FC(The)13 b(non)o(blo)q(c)o(king)h(sends)f(and)g(receiv)o(es)h (can)f(b)q(e)g(replaced)h(b)o(y)f(their)g(primitiv)o(e)h(implemen)o (tation.)166 2591 y(The)f(function)i Fo(MPI)p 519 2591 V 15 w(EX)o(CHANGE)g FC(can)e(b)q(e)h(handled)h(in)f(a)f(similar)i (manner;)e(a)g(temp)q(orary)g(bu\013er)75 2647 y(need)20 b(b)q(e)h(allo)q(cated)f(to)f(replicate)i(the)e(send)h(and)g(receiv)o (e)h(bu\013er.)32 b(A)20 b(size)g(for)f(this)h(bu\013er)g(can)f(b)q(e) 75 2704 y(computed)d(using)g(the)f(function)h Fo(MPI)p 745 2704 V 16 w(TYPE)p 878 2704 V 16 w(EXTENT)p FC(.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 58 64 58 63 bop 75 -100 a FC(58)630 b Fv(SECTION)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fq(3.14.8)49 b(Derived)16 b(datat)o(yp)q(es)75 131 y FC(W)l(e)22 b(ha)o(v)o(e)f (outlined)i(in)f(Section)g(3.13.1)e(ho)o(w)h(eac)o(h)h(datat)o(yp)q(e)e (constructor)h(can)h(b)q(e)g(expressed)g(in)75 187 y(terms)d(of)g(the)g (next)g(one.)32 b(Th)o(us)20 b(all)g(datat)o(yp)q(e)f(constructors)f (can)h(b)q(e)h(expressed)g(in)g(terms)f(of)g(the)75 244 y(constructor)e Fo(MPI)p 403 244 14 2 v 15 w(TYPE)p 535 244 V 17 w(STRUC)p FC(;)h(the)f(functions)h Fo(MPI)p 1088 244 V 16 w(TYPE)p 1221 244 V 17 w(FREE)p FC(,)f Fo(MPI)p 1459 244 V 16 w(TYPE)p 1592 244 V 17 w(EXTENT)h FC(and)75 300 y Fo(MPI)p 160 300 V 16 w(ADDRESS)e FC(are)f(primitiv)o (e.)75 422 y Fq(3.14.9)49 b(Summa)o(ry)75 508 y FC(In)13 b(order)e(to)g(implemen)o(t)j(the)e(MPI)f(p)q(oin)o(t-to-p)q(oin)o(t)i (comm)o(unication)g(one)f(needs)g(16)g(functions;)h(ab)q(out)75 564 y(half)j(of)e(those)h(are)g(trivial)h(macros:)131 671 y(1.)22 b Fo(MPI)p 274 671 V 15 w(COMM)p 431 671 V 17 w(INIT)131 764 y FC(2.)g Fo(MPI)p 274 764 V 15 w(ST)l(ART)131 858 y FC(3.)g Fo(MPI)p 274 858 V 15 w(W)l(AIT)l(ALL)131 952 y FC(4.)g Fo(MPI)p 274 952 V 15 w(TEST)l(ANY)131 1046 y FC(5.)g Fo(MPI)p 274 1046 V 15 w(PROBE)131 1140 y FC(6.)g Fo(MPI)p 274 1140 V 15 w(CANCEL)131 1233 y FC(7.)g Fo(MPI)p 274 1233 V 15 w(GET)p 377 1233 V 17 w(SOURCE)131 1327 y FC(8.)g Fo(MPI)p 274 1327 V 15 w(GET)p 377 1327 V 17 w(T)l(A)o(G)131 1421 y FC(9.)g Fo(MPI)p 274 1421 V 15 w(GET)p 377 1421 V 17 w(LEN)108 1515 y FC(10.)g Fo(MPI)p 274 1515 V 15 w(PROBE)p 437 1515 V 18 w(LEN)108 1609 y FC(11.)g Fo(MPI)p 274 1609 V 15 w(IS)p 327 1609 V 17 w(CANCELLED)108 1702 y FC(12.)g Fo(MPI)p 274 1702 V 15 w(TYPE)p 406 1702 V 17 w(STRUC)108 1796 y FC(13.)g Fo(MPI)p 274 1796 V 15 w(TYPE)p 406 1796 V 17 w(EXTENT)108 1890 y FC(14.)g Fo(MPI)p 274 1890 V 15 w(ADDRESS)108 1984 y FC(15.)g Fo(MPI)p 274 1984 V 15 w(TYPE)p 406 1984 V 17 w(FREE)108 2078 y FC(16.)g Fo(MPI)p 274 2078 V 15 w(COMM)p 431 2078 V 17 w(FREE)166 2184 y FC(On)17 b(the)f(other)g(hand,)h(w)o(e)f(exp)q(ect)h(most)f(implemen) o(tations)h(to)f(use)h(more)e(primitiv)o(es,)j(in)f(order)75 2240 y(to)e(reduce)h(comm)o(unication)g(o)o(v)o(erheads,)e(reduce)i (bu\013ering,)f(and)h(impro)o(v)o(e)f(error)f(rep)q(orting.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 59 65 59 64 bop 75 356 a Fx(Section)35 b(4)75 564 y FE(Collecti)q(v)m(e)42 b(Comm)m(unication)75 805 y Fw(4.1)59 b(Intro)r(duction)75 906 y FC(Collectiv)o(e)13 b(comm)o(unication)f(is)g(de\014ned)h(to)e(b) q(e)h(comm)o(unication)g(that)f(in)o(v)o(olv)o(es)h(a)f(group)g(of)g (pro)q(cesses.)75 963 y(The)k(functions)h(pro)o(vided)g(b)o(y)g(the)f (MPI)g(collectiv)o(e)i(comm)o(unication)f(include:)143 1055 y Fy(\017)23 b FC(Broadcast)14 b(from)g(one)i(mem)o(b)q(er)f(to)g (all)h(mem)o(b)q(ers)f(of)g(a)g(group.)143 1148 y Fy(\017)23 b FC(Barrier)15 b(across)f(all)j(group)d(mem)o(b)q(ers)143 1242 y Fy(\017)23 b FC(Gather)14 b(data)h(from)f(all)i(group)f(mem)o(b) q(ers)g(to)g(one)g(mem)o(b)q(er.)143 1335 y Fy(\017)23 b FC(Scatter)14 b(data)h(from)f(one)i(mem)o(b)q(er)f(to)g(all)h(mem)o (b)q(ers)f(of)g(a)g(group.)143 1428 y Fy(\017)23 b FC(Global)15 b(op)q(erations)g(suc)o(h)h(as)e(sum,)h(max,)f(min,)i(etc.,)e(w)o(ere)h (the)g(result)g(is)h(kno)o(wn)e(b)o(y)h(all)h(group)189 1485 y(mem)o(b)q(ers)e(and)h(a)g(v)m(ariation)g(where)g(the)g(result)g (is)g(kno)o(wn)f(b)o(y)h(only)g(one)g(mem)o(b)q(er.)20 b(The)15 b(abilit)o(y)189 1541 y(to)f(ha)o(v)o(e)h(user)g(de\014ned)i (global)f(op)q(erations.)143 1634 y Fy(\017)23 b FC(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g(called)h (parallel)h(pre\014x\).)143 1728 y Fy(\017)23 b FC(Broadcast)14 b(from)g(all)j(mem)o(b)q(ers)e(to)f(all)j(mem)o(b)q(ers)e(of)g(a)g (group.)143 1821 y Fy(\017)23 b FC(Scatter)c(\(or)g(Gather\))g(data)g (from)g(all)i(mem)o(b)q(ers)f(to)f(all)i(mem)o(b)q(ers)f(of)g(a)f (group)h(\(also)f(called)189 1877 y(complete)d(exc)o(hange)f(or)g (all-to-all\).)75 1970 y(While)j(v)o(endors)f(ma)o(y)f(optimize)i (certain)g(collectiv)o(e)g(routines)g(for)e(their)h(arc)o(hitectures,)h (a)e(complete)75 2026 y(library)d(of)f(the)g(collectiv)o(e)i(comm)o (unication)f(routines)g(can)g(b)q(e)g(written)f(en)o(tirely)h(using)g (p)q(oin)o(t-to-p)q(oin)o(t)75 2083 y(comm)o(unication)j(functions.)166 2139 y(The)d(syn)o(tax)f(and)i(seman)o(tics)f(of)f(the)h(collectiv)o(e) i(op)q(erations)e(are)g(de\014ned)h(so)f(as)g(to)f(b)q(e)i(consisten)o (t)75 2195 y(with)22 b(the)f(syn)o(tax)g(and)h(seman)o(tics)f(of)g(the) h(p)q(oin)o(t-to-p)q(oin)o(t)g(op)q(erations.)39 b(A)21 b(collectiv)o(e)j(op)q(eration)75 2252 y(is)f(executed)h(b)o(y)f(ha)o (ving)g(all)g(pro)q(cesses)h(in)f(the)g(group)g(call)g(the)g(comm)o (unication)h(routine,)h(with)75 2308 y(matc)o(hing)14 b(parameters.)19 b(One)14 b(of)g(the)g(k)o(ey)g(parameters)f(is)i(a)e (comm)o(unicator)h(that)f(de\014nes)i(the)f(group)75 2365 y(of)i(participating)h(pro)q(cesses)f(and)h(pro)o(vides)f(a)g(con) o(text)f(for)h(the)g(op)q(eration.)23 b(The)16 b(reader)g(is)h (referred)75 2421 y(to)12 b(c)o(hapter)g(3)g(for)g(information)g (concerning)i(comm)o(unication)f(bu\013ers)f(and)h(their)g (manipulations)h(and)75 2478 y(t)o(yp)q(e)i(matc)o(hing)h(rules;)g(and) g(to)f(c)o(hapter)g(9)g(for)g(information)g(on)g(ho)o(w)g(to)g (de\014ne)i(groups)e(and)g(create)75 2534 y(comm)o(unicators.)166 2591 y(Collectiv)o(e)h(routines)g(can)f(\(but)g(are)g(not)g(required)h (to\))e(return)i(as)e(so)q(on)h(as)g(their)h(participation)75 2647 y(in)i(the)g(collectiv)o(e)h(comm)o(unication)f(is)g(complete.)31 b(The)19 b(completion)g(of)f(a)h(call)g(indicates)h(that)e(the)75 2704 y(caller)d(is)g(no)o(w)e(free)h(to)g(access)g(the)g(lo)q(cations)h (in)g(the)f(comm)o(unication)g(bu\013er,)g(or)g(an)o(y)g(other)f(lo)q (cation)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 60 66 60 65 bop 75 -100 a FC(60)724 b Fv(SECTION)16 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y FC(that)d(can)g(b)q(e)i (referenced)f(b)o(y)g(the)f(collectiv)o(e)i(op)q(eration.)20 b(It)14 b(do)q(es)f(not)h(indicate)h(that)d(other)i(pro)q(cesses)75 102 y(in)g(the)f(group)f(ha)o(v)o(e)h(started)f(the)h(op)q(eration)g (\(unless)h(otherwise)f(indicated)i(in)f(the)f(description)h(of)f(the) 75 158 y(op)q(eration\).)19 b(The)11 b(successful)i(completion)g(of)e (a)h(collectiv)o(e)h(comm)o(unication)f(call)h(ma)o(y)e(dep)q(end)i(on) f(the)75 214 y(execution)i(of)e(a)h(matc)o(hing)f(call)i(at)e(all)i (pro)q(cesses)f(in)h(the)f(group.)18 b(Th)o(us,)13 b(a)g(collectiv)o(e) h(comm)o(unication)75 271 y(call)h(ma)o(y)l(,)e(or)g(ma)o(y)g(not,)h (ha)o(v)o(e)f(the)h(e\013ect)g(of)f(sync)o(hronizing)i(all)g(calling)g (pro)q(cesses.)20 b(A)14 b(more)f(detailed)75 327 y(discussion)g(of)e (the)g(correct)g(use)g(of)g(the)g(collectiv)o(e)i(routines)f(can)f(b)q (e)h(found)f(at)g(the)g(end)h(of)f(this)h(c)o(hapter.)166 478 y Fu(Discussion:)33 b Ft(The)13 b(collectiv)o(e)g(op)q(erations)g (do)g(not)f(accept)j(a)d(message)h(tag)f(parameter.)17 b(The)d(rationale)75 534 y(for)j(not)g(using)g(tags)g(is)h(that)f(the)h (need)g(for)f(distinguishing)f(collectiv)o(e)h(op)q(erations)h(with)f (the)h(same)e(con)o(text)75 591 y(seldom)g(arises)i(\(since)g(the)g(op) q(erations)f(are)h(blo)q(c)o(king\);)f(the)h(tag)f(\014eld)g(can)h(b)q (e)f(used)i(b)o(y)e(the)g(p)q(oin)o(t-to-p)q(oin)o(t)75 647 y(messages)d(that)g(implemen)o(t)d(the)j(collectiv)o(e)g(comm)o (unication.)75 937 y Fw(4.2)59 b(Communication)18 b(F)n(unctions)75 1060 y FC(The)d(k)o(ey)h(concept)f(of)g(the)g(collectiv)o(e)i (functions)f(is)g(to)f(ha)o(v)o(e)f(a)h(\\group")g(of)g(participating)h (pro)q(cesses.)75 1116 y(The)j(routines)f(do)g(not)g(ha)o(v)o(e)g(a)g (group)g(iden)o(ti\014er)i(as)e(an)g(explicit)j(parameter.)28 b(Instead,)19 b(there)f(is)h(a)75 1173 y(comm)o(unicator)f(parameter.) 29 b(In)19 b(this)g(c)o(hapter)g(a)f(comm)o(unicator)g(can)h(b)q(e)g (though)o(t)f(of)g(as)g(a)g(group)75 1229 y(iden)o(ti\014er)f(merged)e (with)h(a)e(con)o(text.)166 1373 y Fu(Discussion:)38 b Ft(The)16 b(last)e(prop)q(osal)h(con)o(tained)g(t)o(w)o(o)g(la)o(y)o (ers)g(of)f(functions)h(one)g(\(con)o(tiguous\))g(bu\013er)h(for)75 1423 y(the)e(\\con)o(tiguous")g(functions,)f(and)h(an)f(arra)o(y)h(of)f (bu\013ers)j(for)d(the)i(noncon)o(tiguous)e(ones.)166 1484 y(The)e(latest)g(pt2pt)g(c)o(hapter)h(no)f(longer)g(con)o(tains)f (a)h(bu\013er)h(descriptor)g(handle,)f(instead)g(there)h(is)f(a)g (general)75 1533 y(datat)o(yp)q(e)16 b(parameter)f(whic)o(h)h(can)g (describ)q(e)h(arbitrary)f(structures.)26 b(I)16 b(ha)o(v)o(e)f(incorp) q(orated)h(this)g(feature)h(in)o(to)75 1583 y(the)g(collectiv)o(e)f (functions,)g(since)g(w)o(e)h(ha)o(v)o(e)e(agreed)i(that)f(the)h (collectiv)o(e)f(routines)g(m)o(ust)f(b)q(e)i(build-able)e(from)75 1633 y(the)f(pt2pt)g(routines.)75 1923 y Fw(4.3)59 b(Ba)n(rrier)21 b(synchronization)75 2093 y Fo(MPI)p 160 2093 14 2 v 16 w(BARRIER\()16 b(comm)d(\))117 2181 y Ft(IN)155 b Fo(comm)470 b Ft(comm)o(unicator)11 b(handle)166 2317 y Fo(MPI)p 251 2317 V 16 w(BARRIER)k FC(blo)q(c)o(ks)g(the)f(caller)h (un)o(til)g(all)f(group)g(mem)o(b)q(ers)g(ha)o(v)o(e)f(called)j(it;)e (the)g(call)h(returns)75 2373 y(at)g(an)o(y)f(pro)q(cess)i(only)g (after)e(all)i(group)f(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o(tered)h(the)f (call.)75 2580 y Fw(4.4)59 b(Data)19 b(move)g(functions)75 2704 y FC(Figure)c(4.1)g(illustrates)h(the)f(the)g(di\013eren)o(t)h (collectiv)o(e)h(mo)o(v)o(e)d(functions)i(supp)q(orted)g(b)o(y)f(MPI.) 1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 61 67 61 66 bop 75 -100 a Fv(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FC(61)75 2420 y @beginspecial @setspecial @endspecial 98 x(Figure)15 b(4.1:)k(Collectiv)o(e)e(mo)o(v)o(e)d (functions)i(illustrated)h(for)e(a)f(group)h(of)g(six)h(pro)q(cesses.)k (In)c(eac)o(h)f(case,)75 2574 y(eac)o(h)g(ro)o(w)g(of)g(b)q(o)o(xes)g (represen)o(ts)g(data)g(lo)q(cations)h(in)g(one)g(pro)q(cess.)k(Th)o (us,)15 b(in)h(the)f(one-all)i(broadcast,)75 2631 y(initially)k(just)e (the)f(\014rst)h(pro)q(cess)g(con)o(tains)f(the)h(data)f FB(A)1085 2638 y Fl(0)1105 2631 y FC(,)h(but)g(after)e(the)i(broadcast) f(all)i(pro)q(cesses)75 2687 y(con)o(tain)15 b(it.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 62 68 62 67 bop 75 -100 a FC(62)724 b Fv(SECTION)16 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fq(4.4.1)49 b(Broadcast)90 204 y Fo(MPI)p 175 204 14 2 v 16 w(BCAST\()16 b(bu\013er,)f(cnt,)g(t)o(yp)q(e,)h(ro)q(ot,)f(comm)e(\))117 295 y(IN/OUT)38 b(bu\013er)478 b Ft(starting)14 b(address)h(of)f (bu\013er)117 397 y(IN)171 b Fo(cnt)512 b Ft(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(bu\013er)117 499 y(IN)171 b Fo(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f(bu\013er)i(\(p)q (ossibly)f(general\))117 600 y(IN)171 b Fo(ro)q(ot)492 b Ft(rank)14 b(of)f(broadcast)i(ro)q(ot)117 702 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 840 y Fo(MPI)p 251 840 V 16 w(BCAST)g FC(broadcasts)e(a)h(message)g(from)g (the)g(pro)q(cess)h(with)f(rank)21 b Fo(ro)q(ot)10 b FC(to)f(all)j(other)e(pro)q(cesses)75 897 y(of)15 b(the)g(group.)20 b(It)15 b(is)h(called)h(b)o(y)e(all)h(mem)o(b)q(ers)g(of)f(group)g (using)h(the)f(same)g(argumen)o(ts)f(for)30 b Fo(cnt,)16 b(t)o(yp)q(e,)75 953 y(comm,)j(and)i(ro)q(ot)p FC(.)34 b(On)20 b(return)g(the)g(con)o(ten)o(ts)f(of)h(the)g(bu\013er)g(of)g (the)g(pro)q(cess)g(with)g(rank)40 b Fo(ro)q(ot)19 b FC(is)75 1010 y(con)o(tained)d(in)g(the)f(bu\013er)g(of)g(the)g (calling)i(pro)q(cess.)75 1208 y Fq(4.4.2)49 b(Gather)75 1367 y Fo(MPI)p 160 1367 V 16 w(GA)l(THER\()16 b(sendbuf,)h(sendcnt,)h (sendt)o(yp)q(e,)f(recvbuf,)f(recvcnt,)g(recvt)o(yp)q(e,)f(ro)q(ot,)g (comm\))117 1457 y Ft(IN)171 b Fo(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)117 1559 y(IN)171 b Fo(sendcnt)429 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h (bu\013er)f(\(in)o(teger\))117 1661 y(IN)171 b Fo(sendt)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)117 1763 y(OUT)124 b Fo(recvbuf)434 b Ft(address)15 b(of)f(receiv)o(e)h (bu\013er)g({)e(signi\014can)o(t)h(only)f(at)h(ro)q(ot)117 1865 y(IN)171 b Fo(recvcnt)435 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1967 y(IN)171 b Fo(recvt)o(yp)q(e)414 b Ft(data)13 b(t)o(yp)q(e)i(of)d (recv)j(bu\013er)g(elemen)o(ts)e({)g(signi\014can)o(t)g(only)g(at)905 2023 y(ro)q(ot)117 2125 y(IN)171 b Fo(ro)q(ot)492 b Ft(rank)14 b(of)f(receiving)i(pro)q(cess)g(\(in)o(teger\))117 2227 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 2365 y FC(Eac)o(h)17 b(pro)q(cess)g(\(including)i(the)e(ro)q(ot)f(pro)q (cess\))h(sends)g(the)g(con)o(ten)o(ts)g(of)f(its)h(send)h(bu\013er)f (to)f(the)75 2421 y(ro)q(ot)g(pro)q(cess.)25 b(The)17 b(ro)q(ot)f(pro)q(cess)h(places)h(all)f(the)g(incoming)h(messages)f(in) g(the)g(lo)q(cations)h(sp)q(eci\014ed)75 2478 y(b)o(y)e(the)g(recvbuf)h (and)f(recvt)o(yp)q(e.)23 b(The)16 b(receiv)o(e)h(bu\013er)f(is)g (ignored)h(for)e(all)i(non-ro)q(ot)f(pro)q(cesses.)23 b(The)75 2534 y(receiv)o(e)d(bu\013er)f(of)f(the)h(ro)q(ot)f(pro)q (cess)h(is)h(assumed)f(con)o(tiguous)g(and)g(partitioned)g(in)o(to)g Fo(MPI)p 1739 2534 V 16 w(GSIZE)75 2591 y FC(consecutiv)o(e)e(blo)q(c)o (ks,)h(eac)o(h)e(consisting)i(of)32 b Fo(sendcnt)20 b FC(elemen)o(ts.)k(The)17 b(data)f(sen)o(t)g(from)g(pro)q(cess)h(with)75 2647 y(rank)e(i)h(is)f(stored)g(in)h(the)f Fp(i)p FC(-th)g(blo)q(c)o (k.)21 b(The)15 b(function)i(is)e(called)i(with)f(the)f(same)g(v)m (alues)h(for)30 b Fo(sendcnt,)75 2704 y(sendt)o(yp)q(e,)17 b(ro)q(ot)p FC(,)e(and)30 b Fo(comm)14 b FC(at)g(all)i(participating)h (pro)q(cesses.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 63 69 63 68 bop 75 -100 a Fv(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FC(63)75 45 y Fq(4.4.3)49 b(Scatter)75 188 y Fo(MPI)p 160 188 14 2 v 16 w(SCA)l(TTER\()16 b(sendbuf,)h (sendcnt,)h(sendt)o(yp)q(e,)f(recvbuf,)e(recvcnt,)h(recvt)o(yp)q(e,)g (ro)q(ot,)e(comm\))117 270 y Ft(IN)171 b Fo(sendbuf)428 b Ft(address)15 b(of)f(send)g(bu\013er)h({)f(signi\014can)o(t)f(only)g (at)h(ro)q(ot)117 356 y(IN)171 b Fo(sendcnt)429 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 441 y(IN)171 b Fo(sendt)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)117 527 y(OUT)124 b Fo(recvbuf)434 b Ft(address)15 b(of)f(receiv)o(e)h(bu\013er.)117 612 y(IN)171 b Fo(recvcnt)435 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f (in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 697 y(IN)171 b Fo(recvt)o(yp)q(e)414 b Ft(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)117 783 y(IN)171 b Fo(ro)q(ot)492 b Ft(rank)14 b(of)f(sending)h(pro)q(cess)i(\(in)o (teger\))117 868 y(IN)171 b Fo(group)463 b Ft(comm)o(unicator)11 b(handle)166 998 y FC(The)16 b(ro)q(ot)g(pro)q(cess)g(sends)h(the)f Fp(i)p FC(-th)g(p)q(ortion)h(of)e(its)i(send)g(bu\013er)f(to)f(the)i (pro)q(cess)f(with)h(rank)f Fp(i)p FC(;)75 1054 y(eac)o(h)f(pro)q(cess) h(\(including)i(the)e(ro)q(ot)e(pro)q(cess\))h(stores)g(the)h(incoming) g(message)f(in)h(its)g(receiv)o(e)g(bu\013er.)75 1111 y(The)f(send)h(bu\013er)e(of)h(the)g(ro)q(ot)f(pro)q(cess)h(is)h (assumed)f(con)o(tiguous)f(and)i(partitioned)f(in)o(to)g Fo(MPI)p 1739 1111 V 16 w(GSIZE)75 1167 y FC(consecutiv)o(e)d(blo)q(c)o (ks,)h(eac)o(h)e(consisting)i(of)22 b Fo(recvcnt)12 b FC(elemen)o(ts.)19 b(The)12 b Fp(i)p FC(-th)f(blo)q(c)o(k)h(is)g(sen)o (t)f(to)g(the)h(pro)q(cess)75 1224 y(with)g(rank)f(i)h(in)h(the)e (group)h(and)f(stored)h(in)g(its)g(receiv)o(e)g(bu\013er.)19 b(The)12 b(routine)g(is)g(called)h(b)o(y)e(all)i(mem)o(b)q(ers)75 1280 y(of)i(the)g(group)g(using)h(the)f(same)g(argumen)o(ts)f(for)30 b Fo(recvcnt,)16 b(recvt)o(yp)q(e,)g(ro)q(ot)p FC(,)e(and)31 b Fo(comm)p FC(.)166 1342 y(Note)15 b(that)29 b Fo(MPI)p 474 1342 V 16 w(SCA)l(TTER)17 b FC(is)e(the)h(rev)o(erse)f(op)q (eration)g(to)g Fo(MPI)p 1322 1342 V 15 w(GA)l(THER)p FC(.)75 1493 y Fq(4.4.4)49 b(All-to-all)19 b(b)o(roadcast)75 1636 y Fo(MPI)p 160 1636 V 16 w(ALLCAST\()c(sendbuf,)i(sendcnt,)g (sendt)o(yp)q(e,)h(recvbuf,)d(recvcnt,)h(recvt)o(yp)q(e,)g(comm\))117 1718 y Ft(IN)171 b Fo(sendbuf)428 b Ft(starting)14 b(address)h(of)f (send)g(bu\013er)117 1803 y(IN)171 b Fo(sendcnt)429 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in) o(teger\))117 1889 y(IN)171 b Fo(sendt)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)117 1974 y(OUT)124 b Fo(recvbuf)434 b Ft(address)15 b(of)f(receiv)o(e)h (bu\013er.)117 2059 y(IN)171 b Fo(recvcnt)435 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 2145 y(IN)171 b Fo(recvt)o(yp)q(e)414 b Ft(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)117 2230 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 2360 y FC(Eac)o(h)16 b(pro)q(cess)g(in)h(the)f(group)g(broadcasts)g(its)g (en)o(tire)h(send)f(bu\013er)g(to)g(all)h(pro)q(cesses)g(\(including)75 2416 y(itself)t(\);)f(All)h(send)f(bu\013ers)g(ha)o(v)o(e)f(the)h(same) g(n)o(um)o(b)q(er)g(of)f(elemen)o(ts.)23 b(Eac)o(h)15 b(pro)q(cess)h(concatenates)g(the)75 2473 y(incoming)f(messages,)f(in)h (the)f(order)g(of)g(the)g(senders')g(ranks,)g(and)g(stores)g(them)g(in) h(its)f(receiv)o(e)h(bu\013er.)75 2529 y(The)i(routine)g(is)g(called)h (b)o(y)f(all)h(mem)o(b)q(ers)e(of)h(the)f(group)h(using)g(the)g(same)f (argumen)o(ts)g(for)33 b Fo(sendcnt,)75 2586 y(sendt)o(yp)q(e)p FC(,)17 b(and)31 b Fo(comm)p FC(.)166 2647 y Fo(MPI)p 251 2647 V 16 w(ALLCAST)19 b FC(is)h(equiv)m(alen)o(t)h(to)d FB(n)i FC(executions)g(of)f Fo(MPI)p 1220 2647 V 16 w(BCAST)p FC(,)g(with)h(eac)o(h)f(pro)q(cess)h(once)75 2704 y(the)15 b(ro)q(ot.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 64 70 64 69 bop 75 -100 a FC(64)724 b Fv(SECTION)16 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fq(4.4.5)49 b(All-to-all)19 b(scatter-gather)75 180 y Fo(MPI)p 160 180 14 2 v 16 w(ALL)l(TO)o(ALL\(sendbuf,)d(sendcnt,)i(sendt)o(yp)q(e,)f (recvbuf,)f(recvcnt,)f(recvt)o(yp)q(e,)h(comm\))117 259 y Ft(IN)171 b Fo(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g (bu\013er)117 336 y(IN)171 b Fo(sendcnt)429 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 413 y(IN)171 b Fo(sendt)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)117 490 y(OUT)124 b Fo(recvbuf)434 b Ft(address)15 b(of)f(receiv)o(e)h(bu\013er.)117 568 y(IN)171 b Fo(recvcnt)435 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f (in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 645 y(IN)171 b Fo(recvt)o(yp)q(e)414 b Ft(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)117 722 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 848 y FC(Eac)o(h)j(pro)q(cess)h(in)h(the)e(group)g(sends)i(the)e Fp(i)p FC(-th)h(p)q(ortion)g(of)f(its)g(send)i(bu\013er)e(to)g(the)h (pro)q(cess)g(with)75 904 y(rank)21 b Fp(i)g FC(\(itself)h(included\).) 40 b(All)22 b(messages)f(sen)o(t)g(from)f(one)i(pro)q(cess)f(to)f (another)h(ha)o(v)o(e)g(the)g(same)75 961 y(length.)k(The)16 b(send)i(bu\013er)e(of)g(eac)o(h)h(pro)q(cess)g(is)g(partitioned)g(in)o (to)g Fo(MPI)p 1346 961 V 16 w(GSIZE)f FC(consecutiv)o(e)i(blo)q(c)o (ks,)75 1017 y(eac)o(h)13 b(consisting)g(of)25 b Fo(sendcnt)16 b FC(elemen)o(ts.)j(The)13 b Fp(i)p FC(-th)g(blo)q(c)o(k)g(is)g(sen)o (t)g(to)f(the)h Fp(i)p FC(-th)f(pro)q(cess)h(in)h(the)f(group.)75 1074 y(Eac)o(h)k(pro)q(cess)g(concatenates)f(the)h(incoming)h (messages,)e(in)i(the)f(order)f(of)h(the)f(senders')h(ranks,)g(and)75 1130 y(store)c(them)h(in)g(its)g(receiv)o(e)g(bu\013er.)20 b(The)14 b(routine)g(is)g(called)h(b)o(y)f(all)g(mem)o(b)q(ers)g(of)f (the)h(group)f(using)i(the)75 1186 y(same)g(argumen)o(ts)f(for)30 b Fo(sendcnt,)17 b(sendt)o(yp)q(e)p FC(,)h(and)30 b Fo(comm)p FC(.)166 1244 y(An)15 b(all-to-all)h(scatter-gather)d(is)i(the)g(equiv) m(alen)o(t)h(of)e FB(n)h FC(scatters)e(\(or)h FB(n)h FC(gathers\))e(executed)j(with)75 1300 y(eac)o(h)f(pro)q(cess)h(b)q (eing)g(once)g(the)f(ro)q(ot.)166 1441 y Fu(Discussion:)30 b Ft(Do)10 b(w)o(e)h(w)o(an)o(t)f(to)h(ha)o(v)o(e)f(completely)f (general)i(v)o(ersions)g(of)f(the)h(ab)q(o)o(v)o(e)g(data)f(mo)o(v)o(e) f(routines?)75 1497 y(Without)14 b(bu\013er)h(descriptors)i(this)d(is)h (nearly)f(imp)q(ossible)f(in)h(fortran)h(b)q(ecause)h(of)e(v)n(ariable) g(length)g(argumen)o(t)75 1554 y(list)j(or)h(arra)o(y)p 300 1554 13 2 v 15 w(of)p 349 1554 V 14 w(addresses)i(requiremen)o(ts.) 30 b(In)18 b(C)g(it)f(is)h(merely)f(ugly)m(.)29 b(F)m(or)17 b(example)g(the)h(general)g(Alltoall)75 1610 y(w)o(ould)d(b)q(e:)23 b Fr(MPI)p 352 1610 V 15 w(ALL)m(TO)o(ALL\(a)o(rra)o(y)p 667 1610 V 15 w(of)p 716 1610 V 15 w(sendadr,)17 b(a)o(rra)o(y)p 976 1610 V 16 w(of)p 1026 1610 V 14 w(sendcnt,)h(a)o(rra)o(y)p 1285 1610 V 15 w(of)p 1334 1610 V 15 w(sendt)o(yp)q(e,)g(a)o(rra)o(y)p 1613 1610 V 15 w(of)p 1662 1610 V 14 w(recvadr,)g(a)o(r-)75 1666 y(ra)o(y)p 129 1666 V 15 w(of)p 178 1666 V 14 w(recvcnt,)f(a)o (rra)o(y)p 429 1666 V 15 w(of)p 478 1666 V 15 w(recvt)o(yp)q(e,)f (comm\))d Ft(It)i(is)g(not)g(clear)g(if)f(w)o(e)h(w)o(ould)f(w)o(an)o (t)g(to)h(use)h(an)e(arra)o(y)p 1666 1666 V 15 w(of)p 1715 1666 V 14 w(t)o(yp)q(es)i(or)75 1723 y(a)e(more)e(complex)h(datat) o(yp)q(e)h(with)g(end)p 711 1723 V 15 w(of)p 760 1723 V 15 w(bu\013er)h(mark)o(ers)e(inserted.)75 1955 y Fw(4.5)59 b(Global)20 b(Compute)e(Op)r(erations)75 2059 y FC(The)13 b(functions)g(in)h(this)f(section)h(p)q(erform)e(one)h(of)f(the)h (follo)o(wing)g(op)q(erations)g(across)f(all)i(the)f(mem)o(b)q(ers)75 2115 y(of)i(a)g(group:)189 2213 y(global)g(max)g(on)g(in)o(teger)h(and) f(\015oating)g(p)q(oin)o(t)h(data)e(t)o(yp)q(es)189 2311 y(global)h(min)h(on)g(in)o(teger)f(and)g(\015oating)g(p)q(oin)o(t)h (data)f(t)o(yp)q(es)189 2409 y(global)g(sum)h(on)f(in)o(teger)g(and)h (\015oating)f(p)q(oin)o(t)g(data)g(t)o(yp)q(es)189 2507 y(global)g(pro)q(duct)h(on)f(in)o(teger)h(and)f(\015oating)g(p)q(oin)o (t)h(data)e(t)o(yp)q(es)189 2605 y(global)h(AND)h(on)f(logical)h(and)f (in)o(teger)h(data)e(t)o(yp)q(es)189 2704 y(global)h(OR)h(on)g(logical) g(and)f(in)o(teger)h(data)e(t)o(yp)q(es)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 65 71 65 70 bop 75 -100 a Fv(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FC(65)189 45 y(global)15 b(X)o(OR)h(on)f(logical)i(and)e(in)o (teger)h(data)e(t)o(yp)q(es)189 142 y(global)h(max)g(and)h(who)f (\(rank\))f(has)h(it)189 239 y(global)g(min)h(and)g(who)f(\(rank\))f (has)h(it)189 336 y(user)g(de\014ned)i(\(asso)q(ciativ)o(e\))d(op)q (eration)189 434 y(user)h(de\014ned)i(\(asso)q(ciativ)o(e)e(and)g(comm) o(utativ)o(e\))f(op)q(eration)75 560 y Fq(4.5.1)49 b(Reduce)75 695 y Fo(MPI)p 160 695 14 2 v 16 w(REDUCE\()15 b(sendbuf,)j(recvbuf,)d (cnt,)h(t)o(yp)q(e,)g(op,)f(ro)q(ot,)f(comm\))117 773 y Ft(IN)171 b Fo(sendbuf)428 b Ft(address)15 b(of)f(send)g(bu\013er)117 850 y(OUT)124 b Fo(recvbuf)434 b Ft(address)15 b(of)f(receiv)o(e)h (bu\013er)g({)e(signi\014can)o(t)h(only)f(at)h(ro)q(ot)117 926 y(IN)171 b Fo(cnt)512 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f (in)h(input)g(bu\013er)g(\(in)o(teger\))117 1003 y(IN)171 b Fo(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f (input)h(bu\013er)117 1080 y(IN)171 b Fo(op)525 b Ft(op)q(eration)117 1157 y(IN)171 b Fo(ro)q(ot)492 b Ft(rank)14 b(of)f(ro)q(ot)h(pro)q (cess)i(\(in)o(teger\))117 1233 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 1359 y FC(Com)o(bines)19 b(the)f(v)m(alues)i(pro)o(vided)f(in)g(the)g(send)g(bu\013er)f(of)g (eac)o(h)g(pro)q(cess)h(in)g(the)g(group,)f(using)75 1415 y(the)d(op)q(eration)30 b Fo(op)p FC(,)15 b(and)g(returns)g(the)g (com)o(bined)h(v)m(alue)h(in)f(the)f(receiv)o(e)h(bu\013er)f(of)f(the)h (pro)q(cess)h(with)75 1472 y(rank)32 b Fo(ro)q(ot)p FC(.)21 b(Eac)o(h)16 b(pro)q(cess)g(can)g(pro)o(vide)h(one)f(v)m(alue,)h(or)e (a)h(sequence)h(of)e(v)m(alues,)i(in)g(whic)o(h)g(case)f(the)75 1528 y(com)o(bine)j(op)q(eration)g(is)g(executed)g(p)q(oin)o(t-wise)h (on)e(eac)o(h)g(en)o(try)g(of)g(the)h(sequence.)30 b(F)l(or)18 b(example,)h(if)75 1585 y(the)g(op)q(eration)f(is)h Fr(MPI)p 489 1585 13 2 v 15 w(MAX)f FC(and)h(the)f(send)h(bu\013er)g(con)o (tains)g(t)o(w)o(o)e(\015oating)h(p)q(oin)o(t)h(n)o(um)o(b)q(ers,)h (then)75 1641 y(recvbuf\(1\))d(=)h(global)h(max\(sendbuf\(1\)\))d(and)i (recvbuf\(2\))g(=)g(global)g(max\(sendbuf\(2\)\).)26 b(All)19 b(send)75 1697 y(bu\013ers)d(should)i(de\014ne)g(sequences)f (of)f(equal)i(length)f(of)f(en)o(tries)h(all)g(of)f(the)h(same)f(data)g (t)o(yp)q(e,)h(where)75 1754 y(the)d(t)o(yp)q(e)g(is)h(one)f(of)f (those)h(allo)o(w)o(ed)g(for)g(op)q(erands)g(of)28 b Fo(op)p FC(.)19 b(F)l(or)14 b(all)h(op)q(erations)f(except)27 b Fr(MPI)p 1706 1754 V 14 w(MINLOC)75 1810 y FC(and)j Fr(MPI)p 255 1810 V 14 w(MAXLOC)15 b FC(the)h(n)o(um)o(b)q(er)g(and)f (t)o(yp)q(e)h(of)f(elemen)o(ts)h(in)h(the)e(send)i(bu\013er)e(are)g (the)h(same)f(as)h(for)75 1867 y(the)e(receiv)o(e)h(bu\013ers.)k(F)l (or)26 b Fr(MPI)p 635 1867 V 14 w(MINLOC)14 b FC(and)26 b Fr(MPI)p 994 1867 V 15 w(MAXLOC)p FC(,)12 b(the)i(receiv)o(e)h (bu\013er)f(will)h(con)o(tain)28 b Fo(cnt)75 1923 y FC(elemen)o(ts)20 b(of)f(the)h(same)f(t)o(yp)q(e)h(as)f(the)h(elemen)o(ts)g(in)g(the)g (input)g(bu\013er,)g(follo)o(w)o(ed)g(b)o(y)39 b Fo(cnt)21 b FC(in)o(tegers)75 1980 y(\(ranks\).)166 2037 y(The)15 b(op)q(eration)g(de\014ned)h(b)o(y)30 b Fo(op)15 b FC(is)g(asso)q (ciativ)o(e)g(and)g(comm)o(utativ)o(e,)f(and)h(the)g(implemen)o(tation) 75 2093 y(can)h(tak)o(e)f(adv)m(an)o(tage)g(of)h(asso)q(ciativit)o(y)g (and)g(comm)o(utativit)o(y)f(in)i(order)f(to)f(c)o(hange)h(order)f(of)h (ev)m(alua-)75 2150 y(tion.)23 b(The)17 b(routine)f(is)h(called)h(b)o (y)e(all)h(group)f(mem)o(b)q(ers)g(using)h(the)f(same)g(argumen)o(ts)f (for)32 b Fo(cnt,)17 b(t)o(yp)q(e,)75 2206 y(op,)e(ro)q(ot)g FC(and)30 b Fo(comm)p FC(.)166 2264 y(W)l(e)15 b(list)h(b)q(elo)o(w)g (the)f(supp)q(orted)h(options)f(for)30 b Fo(op)p FC(.)117 2367 y Fr(MPI)p 194 2367 V 14 w(MAX)649 b FC(maxim)o(um)117 2423 y Fr(MPI)p 194 2423 V 14 w(MIN)664 b FC(minim)o(um)117 2479 y Fr(MPI)p 194 2479 V 14 w(SUM)653 b FC(sum)117 2535 y Fr(MPI)p 194 2535 V 14 w(PROD)626 b FC(pro)q(duct)117 2592 y Fr(MPI)p 194 2592 V 14 w(AND)654 b FC(and)15 b(\(logical)h(or)f (bit-wise)h(in)o(teger\))117 2648 y Fr(MPI)p 194 2648 V 14 w(OR)683 b FC(or)14 b(\(logical)j(or)d(bit-wise)j(in)o(teger\))117 2704 y Fr(MPI)p 194 2704 V 14 w(X)o(OR)656 b FC(xor)14 b(\(logical)i(or)f(bit-wise)i(in)o(teger\))-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 66 72 66 71 bop 75 -100 a FC(66)724 b Fv(SECTION)16 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)117 45 y Fr(MPI)p 194 45 13 2 v 14 w(MAXLOC)569 b FC(maxim)o(um)14 b(v)m(alue)h(and)f (rank)g(of)g(pro)q(cess)g(with)g(max-)905 102 y(im)o(um)h(v)m(alue)i (\(rank)d(of)g(\014rst)h(pro)q(cess)g(with)g(maxim)o(um)905 158 y(v)m(alue,)h(in)g(case)g(of)e(ties\))117 211 y Fr(MPI)p 194 211 V 14 w(MINLOC)584 b FC(minim)o(um)17 b(v)m(alue)h(and)f(rank)f (of)g(pro)q(cess)h(with)g(min-)905 267 y(im)o(um)g(v)m(alue)g(\(rank)f (of)f(\014rst)h(pro)q(cess)g(with)h(minim)o(um)905 324 y(v)m(alue,)f(in)g(case)g(of)e(ties\))166 407 y(All)19 b(op)q(erations,)f(with)g(the)f(exception)i(of)33 b Fr(MPI)p 1019 407 V 15 w(MAXLOC)16 b FC(and)34 b Fr(MPI)p 1404 407 V 15 w(MINLOC)17 b FC(return)g(a)h(v)m(alue)75 463 y(whic)o(h)11 b(has)f(the)g(same)g(datat)o(yp)q(e)g(as)f(the)i(op)q (erands.)18 b(Eac)o(h)10 b(op)q(erand)h(of)19 b Fr(MPI)p 1389 463 V 14 w(MAXLOC)9 b FC(and)20 b Fr(MPI)p 1752 463 V 14 w(MINLOC)75 520 y FC(can)15 b(b)q(e)h(though)o(t)e(as)g(a)h (pair)g Fp(\(v,)24 b(i\))p FC(:)19 b Fp(i)c FC(is)h(the)f(rank)f(of)h (the)g(calling)i(pro)q(cess)e(that)f(is)h(passed)h(implic-)75 576 y(itly)l(,)f(and)f Fp(v)f FC(is)h(the)g(v)m(alue)h(that)e(is)h (explicitly)j(passed)d(to)f(the)h(call.)38 b Fr(MPI)p 1332 576 V 15 w(MAXLOC)12 b FC(and)27 b Fr(MPI)p 1706 576 V 14 w(MINLOC)75 633 y FC(return)15 b(\(explicitly\))i(a)e(pair)h Fp(\(value,)23 b(rank\))p FC(.)166 689 y(When)33 b Fr(MPI)p 393 689 V 15 w(MINLOC)17 b FC(or)33 b Fr(MPI)p 731 689 V 14 w(MAXLOC)16 b FC(are)h(in)o(v)o(ok)o(ed,)h(the)g(input)g(bu\013er) f(should)i(con)o(tain)e FB(m)75 746 y FC(elemen)o(ts)h(of)f(the)h(same) f(t)o(yp)q(e)h(to)f(whic)o(h)h(the)g(op)q(eration)34 b Fr(MPI)p 1189 746 V 14 w(MIN)18 b FC(or)33 b Fr(MPI)p 1447 746 V 14 w(MAX)17 b FC(can)h(b)q(e)g(applied.)75 802 y(The)f(op)q(eration)f(returns)g(at)g(the)g(ro)q(ot)g FB(m)g FC(elemen)o(ts)h(of)f(the)g(same)g(t)o(yp)q(e)h(as)f(the)g (inputs,)h(follo)o(w)o(ed)g(b)o(y)75 859 y FB(m)e FC(in)o(tegers)g (\(ranks\).)k(The)d(output)f(bu\013er)g(should)h(b)q(e)g(de\014ned)g (accordingly)l(.)166 915 y(The)f(op)q(eration)h(that)e(de\014nes)30 b Fr(MPI)p 801 915 V 15 w(MAXLOC)14 b FC(is)721 967 y Ff( )774 1011 y FB(u)779 1067 y(i)821 967 y Ff(!)864 1039 y Fy(\016)897 967 y Ff( )950 1011 y FB(v)952 1067 y(j)995 967 y Ff(!)1040 1039 y FC(=)1088 967 y Ff( )1142 1011 y FB(w)1146 1067 y(k)1196 967 y Ff(!)75 1163 y FC(where)833 1220 y FB(w)f FC(=)g(max\()p FB(u;)8 b(v)r FC(\))75 1299 y(and)710 1402 y FB(k)13 b FC(=)795 1303 y Ff(8)795 1341 y(>)795 1353 y(<)795 1428 y(>)795 1440 y(:)853 1345 y FB(i)194 b FC(if)15 b FB(u)e(>)g(v)853 1402 y FC(min)q(\()p FB(i;)8 b(j)s FC(\))39 b(if)15 b FB(u)e FC(=)g FB(v)853 1458 y(j)191 b FC(if)15 b FB(u)e(<)g(v)166 1536 y FC(Note)i(that)f (this)i(op)q(eration)f(is)h(asso)q(ciativ)o(e)g(and)f(comm)o(utativ)o (e.)166 1593 y(A)g(similar)i(de\014nition)g(can)e(b)q(e)h(giv)o(en)f (for)29 b Fr(MPI)p 991 1593 V 14 w(MINLOC)p FC(.)166 1725 y Fu(Discussion:)166 1775 y Ft(W)m(e)19 b(de\014ne)40 b Fr(MPI)p 465 1775 V 14 w(MINLOC)19 b Ft(to)g(return)i(a)e(v)o(ector)h (of)f(v)n(alues,)h(follo)o(w)o(ed)e(b)o(y)h(a)g(v)o(ector)h(of)f (ranks.)35 b(The)75 1825 y(alternativ)o(e)13 b(is)g(for)27 b Fr(MPI)p 477 1825 V 14 w(MINLOC)13 b Ft(to)g(return)i(a)e(v)o(ector)h (of)f(\(v)n(alue,)g(rank\))g(pairs,)g(i.e.,)f(a)h(v)o(ector)h(of)f (structures.)75 1875 y(This)18 b(second)i(c)o(hoice)f(is)g(less)g(con)o (v)o(enien)o(t)g(for)f(F)m(ortran.)32 b(Another)20 b(alternativ)o(e)e (is)g(to)h(ha)o(v)o(e)37 b Fr(MPI)p 1706 1875 V 14 w(MINLOC)75 1925 y Ft(return)15 b(t)o(w)o(o)e(output)i(bu\013ers,)f(but)g(then)h (it)e(need)i(b)q(e)g(in)o(v)o(ok)o(ed)e(di\013eren)o(tly)h(than)g(the)h (other)f(op)q(erations.)166 1974 y(The)f(computation)d(can)i(still)g(b) q(e)g(pip)q(elined,)g(pro)o(vided)g(that)g(the)h(lo)q(cation)e(of)h (the)h(\014rst)g(rank)f(en)o(try)h(in)e(the)75 2024 y(output)j (bu\013er)h(can)f(b)q(e)h(computed)e(up)h(fron)o(t.)166 2239 y Fu(Implemen)o(tati)o(on)e(note:)166 2289 y Ft(The)i(op)q (erations)g(can)g(b)q(e)g(applied)g(to)f(op)q(erands)i(of)e(di\013eren) o(t)i(t)o(yp)q(es,)f(in)f(di\013eren)o(t)i(calls:)i(e.g.,)c Fr(MPI)p 1775 2289 V 14 w(SUM)75 2339 y Ft(ma)o(y)h(require)k(an)e(in)o (teger)g(sum)g(in)g(one)g(call,)g(and)g(a)g(complex)f(sum)g(in)h (another.)26 b(Since)17 b(w)o(e)g(require)g(that)f(all)75 2389 y(elemen)o(ts)f(b)q(e)h(of)f(the)h(same)f(datat)o(yp)q(e,)g(it)h (is)f(not)g(necessary)j(to)d(store)h(a)g(full)e(signature)i(with)f(eac) o(h)h(bu\013er:)22 b(It)75 2439 y(is)14 b(only)e(necessary)k(to)e (store)g(the)h(datat)o(yp)q(e)f(of)f(the)h(elemen)o(ts)g(when)g(all)e (elemen)o(ts)i(are)g(of)f(the)h(same)f(t)o(yp)q(e,)h(and)75 2488 y(store)h(a)e(\015ag)h(indicating)f(that)g(the)i(bu\013er)g(is)f (not)f(homogeneous,)g(otherwise.)166 2704 y Fu(Missing:)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 67 73 67 72 bop 75 -100 a Fv(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FC(67)166 45 y Ft(Need)19 b(to)f(de\014ne)h(the)g(t)o(yp)q(es)g (compatible)d(with)i(eac)o(h)h(op)q(eration.)30 b(This)18 b(includes)h Fr(MPI)p 1611 45 13 2 v 14 w(BYTE)f Ft(for)f(the)75 95 y(logical)12 b(op)q(erations,)i(and)g(whatev)o(er)g(F)m(ortran/C)g (allo)o(w)e(for)h(all)g(op)q(erations.)75 281 y Fo(MPI)p 160 281 14 2 v 16 w(USER)p 288 281 V 17 w(REDUCE\()j(sendbuf,)h (recvbuf,)f(cnt,)f(t)o(yp)q(e,)h(function,)h(ro)q(ot,)d(comm\))117 359 y Ft(IN)171 b Fo(sendbuf)428 b Ft(starting)14 b(address)h(of)f (send)g(bu\013er)117 433 y(OUT)124 b Fo(recvbuf)434 b Ft(starting)15 b(address)g(of)f(receiv)o(e)i(bu\013er)f({)f (signi\014can)o(t)g(only)g(at)905 490 y(ro)q(ot)117 565 y(IN)171 b Fo(cnt)512 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h (input)g(bu\013er)g(\(in)o(teger\))117 639 y(IN)171 b Fo(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f (input)h(bu\013er)117 714 y(IN)171 b Fo(function)418 b Ft(user)15 b(de\014ned)g(function)117 789 y(IN)171 b Fo(ro)q(ot)492 b Ft(rank)14 b(of)f(ro)q(ot)h(pro)q(cess)i(\(in)o (teger\))117 864 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 988 y FC(Similar)20 b(to)f(the)g(reduce)h(op)q(eration)f (function)h(ab)q(o)o(v)o(e)f(except)g(that)g(a)f(user)i(supplied)h (function)75 1045 y(is)f(used.)64 b Fo(function)20 b FC(is)g(a)f(function)h(with)f(three)g(argumen)o(ts.)31 b(A)19 b(C)g(protot)o(yp)q(e)f(for)h(suc)o(h)g(function)75 1101 y(is)g Fp(f\()k(invec,)g(inoutvec,)g(*len\))p FC(.)k(Both)36 b Fo(invec)19 b FC(and)36 b Fo(inoutvec)19 b FC(are)f(arra)o(ys)e(with) 37 b Fo(*len)18 b FC(en)o(tries.)75 1158 y(The)h(function)g(computes)g (p)q(oin)o(t-wise)h(a)e(comm)o(utativ)o(e)g(and)h(asso)q(ciativ)o(e)g (op)q(eration)f(on)h(eac)o(h)g(pair)75 1214 y(of)e(en)o(tries)h(and)g (returns)f(the)h(result)g(in)36 b Fo(inoutvec)p FC(.)28 b(A)17 b(pseudo-co)q(de)i(for)35 b Fo(function)19 b FC(is)f(giv)o(en)g (b)q(elo)o(w,)75 1270 y(where)d Fp(op)g FC(is)h(the)f(comm)o(utativ)o (e)g(and)g(asso)q(ciativ)o(e)h(op)q(eration)f(de\014ned)i(b)o(y)30 b Fo(function)p FC(.)361 1374 y Fp(for\(i=0;)23 b(i)h(<)g(*len;)f (i++\))g({)552 1431 y(inoutvec[i])g(op=)g(invec[i])361 1487 y(})166 1591 y FC(The)16 b(t)o(yp)q(e)h(of)f(the)g(elemen)o(ts)h (of)32 b Fo(invec)17 b FC(and)g(of)32 b Fo(inoutvec)18 b FC(matc)o(h)d(the)i(t)o(yp)q(e)f(of)g(the)g(elemen)o(ts)h(of)75 1648 y(the)e(send)h(bu\013ers)f(and)h(the)f(receiv)o(e)h(bu\013er.)75 1751 y Fo(MPI)p 160 1751 V 16 w(USER)p 288 1751 V 17 w(REDUCEA\()g(sendbuf,)h(recvbuf,)f(cnt,)g(t)o(yp)q(e,)g(function,)g (ro)q(ot,)e(comm\))117 1829 y Ft(IN)171 b Fo(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)117 1903 y(OUT)124 b Fo(recvbuf)434 b Ft(starting)15 b(address)g(of)f (receiv)o(e)i(bu\013er)f({)f(signi\014can)o(t)g(only)g(at)905 1960 y(ro)q(ot)117 2035 y(IN)171 b Fo(cnt)512 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g(\(in)o(teger\))117 2109 y(IN)171 b Fo(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(input)h(bu\013er)117 2184 y(IN)171 b Fo(function)418 b Ft(user)15 b(de\014ned)g(function)117 2259 y(IN)171 b Fo(ro)q(ot)492 b Ft(rank)14 b(of)f(ro)q(ot)h(pro)q (cess)i(\(in)o(teger\))117 2334 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 2458 y FC(Iden)o(tical)16 b(to)f Fo(MPI)p 493 2458 V 16 w(USER)p 621 2458 V 17 w(REDUCE)p FC(,)g(except)g(that)g(the)g(op)q(eration)g(de\014ned)h(b)o (y)30 b Fo(function)16 b FC(is)f(not)75 2515 y(required)j(to)e(b)q(e)h (comm)o(utativ)o(e,)f(but)g(only)h(asso)q(ciativ)o(e.)25 b(Th)o(us,)16 b(the)h(order)f(of)g(computation)h(can)f(b)q(e)75 2571 y(mo)q(di\014ed)h(only)e(using)h(asso)q(ciativit)o(y)l(.)166 2704 y Fu(Implemen)o(tati)o(on)c(note:)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 68 74 68 73 bop 75 -100 a FC(68)724 b Fv(SECTION)16 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y Ft(The)j(co)q(de)h(for)e Fr(MPI)p 502 45 13 2 v 14 w(USER)p 620 45 V 14 w(REDUCEA)f Ft(can)i(b)q(e)g(used)h(to)e(pro)o(vide)h(an)f(iden)o(tical)g(implemen) o(tation)d(for)75 102 y Fr(MPI)p 152 102 V 14 w(USER)p 270 102 V 14 w(REDUCE)p Ft(.)166 353 y Fu(Discussion:)166 446 y Ft(The)c(addition)f(of)h(the)g(third)g(parameter,)22 b Fr(*len)12 b Ft(in)21 b Fr(function)12 b Ft(allo)o(w)e(the)h(system)g (to)g(a)o(v)o(oid)e(calling)21 b Fr(function)75 503 y Ft(for)15 b(eac)o(h)g(elemen)o(t)g(in)f(the)i(input)e(bu\013er;)i (rather,)g(the)g(system)e(can)h(c)o(ho)q(ose)h(to)f(apply)29 b Fr(function)16 b Ft(to)f(c)o(h)o(unks)h(of)75 559 y(inputs,)g(where)h (the)f(size)h(of)e(the)h(c)o(h)o(unk)g(is)g(c)o(hosen)h(b)o(y)e(the)i (system)e(so)h(as)g(to)g(optimize)e(comm)o(unicatio)o(n)f(and)75 616 y(computation)f(pip)q(elining.)17 b(E.g.,)27 b Fr(*len)14 b Ft(could)g(b)q(e)h(set)g(to)f(b)q(e)g(the)h(t)o(ypical)e(pac)o(k)o (et)h(size)h(in)f(the)g(comm)o(unication)75 672 y(subsystem.)166 924 y Fu(Missing:)166 1010 y Ft(The)k(last)g(appro)o(v)o(ed)g(draft)f (has)h(an)g(additional)34 b Fr(unitsize)19 b Ft(parameter)e(in)g Fr(MPI)p 1464 1010 V 15 w(USER)p 1583 1010 V 14 w(REDUCE)p Ft(:)f(Eac)o(h)75 1060 y(elemen)o(t)g(of)31 b Fr(invec)17 b Ft(or)32 b Fr(inoutvec)18 b Ft(corresp)q(onds)g(to)32 b Fr(unitsize)17 b Ft(elemen)o(ts)f(of)f(the)i(input)f(\(output\))h (bu\013er.)25 b(This)75 1110 y(allo)o(ws,)16 b(for)i(example,)e(to)h (pass)h(to)f Fr(MPI)p 739 1110 V 15 w(USER)p 858 1110 V 14 w(REDUCE)e Ft(a)j(v)o(ector)g(of)e(real)i(n)o(um)o(b)q(ers,)f(and) g(ha)o(v)o(e)35 b Fr(function)75 1160 y Ft(treat)16 b(eac)o(h)g(pair)e (of)h(real)g(n)o(um)o(b)q(ers)g(as)g(one)g(complex)f(n)o(um)o(b)q(er.) 22 b(I)15 b(am)e(not)i(sure)i(w)o(e)e(need)h(this)f(if)g(w)o(e)g(appro) o(v)o(e)75 1209 y(the)g(prop)q(osal)f(to)g(use)i(bu\013er)f(t)o(yp)q (es:)20 b(It)14 b(w)o(e)h(do)f(so)h(it)f(will)f(b)q(e)i(easy)f(to)h (de\014ne)g(the)g(bu\013er)h(to)e(consists)h(of)f(pairs)75 1259 y(of)f(real)h(n)o(um)o(b)q(ers,)f(sa)o(y)m(.)166 1435 y FC(MPI)j(also)f(includes)j(v)m(arian)o(ts)e(of)f(eac)o(h)h(of)f (the)h(reduce)g(op)q(erations)g(where)g(the)g(result)g(is)g(kno)o(wn)75 1491 y(to)f(all)h(pro)q(cesses)f(in)h(the)g(group)f(on)g(return.)75 1631 y Fo(MPI)p 160 1631 14 2 v 16 w(ALLREDUCE\()g(sendbuf,)i(recvbuf,) f(cnt,)g(t)o(yp)q(e,)f(op,)g(comm\))117 1745 y Ft(IN)171 b Fo(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)117 1893 y(OUT)124 b Fo(recvbuf)434 b Ft(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)117 2042 y(IN)171 b Fo(cnt)512 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g (\(in)o(teger\))117 2190 y(IN)171 b Fo(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er) 117 2338 y(IN)171 b Fo(op)525 b Ft(op)q(eration)117 2486 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 2647 y FC(Same)19 b(as)h(the)f Fo(MPI)p 519 2647 V 16 w(REDUCE)h FC(op)q(eration)g(function)g(except)g(that)f(the)h(result)g (app)q(ears)f(in)i(the)75 2704 y(receiv)o(e)16 b(bu\013er)f(of)g(all)h (the)f(group)g(mem)o(b)q(ers.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 69 75 69 74 bop 75 -100 a Fv(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FC(69)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(USER)p 288 45 V 17 w(ALLREDUCE\()15 b(sendbuf,)j(recvbuf,)d(cnt,)h(t)o(yp)q (e,)g(function,)g(comm\))117 123 y Ft(IN)171 b Fo(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)117 198 y(OUT)124 b Fo(recvbuf)434 b Ft(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)117 274 y(IN)171 b Fo(cnt)512 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g (\(in)o(teger\))117 350 y(IN)171 b Fo(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er) 117 426 y(IN)171 b Fo(function)418 b Ft(user)15 b(de\014ned)g(function) 117 501 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 626 y FC(Same)k(as)g(the)g Fo(MPI)p 506 626 V 16 w(USER)p 634 626 V 18 w(REDUCE)h FC(op)q(eration)f(function)h (except)g(that)f(the)g(result)h(app)q(ears)f(in)75 683 y(the)g(receiv)o(e)h(bu\013er)g(of)e(all)i(the)g(group)f(mem)o(b)q (ers.)75 787 y Fo(MPI)p 160 787 V 16 w(USER)p 288 787 V 17 w(ALLREDUCEA\()h(sendbuf,)h(recvbuf,)e(cnt,)h(t)o(yp)q(e,)g (function,)g(comm\))117 864 y Ft(IN)171 b Fo(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)117 940 y(OUT)124 b Fo(recvbuf)434 b Ft(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)117 1016 y(IN)171 b Fo(cnt)512 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g (\(in)o(teger\))117 1091 y(IN)171 b Fo(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er) 117 1167 y(IN)171 b Fo(function)418 b Ft(user)15 b(de\014ned)g (function)117 1243 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 1367 y FC(Same)16 b(as)f Fo(MPI)p 429 1367 V 16 w(USER)p 557 1367 V 18 w(REDUCEA)p FC(,)h(except)g(that)g(the)g (result)g(app)q(ears)g(in)h(the)f(receiv)o(e)h(bu\013er)e(of)75 1424 y(all)h(the)f(group)g(mem)o(b)q(ers.)166 1557 y Fu(Implemen)o(tati)o(on)d(note:)166 1613 y Ft(The)j(allreduce)h(op)q (erations)f(can)g(b)q(e)h(implem)o(en)o(ted)d(as)i(a)g(reduce,)h(follo) o(w)o(ed)e(b)o(y)h(a)f(broadcast.)22 b(Ho)o(w)o(ev)o(er,)75 1670 y(a)14 b(direct)g(implemen)o(tation)d(can)j(lead)f(to)h(b)q(etter) i(p)q(erformance.)75 1876 y Fq(4.5.2)49 b(Scan)75 2010 y Fo(MPI)p 160 2010 V 16 w(SCAN\()15 b(sendbuf,)j(recvbuf,)d(cnt,)h(t)o (yp)q(e,)g(op,)f(comm)e(\))117 2087 y Ft(IN)171 b Fo(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)117 2163 y(IN)171 b Fo(recvbuf)434 b Ft(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)117 2239 y(IN)171 b Fo(cnt)512 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g (\(in)o(teger\))117 2314 y(IN)171 b Fo(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er) 117 2390 y(IN)171 b Fo(op)525 b Ft(op)q(eration)117 2466 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 2591 y Fo(MPI)p 251 2591 V 16 w(SCAN)23 b FC(is)g(used)h(to)e(p)q (erform)g(a)h(parallel)h(pre\014x)f(with)g(resp)q(ect)g(to)f(an)h(asso) q(ciativ)o(e)g(and)75 2647 y(comm)o(utativ)o(e)e(reduction)i(op)q (eration)g(on)f(data)f(distributed)j(across)d(the)h(group.)40 b(The)23 b(op)q(eration)75 2704 y(returns)17 b(in)h(the)f(receiv)o(e)i (bu\013er)e(of)g(the)g(pro)q(cess)g(with)h(rank)f Fp(i)g FC(the)g(reduction)h(of)f(the)g(v)m(alues)i(in)f(the)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 70 76 70 75 bop 75 -100 a FC(70)729 b Fv(SECTION)16 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y FC(send)22 b(bu\013ers)f(of)f(pro)q(cesses)i(with)g(ranks)e Fp(0,...,i)p FC(.)37 b(The)21 b(t)o(yp)q(e)h(of)e(op)q(erations)i(supp)q(orted,)g (their)75 102 y(seman)o(tics,)15 b(and)g(the)h(constrain)o(ts)e(on)h (send)h(and)g(receiv)o(e)g(bu\013ers)f(are)g(as)g(for)f Fo(MPI)p 1531 102 14 2 v 16 w(REDUCE)p FC(.)75 205 y Fo(MPI)p 160 205 V 16 w(USER)p 288 205 V 17 w(SCAN\()i(sendbuf,)h (recvbuf,)f(cnt,)f(t)o(yp)q(e,)h(function,)h(comm\))117 282 y Ft(IN)171 b Fo(sendbuf)428 b Ft(address)15 b(of)f(input)f (bu\013er)117 357 y(OUT)124 b Fo(recvbuf)434 b Ft(address)15 b(of)f(output)g(bu\013er)117 432 y(IN)171 b Fo(cnt)512 b Ft(n)o(um)o(b)q(er)14 b(of)h(elemen)o(ts)g(in)f(input)h(and)g(output) g(bu\013er)h(\(in)o(te-)905 489 y(ger\))117 564 y(IN)171 b Fo(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f(bu\013er)117 639 y(IN)171 b Fo(function)418 b Ft(user)15 b(pro)o(vided)f(function) 117 714 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 838 y FC(Same)h(as)g(the)h Fo(MPI)p 498 838 V 15 w(SCAN)g FC(op)q(eration)g(function)g(except)g(that)f(a)g (user)g(supplied)j(function)e(is)g(used.)75 894 y Fo(function)k FC(is)f(an)g(asso)q(ciativ)o(e)f(and)h(comm)o(utativ)o(e)f(function)h (with)g(an)g(input)g(v)o(ector,)f(an)g(inout)h(v)o(ector,)75 951 y(and)g(a)g(length)g(argumen)o(t.)21 b(The)16 b(t)o(yp)q(es)g(of)g (the)g(t)o(w)o(o)e(v)o(ectors)h(and)h(of)g(the)g(returned)g(v)m(alues)h (all)g(agree.)75 1007 y(See)f Fo(MPI)p 241 1007 V 16 w(USER)p 369 1007 V 17 w(REDUCE)g FC(for)e(more)h(details.)75 1111 y Fo(MPI)p 160 1111 V 16 w(USER)p 288 1111 V 17 w(SCANA\()h(sendbuf,)h(recvbuf,)f(cnt,)g(t)o(yp)q(e,)g(function,)g (comm\))117 1188 y Ft(IN)171 b Fo(sendbuf)428 b Ft(address)15 b(of)f(input)f(bu\013er)117 1263 y(OUT)124 b Fo(recvbuf)434 b Ft(address)15 b(of)f(output)g(bu\013er)117 1338 y(IN)171 b Fo(cnt)512 b Ft(n)o(um)o(b)q(er)14 b(of)h(elemen)o(ts)g(in)f(input)h (and)g(output)g(bu\013er)h(\(in)o(te-)905 1395 y(ger\))117 1470 y(IN)171 b Fo(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f (bu\013er)117 1545 y(IN)171 b Fo(function)418 b Ft(user)15 b(de\014ned)g(function)117 1619 y(IN)171 b Fo(comm)454 b Ft(comm)o(unicator)11 b(handle)166 1744 y FC(Same)i(as)g Fo(MPI)p 424 1744 V 16 w(USER)p 552 1744 V 17 w(SCAN)p FC(,)h(except)f(that)g(the)g(user-de\014ned)i(op)q(eration)e(need)h (not)f(b)q(e)h(comm)o(u-)75 1800 y(tativ)o(e.)166 1933 y Fu(Implemen)o(tati)o(on)e(note:)166 1989 y Fr(MPI)p 243 1989 13 2 v 14 w(USER)p 361 1989 V 14 w(SCAN)i Ft(can)g(b)q(e)g (implemen)o(ted)e(as)i Fr(MPI)p 1000 1989 V 14 w(USER)p 1118 1989 V 15 w(SCANA)p Ft(.)75 2215 y Fw(4.6)59 b(Co)n(rrectness)75 2316 y FC(A)13 b(correct)f(program)g(should)i(in)o(v)o(ok)o(e)f (collectiv)o(e)h(comm)o(unications)g(so)e(that)g(deadlo)q(c)o(k)i(will) g(not)f(o)q(ccur,)75 2373 y(whether)22 b(collectiv)o(e)h(comm)o (unication)f(is)g(sync)o(hronizing)h(or)d(not.)38 b(The)22 b(follo)o(wing)g(t)o(w)o(o)e(examples)75 2429 y(illustrate)c(dangerous) f(use)h(of)f(collectiv)o(e)i(routines.)j(The)15 b(\014rst)g(example)h (is)g(erroneous.)75 2534 y Fp(/*)24 b(Example)e(A)i(*/)75 2591 y(switch\(MPI_rank\(comm,rank\))o(;)d(rank\))147 2647 y({)147 2704 y(case)i(0:)g({)h(MPI_bcast\(var1,)e(cnt,)h(type,)h (0,)f(comm\);)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 71 77 71 76 bop 75 -100 a Fv(4.6.)29 b(CORRECTNESS)1308 b FC(71)385 45 y Fp(MPI_send\(var2,)22 b(cnt,)h(type,)h(1,)f(tag,)h(comm\);)385 102 y(break;)337 158 y(})147 214 y(case)f(1:)g({)h(MPI_recv\(var2,)e (cnt,)h(type,)h(0,)f(tag,)h(comm\);)385 271 y(MPI_bcast\(var1,)e(cnt,)h (type,)h(0,)f(comm\);)385 327 y(break;)337 384 y(})147 440 y(})166 552 y FC(Pro)q(cess)16 b(zero)f(executes)h(a)g(broadcast,)f (follo)o(w)o(ed)h(b)o(y)f(a)h(blo)q(c)o(king)h(send)f(op)q(eration;)g (pro)q(cess)g(one)75 609 y(\014rst)h(executes)h(a)g(matc)o(hing)f(blo)q (c)o(king)i(receiv)o(e,)g(follo)o(w)o(ed)f(b)o(y)g(the)f(matc)o(hing)h (broadcast)f(call.)28 b(This)75 665 y(program)18 b(ma)o(y)g(deadlo)q(c) o(k.)33 b(The)20 b(broadcast)e(call)i(on)f(pro)q(cess)h(zero)f(ma)o(y)f (blo)q(c)o(k)i(un)o(til)g(pro)q(cess)g(one)75 722 y(executes)c(the)f (matc)o(hing)g(broadcast)g(call,)h(so)f(that)f(the)i(send)f(is)h(not)f (executed.)21 b(Pro)q(cess)15 b(one)g(blo)q(c)o(ks)75 778 y(on)g(the)g(receiv)o(e)i(and)e(nev)o(er)g(executes)h(the)f (broadcast.)166 836 y(The)g(follo)o(wing)h(example)g(is)g(correct,)e (but)i(nondeterministic:)75 949 y Fp(/*)24 b(Example)e(B)i(*/)75 1005 y(switch\(MPI_rank\(comm,rank\))o(;)d(rank\))147 1062 y({)170 1118 y(case)j(0:)f({)h(MPI_bcast\(var1,)e(cnt,)h(type,)g (0,)h(comm\);)409 1175 y(MPI_send\(var2,)e(cnt,)h(type,)h(1,)f(tag,)g (comm\);)409 1231 y(break;)361 1288 y(})170 1344 y(case)h(1:)f({)h (MPI_recv\(var2,)e(cnt,)h(type,)h(MPI_SOURCE_ANY,)d(tag,)j(comm\);)409 1401 y(MPI_bcast\(var1,)e(cnt,)h(type,)g(0,)h(comm\);)409 1457 y(MPI_recv\(var2,)e(cnt,)h(type,)h(MPI_SOURCE_ANY,)d(tag,)j (comm\);)409 1514 y(break;)361 1570 y(})170 1626 y(case)g(2:)f({)h (MPI_send\(var2,)e(cnt,)h(type,)h(1,)f(tag,)g(comm\);)409 1683 y(MPI_bcast\(var1,)f(cnt,)h(type,)g(0,)h(comm\);)409 1739 y(break;)361 1796 y(})170 1852 y(})166 1964 y FC(All)17 b(three)e(pro)q(cesses)h(participate)g(in)g(a)f(broadcast.)20 b(Pro)q(cess)15 b(0)g(sends)h(a)f(message)g(to)g(pro)q(cess)g(1)75 2021 y(after)d(the)h(broadcast,)f(and)g(pro)q(cess)h(2)g(sends)g(a)f (message)g(to)g(pro)q(cess)h(1)f(after)g(the)h(broadcast.)18 b(Pro)q(cess)75 2077 y(1)d(receiv)o(es)h(b)q(efore)f(and)h(after)e(the) i(broadcast,)e(with)h(a)g(wildcard)h(source)g(parameter.)166 2135 y(Tw)o(o)e(p)q(ossible)i(executions,)g(with)f(di\013eren)o(t)g (matc)o(hings)g(of)f(sends)h(and)g(receiv)o(es)h(are)f(illustrated)75 2191 y(b)q(elo)o(w.)337 2305 y Fp(First)24 b(Execution)170 2418 y(0)311 b(1)357 b(2)648 2474 y(/-----)47 b(send)457 2530 y(recv)23 b(<-/)75 2587 y(broadcast)118 b(broadcast)166 b(broadcast)123 2643 y(send)23 b(---\\)337 2700 y(\\-->)h(recv)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 72 78 72 77 bop 75 -100 a FC(72)724 b Fv(SECTION)16 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)337 45 y Fp(Second)24 b(Execution)147 158 y(0)334 b(1)357 b(2)75 214 y(broadcast)123 271 y(send)23 b(---\\)337 327 y(\\-->)48 b(recv)433 384 y(broadcast)166 b(broadcast)719 440 y(/---)47 b(send)481 497 y(recv)23 b(<---/)166 610 y FC(Note)16 b(that)g(the)h(second)g (execution)h(has)f(the)f(p)q(eculiar)j(e\013ect)e(that)f(a)g(send)h (executed)h(after)e(the)75 666 y(broadcast)e(is)i(receiv)o(ed)h(at)d (another)h(no)q(de)h(b)q(efore)f(the)g(broadcast.)166 800 y Fu(Discussion:)166 852 y Ft(An)d(alternativ)o(e)g(design)h(is)f (to)g(require)h(that)g(all)e(collectiv)o(e)h(comm)o(unication)d(calls)j (are)g(sync)o(hronizing.)18 b(In)75 901 y(this)11 b(case,)h(the)g (second)g(program)d(is)i(deterministic)g(and)f(only)h(the)g(\014rst)h (execution)g(ma)o(y)d(o)q(ccur.)18 b(This)11 b(will)e(mak)o(e)75 951 y(a)14 b(di\013erence)h(only)e(for)h(collectiv)o(e)g(op)q(erations) g(where)h(not)e(all)g(pro)q(cesses)j(b)q(oth)e(send)h(and)f(receiv)o(e) h(\(broadcast,)75 1001 y(reduce,)g(scatter,)g(gather\).)166 1142 y FC(It)21 b(is)g(the)g(user's)f(resp)q(onsibilit)o(y)j(to)d(mak)o (e)g(sure)h(that)f(there)h(are)f(no)h(t)o(w)o(o)e(concurren)o(tly)i (exe-)75 1198 y(cuting)g(collectiv)o(e)i(calls)e(that)f(use)h(the)g (same)f(comm)o(unicator)g(on)h(the)g(same)f(pro)q(cess.)36 b(\(Since)22 b(all)75 1254 y(collectiv)o(e)d(comm)o(unication)f(calls)h (are)e(blo)q(c)o(king)i(this)e(restriction)h(only)g(a\013ects)f(m)o (ultithreaded)h(im-)75 1311 y(plemen)o(tations.\))i(On)14 b(the)g(other)g(hand,)g(it)g(is)g(legitimate)h(for)e(one)h(pro)q(cess)h (to)e(start)g(a)g(new)h(collectiv)o(e)75 1367 y(comm)o(unication)j (call)g(ev)o(en)f(though)g(a)f(previous)i(call)g(that)e(uses)h(the)g (same)g(comm)o(unicator)g(has)f(not)75 1424 y(y)o(et)g(terminated)g(on) g(another)g(pro)q(cess.)20 b(As)15 b(illustrated)i(in)f(the)f(follo)o (wing)h(example:)75 1538 y Fp(/*)24 b(Example)e(C)i(*/)99 1595 y(MPI_bcast\(var1,)e(cnt,)h(type,)g(0,)h(comm\);)99 1651 y(MPI_bcast\(var2,)e(cnt,)h(type,)g(1,)h(comm\);)166 1765 y FC(In)17 b(a)e(nonsync)o(hronizing)j(implemen)o(tation)g(of)d (broadcast,)g(pro)q(cess)i(zero)f(ma)o(y)f(start)g(executing)75 1821 y(the)h(second)g(broadcast)f(b)q(efore)h(pro)q(cess)g(one)g (terminated)g(the)g(\014rst)f(broadcast.)21 b(Both)15 b(pro)q(cess)h(zero)75 1877 y(and)h(one)f(ma)o(y)g(terminate)g(their)h (t)o(w)o(o)e(broadcast)g(calls)j(b)q(efore)e(other)g(pro)q(cesses)h(ha) o(v)o(e)f(started)f(their)75 1934 y(calls.)21 b(It)15 b(is)h(the)f(implemen)o(tor's)h(resp)q(onsibilit)o(y)h(to)e(ensure)h (this)f(will)i(not)e(cause)h(an)o(y)e(error.)166 2068 y Fu(Implemen)o(tati)o(on)e(note:)166 2119 y Ft(Assume)e(that)g (broadcast)h(is)f(implemen)o(ted)e(using)i(p)q(oin)o(t-to-p)q(oin)o(t)e (MPI)j(comm)o(unicati)o(on.)j(The)d(follo)o(wing)75 2169 y(t)o(w)o(o)i(rules)i(are)f(satis\014ed:)134 2248 y(1.)22 b(All)13 b(receiv)o(es)i(sp)q(ecify)g(their)f(source)h(explicitly)e (\(no)h(wildcards\).)134 2328 y(2.)22 b(Eac)o(h)12 b(pro)q(cess)i (sends)g(all)d(messages)h(that)h(p)q(ertain)f(to)h(one)f(collectiv)o(e) g(call)g(b)q(efore)h(sending)f(an)o(y)g(message)189 2378 y(that)i(p)q(ertain)g(to)g(a)f(subsequen)o(t)j(collectiv)o(e)e(call.) 166 2457 y(Then)i(messages)g(b)q(elonging)f(to)h(successiv)o(e)i (broadcasts)f(cannot)f(b)q(e)h(confused,)g(as)f(the)h(order)f(of)g(p)q (oin)o(t-)75 2507 y(to-p)q(oin)o(t)d(messages)h(is)g(preserv)o(ed.)20 b(This)14 b(is)f(true,)i(in)e(general,)h(for)f(an)o(y)h(collectiv)o(e)g (library)m(.)166 2647 y FC(A)k(collectiv)o(e)h(comm)o(unication)f(ma)o (y)f(execute)i(in)f(a)g(con)o(text)f(while)i(p)q(oin)o(t-to-p)q(oin)o (t)f(comm)o(uni-)75 2704 y(cations)g(that)g(use)g(the)h(same)f(con)o (text)f(are)h(p)q(ending,)j(or)c(o)q(ccur)i(concurren)o(tly)l(.)30 b(This)19 b(is)g(illustrated)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 73 79 73 78 bop 75 -100 a Fv(4.6.)34 b(CORRECTNESS)1303 b FC(73)75 45 y(in)16 b(example)g(B)g(ab)q(o)o(v)o(e,)f(the)g(\014rst)g(pro)q (cess)h(ma)o(y)f(receiv)o(e)h(a)f(message)g(sen)o(t)g(with)h(the)f(con) o(text)g(of)g(com-)75 102 y(m)o(unicator)32 b Fo(comm)14 b FC(while)j(it)f(is)g(executing)h(a)f(broadcast)f(with)h(the)g(same)g (comm)o(unicator.)22 b(It)15 b(is)i(the)75 158 y(implemen)o(ter)f(resp) q(onsibili)q(t)o(y)h(to)e(ensure)h(this)f(will)i(not)e(cause)g(an)o(y)g (confusion.)166 291 y Fu(Implemen)o(tati)o(on)21 b(note:)65 b Ft(Assume)22 b(that)f(collectiv)o(e)g(comm)o(unications)d(are)k (implemen)o(ted)d(using)75 340 y(p)q(oin)o(t-to-p)q(oin)o(t)e(MPI)i (comm)o(unication.)29 b(Then,)20 b(in)e(order)i(to)e(a)o(v)o(oid)g (confusion,)h(whenev)o(er)h(a)e(comm)o(unica-)75 390 y(tor)g(is)f(created,)i(a)e(\\hidden)h(comm)o(unicator")c(need)19 b(b)q(e)f(created)h(for)e(collectiv)o(e)g(comm)o(unication.)26 b(A)17 b(direct)75 440 y(implemen)o(tatio)o(n)11 b(of)i(MPI)h (collectiv)o(e)g(comm)o(unicatio)o(n)d(can)j(ac)o(hiev)o(e)g(a)g (similar)d(e\013ect)k(more)e(c)o(heaply)m(,)g(e.g.,)f(b)o(y)75 490 y(using)j(a)g(hidden)h(tag)f(or)g(con)o(text)h(bit)g(to)f(indicate) g(whether)i(the)f(comm)o(unicator)d(is)i(used)h(for)f(p)q(oin)o(t-to-p) q(oin)o(t)75 540 y(or)f(collectiv)o(e)g(comm)o(unicatio)o(n.)166 589 y(An)g(alternativ)o(e)h(c)o(hoice)g(is)f(to)g(require)h(that)g(a)f (comm)o(unicator)e(is)i(quiescen)o(t)i(when)f(used)g(in)f(a)g (collectiv)o(e)75 639 y(comm)o(unication:)h(No)f(messages)h(using)f(a)f (con)o(text)i(can)g(b)q(e)g(p)q(ending)f(at)g(a)g(pro)q(cess)i(when)f (this)f(pro)q(cess)i(starts)75 689 y(executing)e(a)f(collectiv)o(e)h (comm)o(unicati)o(on)d(with)i(this)g(con)o(text,)h(nor)f(can)h(an)o(y)f (new)h(message)f(with)g(this)g(con)o(text)75 739 y(arriv)o(e)j(during)h (the)g(execution)g(of)f(this)g(collectiv)o(e)h(comm)o(unicatio)o(n,)d (unless)j(they)g(w)o(ere)h(sen)o(t)f(as)g(part)f(of)g(the)75 789 y(execution)f(of)e(the)h(collectiv)o(e)g(call)f(itself.)166 838 y(This)j(approac)o(h)h(has)f(the)h(adv)n(an)o(tage)f(of)f (simplifying)e(the)k(la)o(y)o(ering)f(of)f(collectiv)o(e)i(comm)o (unicatio)o(ns)d(on)75 888 y(top)d(of)f(p)q(oin)o(t-to-p)q(oin)o(t)f (comm)o(unicatio)o(n)f(\(no)j(need)g(for)g(hidden)f(con)o(texts\).)19 b(Also,)10 b(it)g(imp)q(oses)g(on)h(the)g(collectiv)o(e)75 938 y(collectiv)o(e)k(comm)o(unicatio)o(n)d(library)i(the)i(same)d (restrictions)j(that)f(hold)f(for)h(an)o(y)f(other)i(collectiv)o(e)e (library)m(.)20 b(It)75 988 y(has)14 b(the)h(disadv)n(an)o(tage)e(of)g (restricting)i(the)f(use)h(of)e(collectiv)o(e)h(comm)o(unicatio)o(ns.) 166 1038 y(The)i(question)h(is)f(whether)h(w)o(e)g(w)o(an)o(t)e(to)h (view)g(the)h(collectiv)o(e)f(comm)o(unication)d(op)q(erations)j(as)g (part)h(of)75 1088 y(the)f(basic)g(comm)o(unicati)o(on)d(services)k(of) e(MPI,)g(or)g(whether)i(w)o(e)f(w)o(an)o(t)f(to)g(see)i(them)d(as)i(a)f (library)g(la)o(y)o(ered)g(on)75 1137 y(top)f(of)f(these)i(basic)f (services.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 74 80 74 79 bop 75 391 a Fx(Section)35 b(5)75 633 y FE(Pro)s(cess)40 b(T)-10 b(op)s(ologi)q(es)75 908 y Fw(5.1)59 b(Intro)r(duction)75 1076 y FC(A)14 b(pro)q(cess)f(group)g(in)i(MPI)e(is)h(a)f(collection)j (of)d(n)h(pro)q(cesses.)19 b(Eac)o(h)14 b(pro)q(cess)f(in)i(the)e (group)g(is)h(assigned)75 1133 y(a)g(rank)g(b)q(et)o(w)o(een)h(0)g(and) f(n-1.)20 b(In)15 b(the)g(ma)s(jorit)o(y)e(of)h(parallel)i (applications)g(this)f(linear)h(ordering)f(do)q(es)75 1189 y(not)e(re\015ect)g(the)g(comm)o(unication)h(structure)e(of)h(the) g(pro)q(cesses)g(whic)o(h)h(is)f(usually)i(determined)f(b)o(y)f(the)75 1246 y(underlying)20 b(problem)e(geometry)f(and)h(the)f(n)o(umerical)i (algorithm)f(used.)28 b(Often)18 b(the)f(pro)q(cesses)h(are)75 1302 y(arranged)d(in)h(top)q(ological)g(patterns)e(as)h(simple)i(as)e (a)g(t)o(w)o(o{)e(or)i(three{dimensional)i(grid.)166 1393 y(F)l(or)12 b(the)h(user)g(who)g(programs)f(an)h(application)h(on) f(suc)o(h)g(a)g(pro)q(cess)g(structure)g(the)g(most)f(natural)75 1449 y(w)o(a)o(y)k(of)g(referring)i(to)e(individual)k(pro)q(cesses)d (is)h(to)e(sp)q(ecify)i(their)g(co)q(ordinates)f(in)h(the)f(pro)q(cess) g(grid.)75 1506 y(Usually)f(a)e(pro)q(cess)g(do)q(es)h(not)f(ha)o(v)o (e)g(to)g(kno)o(w)g(the)g(iden)o(ti\014ers)i(of)e Fs(al)r(l)g FC(pro)q(cesses)h(in)g(the)f(grid,)h(b)q(ecause)75 1562 y(comm)o(unication)e(only)f(tak)o(es)g(place)g(with)h(pro)q(cesses)f (in)h(a)f(certain)g(neigh)o(b)q(orho)q(o)q(d,)i(and)e(p)q(erhaps)h (with)75 1619 y(a)18 b(few)g(distan)o(t)f(pro)q(cesses.)29 b(Non{scalable)19 b(features)f(lik)o(e)h(the)f(explicit)i(listing)f(of) f(all)h(participating)75 1675 y(pro)q(cesses)f(in)g(the)f(application)i (could,)f(therefore,)f(b)q(e)h(a)o(v)o(oided,)f(if)h(there)f(w)o(ere)g (a)g(means)g(to)g(sp)q(ecify)75 1732 y(the)e(information)h(whic)o(h)g (is)f(really)i(required.)166 1822 y(Additionally)l(,)g(the)e (user{de\014ned)h(pro)q(cess)f(structure)g(can)g(b)q(e)h(exploited)g(b) o(y)f(the)g(system)f(in)i(the)75 1879 y(assignmen)o(t)d(of)g(pro)q (cesses)h(to)f(hardw)o(are)f(pro)q(cessors,)h(if)h(this)g(helps)h(to)d (impro)o(v)o(e)i(the)f(comm)o(unication)75 1935 y(p)q(erformance)g(on)h (a)f(giv)o(en)h(mac)o(hine.)20 b(Ho)o(w)12 b(this)i(mapping)g(is)g (done)g(in)g(detail)g(is)g(outside)g(the)g(scop)q(e)f(of)75 1992 y(this)k(standardization)h(prop)q(osal,)f(since)h(it)f(is)h(to)q (o)e(m)o(uc)o(h)h(dep)q(enden)o(t)i(on)d(the)h(particular)h(hardw)o (are.)75 2048 y(The)d(description)i(of)d(a)h(\\virtual)g(pro)q(cess)g (top)q(ology",)f(ho)o(w)o(ev)o(er,)g(only)h(dep)q(ends)i(on)e(the)g (application,)75 2105 y(and)k(is,)g(therefore,)g(mac)o(hine{indep)q (endent.)32 b(The)19 b(functions)g(whic)o(h)h(are)e(prop)q(osed)h(in)g (this)g(pap)q(er)75 2161 y(only)d(deal)g(with)f(this)h(side)g(of)f(the) g(mapping)h(problem.)166 2252 y(There)e(are)g(w)o(ell{kno)o(wn)h(tec)o (hniques)g(for)f(mapping)h(grid/torus)e(structures)h(to)g(hardw)o(are)f (top)q(o-)75 2308 y(logies)18 b(suc)o(h)f(as)g(h)o(yp)q(ercub)q(es)h (or)f(grids.)26 b(But)17 b(ev)o(en)g(for)g(more)f(complicated)j(graph)d (structures)h(go)q(o)q(d)75 2365 y(heuristics)i(often)e(yield)i (near{optimal)e(results)h([13)o(].)26 b(On)18 b(the)f(other)g(hand,)h (if)g(there)f(is)h(no)f(w)o(a)o(y)g(for)75 2421 y(the)c(user)g(to)f(sp) q(ecify)j(the)e(logical)h(pro)q(cess)f(arrangemen)o(t)f(as)g(a)h (\\virtual)g(top)q(ology",)f(this)i(information)75 2478 y(is)k(most)f(di\016cult)j(to)d(reco)o(v)o(er)g(later.)28 b(A)18 b(random)g(mapping)g(is)g(most)g(lik)o(ely)h(the)f(consequence.) 29 b(On)75 2534 y(some)15 b(mac)o(hines)h(this)g(will)h(lead)f(to)f (unnecessary)h(con)o(ten)o(tion)g(in)g(the)f(in)o(terconnect)i(net)o(w) o(ork.)i(Some)75 2591 y(details)g(ab)q(out)e(predicted)i(and)f (measured)g(p)q(erformance)f(impro)o(v)o(emen)o(ts)h(whic)o(h)g(result) g(from)f(go)q(o)q(d)75 2647 y(pro)q(cess{to{pro)q(cessor)e(mapping)h (on)f(mo)q(dern)h(w)o(ormhole{routing)f(arc)o(hitectures)h(can)g(b)q(e) g(found)g(in)75 2704 y([5)o(,)f(4].)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 75 81 75 80 bop 75 -100 a Fv(5.2.)34 b(VIR)l(TUAL)17 b(TOPOLOGIES)1119 b FC(75)75 45 y Fw(5.2)59 b(Virtual)20 b(T)-5 b(op)r(ologies)75 148 y FC(In)17 b(the)g(most)e(general)i(case)g(the)f(connection)i (structure)e(of)g(a)g(set)h(of)f(pro)q(cesses)h(can)f(b)q(e)h(represen) o(ted)75 205 y(b)o(y)c(a)g(graph.)19 b(The)14 b(no)q(des)f(stand)h(for) e(the)i(pro)q(cesses,)f(and)h(the)f(edges)h(connect)f(pro)q(cesses)h (whic)o(h)g(com-)75 261 y(m)o(unicate)h(with)g(eac)o(h)g(other.)20 b(MPI)14 b(pro)o(vides)i(message{passing)e(b)q(et)o(w)o(een)h(an)o(y)g (pair)g(of)f(pro)q(cesses)h(in)75 318 y(a)i(group.)27 b(There)18 b(is)g(no)g(requiremen)o(t)g(for)f(op)q(ening)i(a)e(c)o (hannel)i(explicitly)l(.)30 b(Therefore,)18 b(a)f(\\missing)75 374 y(link")g(in)f(the)f(user{de\014ned)i(pro)q(cess)f(graph)f(do)q(es) h(not)f(prev)o(en)o(t)g(the)g(corresp)q(onding)i(pro)q(cesses)f(from)75 431 y(exc)o(hanging)h(messages.)k(It)16 b(only)g(means)g(that)f(this)i (connection)g(is)f(neglected)h(in)g(the)f(virtual)h(top)q(ol-)75 487 y(ogy)l(.)24 b(As)16 b(a)g(consequence,)i(an)f(automatic)f(mapping) h(to)q(ol)g(will)h(not)e(try)g(to)g(assign)h(the)g(pro)q(cesses)g(to)75 544 y(neigh)o(b)q(oring)g(pro)q(cessors.)166 601 y(The)i(graph)g (de\014nition)i(function)g Fo(MPI)p 870 601 14 2 v 15 w(GRAPH)g FC(presen)o(ted)f(in)g(this)f(pap)q(er)h(do)q(es)g(not)f (include)75 657 y(an)o(y)c(w)o(eigh)o(ting)g(of)f(the)h(comm)o (unication)h(v)o(olume)f(along)g(the)g(di\013eren)o(t)g(edges.)20 b(Pro)q(cesses)15 b(are)f(either)75 714 y(neigh)o(b)q(ors)22 b(or)e(not.)37 b(Exp)q(erience)23 b(with)f(similar)g(tec)o(hniques)g (in)g(P)l(ARMA)o(CS)f([10)o(,)g(11])f(sho)o(w)h(that)75 770 y(this)h(information)g(is)g(usually)h(su\016cien)o(t)g(for)e(a)g (go)q(o)q(d)h(mapping.)40 b(Additionally)l(,)25 b(a)d(more)f(precise)75 827 y(sp)q(eci\014cation)16 b(w)o(ould)f(b)q(e)g(more)e(di\016cult)j (for)e(the)g(user)g(to)g(set)g(up,)g(and)h(it)f(w)o(ould)h(mak)o(e)e (the)i(in)o(terface)75 883 y(functions)h(more)f(complicated.)166 941 y(In)i(principle)i(a)c(function)i(for)f(the)g(de\014nition)i(of)e (suc)o(h)g(a)g(graph)g(structure)g(w)o(ould)g(b)q(e)h(su\016cien)o(t)75 997 y(for)c(all)i(applications.)21 b(Ho)o(w)o(ev)o(er,)13 b(in)h(man)o(y)g(applications)h(the)f(graph)g(structure)f(is)h(v)o(ery) g(regular,)g(and)75 1054 y(the)e(detailed)i(set{up)f(of)e(the)i(graph)f (w)o(ould)h(b)q(e)g(incon)o(v)o(enien)o(t)g(for)f(the)g(user)h(and)f (resource{consuming)75 1110 y(during)g(run)g(time.)19 b(A)12 b(large)f(fraction)g(of)g(all)i(parallel)g(applications)g(use)f (pro)q(cess)g(top)q(ologies)f(lik)o(e)i(rings,)75 1167 y(or)g(t)o(w)o(o{)f(or)h(higher{dimensional)k(grids)c(or)h(tori.)19 b(These)14 b(structures)f(are)g(completely)i(de\014ned)g(b)o(y)f(the)75 1223 y(n)o(um)o(b)q(er)g(of)g(dimensions)i(and)e(the)g(n)o(um)o(b)q (ers)h(of)f(pro)q(cesses)g(in)h(eac)o(h)f(co)q(ordinate)h(direction.)21 b(Also,)14 b(the)75 1279 y(mapping)h(of)e(grids)h(and)h(tori)e(is)i (generally)g(an)f(easier)g(problem)h(then)f(that)f(of)h(general)g (graphs.)19 b(Th)o(us,)75 1336 y(it)14 b(is)h(desirable)h(to)d(do)h (the)h(mapping)g(as)e(grids)i(and)f(tori)g(rather)g(than)g(obscuring)h (the)f(information)g(in)75 1392 y(a)h(graph.)166 1450 y(Therefore,)j(in)g(addition)h(to)f(the)f(general)i(graph)e(Cartesian)h (top)q(ologies)g(are)f(supp)q(orted.)29 b(The)75 1506 y(top)q(ology)11 b(de\014nition)j(function)e Fo(MPI)p 716 1506 V 16 w(CART)g FC(can)g(b)q(e)g(used)g(to)f(describ)q(e)j (Cartesian)d(structures)g(of)g(arbi-)75 1563 y(trary)i(dimension.)22 b(F)l(or)13 b(eac)o(h)i(co)q(ordinate)f(direction)i(it)f(is)f(sp)q (eci\014ed)j(whether)d(the)h(pro)q(cess)f(structure)75 1619 y(is)h(p)q(erio)q(dic)h(or)e(not.)19 b(Similar)d(functions)f(are)f (con)o(tained)h(in)g(EXPRESS)g([14)o(])f(and)g(P)l(ARMA)o(CS.)g(Note)75 1676 y(that)e(an)h(n{dimensional)h(h)o(yp)q(ercub)q(e)h(is)e(an)g (n{dimensional)i(torus)d(with)h(2)f(pro)q(cesses)h(p)q(er)h(co)q (ordinate)75 1732 y(direction.)21 b(Th)o(us,)15 b(sp)q(ecial)i(supp)q (ort)e(for)g(h)o(yp)q(ercub)q(e)i(structures)e(is)g(not)g(necessary)l (.)166 1789 y(Based)f(on)g(the)f(p)q(ositions)i(in)g(the)e(virtual)i (top)q(ology)e(a)h(pro)q(cess)g(can)g(lo)q(ok)g(up)g(the)g(ranks)f(of)g (other)75 1846 y(pro)q(cesses)f(b)o(y)g(calling)i Fo(MPI)p 554 1846 V 16 w(INQRANK)p FC(.)e(The)g(pro)q(cesses)h(are)e(sp)q (eci\014ed)j(b)o(y)e(their)h(rectangular)f(pro)q(cess)75 1902 y(co)q(ordinates.)20 b(Th)o(us,)15 b(it)g(is)h(p)q(ossible)h(to)d (inquire)j(for)e(the)g(ranks)g(of)f(only)i(those)f(\(few\))f(pro)q (cesses)i(with)75 1959 y(whic)o(h)e(messages)g(ha)o(v)o(e)f(to)g(b)q(e) h(exc)o(hanged.)20 b(The)14 b(in)o(v)o(erse)g(function)g Fo(MPI)p 1351 1959 V 16 w(INQLOC)g FC(translates)f(a)g(rank)75 2015 y(in)o(to)i(the)g(corresp)q(onding)i(Cartesian)e(co)q(ordinates.) 75 2164 y Fw(5.3)59 b(Emb)r(edding)18 b(in)i(MPI)75 2267 y FC(The)d(supp)q(ort)g(for)f(virtual)i(top)q(ologies)f(has)f(to)h(b)q (e)g(fully)h(in)o(tegrated)f(in)g(the)g(framew)o(ork)f(of)g(MPI.)g(In) 75 2324 y(particular,)21 b(the)f(virtual)g(top)q(ology)f(functions)i (do)e(not)h(in)o(terfere)g(with)g(the)f(functions)i(for)e(pro)q(cess)75 2380 y(group)i(creation)g(and)g(with)g(those)f(for)h(collectiv)o(e)h (comm)o(unication.)38 b(The)21 b(curren)o(t)g(design)g(of)g(the)75 2437 y(pro)q(cess)15 b(top)q(ology)g(functions)h(is)g(based)g(on)f(the) g(follo)o(wing)h(assumptions:)143 2534 y Fy(\017)23 b FC(MPI)15 b(is)h(free)g(to)f(c)o(ho)q(ose)h(an)o(y)f(pro)q(cess)h(in)h (a)e(group)g(for)g(an)o(y)h(lo)q(cation)g(in)h(the)f(top)q(ology)l(.)21 b(There)189 2591 y(are)16 b(no)g(sp)q(ecial)i(pro)q(cesses)f(whic)o(h)g (ha)o(v)o(e)f(to)g(b)q(e)h(aligned)g(with)g(top)q(ology)f (substructures.)23 b(This)189 2647 y(assumption)d(\014ts)g(b)q(est)h (to)e(a)h(\\lo)q(cal)h(SPMD)f(mo)q(del")h(in)g(whic)o(h)g(all)g(pro)q (cesses)g(in)g(the)f(same)189 2704 y(group)15 b(execute)g(the)h(same)f (program.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 76 82 76 81 bop 75 -100 a FC(76)926 b Fv(SECTION)16 b(5.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)143 45 y Fy(\017)23 b FC(One)g(imp)q (ortan)o(t)f(feature)g(of)g(MPI)g(are)g(the)h(pro)q(cess)f(groups.)42 b(The)22 b(treatmen)o(t)f(of)h(virtual)189 102 y(top)q(ologies)f(has)g (to)g(tak)o(e)g(this)g(concept)h(in)o(to)f(accoun)o(t.)38 b(The)22 b(functions)g(presen)o(ted)f(in)i(this)189 158 y(pap)q(er)16 b(are)g(not)g(only)h(compatible)h(with)e(the)h(pro)q (cess)f(groups,)g(they)g(ev)o(en)h(tak)o(e)f(adv)m(an)o(tage)f(of)189 214 y(them)c(for)g(the)g(de\014nition)i(of)e(mapping)h(o)o(v)o(erla)o (ys)e(and)i(dynamic)g(re{mapping)g(during)g(execution)189 271 y(time.)29 b(Virtual)19 b(pro)q(cess)f(top)q(ologies)h(help)h(in)f (the)f(sub)q(division)j(of)d(pro)q(cess)g(groups)g(whic)o(h)h(in)189 327 y(man)o(y)e(applications)j(has)e(to)f(b)q(e)i(done)g(along)f(pro)q (cess)g(co)q(ordinate)h(lines.)30 b(F)l(urthermore,)18 b(the)189 384 y(de\014nition)24 b(of)f(collectiv)o(e)h(gather{t)o(yp)q (e)e(comm)o(unication)i(calls)f(b)q(ene\014ts)h(from)e(the)h(natural) 189 440 y(ordering)15 b(of)g(pro)q(cesses)h(in)g(a)e(virtual)i(top)q (ology)l(.)75 536 y(The)h(virtual)g(top)q(ology)g(handling)h(follo)o (ws)f(the)g(general)g(MPI)g(rule)g(of)g(functional)h(la)o(y)o(ering.)25 b(A)o(t)16 b(the)75 593 y(b)q(ottom)j(it)g(is)h(based)g(on)f(the)h (general)g(pro)q(cess)f(group)g(creation)h(function)g Fo(MPI)p 1522 593 14 2 v 16 w(CREA)l(TE)p 1707 593 V 18 w(GROUP)75 649 y FC(plus)f(a)f(few)g(top)q(ology)g(sp)q(eci\014c)i (mapping)e(functions.)30 b(In)18 b(principle)j(this)e(is)f(su\016cien)o (t)h(for)e(de\014ning)75 706 y(top)q(ologies)f(and)f(use)h(them)f(in)h (applications.)166 763 y(A)f(n)o(um)o(b)q(er)g(of)f(higher{lev)o(el)j (functions)e(pro)o(vide)g(an)g(in)o(terface)g(whic)o(h)h(is)f(similar)h (to)e(what)g(users)75 819 y(of)19 b(pac)o(k)m(ages)g(lik)o(e)h(Express) f(and)g(P)l(ARMA)o(CS)h(are)f(familiar)h(with.)32 b(This)19 b(lev)o(el)i(can)e(b)q(e)h(written)f(in)75 875 y(a)j(p)q(ortable)h(w)o (a)o(y)e(in)i(terms)f(of)g(the)g(lo)o(w{lev)o(el)h(functions.)42 b(Implemen)o(tations)24 b(of)e(the)g(high{lev)o(el)75 932 y(functions)e(whic)o(h)h(b)o(ypass)e(the)g(lo)o(w)h(lev)o(el)h(ma)o (y)d(b)q(e)j(adv)m(an)o(tageous)d(on)i(some)f(systems)g(in)h(terms)f (of)75 988 y(memory)c(and)g(cpu)h(time)f(usage,)g(but)g(this)h(is)g (not)f(a)f(requiremen)o(t.)75 1135 y Fw(5.4)59 b(Overview)19 b(of)h(the)f(p)n(rop)r(osed)g(MPI)h(functions)75 1237 y FC(This)g(c)o(hapter)f(assumes)f(the)i(a)o(v)m(ailabilit)o(y)h(of)d (the)i(follo)o(wing)f(MPI)g(features)g(whic)o(h)h(are)f(de\014ned)h(in) 75 1294 y(other)15 b(c)o(hapters:)131 1390 y(1.)22 b(An)11 b(existing)g(pro)q(cess)g(group)f(\(e.g.,)g(the)h(ALL)h(group\))e(can)g (b)q(e)i(partitioned)f(and/or)f(ren)o(um)o(b)q(ered)189 1446 y(using)189 1569 y Fo(MPI)p 274 1569 V 15 w(CREA)l(TE)p 458 1569 V 18 w(GROUP\(gid)p 703 1569 V 17 w(old,)15 b(gid)p 862 1569 V 17 w(new,)g(group)p 1089 1569 V 17 w(k)o(ey)l(,)g(o)o(rder)p 1289 1569 V 15 w(k)o(ey\))230 1666 y Ft(IN)171 b Fo(gid)p 507 1666 V 17 w(old)441 b Ft(group)14 b(id)f(of)g(paren)o(t)i(group)230 1732 y(OUT)124 b Fo(gid)p 507 1732 V 17 w(new)424 b Ft(group)10 b(id)h(of)f(newly)g (created)j(subgroup)e(of)f(whic)o(h)g(the)1019 1788 y(calling)i(pro)q (cess)k(is)e(a)f(mem)o(b)q(er)230 1853 y(IN)171 b Fo(group)p 558 1853 V 17 w(k)o(ey)385 b Ft(in)o(teger)14 b(v)n(alue)f(used)i(for)f (the)g(partitioning)230 1919 y(IN)171 b Fo(o)o(rder)p 547 1919 V 16 w(k)o(ey)397 b Ft(in)o(teger)15 b(v)n(alue)f(used)h(for)f (determining)g(the)h(ranks)f(in)1019 1975 y(the)g(new)h(group)189 2119 y FC(This)22 b(function)g(is)f(called)i(b)o(y)e(all)i(pro)q (cesses)e(in)i(group)d Fs(gid)p 1265 2119 V 17 w(old)i FC(lo)q(osely)g(sync)o(hronously)l(.)39 b(It)189 2176 y(creates)18 b(as)g(man)o(y)g(di\013eren)o(t)g(subgroups)h(as)f(there)h (are)f(di\013eren)o(t)g(v)m(alues)i(of)e Fs(gr)n(oup)p 1676 2176 V 17 w(key)p FC(.)30 b(The)189 2232 y(ranking)16 b(in)h(the)g(newly)g(created)f(groups)g(is)h(determined)h(b)o(y)e(the)h Fs(or)n(der)p 1464 2232 V 17 w(key)f FC(v)m(alues,)h(with)g(the)189 2289 y(pro)q(cess)e(ranks)g(in)g(the)g(paren)o(t)g(group)g(for)f(tie)h (breaking.)21 b(If)15 b(there)g(is)h(a)e(standardized)i(in)o(ternal)189 2345 y(information)21 b(cac)o(hing)g(facilit)o(y)l(,)j(it)d(can)g(b)q (e)h(used)g(b)o(y)e(MPI)h(routines)h(to)e(store)h(and)g(retriev)o(e)189 2402 y(information)f(related)g(to)f(a)g(pro)q(cess)i(group.)33 b(This)20 b(facilit)o(y)h(w)o(as)e(prop)q(osed)h(in)h(the)f(Con)o(text) 189 2458 y(Sub)q(committee.)f(High{lev)o(el)12 b(top)q(ology)f (functions)g(store)f(structure)h(information)f(for)g(a)h(pro)q(cess)189 2514 y(group,)j(whic)o(h)i(is)g(then)f(later)h(used)f(b)o(y)h(top)q (ology)e(inquiry)j(functions.)189 2591 y(In)12 b(case)f(the)g(cac)o (hing)h(mec)o(hanism)g(do)q(es)f(not)g(b)q(ecome)h(part)f(of)g(MPI,)g (the)g(top)q(ology)g(information)189 2647 y(can)i(b)q(e)g(retriev)o(ed) g(b)o(y)g(group)f(id)i(in)g(an)e(implemen)o(tation{dep)q(enden)o(t)k(w) o(a)o(y)l(,)c(whic)o(h)h(eac)o(h)g(v)o(endor)189 2704 y(can)i(do)g(as)g(is)h(most)e(con)o(v)o(enien)o(t.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 77 83 77 82 bop 75 -100 a Fv(5.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)536 b FC(77)166 45 y(There)17 b(has)g(b)q(een)h(a)e(general)i(decision)g(in)g(MPI)f(to) f(start)f(all)j(n)o(um)o(b)q(ering)g(with)f(0)g(instead)g(of)g(1.)75 102 y(Th)o(us,)f(co)q(ordinates)g(in)g(a)g(Cartesian)g(structure)f(are) h(0,1,)e(...,)h(dim-1.)22 b(In)17 b(the)f(same)f(w)o(a)o(y)l(,)g(no)q (des)i(in)f(a)75 158 y(general)g(graph)f(are)f(n)o(um)o(b)q(ered)i (zero)g(up.)75 279 y Fq(5.4.1)49 b(Lo)o(w{level)18 b(top)q(ology)g (functions)75 365 y FC(On)13 b(the)g(lo)o(w)o(est)f(lev)o(el)i(groups)e (are)h(created)f(b)o(y)h(the)g(general)g(function)g Fo(MPI)p 1384 365 14 2 v 16 w(CREA)l(TE)p 1569 365 V 18 w(GROUP)p FC(.)g(Using)75 422 y(the)k(input)i(argumen)o(t)d Fs(newr)n(ank)h FC(the)g(caller)i(can)e(c)o(hange)g(the)h(pro)q(cess)f(ranks)g(in)h (the)g(new)f(group)g(as)75 478 y(compared)e(to)e(the)i(paren)o(t)f (group.)19 b(This)d(feature)e(can)g(b)q(e)i(used)f(to)e(e\016cien)o (tly)j(map)f(the)f(pro)q(cesses)h(in)75 535 y(the)h(group)g(to)g(the)g (p)q(ositions)h(in)g(the)f(top)q(ology)l(.)23 b(F)l(or)15 b(eac)o(h)i(class)f(of)g(top)q(ologies,)g(therefore,)g(there)g(is)75 591 y(a)g(function)h(whic)o(h)g(computes)f(the)h(rank)f(of)g(the)g (calling)i(pro)q(cess)e(in)h(the)g(new)f(group,)g(based)h(on)f(the)75 648 y(size)g(and)g(structure)g(of)f(the)g(top)q(ology)l(.)21 b(Note)16 b(that)e(it)i(is)g(a)g(v)m(alid)h(implemen)o(tation)g(if)f (the)f(new)h(ranks)75 704 y(are)f(iden)o(tical)j(to)c(the)i(old)g (ranks.)k(So,)c(if)f(for)g(a)h(giv)o(en)g(mac)o(hine)g(the)f(placemen)o (t)i(of)e(pro)q(cesses)h(mak)o(es)75 760 y(no)f(di\013erence,)h(the)f (implemen)o(tation)i(of)e(the)g(rank)g(computation)g(functions)h(is)g (trivial.)166 817 y(The)h(functions)h(in)g(this)f(section)h(do)e(not)h (c)o(hange)g(the)g(state)f(of)h(the)g(calling)h(pro)q(cess,)g(but)f (they)75 873 y(pro)o(vide)f(the)f(information)g(used)h(in)g(a)f(follo)o (wing)h(group)f(creation)g(for)g(pro)q(cess)g(reordering.)166 930 y(F)l(or)f(Cartesian)h(structures)g(the)h(function)g(is)75 1033 y Fo(MPI)p 160 1033 V 16 w(MAP)p 275 1033 V 16 w(CART\(gid)p 485 1033 V 17 w(old,)f(ndims,)g(dims,)g(p)q(erio)q(d,)g(newrank\))117 1111 y Ft(IN)171 b Fo(gid)p 394 1111 V 16 w(old)441 b Ft(group)14 b(id)f(of)h(paren)o(t)g(group)117 1186 y(IN)171 b Fo(ndims)460 b Ft(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of)i (Cartesian)g(structure)117 1260 y(IN)171 b Fo(dims\(*\))424 b Ft(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(eac)o(h)g(co)q (ordinate)g(direction)117 1335 y(IN)155 b Fo(p)q(erio)q(d\(*\))410 b Ft(p)q(erio)q(dicit)o(y)14 b(sp)q(eci\014cation)h(in)e(eac)o(h)h(co)q (ordinate)h(direction)117 1410 y(OUT)124 b Fo(newrank)414 b Ft(reordered)16 b(rank)e(of)f(the)i(calling)d(pro)q(cess)166 1534 y FC(The)j(corresp)q(onding)i(function)f(for)e(general)i(graph)f (structures)g(is)75 1638 y Fo(MPI)p 160 1638 V 16 w(MAP)p 275 1638 V 16 w(GRAPH\(gid)p 516 1638 V 18 w(old,)f(no)q(des,)j(index,) f(nglist,)f(newrank\))117 1715 y Ft(IN)171 b Fo(gid)p 394 1715 V 16 w(old)441 b Ft(group)14 b(id)f(of)h(paren)o(t)g(group)117 1790 y(IN)171 b Fo(no)q(des)464 b Ft(n)o(um)o(b)q(er)13 b(of)h(graph)f(no)q(des)117 1865 y(IN)171 b Fo(index\(*\))414 b Ft(de\014ne)15 b(graph)f(structure,)h(see)h(text)117 1940 y(IN)171 b Fo(nglist\(*\))411 b Ft(de\014ne)15 b(graph)f (structure,)h(see)h(text)117 2015 y(OUT)124 b Fo(newrank)414 b Ft(reordered)16 b(rank)e(of)f(the)i(calling)d(pro)q(cess)166 2139 y FC(De\014nition)k(of)f(graph)g(structure:)k(for)c(eac)o(h)g (graph)g(no)q(de)g(its)g(set)g(of)g(neigh)o(b)q(ors)h(is)f(en)o(tered,) g(i.)h(e.,)75 2195 y(the)g(set)f(of)g(pro)q(cesses)h(with)f(whic)o(h)i (the)e(no)q(de)h(pro)q(cess)g(is)g(going)f(to)g(comm)o(unicate)h(most)f (frequen)o(tly)l(.)75 2252 y(Since)j(there)e(is)h(no)g(w)o(eigh)o(ting) f(of)g(no)q(de)h(connections,)h(it)e(migh)o(t)g(b)q(e)h(sensible)i(to)d (sp)q(ecify)h(only)g(those)75 2308 y(neigh)o(b)q(ors)f(with)f(whic)o(h) i(there)e(is)h(a)e(considerable)j(amoun)o(t)e(of)g(comm)o(unication.) 166 2365 y(F)l(or)21 b(eac)o(h)h(no)q(de)h FB(i)38 b FC(\(0)23 b Fy(\024)i FB(i)e Fy(\024)i FB(nodes)15 b Fy(\000)g FC(1\))21 b(the)h(list)h(of)e(the)h(neigh)o(b)q(or)h(no)q (des)g(is)f(stored)g(at)75 2421 y FB(ng)r(l)q(ist)p FC(\()p FB(j)s FC(\))p FB(;)28 b(index)p FC(\()p FB(i)p FC(\))12 b Fy(\024)h FB(j)i Fy(\024)e FB(index)p FC(\()p FB(i)d FC(+)g(1\))g Fy(\000)g FC(1.)20 b(F)l(or)15 b(the)g(last)g(pro)q(cess)h FB(index)p FC(\()p FB(i)9 b FC(+)i(1\))j(p)q(oin)o(ts)i(to)f(the)75 2478 y(\014rst)i(free)h(p)q(osition)h(in)f(v)o(ector)f(nglist.)28 b(Note)17 b(that)g(the)h(matrix)f(is)i(symmetric,)f(i.)f(e.,)h(if)g(no) q(de)h Fs(i)e FC(is)h(a)75 2534 y(neigh)o(b)q(or)e(of)f(no)q(de)h Fs(j)p FC(,)f(than)g(no)q(de)h Fs(j)f FC(is)h(also)f(a)g(neigh)o(b)q (or)h(of)f(no)q(de)h Fs(i)p FC(.)k(This)15 b(prop)q(ert)o(y)g(can)h(b)q (e)g(used)g(for)75 2591 y(an)f(in)o(ternal)h(c)o(hec)o(king)g(of)f(the) g(input)i(argumen)o(ts.)166 2647 y(The)i(de\014nitions)h(of)e(the)h (argumen)o(ts)f Fp(nodes)p FC(,)g Fp(index)p FC(,)h(and)g Fp(nglist)f FC(are)g(illustrated)i(with)f(the)75 2704 y(follo)o(wing)d(simple)h(example:)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 78 84 78 83 bop 75 -100 a FC(78)926 b Fv(SECTION)16 b(5.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)166 45 y FC(Assume)g(there)h(are)f(four)f (pro)q(cesses)i(0,)e(1,)h(2,)f(3)h(with)h(the)f(follo)o(wing)h (adjacency)g(matrix:)100 98 y(pro)q(cess)50 b(neigh)o(b)q(ors)100 154 y(0)170 b(1,)15 b(3)100 211 y(1)170 b(0)100 267 y(2)g(3)100 323 y(3)g(0,)15 b(2)166 384 y(Then,)g(the)h(input)g(argumen)o(ts)e (are:)100 437 y(no)q(des)i(=)50 b(4)100 493 y(index)16 b(=)56 b(0,)14 b(2,)h(3,)f(4,)h(6)100 550 y(nglist)h(=)54 b(1,)14 b(3,)h(0,)f(3,)h(0,)f(2)166 610 y(If)i(the)g(top)q(ology)g(con) o(tains)g(few)o(er)f(pro)q(cesses)i(than)e(there)h(are)g(pro)q(cesses)g (in)h(the)f(paren)o(t)g(group,)75 667 y(the)g(newrank)h(v)m(alues)g (for)f(the)h(remaining)g(pro)q(cesses)g(are)f(set)g(to)g Fr(MPI)p 1320 667 13 2 v 14 w(PROCNULL)p FC(.)f(It)i(is)g(an)f(error)f (if)75 723 y(the)g(top)q(ology)g(de\014nes)h(more)f(pro)q(cesses)h (than)f(are)g(con)o(tained)h(in)g(the)f(paren)o(t)g(group.)166 864 y Fu(Discussion:)21 b Ft(It)c(cannot)g(b)q(e)h(ruled)f(out)g (completely)f(that)h(top)q(ology)e(de\014nition)i(functions)g(in)o (ternally)75 913 y(require)e(comm)o(unication.)i(This)e(could)f(b)q (ecome)h(a)f(necessit)o(y)i(if)e(a)g(parallel)f(algorithm)f(is)j(used)g (for)g(doing)e(the)75 963 y(mapping.)18 b(If)d(so,)f(then)h(it)g(migh)o (t)d(b)q(e)k(necessary)g(to)f(pass)g(them)f(a)g(comm)o(unicator)e (instead)j(of/in)f(addition)f(to)75 1013 y(the)h(group)g(id.)166 1078 y(In)k(this)g(case)h(w)o(e)f(also)f(ha)o(v)o(e)h(to)g(sa)o(y)g (that)g(a)f(lo)o(w{lev)o(el)g(top)q(ology)f(de\014nition)i(function)g (is)f(a)h(collectiv)o(e)75 1134 y(op)q(eration)f(on)f(the)i(group.)26 b(If)17 b(it)f(is)h(not)g(called)f(b)o(y)h(all)f(group)g(pro)q(cesses)k (lo)q(osely)c(sync)o(hronously)m(,)h(deadlo)q(c)o(k)75 1191 y(migh)o(t)12 b(o)q(ccur.)75 1441 y Fq(5.4.2)49 b(High{level)18 b(top)q(ology)h(functions)75 1542 y FC(In)g(principle,) i(all)e(that)e(is)i(needed)g(to)f(create)g(an)g(MPI)g(pro)q(cess)g (group)g(with)h(top)q(ological)g(ordering)75 1599 y(are)13 b(the)h(t)o(w)o(o)f(mapping)h(functions)g(and)g(the)g(general)g(group)g (creation)f(routine.)20 b(Ho)o(w)o(ev)o(er,)13 b(this)h(is)g(not)75 1655 y(what)19 b(the)g(applications)i(programmer)d(exp)q(ects)i(to)f (see)h(on)f(his)h(lev)o(el)h(of)e(abstraction.)31 b(Therefore,)75 1712 y(another)13 b(la)o(y)o(er)h(of)f(functions)h(is)g(de\014ned)i(on) d(top)g(of)h(the)f(lo)o(w{lev)o(el)i(functions.)20 b(This)14 b(higher)h(lev)o(el)g(can)75 1768 y(b)q(e)g(p)q(ortably)f(implemen)o (ted)i(in)f(terms)f(of)g(the)g(lev)o(el)i(b)q(elo)o(w.)k(Of)14 b(course)h(MPI)f(implemen)o(tors)h(are)f(free)75 1825 y(to)h(b)o(ypass)g(the)g(lo)o(w)o(er)g(lev)o(el)h(for)f(the)g(sak)o(e)g (of)g(b)q(etter)g(p)q(erformance)g(or)g(resource)g(usage.)166 1889 y(F)l(or)h(Cartesian)g(and)h(graph)f(structures)h(the)f(functions) i Fo(MPI)p 1252 1889 14 2 v 16 w(CART)f FC(and)g Fo(MPI)p 1576 1889 V 15 w(GRAPH)h FC(create)75 1946 y(a)c(pro)q(cess)g(group)g (with)h(top)q(ological)g(structure.)k(They)14 b(com)o(bine)h(the)f (call)i(to)d(the)h(mapping)h(function)75 2002 y(with)f(the)g(call)h(to) e(the)g(group)h(creation)g(routine.)20 b(The)14 b(user)f(can)h(decide)i (whether)e(the)f(ranking)h(in)h(the)75 2058 y(new)i(group)g(ma)o(y)f(b) q(e)i(reordered)f(\(for)f(a)g(more)h(e\016cien)o(t)g(mapping\))h(or)e (not.)25 b(In)17 b(the)g(latter)g(case)g(the)75 2115 y(rank)e(of)g(the)g(calling)i(pro)q(cess)e(is)h(copied)g(from)f(the)g (paren)o(t)g(group.)166 2179 y(The)h(option)g(of)f(not)g(reordering)h (the)g(pro)q(cess)f(ranks)h(can)f(b)q(e)i(useful)f(in)h(the)e(case)h (of)f(an)h(o)o(v)o(erla)o(y)75 2236 y(of)f(di\013eren)o(t)h(top)q (ologies.)21 b(The)16 b(user)f(can)h(then)g(fully)h(con)o(trol)e(whic)o (h)h(lo)q(cation)g(in)h(the)e(\014rst)g(top)q(ology)75 2292 y(coincides)i(with)e(whic)o(h)h(lo)q(cation)g(in)f(the)g(second)h (one.)k(The)15 b(price)h(whic)o(h)f(one)h(ob)o(viously)f(has)g(to)f(pa) o(y)75 2349 y(in)i(this)g(case)f(is)h(that)e(for)h(only)g(one)h(top)q (ology)f(the)g(pro)q(cess)g(arrangemen)o(t)f(can)i(b)q(e)g(optimized.) 166 2413 y(As)11 b(sp)q(eci\014ed)j(for)d(the)h(lo)o(w{lev)o(el)g (functions)h(it)f(is)g(an)f(error)g(if)h(one)g(attempts)e(to)h (de\014ne)i(a)e(top)q(ology)75 2470 y(with)18 b(more)g(lo)q(cations)g (than)g(there)g(are)g(pro)q(cesses)g(in)h(the)f(paren)o(t)f(pro)q(cess) i(group.)27 b(If)19 b(the)f(n)o(um)o(b)q(er)75 2526 y(of)f(lo)q (cations)h(is)f(less)h(than)f(the)g(n)o(um)o(b)q(er)h(of)f(a)o(v)m (ailable)h(pro)q(cesses,)g(the)f(remaining)i(pro)q(cesses)e(of)g(the)75 2583 y(paren)o(t)f(group)g(will)h(not)f(b)q(e)h(mem)o(b)q(ers)f(of)g (the)g(new)g(group.)23 b(F)l(or)15 b(those)h(pro)q(cesses)g(the)h(rank) e(inquiry)75 2639 y(function)h Fo(MPI)p 338 2639 V 16 w(RANK)f FC(returns)g Fr(MPI)p 723 2639 13 2 v 15 w(PROCNULL)f FC(if)i(called)h(with)e(the)h(iden)o(ti\014er)g(of)f(the)g(new)h (group.)166 2704 y Fo(MPI)p 251 2704 14 2 v 16 w(CART)f FC(is)h(de\014ned)h(as)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 79 85 79 84 bop 75 -100 a Fv(5.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)536 b FC(79)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(CART\(gid)p 370 45 V 17 w(old,)14 b(gid)p 528 45 V 17 w(ca)o(rt,)g(ndims,)h(dims,)g(p)q (erio)q(d,)h(reo)o(rder\))117 128 y Ft(IN)171 b Fo(gid)p 394 128 V 16 w(old)441 b Ft(group)14 b(id)f(of)h(paren)o(t)g(group)117 215 y(OUT)124 b Fo(gid)p 394 215 V 16 w(ca)o(rt)425 b Ft(group)14 b(id)f(of)h(Cartesian)g(group)117 302 y(IN)171 b Fo(ndims,)15 b(dims\(*\),)e(p)q(erio)q(d\(*\))82 b Ft(de\014nition)17 b(of)g(Cartesian)h(structure,)h(\(for)f(details)f (see)h(the)905 358 y(de\014nition)c(of)f Fr(MPI)p 1214 358 13 2 v 14 w(MAP)p 1319 358 V 15 w(CART)p Ft(\))117 445 y(IN)171 b Fo(reo)o(rder)438 b Ft(ranking)13 b(ma)o(y)f(b)q(e)j (reordered)h(or)e(not)166 575 y FC(Apart)h(from)f(some)h(lo)q(cal)h (computations,)f Fo(MPI)p 1008 575 14 2 v 16 w(CART)h FC(is)f(based)h(on)f(the)g(t)o(w)o(o)f(function)i(calls)166 637 y Fo(MPI)p 251 637 V 16 w(MAP)p 366 637 V 16 w(CART)g(\(gid)p 592 637 V 16 w(old,)f(ndims,)g(dims,)g(p)q(erio)q(d,)g(newrank\))166 699 y FC(and)166 762 y Fo(MPI)p 251 762 V 16 w(CREA)l(TE)p 436 762 V 17 w(GROUP)i(\(gid)p 697 762 V 16 w(old,)e(gid)p 855 762 V 16 w(ca)o(rt,)g(0,)f(newrank\))p FC(.)166 883 y(F)l(or)g(graph)h(structures)g(the)h(corresp)q(onding)g(routine)f(is) 75 992 y Fo(MPI)p 160 992 V 16 w(GRAPH\(gid)p 401 992 V 17 w(old,)g(gid)p 560 992 V 16 w(graph,)g(no)q(des,)i(index,)e (nglist,)h(reo)o(rder\))117 1075 y Ft(IN)171 b Fo(gid)p 394 1075 V 16 w(old)441 b Ft(group)14 b(id)f(of)h(paren)o(t)g(group)117 1162 y(OUT)124 b Fo(gid)p 394 1162 V 16 w(graph)391 b Ft(group)14 b(id)f(of)h(graph)f(group)117 1249 y(IN)171 b Fo(no)q(des,)16 b(index\(*\),)f(nglist\(*\))90 b Ft(de\014nition)14 b(of)g(graph)g(structure,)h(\(for)f(details)g(see)i(the)f(de\014-)905 1305 y(nition)e(of)g(MPI)p 1155 1305 13 2 v 16 w(MAP)p 1268 1305 V 15 w(GRAPH\))117 1392 y(IN)171 b Fo(reo)o(rder)438 b Ft(ranking)13 b(ma)o(y)f(b)q(e)j(reordered)h(or)e(not)166 1581 y FC(The)i(follo)o(wing)h(top)q(ology)f(information)h(is)g(lik)o (ely)h(to)e(b)q(e)g(stored)g(with)h(the)f(pro)q(cess)h(group)f(b)o(y)g (a)75 1637 y(high{lev)o(el)h(top)q(ology)e(de\014nition)i(function:)131 1754 y(1.)22 b(T)o(yp)q(e)15 b(of)g(top)q(ology)g(\(Cartesian/graph\)) 131 1871 y(2.)22 b(F)l(or)14 b(a)h(Cartesian)g(top)q(ology:)243 1988 y Fy(\017)23 b FC(ndims)16 b(\(n)o(um)o(b)q(er)f(of)g (dimensions\),)243 2073 y Fy(\017)23 b FC(dims\(*\))15 b(\(n)o(um)o(b)q(ers)g(of)f(pro)q(cesses)i(p)q(er)g(co)q(ordinate)f (direction\),)243 2157 y Fy(\017)23 b FC(p)q(erio)q(d\(*\))15 b(\(p)q(erio)q(dicit)o(y)i(information\),)243 2242 y Fy(\017)23 b FC(o)o(wn)p 372 2242 14 2 v 16 w(p)q(osition\(*\))c(\(o)o (wn)f(p)q(osition)i(in)g(grid,)h(could)f(also)f(b)q(e)h(computed)f (from)g(rank)g(and)289 2299 y(dims\(*\)\))75 2416 y(F)l(or)d(a)g(graph) g(structure)g(no)g(information)h(has)f(to)g(b)q(e)h(stored,)f(b)q (ecause)h(the)g(graph)f(no)q(de)h(n)o(um)o(b)q(er)f(is)75 2472 y(the)h(rank)f(in)i(the)f(graph)f(group,)h(and)g(the)g(size)g(of)g (the)g(graph)f(is)i(the)e(n)o(um)o(b)q(er)i(of)e(pro)q(cesses)h(in)h (that)75 2528 y(group.)166 2591 y(The)d(actual)g(set)g(of)g(top)q (ology)f(data)h(stored)f(with)i(a)f(pro)q(cess)g(group)g(is)g(implemen) o(tation)h(dep)q(en-)75 2647 y(den)o(t.)k(It)13 b(can)h(con)o(tain)g (more)g(information)g(than)g(giv)o(en)g(ab)q(o)o(v)o(e,)f(whereas)h(in) h(other)e(cases)h(some)g(items)75 2704 y(migh)o(t)h(b)q(e)h(redundan)o (t.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 80 86 80 85 bop 75 -100 a FC(80)926 b Fv(SECTION)16 b(5.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fq(5.4.3)49 b(T)l(op)q(ology)19 b(inquiry)e(functions)75 162 y FC(If)j(a)g(top)q(ology)g(has)f(b)q(een) j(de\014ned)f(with)f(one)g(of)g(the)g(ab)q(o)o(v)o(e)g(high{lev)o(el)i (functions,)f(the)f(top)q(ology)75 218 y(information)c(can)h(b)q(e)f (lo)q(ok)o(ed)h(up)g(using)f(inquiry)i(functions.)23 b(They)17 b(all)g(are)f(lo)q(cal)h(calls,)g(and)f(do)g(not)75 275 y(require)g(an)o(y)f(comm)o(unication.)166 347 y(The)g(function)h Fo(MPI)p 522 347 14 2 v 16 w(INQMAP)f FC(returns)f(the)h(t)o(yp)q(e)g (of)g(mapping)g(whic)o(h)h(is)f(assigned)h(to)e(a)h(group.)75 466 y Fo(MPI)p 160 466 V 16 w(INQMAP\(gid,)g(map\))117 560 y Ft(IN)171 b Fo(gid)514 b Ft(group)14 b(id)117 666 y(OUT)108 b Fo(map)505 b Ft(top)q(ology)13 b(t)o(yp)q(e)h(of)g(group)f (with)h(iden)o(ti\014er)g Fn(gid)166 807 y FC(The)h(output)g(v)m(alue)i Fp(map)d FC(is)i(one)g(of)e(the)i(follo)o(wing:)117 910 y Fr(MPI)p 194 910 13 2 v 14 w(GRAPHTOP)516 b FC(graph)15 b(top)q(ology)117 996 y Fr(MPI)p 194 996 V 14 w(CARTTOP)545 b FC(Cartesian)15 b(top)q(ology)117 1083 y Fr(MPI)p 194 1083 V 14 w(NOTOP)595 b FC(no)15 b(top)q(ology)166 1181 y(F)l(or)i(a)h(pro)q(cess)g(group)f(with)i(a)e(graph)h(structure)f (whic)o(h)i(has)f(b)q(een)h(created)f(b)o(y)f Fo(MPI)p 1698 1181 14 2 v 16 w(GRAPH)p FC(,)75 1238 y(functions)f Fo(MPI)p 356 1238 V 16 w(GRAPHDIMS)g FC(and)f Fo(MPI)p 819 1238 V 16 w(INQGRAPH)h FC(lo)q(ok)g(up)g(top)q(ology)e (information:)75 1357 y Fo(MPI)p 160 1357 V 16 w(GRAPHDIMS\(gid,)h (indexlen,)h(nglistlen\))117 1450 y Ft(IN)171 b Fo(gid)514 b Ft(group)14 b(id)f(of)h(group)f(with)h(graph)g(structure)117 1557 y(OUT)124 b Fo(indexlen)419 b Ft(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(v)o(ector)i Fn(index)117 1664 y Ft(OUT)124 b Fo(nglistlen)416 b Ft(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(v)o(ector)i Fn(nglist)166 1804 y FC(The)h(information)g(pro)o(vided)h(b)o(y)f Fo(MPI)p 844 1804 V 15 w(GRAPHDIMS)h FC(can)f(b)q(e)g(used)h(to)e(dimension)j (the)e(v)o(ectors)75 1861 y Fp(index)i FC(and)i Fp(nglist)e FC(long)h(enough)h(in)g(the)f(follo)o(wing)h(call)g(to)f Fo(MPI)p 1298 1861 V 15 w(INQGRAPH)p FC(.)i(The)e(n)o(um)o(b)q(er)g(of) 75 1917 y(no)q(des)d(in)g(the)f(graph)g(is)h Fp(indexlen)22 b(-)i(1)p FC(.)75 2037 y Fo(MPI)p 160 2037 V 16 w(INQGRAPH\(gid,)16 b(indexdim,)f(nglistdim,)g(index,)h(nglist\))117 2130 y Ft(IN)171 b Fo(gid)514 b Ft(group)14 b(id)f(of)h(group)f(with)h (graph)g(structure)117 2237 y(IN)171 b Fo(indexdim)402 b Ft(length)14 b(of)f(v)o(ector)i(index\(*\))f(in)f(the)i(calling)d (program)117 2344 y(IN)171 b Fo(nglistdim)399 b Ft(length)14 b(of)f(v)o(ector)i(nglist\(*\))e(in)h(the)g(calling)f(program)117 2450 y(OUT)124 b Fo(index\(*\),)14 b(nglist\(*\))227 b Ft(v)o(ectors)15 b(con)o(taining)d(the)h(graph)g(structure)j(\(for)d (details)g(see)905 2507 y(the)i(de\014nition)e(of)g(MPI)p 1291 2507 13 2 v 16 w(MAP)p 1404 2507 V 15 w(GRAPH\))166 2647 y FC(F)l(or)18 b(a)h(pro)q(cess)g(group)f(with)h(Cartesian)g (ordering)g(whic)o(h)h(has)e(b)q(een)i(created)f(b)o(y)g Fo(MPI)p 1730 2647 14 2 v 16 w(CART)p FC(,)75 2704 y(functions)d Fo(MPI)p 356 2704 V 16 w(INQDIM)e FC(and)i Fo(MPI)p 721 2704 V 16 w(INQCART)g FC(lo)q(ok)f(up)h(top)q(ology)f(information:)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 81 87 81 86 bop 75 -100 a Fv(5.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)536 b FC(81)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(INQDIM\(gid,)14 b(dim\))117 123 y Ft(IN)171 b Fo(gid)514 b Ft(group)14 b(id)f(of)h(group)f(with)h (Cartesian)g(structure)117 199 y(OUT)124 b Fo(dim)500 b Ft(dimension)12 b(of)i(the)g(Cartesian)g(structure)75 371 y Fo(MPI)p 160 371 V 16 w(INQCART\(gid,)h(maxdim,)e(size,)i(p)q (erio)q(d,)h(p)q(os\))117 449 y Ft(IN)171 b Fo(gid)514 b Ft(group)14 b(id)f(of)h(group)f(with)h(Cartesian)g(structure)117 525 y(IN)171 b Fo(maxdim)420 b Ft(length)14 b(of)g(v)o(ectors)h (size\(*\),)g(p)q(erio)q(d\(*\),)f(and)g(p)q(os\(*\))h(in)e(the)905 581 y(calling)g(program)117 657 y(OUT)124 b Fo(size\(*\))444 b Ft(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(for)e(eac)o(h)g(co)q (ordinate)g(direction)117 733 y(OUT)124 b Fo(p)q(erio)q(d\(*\))394 b Ft(p)q(erio)q(dicit)o(y)14 b(y)o(es/no)g(for)f(eac)o(h)i(co)q (ordinate)f(direction)117 809 y(OUT)124 b Fo(p)q(os\(*\))448 b Ft(p)q(osition)13 b(of)h(calling)e(pro)q(cess)k(in)d(Cartesian)h (structure)166 934 y FC(The)20 b(inquiry)g(for)f(Cartesian)g(top)q (ology)h(information)f(is)h(th)o(us)f(split)i(up)f(in)o(to)f(t)o(w)o(o) f(calls.)34 b(The)75 991 y(reason)21 b(is)h(that)f(otherwise)h(the)f (user)h(migh)o(t)f(not)g(kno)o(w)g(a)h(sa)o(v)o(e)e(v)m(alue)j(for)e (the)h(dimensioning)h(of)75 1047 y(v)o(ectors)14 b Fp(size,)24 b(period)14 b FC(and)h Fp(pos)g FC(b)q(efore)h(calling)g Fo(MPI)p 1071 1047 V 16 w(INQCART)g FC(\(input)g(argumen)o(t)f Fp(maxdim)p FC(\).)166 1104 y(F)l(or)i(a)g(pro)q(cess)g(group)h(with)f (Cartesian)g(structure,)h(the)f(function)i Fo(MPI)p 1451 1104 V 15 w(INQRANK)f FC(translates)75 1160 y(the)j(logical)h(pro)q (cess)f(co)q(ordinates)g(to)f(pro)q(cess)h(ranks)f(as)g(they)h(are)f (used)i(b)o(y)e(the)h(p)q(oin)o(t{to{p)q(oin)o(t)75 1217 y(routines:)75 1321 y Fo(MPI)p 160 1321 V 16 w(INQRANK\(gid,)15 b(lo)q(cation,)g(rank\))117 1399 y Ft(IN)171 b Fo(gid)514 b Ft(group)14 b(id)f(of)h(group)f(with)h(Cartesian)g(structure)117 1475 y(IN)171 b Fo(lo)q(cation)421 b Ft(co)q(ordinate)14 b(v)o(ector)h(of)e(pro)q(cess)117 1551 y(OUT)124 b Fo(rank)488 b Ft(pro)q(cess)16 b(rank)166 1676 y FC(In)g(a)f(sense,)g(the)g(in)o(v) o(erse)h(op)q(eration)f(to)g Fo(MPI)p 963 1676 V 16 w(INQRANK)g FC(is)h(pro)o(vided)g(b)o(y)f(the)h(function:)75 1780 y Fo(MPI)p 160 1780 V 16 w(INQLOC\(gid,)f(rank,)g(lo)q(cation,)g (maxdim\))117 1857 y Ft(IN)171 b Fo(gid)514 b Ft(group)14 b(id)f(of)h(group)f(with)h(Cartesian)g(structure)117 1933 y(IN)171 b Fo(rank)488 b Ft(rank)14 b(of)f(a)h(pro)q(cess)i (within)d(group)g Fn(gid)117 2009 y Ft(OUT)124 b Fo(lo)q(cation\(*\)) 362 b Ft(co)q(ordinate)14 b(v)o(ector)h(of)e(pro)q(cess)117 2085 y(IN)171 b Fo(maxdim)420 b Ft(length)11 b(of)e(v)o(ector)i Fn(location)e Ft(in)h(the)h(calling)e(program)g(\(only)905 2142 y(for)14 b(error)h(c)o(hec)o(king\))75 2267 y Fo(MPI)p 160 2267 V 16 w(INQLOC)g FC(translates)g(a)g(rank)g(in)h(the)f(group)g (in)o(to)g(a)g(top)q(ology)g(lo)q(cation.)75 2391 y Fq(5.4.4)49 b(P)o(a)o(rtitioning)17 b(of)g(Ca)o(rtesian)e(structures)75 2478 y FC(If)g(a)f(group)h(has)f(b)q(een)i(created)f(with)g Fo(MPI)p 820 2478 V 15 w(CART)p FC(,)g(the)g(function)g Fo(MPI)p 1319 2478 V 16 w(P)l(ARTC)h FC(can)f(b)q(e)g(used)g(to)f(par-) 75 2534 y(tition)i(the)g(group)f(in)o(to)h(subgroups)f(whic)o(h)i(form) e(h)o(yp)q(er{planes)h(of)f(the)h(original)h(pro)q(cess)f(structure.)75 2591 y(As)i(an)g(example,)h(assume)f(that)g Fo(MPI)p 752 2591 V 15 w(CART)h FC(has)f(de\014ned)h(a)f(t)o(w)o(o{dimensional)h (grid)f(structure)g(on)75 2647 y(a)f(pro)q(cess)g(group.)25 b(With)17 b(a)g(call)h(to)f Fo(MPI)p 814 2647 V 15 w(P)l(ARTC)h FC(this)g(group)e(can)i(then)f(b)q(e)h(split)g(up)f(in)o(to)g(disjoin)o (t)75 2704 y(subgroups)e(whic)o(h)h(represen)o(t)g(ro)o(ws)e(or)g (columns)i(of)f(the)g(original)i(group.)i(The)d(function)g(syn)o(tax)e (is:)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 82 88 82 87 bop 75 -100 a FC(82)926 b Fv(SECTION)16 b(5.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(P)l(ARTC\(gid,)g(lostdim,)g(gid)p 646 45 V 16 w(new\))117 122 y Ft(IN)171 b Fo(gid)514 b Ft(group)14 b(id)f(of)h(group)f(with)h(Cartesian)g(structure)117 198 y(IN)171 b Fo(lostdim)433 b Ft(direction)14 b(p)q(erp)q(endicular)h (to)f(resulting)g(h)o(yp)q(er{planes)117 273 y(OUT)124 b Fo(gid)p 394 273 V 16 w(new)424 b Ft(group)12 b(iden)o(ti\014er)g(of) f(the)h(h)o(yp)q(er{plane)g(group)g(in)f(whic)o(h)g(the)905 330 y(calling)i(pro)q(cess)i(is)f(con)o(tained)166 454 y Fo(MPI)p 251 454 V 16 w(P)l(ARTC)22 b FC(uses)f(the)g(function)h Fo(MPI)p 885 454 V 16 w(CREA)l(TE)p 1070 454 V 17 w(GROUP)h FC(to)d(partition)i(the)f(paren)o(t)g(group,)75 511 y(using)h(the)f (pro)q(cess)g(co)q(ordinate)h(in)f(direction)i Fp(lostdim)d FC(as)g(the)i(k)o(ey)e(v)m(alue.)39 b(After)21 b(a)f(group)h(has)75 567 y(b)q(een)16 b(partitioned)f(b)o(y)g Fo(MPI)p 564 567 V 15 w(P)l(ARTC)p FC(,)g(the)g(inquiry)h(functions)f Fo(MPI)p 1262 567 V 16 w(INQCART)h FC(and)e Fo(MPI)p 1660 567 V 16 w(INQRANK)75 624 y FC(consisten)o(tly)i(refer)f(to)g(the) g(grid)g(sections)h(if)g(applied)h(to)d(a)h(subgroup.)166 680 y(If)f(the)h(pro)q(cess)f(top)q(ology)g(is)h(a)e(Cartesian)h (structure,)g(a)g(collectiv)o(e)i(shift)e(op)q(eration)h(most)e(lik)o (ely)75 737 y(is)21 b(used)h(along)e(a)h(co)q(ordinate)g(direction.)37 b(In)22 b(MPI)e(shift)h(op)q(erations)g(are)g(handled)h(b)o(y)f(the)f (more)75 793 y(general)f(send{receiv)o(e)h(functions.)31 b(Those)19 b(tak)o(e)f(as)g(input)i(the)f(rank)f(of)h(a)f(source)h(pro) q(cess)g(for)f(the)75 849 y(receiv)o(e,)e(and)f(the)g(rank)g(of)g(a)g (destination)h(pro)q(cess)g(for)e(the)h(send.)166 906 y(If)d(the)f(function)h Fo(MPI)p 541 906 V 16 w(SHIFT)g FC(is)g(called)h(for)d(a)i(Cartesian)f(pro)q(cess)g(group,)h(it)g(pro)o (vides)g(the)f(calling)75 962 y(pro)q(cess)17 b(with)f(the)h(ab)q(o)o (v)o(e)f(iden)o(ti\014ers,)i(whic)o(h)f(then)g(can)g(b)q(e)g(passed)f (to)g(the)h(send{receiv)o(e)g(function.)75 1019 y(The)e(user)h(sp)q (eci\014es)h(the)e(co)q(ordinate)h(direction)g(for)f(the)g(shift)h(and) f(the)g(step)g(length:)75 1123 y Fo(MPI)p 160 1123 V 16 w(SHIFT)p 303 1123 V 16 w(ID\(gid,)f(direction,)i(steps,)g(rank)p 864 1123 V 17 w(source,)f(rank)p 1111 1123 V 17 w(dest\))117 1200 y Ft(IN)171 b Fo(gid)514 b Ft(group)14 b(id)f(of)h(group)f(with)h (Cartesian)g(structure)117 1275 y(IN)171 b Fo(direction)408 b Ft(co)q(ordinate)14 b(direction)h(of)e(shift)117 1351 y(IN)171 b Fo(steps)478 b Ft(step)22 b(length)f(\()p Fj(>)i Ft(0:)32 b(up)o(w)o(ards)21 b(shift,)h Fj(<)h Ft(0:)32 b(do)o(wn)o(w)o(ards)905 1407 y(shift\))117 1483 y(OUT)124 b Fo(rank)p 420 1483 V 16 w(source)353 b Ft(rank)14 b(of)f(source)j(pro)q(cess)117 1558 y(OUT)124 b Fo(rank)p 420 1558 V 16 w(dest)396 b Ft(rank)14 b(of)f(destination)h (pro)q(cess)166 1683 y FC(Dep)q(ending)k(of)d(the)h(p)q(erio)q(dicit)o (y)j(of)c(the)h(Cartesian)g(group)g(in)h(the)f(sp)q(eci\014ed)i(co)q (ordinate)f(direc-)75 1739 y(tion,)h Fo(MPI)p 269 1739 V 16 w(SHIFT)p 412 1739 V 16 w(ID)f FC(pro)o(vides)i(the)e(iden)o (ti\014ers)j(for)d(a)g(circular)i(or)e(an)h(end{o\013)f(shift.)28 b(In)18 b(the)g(case)75 1795 y(of)d(an)h(end{o\013)f(shift,)g(at)g(the) h(b)q(oundary)g(pro)q(cess)g(one)g(of)f(the)g(iden)o(ti\014ers)i(is)f (set)g(to)f Fr(MPI)p 1637 1795 13 2 v 14 w(PROCNULL)p FC(,)75 1852 y(indicating)i(that)e(there)g(is)h(no)f(pro)q(cess)g(in)h (that)f(direction)h(to)f(comm)o(unicate)g(with.)166 1908 y(In)22 b(a)g(p)q(ortable)g(program)f(the)h(n)o(um)o(b)q(er)g(of)f(a)o (v)m(ailable)j(pro)q(cessors)d(usually)i(is)g(not)e(kno)o(wn)h(at)75 1965 y(compile)h(time.)38 b(In)21 b(the)h(case)f(of)f(a)h(Cartesian)g (top)q(ology)g(the)g(function)h Fo(MPI)p 1494 1965 14 2 v 16 w(MAKDIM)e FC(helps)i(in)75 2021 y(setting)f(the)g(n)o(um)o(b)q (ers)h(of)f(pro)q(cesses)g(p)q(er)h(co)q(ordinate)f(direction,)j(dep)q (ending)g(on)d(the)g(n)o(um)o(b)q(er)g(of)75 2078 y(pro)q(cesses)16 b(in)g(the)f(group)g(and)g(optional)h(constrain)o(ts)f(whic)o(h)h(can)f (b)q(e)h(sp)q(eci\014ed)h(b)o(y)e(the)h(user:)75 2182 y Fo(MPI)p 160 2182 V 16 w(MAKDIM\(np)o(ro)q(cs,)e(ndims,)h(dims\))117 2259 y Ft(IN)171 b Fo(np)o(ro)q(cs)449 b Ft(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g(group)117 2334 y(IN)171 b Fo(ndims)460 b Ft(n)o(um)o(b)q(er)13 b(of)h(Cartesian)g(dimensions) 117 2410 y(INOUT)78 b Fo(dims)483 b Ft(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(eac)o(h)g(co)q(ordinate)g(direction)166 2534 y FC(The)h Fp(nprocs)e FC(pro)q(cesses)i(are)f(divided)j(as)d(ev)o (enly)i(as)e(p)q(ossible)i(among)e Fp(ndims)g FC(dimensions,)h(sub-)75 2591 y(ject)20 b(to)g(the)h(constrain)o(t)e(that)h(all)h(pro)q(cesses)g (are)f(used.)36 b(The)21 b(caller)g(ma)o(y)f(further)g(constrain)h(the) 75 2647 y(op)q(eration)h(of)g(this)h(routine)f(b)o(y)g(sp)q(ecifying)i (elemen)o(ts)f(of)f(arra)o(y)f Fp(dims)p FC(.)40 b(If)22 b Fp(dims\(i\))f FC(is)i(set)f(to)f(a)75 2704 y(p)q(ositiv)o(e)d(n)o (um)o(b)q(er,)g(the)g(routine)f(will)i(\014x)f(the)f(n)o(um)o(b)q(er)h (of)f(pro)q(cesses)h(in)g(co)q(ordinate)g(direction)g Fp(i)g FC(to)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 83 89 83 88 bop 75 -100 a Fv(5.5.)34 b(A)15 b(SIMPLE)h(EXAMPLE)1180 b FC(83)75 45 y Fp(dims\(i\))p FC(.)27 b(Pro)q(cesses)17 b(will)j(then)e(b)q(e)g(distributed)i(among)d(directions)i(where)f Fp(dims\(i\))f FC(is)h(set)g(to)f(0,)75 102 y(sub)s(ject)e(to)g(the)g (ab)q(o)o(v)o(e)g(constrain)o(ts.)166 158 y(Note)g(that)f(an)h(error)g (will)i(o)q(ccur)e(if)h Fp(nprocs)e FC(is)i(not)f(a)g(m)o(ultiple)i(of) 1417 118 y Ff(Y)1352 212 y Fk(i;dims)p Fl(\()p Fk(i)p Fl(\))p Fm(6)p Fl(=0)1543 158 y FB(dims)p FC(\()p FB(i)p FC(\).)166 260 y(Negativ)o(e)i(input)h(v)m(alues)h(of)e Fp(dims\(i\))f FC(will)j(also)e(cause)g(an)g(error.)32 b(All)20 b(elemen)o(ts)g(of)f Fp(dims\(*\))75 316 y FC(m)o(ust)c(b)q(e) g(set)g(to)g(0)g(for)f(completely)j(automatic)e(pro)q(cess)g (distribution.)166 373 y(When)c(the)g(function)h(returns,)f Fp(dims\(i\))f FC(will)j(b)q(e)e(set)g(to)f(the)h(n)o(um)o(b)q(er)g(of) g(pro)q(cesses)g(in)h(co)q(ordinate)75 429 y(direction)20 b Fp(i)p FC(.)29 b(F)l(or)18 b Fp(dims\(i\))g FC(not)g(set)g(b)o(y)h (the)f(user,)h Fp(dims\(i\))f FC(will)i(b)q(e)f(ordered)g(\(MAXIMUM,)e (...,)75 486 y(MINIMUM\).)d(Arra)o(y)h Fp(dims\(*\))f FC(is)i(suitable)g(for)f(use)g(as)g(input)h(to)f(routine)g Fo(MPI)p 1496 486 14 2 v 16 w(CART)p FC(.)75 629 y Fw(5.5)59 b(A)20 b(simple)f(example)75 730 y FC(The)12 b(example)h(in)g(\014gure) f(5.1)f(sho)o(ws)g(ho)o(w)h(the)g(grid)g(de\014nition)i(and)e(inquiry)i (functions)f(can)f(b)q(e)h(used)f(in)75 787 y(an)k(application)h (program.)k(A)16 b(partial)g(di\013eren)o(tial)h(equation,)f(for)f (example)h(the)g(P)o(oisson)g(equation,)75 843 y(is)g(to)f(b)q(e)h (solv)o(ed)f(on)h(a)f(rectangular)g(domain.)21 b(First)15 b(the)g(pro)q(cesses)h(organize)f(themselv)o(es)h(in)h(a)e(t)o(w)o(o{) 75 900 y(dimensional)h(structure.)k(Eac)o(h)14 b(pro)q(cess)h(then)f (inquires)j(for)c(the)i(ranks)f(of)g(its)g(neigh)o(b)q(ors)i(in)f(the)f (four)75 956 y(directions.)29 b(The)18 b(n)o(umerical)h(problem)g(is)f (solv)o(ed)h(b)o(y)f(an)f(iterativ)o(e)h(metho)q(d,)h(the)f(details)h (of)e(whic)o(h)75 1013 y(are)e(hidden)i(in)f(the)f(subroutine)h Fp(relax)p FC(.)166 1069 y(In)e(eac)o(h)f(relaxation)g(step)g(eac)o(h)g (pro)q(cess)h(computes)f(new)g(v)m(alues)h(for)f(the)g(solution)h(grid) f(function)75 1126 y(at)19 b(all)i(p)q(oin)o(ts)f(o)o(wned)f(b)o(y)h (the)g(pro)q(cess.)33 b(Then)21 b(the)e(v)m(alues)i(at)e(in)o(terpro)q (cess)h(b)q(oundaries)h(ha)o(v)o(e)f(to)75 1182 y(b)q(e)h(exc)o(hanged) h(with)f(neigh)o(b)q(oring)h(pro)q(cesses.)37 b(F)l(or)20 b(example,)j(the)d(exc)o(hange)h(subroutine)h(migh)o(t)75 1238 y(con)o(tain)14 b(a)f(call)i(lik)o(e)g Fo(MPI)p 519 1238 V 15 w(SENDC\(...,neigh)p 850 1238 V 16 w(rank\(1\),...\))i FC(to)c(send)h(up)q(dated)h(v)m(alues)g(to)e(the)g(left{hand)75 1295 y(neigh)o(b)q(or)j(\(i-1,j\).)j(Note)c(that)g(there)g(is)h(only)f (lo)q(cal)i(comm)o(unication)f(in)g(this)f(exc)o(hange)h(op)q(eration.) -32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 84 90 84 89 bop 75 -100 a FC(84)926 b Fv(SECTION)16 b(5.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)252 191 y FC(in)o(teger)g(ndims,)h(n)o(um) p 639 191 14 2 v 16 w(neigh)252 247 y(logical)g(reorder)252 304 y(parameter)e(\(ndims=2,)i(n)o(um)p 782 304 V 16 w(neigh=4,)g(reorder=.true.\))252 360 y(in)o(teger)f(gid,)h(gid)p 554 360 V 17 w(cart,)e(dims\(ndims\),)h(neigh)p 1057 360 V 18 w(def\(2\),)f(neigh)p 1325 360 V 17 w(rank\(n)o(um)p 1537 360 V 16 w(neigh\))252 417 y(in)o(teger)h(o)o(wn)p 485 417 V 16 w(p)q(osition\(ndims\),)h(p)q(erio)q(d\(ndims\),)h(i,)e (j,)g(rank)252 473 y(real)p Fy(\003)p FC(8)g(u\(0:101,0:101)o(\),)d (f\(0:101,0:10)o(1\))252 530 y(data)i(dims)i(/)f(ndims)h Fy(\003)f FC(0)g(/)75 586 y(C)89 b(Create)15 b(a)g(grid)g(structure)g (in)h(ALL)g(group)f(and)h(inquire)g(ab)q(out)g(o)o(wn)e(p)q(osition)252 642 y(gid)i(=)f(MPI)p 470 642 V 16 w(ALL)75 699 y(C)89 b(Set)15 b(pro)q(cess)h(grid)f(size)i(and)e(p)q(erio)q(dicit)o(y)252 755 y(call)h(MPI)p 427 755 V 16 w(MAKDIM\(gid,)f(ndims,)h(dims\))252 812 y(p)q(erio)q(d\(1\))f(=)h(MPI)p 594 812 V 16 w(PER)252 868 y(p)q(erio)q(d\(2\))f(=)h(MPI)p 594 868 V 16 w(PER)252 925 y(call)g(MPI)p 427 925 V 16 w(CAR)l(T)g(\(gid,)f(gid)p 754 925 V 17 w(cart,)f(ndims,)i(dims,)f(p)q(erio)q(d,)i(reorder,)d (rank\))252 981 y(call)i(MPI)p 427 981 V 16 w(INQCAR)l(T)h(\(gid)p 752 981 V 17 w(cart,)d(ndims,)h(dims,)h(p)q(erio)q(d,)g(o)o(wn)p 1377 981 V 16 w(p)q(osition\))75 1038 y(C)89 b(Lo)q(ok)15 b(up)h(the)f(ranks)g(for)g(the)g(neigh)o(b)q(ors.)21 b(Own)15 b(pro)q(cess)h(co)q(ordinates)f(are)g(\(i,j\).)75 1094 y(C)89 b(Neigh)o(b)q(ors)16 b(are)f(\(i{1,j\),)f(\(i+1,j\),)g (\(i,j{1\),)g(\(i,j+1\))252 1151 y(i)i(=)f(o)o(wn)p 413 1151 V 16 w(p)q(osition\(1\))252 1207 y(j)g(=)g(o)o(wn)p 414 1207 V 16 w(p)q(osition\(2\))252 1263 y(neigh)p 360 1263 V 17 w(def\(1\))g(=)g(i{1)252 1320 y(neigh)p 360 1320 V 17 w(def\(2\))g(=)g(j)252 1376 y(call)h(MPI)p 427 1376 V 16 w(INQRANK)h(\(gid,)f(neigh)p 893 1376 V 17 w(def,)f(neigh)p 1102 1376 V 18 w(rank\(1\)\))252 1433 y(neigh)p 360 1433 V 17 w(def\(1\))g(=)g(i+1)252 1489 y(neigh)p 360 1489 V 17 w(def\(2\))g(=)g(j)252 1546 y(call)h(MPI)p 427 1546 V 16 w(INQRANK)h(\(gid,)f(neigh)p 893 1546 V 17 w(def,)f(neigh)p 1102 1546 V 18 w(rank\(2\)\))252 1602 y(neigh)p 360 1602 V 17 w(def\(1\))g(=)g(i)252 1659 y(neigh)p 360 1659 V 17 w(def\(2\))g(=)g(j{1)252 1715 y(call)h(MPI)p 427 1715 V 16 w(INQRANK)h(\(gid,)f(neigh)p 893 1715 V 17 w(def,)f(neigh)p 1102 1715 V 18 w(rank\(3\)\))252 1772 y(neigh)p 360 1772 V 17 w(def\(1\))g(=)g(i)252 1828 y(neigh)p 360 1828 V 17 w(def\(2\))g(=)g(j+1)252 1884 y(call)h(MPI)p 427 1884 V 16 w(INQRANK)h(\(gid,)f(neigh)p 893 1884 V 17 w(def,)f(neigh)p 1102 1884 V 18 w(rank\(4\)\))75 1941 y(C)89 b(Initialize)18 b(the)e(grid)f(functions)h(and)g(start)d (the)j(iteration)252 1997 y(call)g(init)g(\(u,)f(f)t(\))252 2054 y(do)g(10)g(it=1,100)322 2110 y(call)i(relax)e(\(u,)g(f)t(\))75 2167 y(C)89 b(Exc)o(hange)15 b(data)g(with)g(neigh)o(b)q(or)h(pro)q (cesses)322 2223 y(call)h(exc)o(hange)e(\(u,)g(neigh)p 779 2223 V 17 w(rank,)g(n)o(um)p 1001 2223 V 16 w(neigh\))75 2280 y(10)131 b(con)o(tin)o(ue)252 2336 y(call)16 b(output)f(\(u\))252 2393 y(end)147 2597 y(Figure)g(5.1:)k(Set{up)d(of)e(pro)q(cess)i (structure)f(for)f(t)o(w)o(o{dimensional)i(parallel)h(P)o(oisson)e (solv)o(er.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 85 91 85 90 bop 75 358 a Fx(Section)35 b(6)75 568 y FE(Language)41 b(Binding)75 811 y Fw(6.1)59 b(Intro)r(duction)75 916 y FC(This)21 b(c)o(hapter)g(de\014nes)h(the)f(rules)g(for)g(MPI)f (language)h(binding)i(in)f(general)f(and)g(for)f(F)l(ortran)g(77,)75 972 y(F)l(ortran)15 b(90,)g(ANSI)i(C,)e(and)i(ANSI)f(C++)h(in)g (particular.)23 b(De\014ned)17 b(here)f(are)g(v)m(arious)g(ob)s(ject)g (repre-)75 1029 y(sen)o(tations,)d(as)h(w)o(ell)g(as)f(the)h(naming)g (con)o(v)o(en)o(tions)g(used)g(for)f(expressing)i(this)f(standard.)19 b(The)14 b(actual)75 1085 y(calling)j(sequences)f(are)f(de\014ned)i (elsewhere.)166 1144 y(It)23 b(is)g(exp)q(ected)h(that)e(an)o(y)g(F)l (ortran)g(90)g(and)h(C++)g(implemen)o(tations)h(use)f(the)g(F)l(ortran) f(77)75 1200 y(and)f(ANSI)h(C)e(bindings,)k(resp)q(ectiv)o(ely)l(.)39 b(Although)22 b(w)o(e)e(consider)i(it)f(premature)g(to)f(de\014ne)i (other)75 1257 y(bindings)e(to)d(F)l(ortran)g(90)g(and)h(C++,)h(the)f (curren)o(t)f(bindings)j(are)e(designed)h(to)e(encourage,)i(rather)75 1313 y(than)c(discourage,)g(exp)q(erimen)o(tation)i(with)e(b)q(etter)g (bindings)i(whic)o(h)f(migh)o(t)f(b)q(e)h(adopted)g(later.)166 1371 y(There)i(are)g(sev)o(eral)g(imp)q(ortan)o(t)f(language)h(binding) i(issues)f(not)e(addressed)i(b)o(y)e(this)i(standard.)75 1428 y(It)f(is)g(b)q(ey)o(ond)h(the)f(scop)q(e)g(of)g(this)g(standard)g (to)f(discuss)i(the)f(in)o(terop)q(erabilit)o(y)i(of)d(message)h (passing)75 1484 y(b)q(et)o(w)o(een)c(languages.)19 b(Although)c(it)e (w)o(ould)i(b)q(e)f(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)h(to)e (guaran)o(tee)g(that)f(messages)75 1541 y(sen)o(t)k(b)q(et)o(w)o(een)h (languages)f(w)o(ere)h(prop)q(erly)g(in)o(terpreted,)g(the)f(matc)o (hing)h(of)f(t)o(yp)q(es)g(across)g(languages)75 1597 y(prev)o(en)o(ts)f(the)h(inclusion)i(of)d(suc)o(h)h(a)f(guaran)o(tee)g (in)i(this)f(standard.)k(It)c(is)g(fully)h(exp)q(ected)g(that)d(man)o (y)75 1654 y(implemen)o(tations)i(will)g(ha)o(v)o(e)f(suc)o(h)g (features,)f(and)h(that)f(suc)o(h)h(features)g(are)f(a)h(sign)g(of)f (the)h(qualit)o(y)g(of)75 1710 y(the)g(implemen)o(tation.)166 1851 y Fu(Discussion:)f Ft(There)f(w)o(as)e(some)g(informal)d (discussion)k(of)f(this)g(issue)h(at)g(the)g(last)f(meeting.)16 b(Is)c(it)f(p)q(ossible)75 1908 y(to)j(sa)o(y)f(something)g(stronger?) 75 2145 y Fw(6.2)59 b(Data)19 b(T)-5 b(yp)r(es)19 b(and)g(Naming)g (Conventions)75 2250 y FC(All)c(named)f(constan)o(ts,)f(routine)h (names,)g(and)g(v)m(ariable)h(names)f(will)h(b)q(egin)g(with)f(the)g (letters)g(\\MPI".)75 2306 y(In)21 b(the)f(C)g(binding,)j(these)d(will) i(b)q(e)f(upp)q(er)g(case,)g(with)f(all)h(remaining)h(letters)e(in)h (lo)o(w)o(er)f(case.)34 b(In)75 2363 y(F)l(ortran,)14 b(the)h(en)o(tire)h(name)f(will)i(b)q(e)f(lo)o(w)o(er)e(case.)166 2421 y(Handles)e(are)g(represen)o(ted)f(as)h(in)o(tegers)f(in)h(F)l (ortran)f(and)g(as)g(p)q(oin)o(ters)h(to)f(structures)g(in)i(C.)d(T)o (yp)q(ed)75 2478 y(comm)o(unication)20 b(routines)g(supp)q(ort)g FD(string)p FC(s,)g FD(double)j(precision)p FC(,)e FD(real)p FC(,)f(and)g FD(in)o(teger)f FC(data)g(in)75 2534 y(F)l(ortran.)34 b(Null-terminated)23 b FD(string)p FC(s,)e FD(double)p FC(,)h FD(real)p FC(,)g FD(short)p FC(,)f FD(c)o(har)p FC(,)g(and)g FD(long)g FC(are)f(supp)q(orted)75 2591 y(in)g(C.)e(In)i(addition,)g(a)f(sp)q(ecial)h FD(b)o(yte)f FC(t)o(yp)q(e)g(is)g(supp)q(orted)g(in)h(b)q(oth)f(languages.)31 b(This)20 b(t)o(yp)q(e)f(allo)o(ws)75 2647 y(comm)o(unication)h(to)f(b) q(e)h(p)q(erformed)g(with)g(no)f(t)o(yp)q(e)h(con)o(v)o(ersion)g(at)f (all.)33 b(In)21 b(C,)d(the)i(de\014nitions)h(of)75 2704 y(named)14 b(constan)o(ts,)f(function)i(protot)o(yp)q(es,)f(and)g (aggregate)f(t)o(yp)q(e)h(information)g(m)o(ust)g(b)q(e)h(supplied)h (in)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 86 92 86 91 bop 75 -100 a FC(86)976 b Fv(SECTION)16 b(6.)34 b(LANGUA)o(GE)15 b(BINDING)290 81 y Fp(double)23 b(precision)f(a)290 138 y(call)h(foo\(a\))385 194 y(.)385 251 y(.)385 307 y(.)290 363 y(subroutine)f(foo\(b\))290 420 y(integer)h(b)75 568 y FC(Figure)e(6.1:)29 b(This)21 b(is)g(an)f(example)h(of)f(calling) i(a)e(routine)h(with)g(mismatc)o(hed)f(formal)g(and)h(actual)75 624 y(parameters.)75 765 y(an)15 b(include)j(\014le)e Fp(mpi.h)p FC(.)75 931 y Fw(6.3)59 b(F)n(o)n(rtran)21 b(77)e(Binding)h(Issues)75 1040 y FC(There)d(are)g(sev)o(eral)g(p)q (oin)o(ts)h(w)o(ere)f(this)g(standard)g(div)o(erges)h(from)e(the)h (ANSI)h(F)l(ortran)e(77)h(standard.)75 1097 y(Unless)j(explicitly)i (stated,)d(the)h(MPI)f(F77)f(binding)j(is)f(consisten)o(t)f(with)g (ANSI)h(standard)f(F)l(ortran)75 1153 y(77.)26 b(These)18 b(exceptions)g(are)g(consisten)o(t)f(with)h(common)f(practice)h(in)h (the)e(F)l(ortran)f(comm)o(unit)o(y)l(.)27 b(In)75 1210 y(particular:)143 1320 y Fy(\017)c FC(Iden)o(ti\014ers)16 b(are)f(limited)i(to)e(31,)f(not)h(six,)g(signi\014can)o(t)h(c)o (haracters.)143 1429 y Fy(\017)23 b FC(Iden)o(ti\014ers)16 b(ma)o(y)f(con)o(tain)g(underscores)h(after)e(the)h(\014rst)g(c)o (haracter.)143 1539 y Fy(\017)23 b FC(The)14 b(t)o(yp)q(es)f(of)g(the)h (formal)f(parameter)g(list)i(ma)o(y)e(not)g(matc)o(h)g(the)h(calling)h (parameter)e(list.)20 b(\(An)189 1595 y(example)c(is)f(giv)o(en)h(in)g (\014gure)f(6.3.\))143 1705 y Fy(\017)23 b FC(Although)16 b(not)g(required,)g(it)h(is)f(strongly)g(suggested)f(that)h(named)g (constan)o(ts)f(b)q(e)h(pro)o(vided)h(in)189 1761 y(an)e(include)i (\014le.)75 1977 y Fw(6.4)59 b(C)20 b(Binding)g(Issues)75 2086 y FC(W)l(e)j(use)g(the)f(ANSI)i(C)e(declaration)h(format.)41 b(All)24 b(MPI)f(names)f(ha)o(v)o(e)g(an)h Fp(MPI)p 1548 2086 15 2 v 39 w FC(pre\014x,)i(de\014ned)75 2142 y(constan)o(ts)12 b(are)h(in)h(all)h(capital)f(letters,)f(and)g(de\014ned)i(t)o(yp)q(es)e (ha)o(v)o(e)g(one)h(capital)g(letter)f(after)g(the)g(pre\014x.)75 2308 y Fw(6.5)59 b(Sp)r(eci\014c)19 b(Bindings)75 2500 y Fu(Discussion:)13 b Ft(What)c(w)o(ould)g(b)q(e)h(the)g(b)q(est)h(w)o (a)o(y)e(to)g(presen)o(t)i(the)f(follo)o(wing)d(information?)14 b(One)c(big)f(alphab)q(etical)75 2556 y(list?)16 b(In)10 b(order)h(of)e(presen)o(tation)i(in)e(the)i(individual)d(c)o(hapters?) 18 b(By)10 b(c)o(hapter)h(but)f(alphab)q(etical)f(within)g(c)o(hapter?) 75 2613 y(C)14 b(and)g(F)m(ortran)f(mixed)g(or)g(separate?)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 87 93 87 92 bop 75 -100 a Fv(6.5.)34 b(SPECIFIC)15 b(BINDINGS)1191 b FC(87)75 45 y Fq(6.5.1)49 b(C)16 b(bindings)i(fo)o(r)e(P)o(oint-to-P) o(oint)h(Communication)75 138 y FC(These)f(are)e(in)i(alphab)q(etical)i (order.)75 199 y Fp(int)23 b(MPI)p 245 199 15 2 v 17 w(append)p 406 199 V 17 w(contiguous\()f(MPI)p 781 199 V 17 w(Bd)h(bd,)h(void*)f(start,)g(int)g(count,)393 255 y(MPI)p 468 255 V 17 w(Datatype)g(datatype)f(\))75 345 y(int)h(MPI)p 245 345 V 17 w(append)p 406 345 V 17 w(hindexed\()f(MPI)p 733 345 V 17 w(Bd)i(bd,)f(void*)g(start,)g(int)h(count,)393 402 y(int)g(*index)p 636 402 V 16 w(array,)f(MPI)p 891 402 V 17 w(Datatype)g(datatype)f(\))75 492 y(int)h(MPI)p 245 492 V 17 w(append)p 406 492 V 17 w(hvec\()g(MPI)p 638 492 V 17 w(Bd)g(bd,)h(void*)f(start,)g(int)g(count,)g(int)h (stride,)393 548 y(int)g(lenblk,)f(MPI)p 755 548 V 16 w(Datatype)g(datatype)g(\))75 639 y(int)g(MPI)p 245 639 V 17 w(append)p 406 639 V 17 w(indexed\()f(MPI)p 709 639 V 17 w(Bd)i(bd,)f(void*)g(start,)g(int)h(count,)f(int)g(*index)p 1705 639 V 17 w(array,)393 695 y(MPI)p 468 695 V 17 w(Datatype)g (datatype)f(\))75 785 y(int)h(MPI)p 245 785 V 17 w(append)p 406 785 V 17 w(vec\()g(MPI)p 614 785 V 17 w(Bd)g(bd,)h(int)f(count,)g (int)h(stride,)f(int)g(lenblk,)393 842 y(MPI)p 468 842 V 17 w(Datatype)g(datatype)f(\))75 932 y(int)h(MPI)p 245 932 V 17 w(associated\(MPI)p 598 932 V 15 w(Handle)h(handle,)e(MPI) p 1043 932 V 17 w(Handle)h(handle)p 1371 932 V 17 w(type)g(\))75 1022 y(int)g(MPI)p 245 1022 V 17 w(cancel\()g(MPI)p 525 1022 V 17 w(Handle)g(handle,)g(int)g(flag)h(\))75 1113 y(int)f(MPI)p 245 1113 V 17 w(commit)p 406 1113 V 17 w(buffer\()f(MPI)p 685 1113 V 17 w(Bd)i(bd)f(\))75 1203 y(int)g(MPI)p 245 1203 V 17 w(create)p 406 1203 V 17 w(status\()f(MPI)p 685 1203 V 17 w(Handle)h(handle)g(\))75 1293 y(int)g(MPI)p 245 1293 V 17 w(create)p 406 1293 V 17 w(buffer\()f(MPI)p 685 1293 V 17 w(Bd)i(*bd,)f(MPI)p 965 1293 V 17 w(LIFETIME)g(persistence)f(\))75 1383 y(int)h(MPI)p 245 1383 V 17 w(free\()g(MPI)p 477 1383 V 17 w(Handle)g(handle)g(\))75 1474 y(int)g(MPI)p 245 1474 V 17 w(get)p 334 1474 V 17 w(len\()g(int)h(count,)f(MPI)p 805 1474 V 17 w(Status)g(*status,)f(MPI) p 1275 1474 V 17 w(Bd)i(bd)f(\))75 1564 y(int)g(MPI)p 245 1564 V 17 w(init)p 358 1564 V 17 w(recv)g(\()h(MPI)p 614 1564 V 17 w(Handle)f(handle,)g(MPI)p 1061 1564 V 16 w(Bd)h(bd,)f(int)h(source,)f(MPI)p 1603 1564 V 17 w(Tag)g(tag,)393 1620 y(MPI)p 468 1620 V 17 w(Communicator)f(context,)h (MPI)p 1082 1620 V 17 w(LIFETIME)f(persistence)h(\))75 1710 y(int)g(MPI)p 245 1710 V 17 w(init)p 358 1710 V 17 w(rsend)g(\()h(MPI)p 638 1710 V 17 w(Handle)f(handle,)f(MPI)p 1084 1710 V 17 w(Bd)i(bd,)f(int)h(dest,)f(MPI)p 1579 1710 V 17 w(Tag)g(tag,)393 1767 y(MPI)p 468 1767 V 17 w(Communicator)f(context,)h(MPI)p 1082 1767 V 17 w(LIFETIME)f (persistence)h(\))75 1857 y(int)g(MPI)p 245 1857 V 17 w(init)p 358 1857 V 17 w(send)g(\()h(MPI)p 614 1857 V 17 w(Handle)f(handle,)g(MPI)p 1061 1857 V 16 w(Bd)h(bd,)f(int)h(dest,)f (MPI)p 1555 1857 V 17 w(Tag)g(tag,)393 1914 y(MPI)p 468 1914 V 17 w(COMunicator)f(context,)h(MPI)p 1058 1914 V 17 w(LIFETIME)g(persistence)f(\))75 2004 y(int)h(MPI)p 245 2004 V 17 w(init)p 358 2004 V 17 w(ssend)g(\()h(MPI)p 638 2004 V 17 w(Handle)f(handle,)f(MPI)p 1084 2004 V 17 w(Bd)i(bd,)f(int)h(dest,)f(MPI)p 1579 2004 V 17 w(Tag)g(tag,)393 2060 y(MPI)p 468 2060 V 17 w(Communicator)f(context,)h(MPI)p 1082 2060 V 17 w(LIFETIME)f(persistence)h(\))75 2151 y(int)g(MPI)p 245 2151 V 17 w(iprobe\()g(int)h(source,)e(MPI)p 811 2151 V 17 w(Tag)i(tag,)f(MPI)p 1115 2151 V 17 w(Communicator)f (context,)h(int)g(flag,)393 2207 y(type,)g(MPI)p 611 2207 V 17 w(Status)g(*status)g(\))75 2297 y(int)g(MPI)p 245 2297 V 17 w(irecv\()g(MPI)p 501 2297 V 17 w(Handle)g(handle,)g(MPI) p 948 2297 V 17 w(Bd)g(bd,)h(int)f(source,)g(MPI)p 1490 2297 V 17 w(Tag)g(tag,)393 2354 y(MPI)p 468 2354 V 17 w(Communicator)f(context)h(\))75 2444 y(int)g(MPI)p 245 2444 V 17 w(irecvc\()g(MPI)p 525 2444 V 17 w(Handle)g(handle,)g(void*)g (start,)g(int)h(count,)393 2500 y(MPI)p 468 2500 V 17 w(Datatype)f(datatype,)f(int)i(source,)f(MPI)p 1297 2500 V 16 w(Tag)h(tag,)393 2557 y(MPI)p 468 2557 V 17 w(Communicator)e (context,)h(MPI)p 1082 2557 V 17 w(Status)g(*status)g(\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(irsend)g(\()h(MPI)p 549 2647 V 17 w(Handle)f(handle,)g(MPI)p 996 2647 V 16 w(Bd)h(bd,)g(int)f(dest,)g(MPI)p 1490 2647 V 17 w(Tag)g(tag,)393 2704 y(MPI)p 468 2704 V 17 w(Communicator)f(context)h(\))-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 88 94 88 93 bop 75 -100 a FC(88)976 b Fv(SECTION)16 b(6.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fp(int)23 b(MPI)p 245 45 15 2 v 17 w(irsendc)g(\()h(MPI)p 573 45 V 17 w(Handle)f(handle,) g(void*)g(start,)g(int)g(count,)393 102 y(MPI)p 468 102 V 17 w(Datatype)g(datatype,)f(int)i(dest,)f(MPI)p 1249 102 V 17 w(Tag)g(tag,)393 158 y(MPI)p 468 158 V 17 w(Communicator)f (context)h(\))75 246 y(int)g(MPI)p 245 246 V 17 w(isend)g(\()h(MPI)p 525 246 V 17 w(Handle)f(handle,)g(MPI)p 972 246 V 17 w(Bd)g(bd,)h(int)f(dest,)g(MPI)p 1466 246 V 17 w(Tag)h(tag,)393 302 y(MPI)p 468 302 V 17 w(Communicator)e(context)h(\))75 390 y(int)g(MPI)p 245 390 V 17 w(isendc)g(\()h(MPI)p 549 390 V 17 w(Handle)f(handle,)g(void*)g(start,)g(int)g(count,)393 446 y(MPI)p 468 446 V 17 w(Datatype)g(datatype,)f(int)i(dest,)f(MPI)p 1249 446 V 17 w(Tag)g(tag,)393 503 y(MPI)p 468 503 V 17 w(Communicator)f(context)h(\))75 591 y(int)g(MPI)p 245 591 V 17 w(issend)g(\()h(MPI)p 549 591 V 17 w(Handle)f(handle,)g (MPI)p 996 591 V 16 w(Bd)h(bd,)g(int)f(dest,)g(MPI)p 1490 591 V 17 w(Tag)g(tag,)393 647 y(MPI)p 468 647 V 17 w(Communicator)f(context)h(\))75 735 y(int)g(MPI)p 245 735 V 17 w(issendc)g(\()h(MPI)p 573 735 V 17 w(Handle)f(handle,)g (void*)g(start,)g(int)g(count,)393 791 y(MPI)p 468 791 V 17 w(Datatype)g(datatype,)f(int)i(dest,)f(MPI)p 1249 791 V 17 w(Tag)g(tag,)393 848 y(MPI)p 468 848 V 17 w(Communicator)f (context)h(\))75 936 y(int)g(MPI)p 245 936 V 17 w(probe\()g(int)h (source,)f(MPI)p 788 936 V 16 w(Tag)h(tag,)f(MPI)p 1091 936 V 17 w(Communicator)f(context,)393 992 y(MPI)p 468 992 V 17 w(Datatype)h(datatype,)f(MPI)p 1010 992 V 17 w(Status)h(*status)g(\))75 1080 y(int)g(MPI)p 245 1080 V 17 w(query\()g(MPI)p 501 1080 V 17 w(Handle)g(handle,)g(int)g(count,) g(int)h(source,)f(MPI)p 1497 1080 V 17 w(Tag)g(tag\))75 1168 y(int)g(MPI)p 245 1168 V 17 w(recv\()g(MPI)p 477 1168 V 17 w(Bd)h(bd,)f(int)h(source,)f(MPI)p 1020 1168 V 16 w(Tag)h(tag,)f(MPI)p 1323 1168 V 17 w(Communicator)f(context,)393 1224 y(MPI)p 468 1224 V 17 w(Status)h(*status)g(\))75 1312 y(int)g(MPI)p 245 1312 V 17 w(recvc)g(\()h(void*)f(start,)g(int)h (count,)f(MPI)p 1098 1312 V 17 w(Datatype)f(datatype,)h(int)g(source,) 393 1368 y(MPI)p 468 1368 V 17 w(Tag)g(tag,)h(MPI)p 772 1368 V 16 w(Communicator)f(context,)f(MPI)p 1385 1368 V 17 w(Status)h(*status)g(\))75 1456 y(int)g(MPI)p 245 1456 V 17 w(rsend)g(\()h(MPI)p 525 1456 V 17 w(Bd)f(bd,)h(int)f(dest,)h (MPI)p 1020 1456 V 16 w(Tag)g(tag,)f(MPI)p 1323 1456 V 17 w(Communicator)f(context)h(\))75 1544 y(int)g(MPI)p 245 1544 V 17 w(rsendc)g(\()h(void*)f(start,)g(int)h(count,)f(MPI)p 1122 1544 V 16 w(Datatype)g(datatype,)g(int)g(dest,)393 1600 y(MPI)p 468 1600 V 17 w(Tag)g(tag,)h(MPI)p 772 1600 V 16 w(Communicator)f(context)g(\))75 1688 y(int)g(MPI)p 245 1688 V 17 w(send)h(\()f(MPI)p 501 1688 V 17 w(Bd)h(bd,)f(int)h (dest,)f(MPI)p 996 1688 V 16 w(Tag)h(tag,)f(MPI)p 1299 1688 V 17 w(Communicator)f(context)h(\))75 1776 y(int)g(MPI)p 245 1776 V 17 w(sendc)g(\()h(void*)f(start,)g(int)h(count,)f(MPI)p 1098 1776 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 1832 y(MPI)p 468 1832 V 17 w(Tag)g(tag,)h(MPI)p 772 1832 V 16 w(Communicator)f(context)g(\))75 1920 y(int)g(MPI)p 245 1920 V 17 w(ssend)g(\()h(MPI)p 525 1920 V 17 w(Bd)f(bd,)h(int)f (dest,)h(MPI)p 1020 1920 V 16 w(Tag)g(tag,)f(MPI)p 1323 1920 V 17 w(Communicator)f(context)h(\))75 2008 y(int)g(MPI)p 245 2008 V 17 w(ssendc)g(\()h(void*)f(start,)g(int)h(count,)f(MPI)p 1122 2008 V 16 w(Datatype)g(datatype,)g(int)g(dest,)393 2064 y(MPI)p 468 2064 V 17 w(Tag)g(tag,)h(MPI)p 772 2064 V 16 w(Communicator)f(context)g(\))75 2152 y(int)g(MPI)p 245 2152 V 17 w(start\()g(MPI)p 501 2152 V 17 w(Handle)g(handle)g(\))75 2240 y(int)g(MPI)p 245 2240 V 17 w(Status)g(\()h(MPI)p 549 2240 V 17 w(Handle)f(handle,)g(flag,)g(MPI)p 1139 2240 V 17 w(Status)g(*status)g(\))75 2327 y(int)g(MPI)p 245 2327 V 17 w(Statusany)g(\()h(MPI)p 621 2327 V 16 w(Handle)f(*handle)p 972 2327 V 17 w(list,)g(int)g(count,)g(int)h (index,)393 2384 y(MPI)p 468 2384 V 17 w(Status)f(*status)g(\))75 2472 y(int)g(MPI)p 245 2472 V 17 w(wait)h(\()f(MPI)p 501 2472 V 17 w(Handle)g(handle,)g(MPI)p 948 2472 V 17 w(Status)g(*status)g(\))75 2559 y(int)g(MPI)p 245 2559 V 17 w(waitall\()g(MPI)p 549 2559 V 17 w(Handle)g(*handle)p 901 2559 V 16 w(list,)g(int)h(count,)f(list)p 1419 2559 V 16 w(of)p 1483 2559 V 17 w(return)p 1644 2559 V 17 w(handles\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(waitany)g(\()h(MPI)p 573 2647 V 17 w(Handle)f(*handle)p 925 2647 V 16 w(list,)g(int)h (count,)f(int)g(index,)393 2704 y(MPI)p 468 2704 V 17 w(Status)g(*status)g(\))1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 89 95 89 94 bop 75 -100 a Fv(6.5.)34 b(SPECIFIC)15 b(BINDINGS)1191 b FC(89)75 45 y Fq(6.5.2)49 b(C)16 b(Bindings)i(fo)o(r)e(Collective)h (Communication)g(Chapter)75 135 y FC(These)f(are)e(in)i(order)f(of)g (presen)o(tation)g(in)h(the)g(Collectiv)o(e)g(Comm)o(unication)g(c)o (hapter.)75 194 y Fp(int)23 b(MPI)p 245 194 15 2 v 17 w(comm)p 358 194 V 17 w(init\()g(MPI)p 590 194 V 17 w(Group)g(group)g (\))75 283 y(int)g(MPI)p 245 283 V 17 w(rank\()g(MPI)p 477 283 V 17 w(Group)g(group,)g(int)h(rank)f(\))75 372 y(int)g(MPI)p 245 372 V 17 w(gsize\()g(MPI)p 501 372 V 17 w(Group)g(group,)g(int)h(size)f(\))75 461 y(int)g(MPI)p 245 461 V 17 w(barrier\()g(MPI)p 549 461 V 17 w(Group)g(group)g(\))75 549 y(int)g(MPI)p 245 549 V 17 w(synch\()g(MPI)p 501 549 V 17 w(Group)g(group,)g(MPI)p 900 549 V 17 w(MAP)g(map)h(\))75 638 y(int)f(MPI)p 245 638 V 17 w(bcast\()g(MPI)p 501 638 V 17 w(Bd)h(bd,)f(MPI)p 757 638 V 17 w(Group)g(group,)g(int)h(root) f(\))75 727 y(int)g(MPI)p 245 727 V 17 w(bcastc\()g(buf,)g(int)h(len,)f (MPI)p 859 727 V 17 w(Datatype)g(datatype,)f(MPI)p 1401 727 V 17 w(Group)h(group,)393 783 y(int)h(root)f(\))75 872 y(int)g(MPI)p 245 872 V 17 w(gather\()g(MPI)p 525 872 V 17 w(Bd)g(inbd,)h(list)p 853 872 V 16 w(of)p 917 872 V 17 w(MPI)p 1006 872 V 17 w(Bd)g(outbds,)e(MPI)p 1357 872 V 17 w(Group)h(group,)g(int)h(root)393 929 y(\))75 1017 y(int)f(MPI)p 245 1017 V 17 w(gatherc\()g(void*)g(inbuf,)g(void*)g (outbuf,)g(int)h(len,)f(MPI)p 1408 1017 V 17 w(Datatype)g(datatype,)393 1074 y(MPI)p 468 1074 V 17 w(Group)g(group,)g(int)h(root)f(\))75 1163 y(int)g(MPI)p 245 1163 V 17 w(scatter\()g(list)p 573 1163 V 17 w(of)p 638 1163 V 17 w(MPI)p 727 1163 V 16 w(Bd)h(inbds,)f(MPI)p 1054 1163 V 17 w(Bd)g(outbd,)g(MPI)p 1381 1163 V 17 w(Group)g(group,)393 1219 y(int)h(root)f(\))75 1308 y(int)g(MPI)p 245 1308 V 17 w(scatterc\()g(void*)g(inbuf,)g(void*) g(outbuf,)g(int)h(len,)393 1364 y(MPI)p 468 1364 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1364 V 17 w(Group)h(group,)g(int)h (root)f(\))75 1453 y(int)g(MPI)p 245 1453 V 17 w(allscatter\()g(MPI)p 621 1453 V 16 w(Bd)h(inbds[],)f(MPI)p 996 1453 V 16 w(Bd)h(outbds[],)f (MPI)p 1395 1453 V 16 w(Group)h(group)f(\))75 1542 y(int)g(MPI)p 245 1542 V 17 w(allscatterc\()f(void*)i(inbuf,)f(void*)g(outbuf,)g(int) g(len,)393 1598 y(MPI)p 468 1598 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 1598 V 17 w(Group)h(group)g(\))75 1687 y(int)g(MPI)p 245 1687 V 17 w(allcast\()g(MPI)p 549 1687 V 17 w(Bd)g(inbds[],)g(MPI)p 924 1687 V 17 w(Bd)g(outbds[],)g(MPI)p 1323 1687 V 17 w(Group)g(group)g(\))75 1776 y(int)g(MPI)p 245 1776 V 17 w(allcastc\()g(void*)g(inbuf,)g(void*)g(outbuf,)g(int)h(len,)393 1832 y(MPI)p 468 1832 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1832 V 17 w(Group)h(group)g(\))75 1921 y(int)g(MPI)p 245 1921 V 17 w(reduce\()g(MPI)p 525 1921 V 17 w(Bd)g(inbd,)h(MPI)p 829 1921 V 16 w(Bd)g(outbd,)f(MPI)p 1156 1921 V 17 w(Group)g(group,)g (int)h(root,)393 1977 y(MPI)p 468 1977 V 17 w(Op)g(op)f(\))75 2066 y(int)g(MPI)p 245 2066 V 17 w(reducec\()g(void*)g(inbuf,)g(void*)g (outbuf,)g(int)h(len,)f(MPI)p 1408 2066 V 17 w(Datatype)g(datatype,)393 2123 y(MPI)p 468 2123 V 17 w(Group)g(group,)g(int)h(root,)f(MPI)p 1106 2123 V 17 w(Op)g(op)h(\))75 2211 y(int)f(MPI)p 245 2211 V 17 w(user)p 358 2211 V 17 w(reduce\()g(MPI)p 638 2211 V 17 w(Bd)g(inbd,)g(MPI)p 941 2211 V 17 w(Bd)h(outbd,)f(MPI)p 1269 2211 V 16 w(Group)h(group,)f(int)g(root,)393 2268 y(void)g(*userfunc\(\),)g(int)g(unitsize)g(\))75 2357 y(int)g(MPI)p 245 2357 V 17 w(user)p 358 2357 V 17 w(reducec\()g(void*) g(inbuf,)g(void*)g(outbuf,)g(int)g(len,)393 2413 y(MPI)p 468 2413 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2413 V 17 w(Group)h(group,)g(int)h(root,)393 2470 y(void)f(*userfunc\(\),)g (unitsize)f(\))75 2558 y(int)h(MPI)p 245 2558 V 17 w(allreduce\()g(MPI) p 597 2558 V 16 w(Bd)h(inbd,)f(MPI)p 900 2558 V 17 w(Bd)h(outbd,)f(MPI) p 1228 2558 V 16 w(Group)h(group,)f(MPI)p 1627 2558 V 16 w(Op)h(op)g(\))75 2647 y(int)f(MPI)p 245 2647 V 17 w(allreducec\()g(void*)g(inbuf,)g(void*)g(outbuf,)g(int)g(len,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2704 V 17 w(Group)h(group,)g(MPI)p 1409 2704 V 17 w(Op)h(op)f(\))-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 90 96 90 95 bop 75 -100 a FC(90)976 b Fv(SECTION)16 b(6.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fp(int)23 b(MPI)p 245 45 15 2 v 17 w(user)p 358 45 V 17 w(allreduce\()f(MPI)p 709 45 V 17 w(Bd)i(inbd,)f(MPI)p 1013 45 V 17 w(Bd)g(outbd,)g(MPI)p 1340 45 V 17 w(Group)g(group,)393 102 y(void)g(*userfunc\(\)\))75 189 y(int)g(MPI)p 245 189 V 17 w(user)p 358 189 V 17 w(allreducec\()f(void*)h(inbuf,)g(void*)h(outbuf,)f(int)g(len,)393 246 y(MPI)p 468 246 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 246 V 17 w(Group)h(group,)g(void)h(*userfunc\(\)\))75 333 y(int)f(MPI)p 245 333 V 17 w(scan\()g(MPI)p 477 333 V 17 w(Bd)h(inbd,)f(MPI)p 781 333 V 17 w(Bd)g(outbd,)g(MPI)p 1108 333 V 17 w(Group)g(group,)g(MPI)p 1507 333 V 17 w(Op)h(op)f(\))75 421 y(int)g(MPI)p 245 421 V 17 w(scanc\()g(void*)g (inbuf,)g(void*)h(outbuf,)f(int)g(len,)g(MPI)p 1360 421 V 17 w(Datatype)g(datatype,)393 477 y(MPI)p 468 477 V 17 w(Group)g(group,)g(MPI)p 867 477 V 17 w(Op)h(op)f(\))75 565 y(int)g(MPI)p 245 565 V 17 w(user)p 358 565 V 17 w(scan\()g(MPI)p 590 565 V 17 w(Bd)g(inbd,)h(MPI)p 894 565 V 16 w(Bd)g(outbd,)f(MPI)p 1221 565 V 17 w(Group)g(group,)393 621 y(void)g(*userfunc\(\))g(\))75 709 y(int)g(MPI)p 245 709 V 17 w(user)p 358 709 V 17 w(scanc\()g(void*)g(inbuf,)g(void*)g (outbuf,)g(int)h(len,)393 765 y(MPI)p 468 765 V 17 w(Datatype)f (datatype,)f(MPI)p 1010 765 V 17 w(Group)h(group,)g(void)h (*userfunc\(\)\))75 924 y Fq(6.5.3)49 b(F)o(o)o(rtran)15 b(Bindings)j(fo)o(r)d(P)o(oint-to-P)o(oint)i(Routines)75 1012 y Fp(mpi)p 150 1012 V 17 w(append)p 311 1012 V 16 w(contiguous\()22 b(bd,)i(start,)f(count,)g(datatype,)g(rc)g(\))170 1068 y(real)h(start\(*\))170 1125 y(integer)f(bd,)h(count,)f(datatype,) f(rc)75 1212 y(mpi)p 150 1212 V 17 w(append)p 311 1212 V 16 w(hindexed\()h(bd,)g(start,)g(count,)g(index)p 1115 1212 V 17 w(array,)g(datatype,)f(rc)i(\))170 1269 y(real)g(start\(*\)) 170 1325 y(integer)f(index)p 484 1325 V 17 w(array\(*\))170 1382 y(integer)g(bd,)h(count,)f(datatype,)f(rc)75 1469 y(mpi)p 150 1469 V 17 w(append)p 311 1469 V 16 w(hvec\()h(bd,)h(start,) f(count,)g(stride,)g(lenblk,)g(datatype,)f(rc)i(\))170 1526 y(real)g(start\(*\))170 1582 y(integer)f(bd,)h(count,)f(stride,)g (lenblk,)g(datatype,)f(rc)75 1670 y(mpi)p 150 1670 V 17 w(append)p 311 1670 V 16 w(indexed\()h(bd,)g(start,)g(count,)g (index)p 1091 1670 V 17 w(array,)g(datatype,)g(rc)g(\))170 1726 y(real)h(start\(*\))170 1783 y(integer)f(index)p 484 1783 V 17 w(array\(*\))170 1839 y(integer)g(bd,)h(count,)f (datatype,)f(rc)75 1927 y(mpi)p 150 1927 V 17 w(append)p 311 1927 V 16 w(vec\()h(bd,)h(count,)f(stride,)g(lenblk,)g(datatype,)f (rc)i(\))170 1983 y(integer)f(bd,)h(count,)f(stride,)g(lenblk,)g (datatype,)f(rc)75 2071 y(mpi)p 150 2071 V 17 w(associated\()g(handle,) h(handle)p 788 2071 V 16 w(type,)h(rc)f(\))170 2127 y(integer)g (handle,)g(handle)p 699 2127 V 17 w(type,)g(rc)75 2215 y(mpi)p 150 2215 V 17 w(cancel\()g(handle,)g(flag,)g(rc)g(\))170 2271 y(integer)g(handle,)g(flag,)g(rc)75 2359 y(mpi)p 150 2359 V 17 w(commit)p 311 2359 V 16 w(buffer\()g(bd,)g(rc)h(\))170 2415 y(integer)f(bd,)h(rc)75 2503 y(mpi)p 150 2503 V 17 w(create)p 311 2503 V 16 w(buffer\()f(bd,)g(persistence,)g(rc)g(\)) 170 2560 y(integer)g(bd,)h(persistence,)e(rc)75 2647 y(mpi)p 150 2647 V 17 w(create)p 311 2647 V 16 w(status\()h(handle,)g (rc)h(\))170 2704 y(integer)f(handl,)g(rce)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 91 97 91 96 bop 75 -100 a Fv(6.5.)34 b(SPECIFIC)15 b(BINDINGS)1191 b FC(91)75 45 y Fp(mpi)p 150 45 15 2 v 17 w(free\()23 b(handle,)g(rc)g(\))170 102 y(integer)g(handl,)g(rce)75 190 y(mpi)p 150 190 V 17 w(get)p 239 190 V 17 w(len\()g(count,)g (status,)g(bd,)g(rc)h(\))170 247 y(integer)f(count,)g(status,)g(bd,)h (rc)75 335 y(mpi)p 150 335 V 17 w(init)p 263 335 V 16 w(recv)g(\()f(handle,)g(bd,)h(source,)f(tag,)g(context,)g(persistence,) f(rc)i(\))170 392 y(integer)f(handle,)g(bd,)h(source,)f(tag,)g (context,)g(persistence,)f(rc)75 480 y(mpi)p 150 480 V 17 w(init)p 263 480 V 16 w(rsend)i(\()f(handle,)g(bd,)h(dest,)f(tag,) g(context,)g(persistence,)f(rc)i(\))170 537 y(integer)f(handle,)g(bd,)h (dest,)f(tag,)g(context,)g(persistence,)f(rc)75 625 y(mpi)p 150 625 V 17 w(init)p 263 625 V 16 w(send)i(\()f(handle,)g(bd,)h(dest,) f(tag,)g(context,)g(persistence,)f(rc)i(\))170 681 y(integer)f(handle,) g(bd,)h(dest,)f(tag,)g(context,)g(persistence,)f(rc)75 770 y(mpi)p 150 770 V 17 w(init)p 263 770 V 16 w(ssend)i(\()f(handle,)g (bd,)h(dest,)f(tag,)g(context,)g(persistence,)f(rc)i(\))170 826 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g(persistence,) f(rc)75 915 y(mpi)p 150 915 V 17 w(iprobe\()h(source,)g(tag,)g (context,)g(flag,)g(type,)g(status,)g(rc)h(\))170 971 y(integer)f(source,)g(tag,)h(context,)e(flag,)i(type,)f(status,)g(rc)75 1060 y(mpi)p 150 1060 V 17 w(irecv\()g(handle,)g(bd,)g(source,)g(tag,)g (context,)g(rc)h(\))170 1116 y(integer)f(handle,)g(bd,)h(source,)f (tag,)g(context,)g(rc)75 1205 y(mpi)p 150 1205 V 17 w(irecvc\()g (handle,)g(start,)g(count,)g(datatype,)f(source,)h(tag,)h(context,)e (status,)393 1261 y(rc)i(\))170 1318 y(real)g(start\(*\))170 1374 y(integer)f(handle,)g(count,)g(datatype,)g(source,)g(tag,)g (context,)g(status,)g(rc)75 1463 y(mpi)p 150 1463 V 17 w(irsend)g(\()h(handle,)e(bd,)i(dest,)f(tag,)g(context,)g(rc)h(\))170 1519 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g(rc)75 1608 y(mpi)p 150 1608 V 17 w(irsendc)g(\()g(handle,)g(start,)g(count,)g (datatype,)g(dest,)g(tag,)h(context,)e(rc)i(\))170 1664 y(real)g(start\(*\))170 1721 y(integer)f(handle,)g(count,)g(datatype,)g (dest,)g(tag,)g(context,)g(rc)75 1809 y(mpi)p 150 1809 V 17 w(isend)g(\()h(handle,)f(bd,)g(dest,)g(tag,)h(context,)e(rc)i(\)) 170 1866 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g(rc)75 1954 y(mpi)p 150 1954 V 17 w(isendc)g(\()h(handle,)e(start,)h(count,)h (datatype,)e(dest,)h(tag,)h(context,)e(rc)i(\))170 2011 y(real)g(start\(*\))170 2067 y(integer)f(handle,)g(count,)g(datatype,)g (dest,)g(tag,)g(context,)g(rc)75 2156 y(mpi)p 150 2156 V 17 w(issend)g(\()h(handle,)e(bd,)i(dest,)f(tag,)g(context,)g(rc)h(\)) 170 2212 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(contex,)g(rct)75 2301 y(mpi)p 150 2301 V 17 w(issendc)g(\()g(handle,)g(start,)g(count,)g (datatype,)g(dest,)g(tag,)h(context,)e(rc)i(\))170 2357 y(real)g(start\(*\))170 2414 y(integer)f(handle,)g(count,)g(datatype,)g (dest,)g(tag,)g(context,)g(rc)75 2502 y(mpi)p 150 2502 V 17 w(probe\()g(source,)g(tag,)g(context,)g(datatype,)f(status,)h(rc)h (\))170 2559 y(integer)f(source,)g(tag,)h(context,)e(datatype,)h (status,)g(rc)75 2647 y(mpi)p 150 2647 V 17 w(query\()g(handle,)g (count,)g(source,)g(tag,)g(rc)h(\))170 2704 y(integer)f(handle,)g (count,)g(source,)g(tag,)h(rc)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 92 98 92 97 bop 75 -100 a FC(92)976 b Fv(SECTION)16 b(6.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fp(mpi)p 150 45 15 2 v 17 w(recv\()23 b(bd,)g(source,)g(tag,)h(context,)e(status,)h(rc)h (\))170 102 y(integer)f(bd,)h(source,)f(tag,)g(context,)g(status,)g(rc) 75 188 y(mpi)p 150 188 V 17 w(recvc)g(\(start,)g(count,)g(datatype,)g (source,)f(tag,)i(context,)f(statu,)g(rcs\))170 244 y(real)h (start\(*\))170 301 y(integer)f(count,)g(datatype,)g(source,)g(tag,)g (context,)g(status,)g(rc)75 387 y(mpi)p 150 387 V 17 w(rsend)g(\()h(bd,)f(dest,)g(tag,)h(context,)e(rc)i(\))170 444 y(integer)f(bd,)h(dest,)f(tag,)g(context,)g(rc)75 530 y(mpi)p 150 530 V 17 w(rsendc)g(\(start,)g(count,)g(datatype,)f (dest,)i(tag,)f(context,)g(rc)g(\))170 587 y(real)h(start\(*\))170 643 y(integer)f(count,)g(datatype,)g(dest,)g(tag,)h(context,)e(rc)75 730 y(mpi)p 150 730 V 17 w(send)h(\()h(bd,)f(dest,)g(tag,)h(context,)f (rc)g(\))170 786 y(integer)g(bd,)h(dest,)f(tag,)g(context,)g(rc)75 873 y(mpi)p 150 873 V 17 w(sendc)g(\(start,)g(count,)g(datatype,)g (dest,)g(tag,)g(context,)g(rc)g(\))170 929 y(real)h(start\(*\))170 985 y(integer)f(count,)g(datatype,)g(dest,)g(tag,)h(context,)e(rc)75 1072 y(mpi)p 150 1072 V 17 w(ssend)h(\()h(bd,)f(dest,)g(tag,)h (context,)e(rc)i(\))170 1128 y(integer)f(bd,)h(dest,)f(tag,)g(context,) g(rc)75 1215 y(mpi)p 150 1215 V 17 w(ssendc)g(\(start,)g(count,)g (datatype,)f(dest,)i(tag,)f(context,)g(rc)g(\))170 1271 y(real)h(start\(*\))170 1328 y(integer)f(count,)g(datatype,)g(dest,)g (tag,)h(context,)e(rc)75 1414 y(mpi)p 150 1414 V 17 w(start\()h (handle,)g(rc)g(\))170 1471 y(integer)g(handle,)g(rc)75 1557 y(mpi)p 150 1557 V 17 w(status)g(\()h(handle,)e(flag,)i(status,)f (rc)g(\))170 1614 y(integer)g(handle,)g(flag,)g(status,)g(rc)75 1700 y(mpi)p 150 1700 V 17 w(statusany)f(\()i(handle)p 597 1700 V 16 w(list,)g(count,)f(index,)g(status,)g(rc)g(\))170 1756 y(integer)g(handle)p 508 1756 V 17 w(list\(*\))170 1813 y(integer)g(count,)g(index,)g(status,)g(rc)75 1899 y(mpi)p 150 1899 V 17 w(wait)g(\()h(handle,)f(status,)g(rc)g(\))170 1956 y(integer)g(handle,)g(status,)g(rc)75 2042 y(mpi)p 150 2042 V 17 w(waitall\()g(handle)p 526 2042 V 16 w(list,)g(count,)g (list)p 948 2042 V 17 w(of)p 1013 2042 V 17 w(return)p 1174 2042 V 16 w(handles,)g(rc)g(\))170 2099 y(integer)g(handle)p 508 2099 V 17 w(list\(*\))170 2155 y(integer)g(count)170 2212 y(integer)g(list)p 460 2212 V 17 w(of)p 525 2212 V 17 w(return)p 686 2212 V 16 w(handles\(*,)g(rc\))75 2298 y(mpi)p 150 2298 V 17 w(waitany)g(\()g(handle)p 549 2298 V 17 w(list,)g(count,)g(index,)g(status,)g(rc)h(\))170 2355 y(integer)f(handle)p 508 2355 V 17 w(list\(*\))170 2411 y(integer)g(count,)g(index,)g(status,)g(rc)75 2561 y Fq(6.5.4)49 b(F)o(o)o(rtran)15 b(Bindings)j(fo)o(r)d(Collective)i (Communciation)g(Chapter)75 2647 y FC(These)f(are)e(not)h(done)h(y)o (et.)75 2704 y Fp(mpi)p 150 2704 V 17 w(comm)p 263 2704 V 16 w(init\()24 b(group,)f(rc)g(\))1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 93 99 93 98 bop 75 -100 a Fv(6.5.)34 b(SPECIFIC)15 b(BINDINGS)1191 b FC(93)75 45 y Fp(mpi)p 150 45 15 2 v 17 w(rank\()23 b(group,)g(rank,)g(rc)h(\))75 132 y(mpi)p 150 132 V 17 w(gsize\()f(group,)g(size,)g(rc)h(\))75 218 y(mpi)p 150 218 V 17 w(barrier\()f(group,)g(rc)g(\))75 304 y(mpi)p 150 304 V 17 w(synch\()g(group,)g(map,)g(rc)h(\))75 391 y(mpi)p 150 391 V 17 w(bcast\()f(buffer)p 478 391 V 16 w(handle,)g(group,)g(root,)g(rc)h(\))75 477 y(mpi)p 150 477 V 17 w(bcastc\()f(buf,)g(len,)g(type,)g(group,)h(root,)f(rc)g(\))75 564 y(mpi)p 150 564 V 17 w(gather\()g(inhandle,)f(list)p 692 564 V 17 w(of)p 757 564 V 17 w(outhandles,)g(group,)h(root,)g(rc)h (\))75 650 y(mpi)p 150 650 V 17 w(gatherc\()f(inbuf,)g(outbuf,)f (inlen,)i(type,)f(group,)g(root,)g(rc)h(\))75 737 y(mpi)p 150 737 V 17 w(scatter\()f(list)p 478 737 V 16 w(of)p 542 737 V 17 w(inhandles,)g(outhandle,)f(group,)h(root,)g(rc)h(\))75 823 y(mpi)p 150 823 V 17 w(scatterc\()e(inbuf,)h(outbuf,)g(len,)h (type,)f(group,)g(root,)g(rc)h(\))75 910 y(mpi)p 150 910 V 17 w(allscatter\()e(list)p 549 910 V 17 w(of)p 614 910 V 17 w(inhandles,)g(list)p 989 910 V 17 w(of)p 1054 910 V 17 w(outhandles,)g(group,)h(rc)h(\))75 996 y(mpi)p 150 996 V 17 w(allscatterc\()e(inbuf,)h(outbuf,)g(len,)g(type,) g(grou,)h(rcp\))75 1083 y(mpi)p 150 1083 V 17 w(allcast\()f(inhandle,)f (list)p 716 1083 V 17 w(of)p 781 1083 V 17 w(outhandles,)g(group,)h(rc) h(\))75 1169 y(mpi)p 150 1169 V 17 w(allcastc\()e(inbuf,)h(outbuf,)g (len,)h(type,)f(grou,)g(rcp\))75 1255 y(mpi)p 150 1255 V 17 w(reduce\()g(inhandle,)f(outhandle,)h(group,)g(root,)g(op,)g(rc)h (\))75 1342 y(mpi)p 150 1342 V 17 w(reducec\()f(inbuf,)g(outbuf,)f (len,)i(type,)f(group,)g(root,)g(op,)h(rc)f(\))75 1428 y(mpi)p 150 1428 V 17 w(user)p 263 1428 V 16 w(reduce\()g(inhandle,)g (outhandle,)f(group,)h(root,)h(function,)e(unitsize,)h(rc)h(\))75 1515 y(mpi)p 150 1515 V 17 w(user)p 263 1515 V 16 w(reducec\()f(inbuf,) g(outbuf,)g(len,)g(type,)h(group,)f(root,)g(function,)393 1571 y(unitsize,)g(rc)g(\))75 1658 y(mpi)p 150 1658 V 17 w(allreduce\()f(inhandle,)h(outhandle,)f(group,)h(op,)h(rc)f(\))75 1744 y(mpi)p 150 1744 V 17 w(allreducec\()f(inbuf,)h(outbuf,)g(len,)g (type,)h(group,)f(op,)g(rc)h(\))75 1831 y(mpi)p 150 1831 V 17 w(user)p 263 1831 V 16 w(allreduce\()f(inhandle,)g(outhandle,)f (group,)h(function,)g(rc)g(\))75 1917 y(mpi)p 150 1917 V 17 w(user)p 263 1917 V 16 w(allreducec\()g(inbuf,)g(outbuf,)g(len,)g (type,)g(group,)g(function,)g(rc)g(\))75 2004 y(mpi)p 150 2004 V 17 w(scan\()g(inhandle,)g(outhandle,)f(group,)h(op,)h(rc)f (\))75 2090 y(mpi)p 150 2090 V 17 w(scanc\()g(inbuf,)g(outbuf,)g(len,)g (type,)g(group,)g(op,)h(rc)f(\))75 2177 y(mpi)p 150 2177 V 17 w(user)p 263 2177 V 16 w(scan\()h(inhandle,)e(outhandle,)h(group,) g(function,)f(rc)i(\))75 2263 y(mpi)p 150 2263 V 17 w(user)p 263 2263 V 16 w(scanc\()f(inbuf,)g(outbuf,)g(len,)h(type,)f(group,)g (function,)g(rc)g(\))-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 94 100 94 99 bop 75 357 a Fx(Section)35 b(7)75 565 y FE(En)m(vironmen)m(tal)41 b(Managemen)m(t)f(and)75 690 y(Inquiry)75 932 y Fw(7.1)59 b(Intro)r(duction)75 1034 y FC(This)17 b(c)o(hapter)e(con)o(tains)h (MPI-sp)q(eci\014c)i(routines)f(for)e(in)o(teracting)i(with)f(the)g(en) o(vironmen)o(t)g(in)h(whic)o(h)75 1091 y(MPI)12 b(programs)e(run.)19 b(The)12 b(c)o(hapter)g(op)q(ens)g(with)g(a)g(discussion)h(of)f(MPI)f (routines)i(for)e(initialization)j(of)75 1147 y(the)g(MPI)h(en)o (vironmen)o(t.)20 b(The)14 b(c)o(hapter)h(includes)h(a)e(large)h(n)o (um)o(b)q(er)f(of)g(enquiry)i(functions,)f(b)o(y)f(whic)o(h)75 1204 y(an)h(MPI)f(program)g(can)h(determine)g(features)g(of)f(the)h(en) o(vironmen)o(t)g(in)g(whic)o(h)h(it)f(is)g(running)h(in)f(order)75 1260 y(to)g(tune)i(itself.)23 b(There)17 b(is)f(also)h(a)e(discussion)j (of)e(functions)h(that)e(are)h(not)g(sp)q(eci\014c)i(to)d(MPI,)h(suc)o (h)g(as)75 1317 y(timing)d(functions,)g(that)e(ma)o(y)h(b)q(e)h(pro)o (vided)g(b)o(y)f(other)g(standards)f(\(e.g.)18 b(POSIX\).)13 b(F)l(or)e(completeness,)75 1373 y(w)o(e)k(discuss)h(signals,)g (although)f(p)q(erhaps)h(this)g(b)q(elongs)g(in)g(the)f(c)o(hapter)g (on)g(correctness.)75 1499 y Fq(7.1.1)49 b(Program)16 b(Sta)o(rtup)f(Issues)75 1586 y FC(The)21 b(MPI)f(standard)g(should)i (pro)o(vide)f(enough)g(information)g(to)e(sp)q(ecify)j(an)f(en)o(tire,) h(source)e(co)q(de)75 1643 y(p)q(ortable)12 b(program.)18 b(T)l(o)12 b(do)f(this,)i(it)f(is)g(necessary)g(to)g(sp)q(ecify)h(ho)o (w)e(parallel)i(programs)e(start)g(and)h(end.)75 1699 y(This)f(is)f(the)h(analogue)f(of)g Fp(PROGRAM)23 b(MAIN)g(...)47 b(END)10 b FC(in)h(F)l(ortran)e(and)h Fp(main\(argc,argv\))p Fy(f)p Fp(...exit\(rc\))o(;)p Fy(g)p FC(.)75 1756 y(This)18 b(prop)q(osal)h(do)q(es)f(not)f(deal)i(with)f(metho)q(ds)g(for)g(spa)o (wning)g(tasks)f(during)i(the)f(execution)h(of)e(an)75 1812 y(MPI)g(program,)e(or)h(for)g(handling)i(explicitly)i(MIMD)c(mo)q (dels)h(\(b)q(ey)o(ond)g(the)g(usual)g(w)o(a)o(y)e(of)i(starting)75 1869 y(a)h(T)l(uring)i(mac)o(hine)f(on)g(all)h(no)q(des)f(and)g (sending)h(the)f(appropriate)f(tap)q(e\).)31 b(This)19 b(is)g(not)f(mean)o(t)h(to)75 1925 y(suggest)11 b(that)h(MPI)f(not)h (ha)o(v)o(e)f(suc)o(h)i(in)o(terfaces,)f(just)g(that)f(w)o(e)h(are)f (only)i(prop)q(osing)f(an)g(SPMD)f(mo)q(del.)166 1982 y(It)g(should)i(b)q(e)f(p)q(ossible)h(to)e(write)h(a)f(program)f(that)h (is)h(source-language)g(p)q(ortable)g(to)f(an)o(y)g(system,)75 2039 y(particularly)k(if)f(it)g(requires)h(only)f(a)g(simple)h(in)o (terface)f(to)f(the)h(op)q(erating)g(system.)19 b(An)14 b(example)h(is)f(an)75 2095 y(en)o(vironmen)o(t)i(that)f(starts)g(up)h (a)f(program)g(with)h(a)g(\014xed)g(n)o(um)o(b)q(er)g(of)g(pro)q (cessors.)21 b(Without)16 b(this,)g(it)75 2152 y(is)g(not)f(p)q (ossible)h(to)f(write)g Fs(any)k FC(p)q(ortable)d(programs.)166 2209 y(Ho)o(w)o(ev)o(er,)d(no)i(standard)g(should)h(restrict)e (programs)g(to)g(a)h(particular)g(in)o(terface.)20 b(F)l(or)14 b(example,)75 2265 y(some)h(applications)i(ma)o(y)e(w)o(an)o(t)f(to)h (dynamically)i(add)f(and)g(remo)o(v)o(e)e(tasks)h(during)h(the)g (execution)g(of)75 2322 y(an)d(application.)21 b(W)l(e)13 b(lea)o(v)o(e)h(it)f(to)f(others)h(on)g(the)h(committee)f(to)f(propse)i (a)e(mec)o(hanism)i(for)f(this)g(case.)166 2379 y(Belo)o(w,)i(w)o(e)g (discuss)h(three)f(di\013eren)o(t)g(approac)o(hes;)f(w)o(e)h(note)g (that)f(all)i(of)f(these)g(ha)o(v)o(e)f(b)q(een)i(tried)75 2436 y(in)g(some)f(existing)h(system)f(or)g(systems.)75 2560 y Fo(MPI)p 160 2560 14 2 v 16 w(INIT)75 2647 y FC(The)c(seemingly) i(simplest)f(approac)o(h)f(is)g(to)f(use)i(an)f(explicit)i (initialization)h(routine,)e Fp(MPI)p 1618 2647 15 2 v 17 w(Init)p FC(.)18 b(There)75 2704 y(are)h(sev)o(eral)g(problems)h (with)f(this)h(approac)o(h.)31 b(The)20 b(ma)s(jor)d(one)j(is)f(that)g (the)g(parallel)i(part)d(of)h(the)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 95 101 95 100 bop 75 -100 a Fv(7.1.)34 b(INTR)o(ODUCTION)1279 b FC(95)75 45 y(program)15 b(is)j(the)e(co)q(de)h(that)f(\\follo)o(ws") g(the)h Fp(MPI)p 942 45 15 2 v 17 w(Init)p FC(.)23 b(This)17 b(is)h(p)q(oten)o(tially)g(error)d(prone;)i(further,)75 102 y(it)e(can)h(b)q(e)g(di\016cult)g(to)f(implemen)o(t)h(on)f(some)g (systems.)75 227 y Fo(MPI)p 160 227 14 2 v 16 w(MAIN)75 315 y FC(Another)i(approac)o(h)g(is)h(to)f(insist)h(that)e(the)i(en)o (tire)f(program)g(b)q(e)h(run)f(in)h(parallel;)i(a)d(simple)h(w)o(a)o (y)f(to)75 371 y(do)h(this)h(is)h(to)d(replace)j(the)f(\\main")f (program)f(with)i(a)g(routine.)30 b(In)19 b(C,)f(this)h(could)h(b)q(e)f Fp(MPI)p 1742 371 15 2 v 17 w(Main\()75 428 y(argc,)k(argv)g(\))p FC(.)d(The)15 b(linking)i(step)d(then)h(pro)o(vides)g(a)g Fp(main)f FC(that)g(handles)h(all)h(appropriate)f(startup)75 484 y(\(including)h(the)e(pro)q(cessing)g(of)f(command)g(line)j (argumen)o(ts)c(where)i(appropriate)f(for)g(information)h(on)75 541 y(the)h(n)o(um)o(b)q(er)h(of)f(tasks)f(etc.\))20 b(and)15 b(then)g(calls)i(the)e(user's)g Fp(MPI)p 1168 541 V 17 w(main)f FC(routine.)166 598 y(Note)h(that)f(the)i Fp(MPI)p 528 598 V 16 w(main)f FC(in)o(terface)g(can)h(b)q(e)g (implemen)o(ted)h(with)e Fp(MPI)p 1429 598 V 17 w(call)f FC(through)147 710 y Fp(int)23 b(main\()g(argc,)g(argv)h(\))147 767 y(int)f(argc;)147 823 y(char)g(**argv;)147 880 y({)147 936 y(return)g(MPI_call\()f(argc,)h(argv,)h(MPI_main)e(\);)147 993 y(})75 1118 y Fo(MPI)p 160 1118 14 2 v 16 w(CALL)75 1206 y FC(Another)17 b(approac)o(h)f(is)h(to)f(pro)o(vide)h(a)f (routine)h(that)f(runs)h(a)f(user-de\014ned)j(routine)e(in)g(parallel.) 26 b(F)l(or)75 1262 y(example,)19 b Fp(MPI)p 344 1262 15 2 v 17 w(Call\()k(argc,)g(argv,)g(routine)g(\))18 b FC(can)g(b)q(e)h(used)f(to)f(run)i Fp(routine\()j(argc,)i(argv)75 1319 y(\))15 b FC(in)i(parallel.)23 b(A)16 b(generalization)h(of)e (this)h(w)o(ould)g(allo)o(w)g(the)g(running)h(of)e(a)g(particular)h (routine)h(on)e(a)75 1375 y(de\014ned)i(group:)k Fp(MPI)p 455 1375 V 17 w(Call\()i(groupid,)g(argc,)g(argv,)g(routine)g(\))p FC(.)f(This)16 b(is)h(more)e(\015exible)j(than)75 1432 y(the)g Fp(MPI)p 231 1432 V 17 w(Main)g FC(approac)o(h)g(but)g(retains) h(the)f(simplicit)o(y)j(of)d(running)h(a)f(w)o(ell-de\014ned)j(program) c(unit)75 1488 y(\(the)e(routine\))g(in)h(parallel.)166 1546 y(Note)g(that)f(in)i(this)f(case,)g(program)f(termination)h(is)g (handled)i(b)o(y)e(ha)o(ving)g(\(a\))f(the)h(routine)h(exit)75 1602 y(and)e(\(b\))g(the)g(calling)i(routine)f(exit.)k(F)l(or)15 b(example)170 1701 y Fp(irc)24 b(=)g(MPI_Call\()e(argc,)h(argv,)h (worker)f(\);)170 1757 y(exit\()h(irc)f(\);)75 1814 y(...)170 1870 y(int)h(worker\()f(argc,)g(argv)g(\))170 1926 y(int)h(argc;)f (char)g(**argv;)170 1983 y({)75 2039 y(...)170 2096 y(if)h(\(error)f (condition\))g(return)g(1;)170 2152 y(...)170 2209 y(return)g(0;)170 2265 y(})166 2363 y FC(All)16 b(three)g(of)e(these)i(approac)o(hes)f (are)g(curren)o(tly)g(in)h(use.)166 2420 y(These)i(are)g(not)g(the)g (most)f(general)i(metho)q(ds.)29 b(F)l(or)17 b(example,)i(these)g(pro)o (vide)f(no)g(mec)o(hanism)75 2477 y(for)c(dynamically)i(creating)f (\(remote)e(spa)o(wn\))h(tasks)f(on)i(other)f(pro)q(cessors.)19 b(Also,)c(there)f(is)h(no)f(direct)75 2533 y(supp)q(ort)e(for)f(MIMD)g (programming.)18 b(Finally)l(,)c(there)e(is)g(no)f(supp)q(ort)h(for)f (adding)i(an)e(already-running)75 2590 y(pro)q(cess)k(to)g(the)g(MPI)g (collection)i(of)e(pro)q(cesses.)166 2647 y(In)f(the)f(case)g(of)g (heterogeneous)g(en)o(vironmen)o(ts,)g(it)h(is)f(necessary)h(to)e(sp)q (ecify)j(the)e(set)g(of)f(w)o(orksta-)75 2704 y(tions)i(to)f(b)q(e)h (used.)20 b(Our)14 b(only)h(requiremen)o(t)f(is)g(that)f(this)i(b)q(e)f (done,)g(in)h(the)e(C/POSIX)i(case,)e(through)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 96 102 96 101 bop 75 -100 a FC(96)348 b Fv(SECTION)16 b(7.)34 b(ENVIR)o(ONMENT)l(AL)17 b(MANA)o(GEMENT)d(AND)h(INQUIR)l(Y)75 45 y FC(command-line)j(argumen)o(ts)e(\(th)o(us)g(main)o(taining)h (source-co)q(de)g(p)q(ortabilit)o(y\).)24 b(W)l(e)17 b(could)g(sp)q(ecify)h(an)75 102 y(extensible,)j(common)d(format)f(for) h(sp)q(ecifying)i(the)f(mac)o(hines,)h(executables,)g(and)e(resource)h (limits,)75 158 y(but,)h(to)f(a)o(v)o(oid)g(additional)i(debate,)f(w)o (e)f(suggest)g(only)h(that)e(the)i(command)f(line)i(option)f Fp(-mpi)e FC(b)q(e)75 214 y(reserv)o(ed)d(for)g(future)g(extensions.)75 358 y Fw(7.2)59 b(MPI)20 b(Program)g(Sta)n(rtup)g(Prop)r(osal{MPI)p 1137 358 18 2 v 21 w(INIT)75 459 y FC(One)13 b(goal)g(of)f(MPI)h(is)g (to)f(ac)o(hiev)o(e)h Fs(sour)n(c)n(e)h(c)n(o)n(de)g(p)n(ortability)p FC(.)19 b(By)13 b(this)g(w)o(e)f(mean)h(that)f(a)g(program)g(writ-)75 516 y(ten)j(using)i(MPI)e(and)g(complying)i(with)f(the)f(relev)m(an)o (t)h(language)g(standards)f(is)g(p)q(ortable)h(as)f(written,)75 572 y(and)k(m)o(ust)e(not)h(require)i(an)o(y)e(source)g(co)q(de)h(c)o (hanges)f(when)h(mo)o(v)o(ed)f(from)g(one)g(system)g(to)g(another.)75 629 y(This)h(explicitly)i(do)q(es)e Fs(not)f FC(sa)o(y)g(an)o(ything)g (ab)q(out)g(ho)o(w)g(an)h(MPI)f(program)f(is)i(started)f(or)f(launc)o (hed)75 685 y(from)h(the)i(command)f(line,)i(nor)e(what)f(the)i(user)f (m)o(ust)f(do)i(to)e(set)h(up)h(the)f(en)o(vironmen)o(t)g(in)h(whic)o (h)75 742 y(an)h(MPI)g(program)f(will)j(run.)38 b(Ho)o(w)o(ev)o(er,)21 b(an)h(implemen)o(tation)g(ma)o(y)e(require)i(some)f(setup)h(to)e(b)q (e)75 798 y(p)q(erformed)c(b)q(efore)h(other)f(MPI)g(routines)g(ma)o(y) g(b)q(e)h(called.)24 b(T)l(o)16 b(pro)o(vide)g(for)g(this,)g(MPI)h (includes)h(an)75 854 y(initialization)g(routine)e Fp(MPI)p 572 854 15 2 v 17 w(init)p FC(.)166 911 y(All)g(MPI)f(programs)f(m)o (ust)h(con)o(tain)g(a)g(call)h(to)f Fp(MPI)p 1069 911 V 16 w(init)p FC(;)g(this)g(routine)h(m)o(ust)e(b)q(e)i(called)h(b)q (efore)75 967 y(an)o(y)e(other)g(MPI)g(routines)g(are)g(called.)22 b(The)15 b(format)f(is:)75 1061 y Fp(start_ctx)23 b(=)g(MPI_init\()g (...)g(args)h(...)f(\);)75 1155 y FC(The)13 b Fp(start)p 289 1155 V 16 w(ctx)f FC(can)h(then)g(b)q(e)g(used)g(in)h(place)f(of)f (the)h Fp(all)p 1092 1155 V 17 w(ctx)f FC(\(con)o(text)f(of)i(all)g (pro)q(cesses\).)19 b(Multiple)75 1212 y(calls)h(to)e Fp(MPI)p 314 1212 V 17 w(init)g FC(\(sa)o(y)f(\014rst)i(b)o(y)f(a)h (user-co)q(de)g(and)g(then)g(b)o(y)g(a)f(soft)o(w)o(are)f(pac)o(k)m (age)i(called)h(b)o(y)f(the)75 1268 y(user's)d(co)q(de\))h(ha)o(v)o(e)g (meaning;)h(they)e(set)h(up)g(the)g(con)o(text)f(that)g(will)j(b)q(e)e (used)g(as)g(the)g(con)o(text)f(of)g(all)75 1324 y(pro)q(cesses.)166 1457 y Fu(Discussion:)g Ft(Sev)o(eral)e(other)h(approac)o(hes)g(w)o (ere)h(discussed,)f(including)e(a)h(replacemen)o(t)g(main)f(program)75 1507 y(and)f(a)g(subroutine-call)g(in)o(terface.)18 b(I'v)o(e)12 b(included)h(the)g Fn(MPI)p 1033 1507 14 2 v 15 w(init)e Ft(v)o(ersion)i(b)q(ecause)h(it)e(seemed)h(to)f(b)q(e)h(the)g(least)75 1557 y(ob)r(jectionable)h(and)g(since)g(w)o(ork)n(able)f(v)o(ersions)i (of)e(the)h(other)h(t)o(w)o(o)e(ma)o(y)f(b)q(e)j(written)f(using)g Fn(MPI)p 1618 1557 V 15 w(init)p Ft(.)166 1606 y(I'm)8 b(not)i(sure)h(what)f(argumen)o(ts)f Fn(MPI)p 758 1606 V 15 w(init)g Ft(should)h(tak)o(e.)17 b(One)11 b(argumen)o(t)e(should)g (b)q(e)i(the)g(initial)d(con)o(text;)75 1656 y(if)j(n)o(ull,)g(then)h (the)h(\\all)d(con)o(text")j(is)e(assumed.)17 b(There)c(is)f(some)f (reason)i(to)e(pass)i(the)f(command)d(line)j(argumen)o(ts)75 1706 y Fn(argc,argv)f Ft(to)i Fn(MPI)p 403 1706 V 15 w(init)f Ft(in)h(order)g(to)g(allo)o(w)e(it)i(to)g(lo)q(ok)f(for)g (MPI-sp)q(eci\014c)j(argumen)o(ts)d(\(suc)o(h)i(as)f(the)h(n)o(um)o(b)q (er)75 1756 y(and)g(name)e(of)i(pro)q(cessors)i(to)e(use)g(in)g(a)f (heterogeneous)j(en)o(vironmen)o(t\),)d(but)h(this)g(ma)o(y)e(b)q(e)i (to)q(o)g(con)o(tro)o(v)o(ersial.)166 1812 y(W)m(e)g(should)g(also)g (discuss)h(the)g(other)g(p)q(ossibilities.)k(One,)c Fn(MPI)p 1184 1812 V 15 w(main)p Ft(,)e(is)h(a)g(main)e(program)h(replacemen)o (t)75 1869 y(\(users)21 b(use)f Fn(MPI)p 347 1869 V 15 w(main)e Ft(instead)i(of)e Fn(main)h Ft(in)f(C)i(and)f Fn(PROGRAM)h(name)e Ft(in)h(F)m(ortran\).)34 b(Another)20 b(approac)o(h)f(is)75 1925 y Fn(MPI)p 144 1925 V 15 w(call)13 b Ft(that)h(calls)f(a)h(sp)q(eci\014ed)i(routine.)75 2151 y Fw(7.3)59 b(MPI-Sp)r(eci\014c)19 b(Enquiry)g(F)n(unctions)75 2253 y FC(These)d(routines)f(are)g(sp)q(eci\014c)i(to)e(the)g(op)q (eration)g(of)g(MPI.)166 2309 y(The)g(\014rst)g(t)o(w)o(o)f(routines)i (are)f(absolutely)h(essen)o(tial;)f(they)h(are:)75 2403 y FD(MPI)p 184 2403 16 2 v 18 w(rank)22 b FC(My)15 b(pro)q(cess)h(id)g (relativ)o(e)f(to)g(a)g(group)75 2497 y FD(MPI)p 184 2497 V 18 w(size)23 b FC(The)15 b(n)o(um)o(b)q(er)h(of)f(pro)q(cess)g (id's)h(in)g(a)f(group)166 2591 y(MPI)f(also)g(con)o(tain)h(routines)g (for)e(managing)i(implemen)o(tation)g(limits)h(in)f(MPI.)f(These)h (routines)75 2647 y(will)i(tak)o(e)d(a)h(requested)h(v)m(alue)g(and)f (return)h(the)f(actual)g(v)m(alue,)h(p)q(ossibly)h(including)h(a)d(v)m (alue)h(for)f(\\not)75 2704 y(applicable.")22 b(Suc)o(h)16 b(routines)g(ma)o(y)e(include)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 97 103 97 102 bop 75 -100 a Fv(7.3.)34 b(MPI-SPECIFIC)16 b(ENQUIR)l(Y)g (FUNCTIONS)811 b FC(97)75 45 y FD(MPI)p 184 45 16 2 v 18 w(V)l(alidT)l(ags)24 b FC(The)15 b(range)g(\(or)f(other)h (description\))h(of)f(v)m(alid)i(tags)75 139 y FD(MPI)p 184 139 V 18 w(NumGroups)22 b FC(Maxim)o(um)15 b(n)o(um)o(b)q(er)g(of)g (groups)75 232 y FD(MPI)p 184 232 V 18 w(NumCn)o(txs)21 b FC(Maxim)o(um)15 b(n)o(um)o(b)q(er)h(of)e(con)o(texts)75 326 y FD(MPI)p 184 326 V 18 w(Bu\013erSize)23 b FC(Bu\013er)15 b(space)g(for)g(unsen)o(t/unreceiv)o(ed)i(messages)75 419 y FD(MPI)p 184 419 V 18 w(Bu\013erManagemen)o(t)22 b FC(Bu\013er)15 b(managemen)o(t)f(t)o(yp)q(e)166 513 y(Note)h(that)f(all)i(of)f(these)h(could)g(return)f(\\not)f (applicable.")166 652 y Fu(Discussion:)36 b Ft(These)16 b(need)f(to)f(refer)i(to)e(a)g(diagram)e(of)i(p)q(ossible)h(bu\013er)g (usages.)20 b(W)m(e)14 b(m)o(ust)g(b)q(e)h(careful)75 708 y(to)f(handle)f(the)i(case)g(of)e(\(a\))h(no)g(bu\013ering)g(and)g (\(b\))g(bu\013ers)h(on)f(the)g(sender.)166 847 y FC(More)21 b(detailed)i(routines)f(for)g(bu\013er)f(managemen)o(t)g(ma)o(y)g(also) h(b)q(e)g(pro)o(vided.)41 b(These)22 b(could)75 904 y(include:)75 997 y FD(MPI)p 184 997 V 18 w(Bu\013erSizeP)o(airs)g FC(Bet)o(w)o(een)15 b(pairs)h(of)e(pro)q(cessors)75 1090 y FD(MPI)p 184 1090 V 18 w(Bu\013erSizeAll)23 b FC(Bet)o(w)o(een)15 b(\\me")g(and)g(all)i(others)75 1184 y FD(MPI)p 184 1184 V 18 w(Exclusiv)o(eBu\013er)22 b FC(Dedicate)g(a)f(bu\013er)g(of)g(a)g (sp)q(eci\014ed)j(size)e(b)q(et)o(w)o(een)g(a)f(selected)i(pair)e(of) 629 1241 y(pro)q(cessors)166 1410 y Fu(Discussion:)h Ft(Another)c(approac)o(h)f(is)g(to)g(use)h(a)f(single)g(routine)g(that) g(tak)o(es)h(a)f(query)g(parameter)g(and)75 1460 y(returns)e(the)g (requested)h(data)d(\(or)h(a)g(\\not)g(a)o(v)n(ailable"\).)166 1509 y(The)h(routines)h(returning)f(the)g(maxim)o(um)10 b(n)o(um)o(b)q(er)k(of)g(things)h(\(lik)o(e)f(groups\))i(could)e (instead)h(return)h(the)75 1559 y(maxim)n(um)g(n)o(um)o(b)q(er)j(of)g Fb(new)24 b Ft(things)c(that)g(could)f(b)q(e)i(created.)37 b(All)19 b(of)g(these)i(routines)g(are)f(mean)o(t)f(to)g(b)q(e)75 1609 y(advisory)m(,)12 b(particularly)i(in)f(a)h(m)o(ultithreaded)e(en) o(vironmen)o(t,)h(since)h(they)h(ma)o(y)d(b)q(e)i(allo)q(cated)g (dynamically)l(.)166 1748 y FC(There)h(is)h(also)f(some)g(desire)h(for) 75 1841 y FD(MPI)p 184 1841 V 18 w(IOmo)q(de)23 b FC(Whether)15 b(IO)h(is)g(p)q(er)g(no)q(de)f(or)g(p)q(er)h(group/con)o(text)75 1935 y FD(MPI)p 184 1935 V 18 w(ErrorMo)q(de)22 b FC(V)l(arious)16 b(c)o(hoices)g(of)e(error)h(handling)166 2028 y(Both)j(of)f(these,)h (since)h(they)f(ha)o(v)o(e)f(global)i(e\013ect,)f(are)f(somewhat)g (undesirable.)30 b(They)18 b(should)75 2085 y(return)j(the)g(previous)h (mo)q(de;)j(m)o(ultithreaded)d(applications)h(will)g(ha)o(v)o(e)d(to)h (b)q(e)h(v)o(ery)f(careful)h(with)75 2141 y(these.)166 2197 y(It)16 b(is)h(imp)q(ortan)o(t)f(to)g(b)q(e)g(able)h(to)f (determine,)h(b)q(oth)g(at)e(run)i(time)g(and)f(compile)i(time,)e(whic) o(h)h(of)75 2254 y(the)e(\\optional")h(capabilities)h(the)e(MPI)h (implemen)o(tation)g(supp)q(orts.)k(These)c(include)75 2347 y FD(MPI)p 184 2347 V 18 w(Has)p 289 2347 V 19 w(Ready)p 453 2347 V 19 w(Receiv)o(er)22 b FC(Whether)10 b(ready-receiv)o(er)h (comm)o(unication)g(is)g(sim)o(ulated)g(with)f(reg-)629 2404 y(ular)16 b(sends)g(and)f(receiv)o(es.)75 2497 y FD(MPI)p 184 2497 V 18 w(Has)p 289 2497 V 19 w(Non)o(blo)q(c)o(king)23 b FC(Whether)h(non)o(blo)q(c)o(king)h(op)q(erations)e(are)g(sim)o (ulated)i(with)f(blo)q(c)o(king)629 2554 y(sends)16 b(and)f(receiv)o (es)75 2647 y FD(MPI)p 184 2647 V 18 w(Has)p 289 2647 V 19 w(Heterogeneous)22 b FC(Whether)16 b(heterogeneous)g(pro)q (cessing)g(is)h(supp)q(orted)f(\(that)f(is,)h(this)629 2704 y(MPI)f(w)o(orks)g(with)g(heterogeneous)g(systems\))-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 98 104 98 103 bop 75 -100 a FC(98)348 b Fv(SECTION)16 b(7.)34 b(ENVIR)o(ONMENT)l(AL)17 b(MANA)o(GEMENT)d(AND)h(INQUIR)l(Y)166 45 y FC(In)f(addition,)h(it)f(is)g(useful)h(to)e(b)q(e)h(able)h(to)e (unpac)o(k)h(the)g(v)m(arious)g(opaque)g(ob)s(jects)f(in)o(to)h (individual)75 102 y(v)m(ariables,)25 b(suc)o(h)d(as)f(bu\013er)h (descriptors)h(and)f(con)o(texts,)g(and)h(the)f(message)f(queue.)41 b(Op)q(erations)75 158 y(should)16 b(include)75 251 y FD(MPI)p 184 251 16 2 v 18 w(Unpac)o(kBu\013erDescriptor)22 b FC(T)l(ak)o(es)d(a)f(bu\013er)h(and)h(returns)f(a)f(con)o(text)h(and) g(\(p)q(ossibly\))h(a)629 307 y(n)o(um)o(b)q(er)c(of)f(mem)o(b)q(ers.)k (Individual)g(routines)c(can)629 413 y FD(Bu\013erDescT)o(yp)q(e)22 b FC(Giv)o(en)d(a)g(con)o(text,)f(return)h(the)g(kind)h(of)e(bu\013er) 729 470 y(descriptor)e(elemen)o(t)g(\(con)o(tiguous)f(etc\).)629 542 y FD(MPI)p 738 542 V 18 w(Unpac)o(kCon)o(tig)23 b FC(Giv)o(en)18 b(a)f(con)o(text,)g(return)g(the)g(data.)26 b(Ad-)729 599 y(v)m(ance)16 b(the)g(con)o(text.)629 672 y FD(MPI)p 738 672 V 18 w(Unpac)o(kIndex)22 b FC(Ditto)15 b(of)g(indexed)629 744 y FD(MPI)p 738 744 V 18 w(Unpac)o(kBlo)q(c)o(k) 24 b FC(Ditto)14 b(for)h(blo)q(c)o(ks.)75 850 y FD(MPI)p 184 850 V 18 w(DumpMessageQueue)22 b FC(T)l(ak)o(es)14 b(a)g(length)h(and)g(a)f(user-allo)q(cated)i(arra)o(y)l(,)d(it)h (return)h(n)o(um)o(b)q(er)629 906 y(of)j(elemen)o(ts)i(a)o(v)m(ailable) g(\(ma)o(y)d(b)q(e)j(greater)d(than)i(length\))g(and)g(infor-)629 963 y(mation)g(on)g(the)g(messages)g(in)h(the)f(format)f(includes)j (\(tag,)e(con)o(text,)629 1019 y(source,)c(size\))75 1113 y FD(MPI)p 184 1113 V 18 w(Unpac)o(kCon)o(text)22 b FC(T)l(ak)o(es)15 b(a)g(con)o(text)g(and)g(returns)g(mem)o(b)q(ers)h (of)e(group/con)o(text)75 1206 y FD(MPI)p 184 1206 V 18 w(StatusOfHandles)24 b FC(I'v)o(e)15 b(forgotten)f(what)h(this)g(is) h(for.)166 1299 y(Note)f(that)f(some)h(of)g(these)g(are)g(tough)g(to)g (implemen)o(t)h(directly)h(or)d(e\016cien)o(tly)j(in)f(F)l(ortran.)166 1438 y Fu(Discussion:)i Ft(An)e(alternate)f(approac)o(h)h(for)f(the)g (queue)i(and)e(con)o(text)h(inquiry)e(functions)i(\(at)f(least)g(for)75 1495 y(C\))f(is)g(for)f(the)i(routine)f(to)g(allo)q(cate)f(the)h(space) h(rather)g(than)f(ha)o(ving)f(it)g(passed)i(in.)75 1699 y Fq(7.3.1)49 b(P)o(a)o(rallel)17 b(p)o(rogramming)75 1785 y FC(This)d(section)f(includes)i(routines)f(that)e(relate)h(to)f (parallel)j(programming)d(but)h(are)g(not)f(MPI-sp)q(eci\014c.)75 1841 y(These)k(ma)o(y)e(include)k(suc)o(h)d(things)h(as:)75 1934 y FD(MPI)p 184 1934 V 18 w(GetNbrs)22 b FC(Return)16 b(the)f(\\neigh)o(b)q(ors")h(in)g(the)f(underlying)i(ph)o(ysical)g(top) q(ology)75 2028 y FD(MPI)p 184 2028 V 18 w(GetPh)o(ysNo)q(de)22 b FC(Return)i(a)f(unique)h(sp)q(eci\014er)h(for)e(the)g(actual)g(\(as)f (opp)q(osed)i(to)f(virtual\))629 2084 y(no)q(de.)h(This)17 b(should)h(b)q(e)f(a)f(c)o(haracter)g(string)g(for)g(maxim)o(um)g (\015exibil-)629 2141 y(it)o(y)l(.)26 b(>F)l(rom)17 b(this)g(v)m(alue)h (it)g(m)o(ust)e(b)q(e)i(p)q(ossible)h(to)d(iden)o(tify)i(a)f(sp)q (eci\014c)629 2197 y(piece)h(of)e(hardw)o(are;)g(p)q(ossible)i(v)m (alues)f(include)i(\\pro)q(cessor)c(9)h(in)i(rac)o(k)629 2254 y(4)e(of)g(mpp.cs.org")f(and)h(\\231")f(\(where)h(231)f(is)i(the)f (actual)g(pro)q(cessor)629 2310 y(n)o(um)o(b)q(er)i(in)h(the)f(running) g(homogeneous)g(system\).)26 b(A)18 b(return)g(v)m(alue)629 2367 y(of)d(\\not)g(applicable")i(is)f(allo)o(w)o(ed)f(but)h(strongly)e (discouraged.)75 2460 y FD(GetT)l(op)q(ology)24 b FC(Return)18 b(ph)o(ysical)g(net)o(w)o(ork)f(top)q(ology)l(.)25 b(Prede\014ned)19 b(v)m(alues)f(include)h Fp(MPI)p 1705 2460 15 2 v 17 w(mesh2d)p FC(,)629 2517 y Fp(MPI)p 704 2517 V 17 w(hypercube)p FC(,)14 b(and)h Fp(MPI)p 1124 2517 V 17 w(omeganetwork)p FC(.)75 2610 y FD(MPI)p 184 2610 16 2 v 18 w(GetDistance)24 b FC(Return)16 b(the)f(\\distance")h(from)e(one)i(pro)q(cessor)f(to)f (another)75 2704 y FD(MPI)p 184 2704 V 18 w(GetDiameter)23 b FC(Return)16 b(the)g(maxim)o(um)f(\\distance")g(from)g(one)g(pro)q (cessor)g(to)f(another.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 99 105 99 104 bop 75 -100 a Fv(7.3.)34 b(MPI-SPECIFIC)16 b(ENQUIR)l(Y)g (FUNCTIONS)811 b FC(99)166 45 y(Eac)o(h)13 b(of)g(these)h(routines)g (ma)o(y)e(return)i(\\not)e(applicable;")k(it)e(is)g(hop)q(ed)g(that)f (qualit)o(y)h(implemen-)75 102 y(tations)g(will)j(try)d(and)g(pro)o (vide)i(as)e(m)o(uc)o(h)h(information)f(as)h(p)q(ossible)h(to)e(the)h (user.)20 b(In)15 b(addition,)g(some)75 158 y(of)g(the)g(\\get")g (routines)g(m)o(ust)g(b)q(e)h(in)o(terpreted)g(as)f(pro)o(viding)h (appro)o(ximate)f(answ)o(ers.)20 b(F)l(or)14 b(example,)75 214 y(if)k(groups)f(and)h(con)o(texts)f(are)h(allo)q(cated)g(from)f (the)h(user's)f(address)h(space,)g(then)g(an)o(y)f(return)h(v)m(alue)75 271 y(will)f(b)q(e)f(appro)o(ximate.)166 327 y(The)g(exact)f(meaning)h (of)f(\\distance")h(is)g(left)g(to)f(the)g(implemen)o(tation.)22 b(In)16 b(a)g(system)f(with)g(only)75 384 y(lo)q(cal)23 b(connections,)g(suc)o(h)f(as)g(a)f(mesh)h(or)f(h)o(yp)q(ercub)q(e,)j (the)e(distance)g(should)h(b)q(e)f(the)f(n)o(um)o(b)q(er)h(of)75 440 y(edges)17 b(in)h(the)e(connection)i(graph)f(that)f(a)g(message)h (w)o(ould)g(tra)o(v)o(el)f(along.)25 b(In)17 b(a)f(system)h(with)g(a)f (\015at)75 497 y(in)o(terconnect,)d(suc)o(h)g(as)g(an)f(omega)g(or)g (butter\015y)g(net)o(w)o(ork,)g(the)h(distance)g(could)h(b)q(e)f(zero)f (from)g(a)g(no)q(de)75 553 y(to)j(itself)h(and)f(either)h(one)f(or)g (the)g(n)o(um)o(b)q(er)h(of)f(stages)f(in)i(the)f(net)o(w)o(ork)f(for)h (all)h(other)f(no)q(des.)75 675 y Fq(7.3.2)49 b(non-MPI)75 761 y FC(This)21 b(section)h(con)o(tains)e(routines)h(that)f(are)h Fs(not)k FC(part)20 b(of)g(MPI)h(but)f(are)h(included)i(for)d (complete-)75 817 y(ness.)k(These)17 b(will)h(include)h(v)m(arious)e (routines)g(for)f(resource)g(usage,)h(time-of-da)o(y)l(,)f(etc.)24 b(The)17 b(POSIX)75 874 y(in)o(terfaces)e(will)i(b)q(e)f(summarized)g (here.)75 994 y Fo(Interaction)g(with)g(signals)75 1080 y FC(The)f(in)o(ten)o(t)g(of)f(MPI)h(is)g(to)f(b)q(e)h(thread)g(and)f (signal)i(safe.)j(This)d(has)e(a)g(n)o(um)o(b)q(er)h(of)g(subtle)g (e\013ects.)20 b(F)l(or)75 1136 y(example,)g(on)e(Unix)h(systems,)g(a)f (catc)o(hable)h(signal)g(suc)o(h)g(as)f(SIGALRM)h(\(an)f(alarm)h (signal\))g(m)o(ust)75 1192 y(not)f(cause)g(an)h(MPI)f(routine)g(to)g (b)q(eha)o(v)o(e)h(di\013eren)o(tly)g(than)f(it)g(w)o(ould)h(ha)o(v)o (e)f(in)h(the)f(absence)h(of)f(the)75 1249 y(signal.)37 b(Of)21 b(course,)h(if)f(the)g(signal)h(handler)f(issues)h(MPI)f(calls) g(or)g(c)o(hanges)f(the)h(en)o(vironmen)o(t)g(in)75 1305 y(whic)o(h)c(the)g(MPI)g(routine)g(is)g(op)q(erating)f(\(for)g (example,)h(consuming)h(all)f(a)o(v)m(ailable)h(memory)e(space\),)75 1362 y(the)e(MPI)f(routine)h(should)h(b)q(eha)o(v)o(e)f(as)f (appropriate)h(for)f(that)g(situation)h(\(in)g(particular,)g(in)g(this) g(case,)75 1418 y(the)h(b)q(eha)o(vior)h(should)g(b)q(e)g(the)f(same)g (as)g(for)g(a)g(m)o(ultithreaded)h(MPI)f(implemen)o(tation\).)166 1475 y(A)j(second)i(e\013ect)e(is)h(that)f(a)g(signal)h(handler)h(that) e(do)q(es)h(MPI)f(calls)i(m)o(ust)e(not)g(in)o(terfere)h(with)75 1531 y(the)c(op)q(eration)f(of)g(MPI.)g(F)l(or)g(example,)h(an)g(MPI)f (receiv)o(e)i(of)e(an)o(y)g(t)o(yp)q(e)h(that)e(o)q(ccurs)i(within)h(a) e(signal)75 1588 y(handler)22 b(m)o(ust)f(not)g(cause)h(erroneous)f(b)q (eha)o(vior)h(b)o(y)f(the)h(MPI)f(implemen)o(tation.)39 b(Note)21 b(that)g(an)75 1644 y(implemen)o(tation)16 b(is)g(p)q(ermitted)g(to)e(prohibit)i(the)f(use)h(of)e(MPI)h(calls)h (from)e(within)j(a)d(signal)i(handler.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 100 106 100 105 bop 75 359 a Fx(Section)35 b(8)75 570 y FE(Pro\014ling)75 814 y Fw(8.1)59 b(Requirements)75 921 y FC(T)l(o)15 b(meet)g(the)g(MPI) g(pro\014ling)i(in)o(terface,)e(an)g(implemen)o(tation)i(of)d(the)i (MPI)f(functions)h Fs(must)131 1027 y FC(1.)22 b(pro)o(vide)14 b(a)f(mec)o(hanism)h(through)f(whic)o(h)h(all)g(of)f(the)g(MPI)h (de\014ned)h(functions)f(ma)o(y)e(b)q(e)i(accessed)189 1083 y(with)e(a)g(name)f(shift.)20 b(Th)o(us)11 b(all)i(of)f(the)g(MPI) g(functions)h(\(whic)o(h)f(normally)g(start)f(with)h(the)g(pre\014x)189 1140 y(\\)p Fp(MPI)p 287 1140 15 2 v 16 w FC("\))j(should)h(also)f(b)q (e)h(accessable)g(with)g(the)f(pre\014x)h(\\)p Fp(PMPI)p 1300 1140 V 16 w FC(".)131 1246 y(2.)22 b(ensure)c(that)f(those)h(MPI)g (functions)h(whic)o(h)f(are)g(not)f(replaced)i(ma)o(y)f(still)h(b)q(e)g (link)o(ed)g(in)o(to)f(an)189 1302 y(executable)e(image)f(without)h (causing)f(name)h(clashes.)131 1408 y(3.)22 b(do)q(cumen)o(t)c(the)f (implemen)o(tation)i(of)e(di\013eren)o(t)h(language)g(bindings)h(of)e (the)h(MPI)g(in)o(terface)f(if)189 1464 y(they)g(are)h(la)o(y)o(ered)g (on)f(top)g(of)h(eac)o(h)f(other,)h(so)f(that)g(the)h(pro\014ler)g(dev) o(elop)q(er)h(kno)o(ws)e(whether)189 1521 y(she)d(m)o(ust)f(implemen)o (t)i(the)e(pro\014le)i(in)o(terface)f(for)f(eac)o(h)h(binding,)h(or)e (can)h(economise)h(b)o(y)e(imple-)189 1577 y(men)o(ting)i(it)h(only)f (for)g(the)g(lo)o(w)o(est)g(lev)o(el)h(routines.)75 1737 y Fw(8.2)59 b(Discussion)75 1845 y FC(The)13 b(ob)s(jectiv)o(e)g(of)g (the)g(MPI)g(pro\014ling)h(in)o(terface)g(is)f(to)f(ensure)i(that)e(it) i(is)f(relativ)o(ely)h(easy)f(for)g(authors)75 1901 y(of)18 b(pro\014ling)i(\(and)f(other)f(similar\))h(to)q(ols)g(to)f(in)o (terface)h(their)g(co)q(des)g(to)f(MPI)h(implemen)o(tations)h(on)75 1958 y(di\013eren)o(t)15 b(mac)o(hines.)166 2017 y(Since)20 b(MPI)e(is)h(a)f(mac)o(hine)h(indep)q(enden)o(t)h(standard)e(with)h (man)o(y)f(di\013eren)o(t)g(implemen)o(tations,)75 2074 y(it)e(is)h(unreasonable)g(to)e(exp)q(ect)i(that)f(the)g(authors)f(of)h (pro\014ling)h(to)q(ols)f(for)g(MPI)g(will)i(ha)o(v)o(e)d(access)i(to) 75 2130 y(the)c(source)h(co)q(de)f(whic)o(h)h(implemen)o(ts)h(MPI)e(on) g(an)o(y)g(particular)h(mac)o(hine.)20 b(It)13 b(is)h(therefore)f (necessary)75 2187 y(to)21 b(pro)o(vide)i(a)f(mec)o(hanism)g(b)o(y)g (whic)o(h)h(the)f(implemen)o(tors)h(of)f(suc)o(h)g(to)q(ols)g(can)g (collect)i(whatev)o(er)75 2243 y(p)q(erformance)15 b(information)h (they)f(wish)h Fs(without)g FC(access)f(to)g(the)g(underlying)i (implemen)o(tation.)166 2302 y(W)l(e)d(b)q(eliev)o(e)i(that)e(ha)o (ving)g(suc)o(h)g(an)h(in)o(terface)f(is)h(imp)q(ortan)o(t)e(if)i(MPI)f (is)g(to)g(b)q(e)h(attractiv)o(e)e(to)g(end)75 2359 y(users,)i(since)i (the)e(a)o(v)m(ailabilit)o(y)j(of)d(man)o(y)g(di\013eren)o(t)h(to)q (ols)f(will)i(b)q(e)g(a)e(signi\014can)o(t)h(factor)f(in)h(attracting) 75 2415 y(users)f(to)g(the)g(MPI)g(standard.)166 2475 y(The)e(pro\014ling)h(in)o(terface)e(is)h(just)g(that,)f(an)g(in)o (terface.)20 b(It)12 b(sa)o(ys)g Fs(nothing)g FC(ab)q(out)g(the)h(w)o (a)o(y)f(in)h(whic)o(h)75 2531 y(it)h(is)h(used.)20 b(There)14 b(is)h(therefore)f(no)g(attempt)f(to)g(la)o(y)h(do)o(wn)g(what)g (information)g(is)h(collected)g(through)75 2588 y(the)g(in)o(terface,)g (or)g(ho)o(w)g(the)g(collected)i(information)e(is)h(sa)o(v)o(ed,)e (\014ltered,)i(or)f(displa)o(y)o(ed.)166 2647 y(While)g(the)g(initial)h (imp)q(etus)f(for)f(the)g(dev)o(elopmen)o(t)h(of)e(this)i(in)o(terface) f(arose)g(from)f(the)h(desire)i(to)75 2704 y(p)q(ermit)g(the)f (implemen)o(tation)i(of)d(pro\014ling)j(to)q(ols,)d(it)i(is)f(clear)h (that)f(an)g(in)o(terface)g(lik)o(e)h(that)f(sp)q(eci\014ed)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 101 107 101 106 bop 75 -100 a Fv(8.3.)34 b(LOGIC)16 b(OF)f(THE)g(DESIGN)1092 b FC(101)75 45 y(ma)o(y)17 b(also)h(pro)o(v)o(e)f(useful)i(for)e(other) h(purp)q(oses,)g(suc)o(h)g(as)g(\\in)o(ternet)o(w)o(orking")f(m)o (ultiple)i(MPI)f(imple-)75 102 y(men)o(tations.)j(Since)c(all)f(that)f (is)h(de\014ned)h(is)f(an)g(in)o(terface,)f(there)h(is)g(no)f(ob)s (jection)h(to)f(its)h(b)q(eing)g(used)75 158 y(wherev)o(er)f(it)h(is)f (useful.)166 214 y(As)j(the)g(issues)h(b)q(eing)g(addressed)f(here)h (are)f(in)o(timately)h(tied)f(up)h(with)f(the)g(w)o(a)o(y)f(in)i(whic)o (h)g(ex-)75 271 y(ecutable)j(images)e(are)g(built,)j(whic)o(h)e(ma)o(y) f(di\013er)h(greatly)f(on)g(di\013eren)o(t)h(mac)o(hines,)h(the)e (examples)75 327 y(giv)o(en)c(b)q(elo)o(w)g(should)h(b)q(e)f(treated)g (solely)g(as)f(one)h(w)o(a)o(y)f(of)g(implemen)o(ting)j(the)d(ob)s (jectiv)o(e)h(of)f(the)h(MPI)75 384 y(pro\014ling)h(in)o(terface.)k (The)16 b(actual)g(requiremen)o(ts)g(made)g(of)f(an)h(implemen)o (tation)h(are)e(those)g(detailed)75 440 y(in)i(the)f(Requiremen)o(ts)i (section)e(ab)q(o)o(v)o(e,)g(the)g(whole)h(of)f(the)g(rest)g(of)f(this) i(c)o(hapter)f(is)g(only)h(presen)o(t)f(as)75 497 y(justi\014cation)g (and)f(discussion)i(of)e(the)g(logic)h(for)f(those)g(requiremen)o(ts.) 166 553 y(The)i(examples)g(b)q(elo)o(w)h(sho)o(w)e(one)h(w)o(a)o(y)f (in)h(whic)o(h)h(an)f(implemen)o(tation)h(could)f(b)q(e)h(constructed) 75 610 y(to)h(meet)h(the)g(requiremen)o(ts)g(on)g(a)g(Unix)h(system)e (\(there)h(are)f(doubtless)i(others)e(whic)o(h)i(w)o(ould)f(b)q(e)75 666 y(equally)d(v)m(alid\).)75 806 y Fw(8.3)59 b(Logic)20 b(of)g(the)f(design)75 907 y FC(Pro)o(vided)d(that)f(an)g(MPI)g (implemen)o(tation)i(meets)e(the)h(requiremen)o(ts)g(ab)q(o)o(v)o(e,)e (it)i(is)g(p)q(ossible)h(for)e(the)75 964 y(implemen)o(tor)20 b(of)e(the)h(pro\014ling)h(system)f(to)f(in)o(tercept)i(all)f(of)g(the) g(MPI)g(calls)h(whic)o(h)f(are)g(made)g(b)o(y)75 1020 y(the)g(user)h(program.)30 b(She)20 b(can)f(then)h(collect)g(whatev)o (er)f(information)g(she)h(requires)g(b)q(efore)f(calling)75 1076 y(the)c(underlying)i(MPI)d(implemen)o(tation)i(\(through)f(its)g (name)f(shifted)i(en)o(try)e(p)q(oin)o(ts\))h(to)f(ac)o(hiev)o(e)i(the) 75 1133 y(desired)g(e\013ects.)75 1272 y Fw(8.4)59 b(Examples)75 1376 y Fq(8.4.1)49 b(Pro\014ler)16 b(implementation)75 1461 y FC(Supp)q(ose)25 b(that)e(the)h(pro\014ler)h(wishes)g(to)e (accum)o(ulate)i(the)f(total)f(amoun)o(t)g(of)h(data)f(sen)o(t)h(b)o(y) g(the)75 1518 y(MPI)p 167 1518 14 2 v 16 w(SEND)18 b(function,)g(along) f(with)h(the)f(total)g(elapsed)h(time)f(sp)q(en)o(t)h(in)g(the)f (function.)27 b(This)18 b(could)75 1574 y(trivially)f(b)q(e)f(ac)o (hiev)o(ed)g(th)o(us)f(:{)166 1631 y Fa(W)-6 b(ARNING)18 b(:)d FC(syn)o(tax)f(correction)h(needed)i(in)f(examples)g(when)g(MPI)f (syn)o(tax)f(is)i(de\014ned.)75 1709 y Fp(static)23 b(int)g (totalBytes;)75 1766 y(static)g(int)g(totalTime;)75 1822 y(extern)g(int)g(now\(void\);)71 b(/*)23 b(Some)h(suitable)e(timer)i (function)e(!)i(*/)75 1935 y(int)f(MPI_SEND\(const)f(int)i(tag,)f (const)g(int)h(bytes,)f(void)g(*)h(buffer,)f(const)g(int)g(who\))75 1992 y({)147 2048 y(int)g(tstart)g(=)h(now\(\);)147 2105 y(int)f(result)g(=)h(PMPI_SEND\(tag,bytes,buffe)o(r,who\);)68 b(/*)24 b(Pass)f(on)h(all)f(the)g(arguments)g(*/)147 2217 y(totalBytes)f(+=)i(bytes;)405 b(/*)23 b(Accumulate)g(byte)g (count)g(*/)147 2274 y(totalTime)46 b(+=)24 b(now\(\))f(-)h(tstart;)190 b(/*)23 b(and)h(time)333 b(*/)147 2387 y(return)23 b(result;)75 2443 y(})75 2561 y Fq(8.4.2)49 b(MPI)17 b(lib)o(ra)o(ry)f (implementation)75 2647 y FC(On)i(a)f(Unix)h(system,)f(in)h(whic)o(h)g (the)g(MPI)f(library)h(is)g(implemen)o(ted)h(in)f(C,)f(then)g(there)h (are)e(v)m(arious)75 2704 y(p)q(ossible)22 b(options,)f(of)e(whic)o(h)i (t)o(w)o(o)e(of)g(the)h(most)g(ob)o(vious)g(are)g(presen)o(ted)g(here.) 35 b(Whic)o(h)20 b(is)h(b)q(etter)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 102 108 102 107 bop 75 -100 a FC(102)1178 b Fv(SECTION)16 b(8.)35 b(PR)o(OFILING)75 45 y FC(dep)q(ends)17 b(on)e(whether)g(the)h(link)o (er)g(and)f(compiler)i(supp)q(ort)e(w)o(eak)g(sym)o(b)q(ols.)75 165 y Fo(Systems)h(with)g(w)o(eak)f(symb)q(ols)75 250 y FC(If)k(the)f(compiler)i(and)f(link)o(er)h(supp)q(ort)f(w)o(eak)f (external)h(sym)o(b)q(ols)f(\(e.g.)30 b(Solaris)19 b(2.x,)f(other)g (system)75 307 y(V.4)13 b(mac)o(hines\),)i(then)f(only)g(a)g(single)h (library)g(is)f(required)h(through)f(the)g(use)g(of)g Fp(#pragma)23 b(weak)13 b FC(th)o(us)75 363 y(:{)75 465 y Fp(#pragma)23 b(weak)g(MPI_EXAMPLE)g(=)g(PMPI_EXAMPLE)75 578 y(int)g(PMPI_EXAMPLE\(/*)f(appropriate)h(args)g(*/\))75 634 y({)170 691 y(/*)h(Useful)f(content)g(*/)75 747 y(})166 849 y FC(The)d(e\013ect)g(of)g(this)g Fp(#pragma)f FC(is)i(to)e (de\014ne)i(the)f(external)h(sym)o(b)q(ol)f Fp(MPI)p 1472 849 15 2 v 17 w(EXAMPLE)f FC(as)h(a)g(w)o(eak)75 906 y(de\014nition,)d(this)f(means)g(that)f(the)g(link)o(er)i(will)g (not)e(complain)i(if)f(there)g(is)g(another)f(de\014nition)j(of)d(the) 75 962 y(sym)o(b)q(ol)g(\(for)e(instance)i(in)g(the)f(pro\014ling)i (library\),)e(ho)o(w)o(ev)o(er)f(if)i(no)f(other)g(de\014nition)i (exists,)e(then)h(the)75 1019 y(link)o(er)h(will)h(use)f(the)f(w)o(eak) g(de\014nition.)75 1138 y Fo(Systems)h(without)h(w)o(eak)e(symb)q(ols) 75 1224 y FC(In)k(the)g(absence)g(of)f(w)o(eak)g(sym)o(b)q(ols)h(then)g (one)f(p)q(ossible)j(solution)e(w)o(ould)g(b)q(e)g(to)f(use)h(the)f(C)h (macro)75 1280 y(pre-pro)q(cessor)c(th)o(us)g(:{)75 1382 y Fp(#ifdef)23 b(PROFILELIB)75 1438 y(#)95 b(ifdef)23 b(__STDC__)75 1495 y(#)191 b(define)23 b(FUNCTION\(name\))f(P##name)75 1551 y(#)95 b(else)75 1608 y(#)191 b(define)23 b(FUNCTION\(name\))f (P/**/name)75 1664 y(#)95 b(endif)75 1721 y(#else)75 1777 y(#)g(define)23 b(FUNCTION\(name\))f(name)75 1834 y(#endif)166 1935 y FC(Eac)o(h)15 b(of)g(the)g(user)g(visible)j (functions)e(in)g(the)f(library)h(w)o(ould)g(then)f(b)q(e)h(declared)g (th)o(us)75 2037 y Fp(int)23 b(FUNCTION\(MPI_EXAMPLE\)\(/*)e (appropriate)h(args)i(*/\))75 2094 y({)170 2150 y(/*)g(Useful)f (content)g(*/)75 2207 y(})166 2308 y FC(The)d(same)g(source)g(\014le)h (can)g(then)f(b)q(e)h(compiled)h(to)d(pro)q(duce)i(b)q(oth)f(v)o (ersions)g(of)g(the)g(library)l(,)75 2365 y(dep)q(ending)d(on)f(the)f (state)f(of)h(the)g Fp(PROFILELIB)f FC(macro)h(sym)o(b)q(ol.)166 2421 y(It)e(is)g(required)h(that)e(the)h(standard)f(MPI)h(library)g(b)q (e)h(built)g(in)f(suc)o(h)g(a)g(w)o(a)o(y)e(that)h(the)h(inclusion)i (of)75 2478 y(MPI)g(functions)g(can)g(b)q(e)h(ac)o(hiev)o(ed)f(one)g (at)g(a)f(time.)20 b(This)c(is)f(a)f(somewhat)g(unpleasan)o(t)i (requiremen)o(t,)75 2534 y(since)22 b(it)f(ma)o(y)f(mean)h(that)f(eac)o (h)g(external)i(function)f(has)g(to)f(b)q(e)h(compiled)i(from)d(a)g (separate)g(\014le.)75 2591 y(Ho)o(w)o(ev)o(er)14 b(this)i(is)g (necessary)g(so)f(that)g(the)g(author)g(of)g(the)h(pro\014ling)g (library)g(need)h(only)f(de\014ne)g(those)75 2647 y(MPI)f(functions)g (whic)o(h)g(she)g(wishes)h(to)e(in)o(tercept,)h(references)g(to)f(an)o (y)g(others)g(b)q(eing)i(ful\014lled)h(b)o(y)e(the)75 2704 y(normal)g(MPI)g(library)l(.)21 b(Therefore)15 b(the)h(link)g (step)f(can)h(lo)q(ok)f(something)h(lik)o(e)g(this)g(:{)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 103 109 103 108 bop 75 -100 a Fv(8.5.)34 b(MUL)l(TIPLE)16 b(LEVELS)g(OF)f (INTER)o(CEPTION)739 b FC(103)75 45 y Fp(\045)24 b(cc)f(...)h(-lmyprof) f(-lpmpi)g(-lmpi)166 151 y FC(Here)e Fp(libmyprof.a)e FC(con)o(tains)i(the)g(pro\014ler)g(functions)h(whic)o(h)f(in)o (tercept)g(some)g(of)f(the)h(MPI)75 208 y(functions.)j Fp(libpmpi.a)15 b FC(con)o(tains)i(the)f(\\name)h(shifted")g(MPI)f (functions,)h(and)g Fp(libmpi.a)e FC(con)o(tains)75 264 y(the)g(normal)g(de\014nitions)i(of)e(the)g(MPI)h(functions.)75 407 y Fw(8.5)59 b(Multiple)20 b(levels)f(of)h(interception)75 509 y FC(The)15 b(sc)o(heme)h(giv)o(en)g(here)f(do)q(es)h(not)f (directly)h(supp)q(ort)f(the)h(nesting)g(of)e(pro\014ling)j(functions,) e(since)i(it)75 565 y(pro)o(vides)f(only)f(a)g(single)h(alternativ)o(e) g(name)f(for)f(eac)o(h)h(MPI)g(function.)21 b(Consideration)16 b(w)o(as)e(giv)o(en)i(to)75 622 y(an)f(implemen)o(tation)i(whic)o(h)f (w)o(ould)g(allo)o(w)g(m)o(ultiple)h(lev)o(els)g(of)e(call)h(in)o (terception,)h(ho)o(w)o(ev)o(er)d(w)o(e)h(w)o(ere)75 678 y(unable)j(to)e(construct)g(an)g(implemen)o(tation)i(of)e(this)h (whic)o(h)h(did)f(not)g(ha)o(v)o(e)f(the)g(follo)o(wing)i(disadv)m(an-) 75 735 y(tages)143 829 y Fy(\017)23 b FC(assuming)15 b(a)g(particular)h(implemen)o(tation)g(language.)143 922 y Fy(\017)23 b FC(imp)q(osing)16 b(a)f(run)g(time)h(cost)f(ev)o(en) g(when)h(no)f(pro\014ling)h(w)o(as)f(taking)g(place.)75 1016 y(Since)i(one)e(of)g(the)h(ob)s(jectiv)o(es)f(of)g(MPI)h(is)f(to)g (p)q(ermit)h(e\016cien)o(t,)g(lo)o(w)f(latency)h(implemen)o(tations,)h (and)75 1073 y(it)j(is)f(not)g(the)h(business)g(of)f(a)g(standard)g(to) g(require)h(a)f(particular)h(implemen)o(tation)g(language,)g(w)o(e)75 1129 y(decided)d(to)e(accept)g(the)g(sc)o(heme)h(outlined)h(ab)q(o)o(v) o(e.)166 1186 y(Note,)f(ho)o(w)o(ev)o(er,)f(that)h(it)h(is)f(p)q (ossible)j(to)c(use)i(the)g(sc)o(heme)f(ab)q(o)o(v)o(e)g(to)g(implemen) o(t)i(a)e(m)o(ulti-lev)o(el)75 1242 y(system,)k(since)g(the)g(function) g(called)i(b)o(y)d(the)h(user)f(ma)o(y)g(call)i(man)o(y)e(di\013eren)o (t)g(pro\014ling)i(functions)75 1299 y(b)q(efore)15 b(calling)i(the)f (underlying)h(MPI)e(function.)166 1355 y(Unfortunately)22 b(suc)o(h)g(an)g(implemen)o(tation)h(ma)o(y)e(require)h(more)g(co-op)q (eration)g(b)q(et)o(w)o(een)g(the)75 1411 y(di\013eren)o(t)g (pro\014ling)i(libraries)g(than)e(is)h(required)g(for)e(the)i(single)g (lev)o(el)h(implemen)o(tation)f(detailed)75 1468 y(ab)q(o)o(v)o(e.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 104 110 104 109 bop 75 362 a Fx(Section)35 b(9)75 576 y FE(Maps,)k(Groups)h (and)f(Con)m(texts)75 823 y Fw(9.1)59 b(Intro)r(duction)75 936 y FC(MPI)20 b(pro)o(vides)h(supp)q(ort)g(for)f(the)g(execution)i (of)e FD(parallel)25 b(pro)q(cedures)p FC(.)36 b(A)20 b(parallel)i(pro)q(cedure)75 992 y(is)f(executed)g(collectiv)o(ely)h(b) o(y)e(a)g(set)g(of)g(comm)o(unicating)h(pro)q(cesses.)35 b(T)l(ransfer)20 b(of)f(con)o(trol)h(to)g(the)75 1049 y(parallel)d(pro)q(cedure,)e(and)h(bac)o(k,)e(is)i(ac)o(hiev)o(ed)g(b)o (y)f(ha)o(ving)g(eac)o(h)g(executing)i(pro)q(cess)e(transfer)f(con)o (trol)75 1105 y(to)h(the)i(lo)q(cal)g(pro)q(cedure)g(co)q(de,)f(and)h (return)f(from)f(it.)23 b(Not)15 b(all)i(pro)q(cesses)g(need)g (transfer)e(con)o(trol)h(to)75 1161 y(the)f(same)g(parallel)h(pro)q (cedure;)g(the)f(parallel)h(pro)q(cedure)g(ma)o(y)f(in)o(v)o(olv)o(e)g (only)h(a)e(subset)h(of)g(pro)q(cesses,)75 1218 y(or)g(di\013eren)o(t)i (parallel)g(calls)g(ma)o(y)f(b)q(e)g(executed)h(b)o(y)f(di\013eren)o(t) g(subsets)h(of)e(pro)q(cesses.)23 b(The)16 b(collectiv)o(e)75 1274 y(comm)o(unication)h(calls)h(pro)o(vided)f(b)o(y)g(MPI)f(are)g (examples)i(of)e(suc)o(h)h(parallel)h(pro)q(cedures.)25 b(Libraries)75 1331 y(suc)o(h)16 b(as)e(parallel)j(scien)o(ti\014c)g (libraries)g(will)g(b)q(e)e(another)g(example.)166 1393 y(It)d(is)h(highly)h(desirable)g(to)e(allo)o(w)g(the)h(pro)q(cesses)g (that)e(execute)j(a)e(parallel)i(pro)q(cedure)f(use)g(a)f(\\vir-)75 1450 y(tual)k(pro)q(cess)g(name)f(space")h(lo)q(cal)g(to)f(the)h(in)o (v)o(ok)m(ation.)21 b(Th)o(us,)15 b(the)h(co)q(de)g(of)f(the)h (parallel)h(pro)q(cedure)75 1506 y(will)j(lo)q(ok)e(iden)o(tical,)i (irresp)q(ectiv)o(e)f(of)f(the)g(absolute)g(addresses)g(of)f(the)h (executing)h(pro)q(cesses.)29 b(It)17 b(is)75 1563 y(often)e(the)f (case)h(that)f(parallel)j(application)f(co)q(de)g(is)f(built)h(b)o(y)f (comp)q(osing)g(sev)o(eral)g(parallel)i(mo)q(dules)75 1619 y(\(e.g.,)e(a)h(n)o(umerical)i(solv)o(er,)e(and)h(a)f(graphic)h (displa)o(y)g(mo)q(dule\).)24 b(Supp)q(ort)17 b(of)f(a)g(virtual)h (name)g(space)75 1675 y(for)j(eac)o(h)g(mo)q(dule)i(will)g(allo)o(w)f (to)e(comp)q(ose)i(mo)q(dules)g(that)f(w)o(ere)g(dev)o(elop)q(ed)i (separately)f(without)75 1732 y(c)o(hanging)d(all)g(message)f(passing)g (calls)h(within)h(eac)o(h)e(mo)q(dule.)27 b(The)17 b(set)g(of)g(pro)q (cesses)h(that)e(execute)75 1788 y(a)i(parallel)i(pro)q(cedure)f(ma)o (y)f(b)q(e)h(\014xed,)g(or)f(ma)o(y)g(b)q(e)h(determined)g(dynamically) i(b)q(efore)d(the)h(in)o(v)o(ok)m(a-)75 1845 y(tion.)g(Th)o(us,)13 b(MPI)h(has)e(to)h(pro)o(vide)g(a)g(mec)o(hanism)h(for)e(creating)i (dynamically)g(sets)f(of)g(lo)q(cally)i(named)75 1901 y(pro)q(cesses.)35 b(W)l(e)20 b(alw)o(a)o(ys)f(n)o(um)o(b)q(er)h(pro)q (cesses)h(that)e(execute)h(a)g(parallel)i(pro)q(cedure)f(consecutiv)o (ely)l(,)75 1958 y(starting)15 b(form)g(zero.)21 b(Th)o(us,)16 b(a)f FD(group)h FC(is)g(an)f(ordered)h(set)g(of)f(pro)q(cesses.)22 b(Eac)o(h)15 b(pro)q(cess)h(in)h(a)e(group)75 2014 y(is)h(asso)q (ciated)f(with)h(a)e FD(rank)p FC(,)h(starting)f(from)h(zero.)20 b(Pro)q(cesses)15 b(are)g(iden)o(ti\014ed)i(b)o(y)e(their)h(ranks)e (when)75 2071 y(comm)o(unication)i(o)q(ccurs)f(within)i(the)e(group.) 166 2133 y(Another)g(imp)q(ortan)o(t)g(goal)g(is)g(the)h(supp)q(ort)f (of)g(a)f FD(lo)q(osely)19 b(sync)o(hronous)13 b FC(transfer)i(of)g (con)o(trol:)75 2190 y(no)f(sync)o(hronization)g(o)q(ccurs)h(either)f (b)q(efore)g(or)f(after)h(the)f(call.)21 b(Th)o(us,)14 b(a)f(pro)q(cess)h(ma)o(y)f(start)g(sending)75 2246 y(messages)g(that)g (p)q(ertain)h(to)f(the)g(execution)i(of)e(a)g(parallel)i(pro)q(cedure)g (b)q(efore)e(all)i(other)e(participating)75 2302 y(pro)q(cesses)k(ha)o (v)o(e)f(joined)i(the)e(execution;)i(a)e(pro)q(cess)h(ma)o(y)f(receiv)o (e)i(a)e(message)g(that)g(p)q(ertains)h(to)f(the)75 2359 y(execution)e(of)e(a)g(parallel)i(pro)q(cedure,)g(while)g (participating)f(in)h(another)e(parallel)i(execution.)20 b(A)13 b FD(com-)75 2415 y(m)o(unication)20 b(con)o(text)c FC(mec)o(hanism)h(is)f(needed)i(to)d(distinguish)j(comm)o(unication)f (that)f(b)q(elongs)h(to)75 2472 y(distinct)f(parallel)h(pro)q(cedure)f (executions.)21 b(\(Ev)o(en)15 b(if)g(parallel)i(transfer)d(of)h(con)o (trol)g(is)h(executed)g(syn-)75 2528 y(c)o(hroneously)l(,)f(one)g (still)i(needs)e(a)g(con)o(text)f(mec)o(hanism)h(for)g(the)f(sync)o (hronization)i(calls)g(themselv)o(es.\))166 2591 y(Normally)l(,)24 b(a)e(parallel)h(pro)q(cedure)g(is)g(written)f(so)g(that)f(all)i (messages)f(pro)q(duced)h(during)g(its)75 2647 y(execution)17 b(are)e(also)h(consumed)g(b)o(y)g(the)g(pro)q(cesses)g(that)f(execute)h (the)g(pro)q(cedure.)22 b(Ho)o(w)o(ev)o(er,)15 b(if)h(one)75 2704 y(parallel)k(pro)q(cedure)g(calls)f(another,)g(then)g(it)g(migh)o (t)f(b)q(e)i(desirable)g(to)e(allo)o(w)g(suc)o(h)h(call)h(to)e(pro)q (ceed)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 105 111 105 110 bop 75 -100 a Fv(9.2.)34 b(MAPS)1494 b FC(105)75 45 y(while)22 b(messages)e(are)h(p)q(ending)h(\(the)e(messages)g(will)j (b)q(e)e(consumed)g(b)o(y)g(the)f(pro)q(cedure)i(after)e(the)75 102 y(call)e(returns\).)24 b(In)18 b(suc)o(h)f(case,)f(a)h(new)g(comm)o (unication)g(con)o(text)g(is)g(needed)h(for)e(the)h(called)h(parallel) 75 158 y(pro)q(cedure,)e(ev)o(en)f(if)h(the)f(transfer)g(of)f(con)o (trol)h(is)h(sync)o(hronized.)166 222 y(A)d FD(con)o(text)h FC(is)g(the)f(MPI)g(mec)o(hanism)h(for)f(partitioning)h(comm)o (unication)g(space.)20 b(A)13 b(send)h(made)75 279 y(in)k(a)f(con)o (text)g(cannot)g(b)q(e)i(receiv)o(ed)f(in)g(another)g(con)o(text.)26 b(Con)o(texts)16 b(are)h(iden)o(ti\014ed)j(in)e(MPI)f(using)75 335 y(in)o(teger-v)m(alued)g FD(con)o(text)p 540 335 16 2 v 19 w(id)p FC('s.)166 399 y(The)d(comm)o(unication)i(domain)e (used)h(b)o(y)f(a)g(parallel)i(pro)q(cedure)g(is)e(iden)o(ti\014ed)j(b) o(y)d(a)g FD(comm)o(uni-)75 456 y(cator)p FC(.)29 b(Comm)o(unicators)17 b(bring)h(together)f(the)h(concepts)h(of)e(pro)q(cess)h(group)g(and)g (comm)o(unication)75 512 y(con)o(text.)29 b(A)18 b(comm)o(unicator)g (is)h(an)f(explicit)j(parameter)c(in)j(eac)o(h)e(p)q(oin)o(t-to-p)q (oin)o(t)h(comm)o(unication)75 569 y(op)q(eration.)31 b(The)19 b(comm)o(unicator)f(iden)o(ti\014es)j(the)d(comm)o(unication)i (con)o(text)e(of)g(that)g(op)q(eration;)j(it)75 625 y(iden)o(ti\014es)f (the)f(group)f(of)g(pro)q(cesses)g(that)g(can)g(b)q(e)h(in)o(v)o(olv)o (ed)h(in)f(this)g(comm)o(unication;)h(and)e(it)h(pro-)75 681 y(vides)g(the)g(translation)f(from)g(virtual)h(pro)q(cess)g(names,) g(whic)o(h)g(are)f(ranks)g(within)i(the)e(group,)h(in)o(to)75 738 y(absolute)13 b(addresses.)20 b(Collectiv)o(e)14 b(comm)o(unication)g(calls)g(also)f(tak)o(e)f(a)h(comm)o(unicator)f(as) h(parameter;)75 794 y(it)k(is)h(exp)q(ected)g(that)e(parallel)j (libraries)g(will)f(b)q(e)g(built)g(to)f(accept)g(a)g(comm)o(unicator)f (as)h(parameter.)75 851 y(Comm)o(unicators)d(are)h(represen)o(ted)h(b)o (y)f(opaque)g(MPI)h(ob)s(jects.)166 915 y(MPI)g(do)q(es)h(not)e(pro)o (vide)i(for)f(absolute)g(pro)q(cess)h(names.)22 b(Rather,)17 b(pro)q(cesses)f(are)g(alw)o(a)o(ys)g(iden-)75 971 y(ti\014ed)h(b)o(y)g (their)g(rank)f(inside)i(a)e(group.)23 b(A)16 b(univ)o(ersal)i(group)e (that)g(includes)i(all)g(pro)q(cesses)e(a)o(v)m(ailable)75 1028 y(when)k(computation)f(starts)f(is)i(the)g(nearest)f(equiv)m(alen) o(t)i(to)e(an)g(absolute)h(pro)q(cess)g(name)f(space)h(in)75 1084 y(MPI.)i(Relativ)o(e)i(naming)e(is)h(consisten)o(t)g(with)f(MPI)h (implemen)o(tations)g(where)g(pro)q(cesses)g(can)f(b)q(e)75 1141 y(added)16 b(dynamically)l(.)166 1205 y(New)j(pro)q(cess)g(groups) f(are)h(built)h(b)o(y)f(subsetting)g(and)g(reordering)g(pro)q(cesses)g (within)h(existing)75 1261 y(groups)i(\(as)f(de\014ned)j(b)o(y)e(comm)o (unicators\).)40 b(A)23 b(new)f(group)g(is)h(de\014ned)h(from)d(an)h (old)h(group)f(b)o(y)75 1318 y(sp)q(ecifying)d(the)f(corresp)q(ondence) h(b)q(et)o(w)o(een)e(the)h(rank)f(of)g(eac)o(h)h(pro)q(cess)f(in)i(the) e(new)h(group)f(and)h(its)75 1374 y(rank)f(in)h(the)f(old)h(group.)26 b(Suc)o(h)18 b(corresp)q(ondence)g(is)g(called)h(a)e FD(map)p FC(;)h(it)f(is)h(represen)o(ted)g(in)g(MPI)f(b)o(y)75 1431 y(an)e(opaque)g(ob)s(ject.)75 1618 y Fw(9.2)59 b(Maps)75 1734 y FC(A)19 b FD(map)g FC(is)g(a)g(one-to)f(one-mapping)i(from)e(0)p FB(:::m)11 b Fy(\000)i FC(1)19 b(in)o(to)g(the)g(set)f(of)h(non)g (negativ)o(e)g(in)o(tegers;)h FB(m)75 1791 y FC(is)e(the)g FD(size)g FC(of)f(map)g FB(f)5 b FC(.)28 b(A)17 b(p)q(ossible)j (represen)o(tation)d(for)g(suc)o(h)h(map)g(is)g(a)f(list)h(of)g FB(m)f FC(in)o(tegers;)i(it)e(is)75 1847 y(often)d(con)o(v)o(enien)o(t) h(to)e(think)j(of)d(a)h(map)g(as)g(b)q(eing)i(suc)o(h)f(list.)20 b(Another)14 b(p)q(ossible)i(represen)o(tation)f(is)f(as)75 1904 y(an)j(algorithmic)g(sp)q(eci\014cation)i(of)d(the)h(map)f (\(e.g.,)g(hash)h(function\).)25 b(A)16 b(map)h(is)g(represen)o(ted)g (b)o(y)g(an)75 1960 y(opaque)e(map)g(ob)s(ject.)166 2024 y(Maps)e(are)g(used)h(in)h(order)e(to)g(represen)o(t)h(groups.)19 b(If)14 b(pro)q(cesses)g(are)f(n)o(um)o(b)q(ered)h(from)f(0)g(to)g FB(n)7 b Fy(\000)g FC(1,)75 2081 y(then)15 b(a)f(map)g FB(f)k FC(:)12 b(0)p FB(:::m)7 b Fy(\000)i FC(1)j Fy(!)h FC(0)p FB(:::n)7 b Fy(\000)i FC(1)14 b(represen)o(ts)h(a)f(subset)g (group)g(of)g FB(m)g FC(pro)q(cesses:)20 b(pro)q(cess)15 b FB(f)5 b FC(\()p FB(i)p FC(\))75 2137 y(has)16 b(rank)f FB(i)h FC(in)h(the)f(new)g(group.)22 b(A)16 b(map,)f(b)o(y)h(itself,)h (do)q(es)f(not)f(represen)o(t)h(a)g(group;)f(it)i(do)q(es)f(so)f(only) 75 2194 y(relativ)o(e)g(to)e(a)h(pre-de\014ned)i(con)o(taining)f (group,)f(i.e.,)g(relativ)o(e)h(to)e(another)h(ranking)h(of)f(the)g (pro)q(cesses.)166 2334 y Fu(Discussion:)51 b Ft(Should)17 b(decide)i(if)f(opaque)g(ob)r(jects)h(are)g(justi\014ed,)g(or)f (whether)h(an)f(arra)o(y)g(of)g(indices)75 2384 y(is)f(OK.)h(On)g(the)g (\\opaque)f(ob)r(ject")h(side:)26 b(Ma)o(y)17 b(ha)o(v)o(e)g (algorithmic)e(map)h(de\014nitions)h(and)g(ma)o(y)f(spread)i(the)75 2434 y(de\014nition)f(o)o(v)o(er)g(more)g(than)g(one)h(no)q(de,)g(on)f (a)g(v)o(ery)g(large)h(system.)28 b(On)17 b(the)h(explicit)f(represen)o (tation)i(side:)75 2483 y(The)g(represen)o(tation)h(is)e(simple,)g(w)o (e)h(a)o(v)o(oid)e(an)h(additional)f(ob)r(ject,)i(and)g(there)h(is)e (no)g(m)o(uc)o(h)f(w)o(aste.)33 b(Also,)75 2533 y(with)13 b(an)g(explicit)f(represen)o(tation,)j(the)f(user)g(can)f(easily)g(add) g(his/her)g(o)o(wn)g(map)f(constructors,)i(and)f(building)75 2583 y(comm)o(unicators)e(is)j(faster.)19 b(Need)c(to)f(understand)h (what)e(an)h(opaque)g(map)e(w)o(ould)h(b)q(e)i(in)e(F)m(ortran.)166 2647 y(If)j(maps)f(are)i(explictly)e(represen)o(ted)20 b(as)c(arra)o(ys,)h(then)g(one)f(needs)i(map)d(size)i(to)f(b)q(e)h(an)f (additional)e(pa-)75 2704 y(rameter.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 106 112 106 111 bop 75 -100 a FC(106)693 b Fv(SECTION)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)166 45 y FC(F)l(or)h(eac)o(h)g Fp(m)p FC(,)h(an)f(iden)o(tit)o(y)i(map)e Fp(MPI)p 817 45 15 2 v 17 w(IDENT\(m\))f FC(of)h(size)i Fp(m)e FC(is)h (prede\014ned:)24 b Fp(MPI)p 1581 45 V 17 w(IDENT)p FC(\()p Fp(m)p FC(\))o(\()p FB(i)p FC(\))14 b(=)75 102 y FB(i)h(;)8 b(i)j FC(=)i(0)p FB(;)8 b(:::;)g(m)f Fy(\000)k FC(1.)166 241 y Fu(Discussion:)43 b Ft(If)16 b(maps)f(are)h(non)g(opaque)g(then)h (the)g(iden)o(t)f(map)e(is)i(just)h(a)f(long)f(arra)o(y)h(with)g Fj(i)g Ft(stored)75 297 y(at)h(the)g Fj(i)p Ft(-th)g(en)o(try)m(,)g (and)f(it)h(is)f(just)h Fn(MPI)p 733 297 14 2 v 15 w(IDENT)p Ft(.)e(If)h(maps)g(are)h(opaque,)g(w)o(e)g(need)g(a)f(di\013eren)o(t)i (map)d(for)h(eac)o(h)75 354 y(domain)c(size)i(\(a)g(function)g(from)e (in)o(tegers)i(to)g(maps?\))j(Need)e(to)f(think)f(of)h(F)m(ortran)f (implemen)o(tation.)75 554 y Fq(9.2.1)49 b(Op)q(erations)16 b(on)h(maps)75 640 y FC(The)12 b(follo)o(wing)h(op)q(erations)f(are)g (de\014ned)h(on)f(maps.)19 b(Eac)o(h)12 b(function)h(is)f(in)o(v)o(ok)o (ed)h(lo)q(cally)g(b)o(y)f(a)g(pro)q(cess.)166 744 y FD(MPI)p 275 744 16 2 v 18 w(MAP)p 419 744 V 18 w(SIZE\(map,)17 b(size\))75 869 y(IN)g(map)23 b FC(handle)16 b(to)f(map)g(ob)s(ject)75 955 y FD(OUT)j(size)23 b FC(size)16 b(of)f(map)g(\(in)o(teger\))166 1080 y FD(MPI)p 275 1080 V 18 w(MAP)p 419 1080 V 18 w(APPL)l(Y\()h (map,)h(argumen)o(t,)f(v)m(alue\))75 1205 y(IN)h(map)23 b FC(handle)16 b(to)f(map)g(ob)s(ject)75 1291 y FD(IN)i(argumen)o(t)22 b FC(map)15 b(argumen)o(t)g(\(in)o(teger\))75 1377 y FD(OUT)j(v)m(alue)24 b FC(image)15 b(of)g Fp(argument)f FC(under)i Fp(map)e FC(\(in)o(teger\))166 1502 y FD(MPI)p 275 1502 V 18 w(MAP)p 419 1502 V 18 w(INVERSE\()k(map,)e(argumen)o(t,)h (v)m(alue\))75 1627 y(IN)g(map)23 b FC(handle)16 b(to)f(map)g(ob)s (ject)75 1713 y FD(OUT)j(argumen)o(t)23 b FC(in)o(v)o(erse)14 b(image)g(of)g Fp(value)f FC(under)i Fp(map)p FC(;)e(-1)h(if)h Fp(value)e FC(is)i(not)e(in)i(the)f(range)g(of)g Fp(map)p FC(.)75 1799 y FD(IN)j(v)m(alue)23 b FC(map)16 b(v)m(alue)g(\(in)o (teger\))166 1924 y FD(MPI)p 275 1924 V 18 w(MAP)p 419 1924 V 18 w(LIST\(map,)i(len,)f(arra)o(y)p 924 1924 V 18 w(of)p 984 1924 V 19 w(in)o(teger,)g(size\))166 2028 y FC(Creates)d(an)i(explicit)h(list)f(represen)o(tation)f(of)g(a)g (map.)75 2101 y FD(IN)i(map)23 b FC(handle)16 b(to)f(map)g(ob)s(ject)75 2187 y FD(IN)i(LEN)24 b FC(length)15 b(of)g(arra)o(y)f(\(in)o(teger\)) 75 2273 y FD(OUT)k(arra)o(y)p 332 2273 V 18 w(of)p 392 2273 V 19 w(in)o(teger)23 b FC(list)16 b(of)f(v)m(alues)h(in)g(the)f (range)g(of)g Fp(map)75 2358 y FD(OUT)j(size)23 b FC(length)16 b(of)f(returned)h(list)g({)e(size)j(of)d(map)h(\(in)o(teger\).)166 2508 y Fu(Discussion:)166 2564 y Ft(I)i(c)o(hanged)h(terminology)c (from)i(\\elemen)o(t,)h(rank")f(to)h(\\argumen)o(t,)g(v)n(alue")f (since)i(b)q(oth)f(argumen)o(t)f(and)75 2621 y(v)n(alue)d(are)i(ranks.) 1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 107 113 107 112 bop 75 -100 a Fv(9.2.)34 b(MAPS)1494 b FC(107)75 45 y Fq(9.2.2)49 b(Map)17 b(constructo)o(rs)75 131 y FC(The)e(map)g(constructors)g(ma)o(y)f(b)q(e)i(either)g(lo)q(cal)g(or)f (ma)o(y)g(require)h(comm)o(unication.)166 235 y FD(MPI)p 275 235 16 2 v 18 w(MAP)p 419 235 V 18 w(BUILD\(map,)i(arra)o(y)p 877 235 V 17 w(of)p 936 235 V 19 w(in)o(teger,)f(size\))166 338 y FC(Build)g(a)d(map)g(from)g(an)h(explicit)i(list)f(represen)o (tation.)j(This)d(function)f(is)g(called)i(lo)q(cally)f(b)o(y)f(one)75 395 y(pro)q(cess.)75 485 y FD(OUT)j(map)23 b FC(handle)16 b(to)f(map)g(ob)s(ject)75 577 y FD(IN)i(arra)o(y)p 277 577 V 18 w(of)p 337 577 V 19 w(in)o(teger)23 b FC(list)16 b(of)e(v)m(alues)j(in)f(the)f(range)g(of)g Fp(map)75 669 y FD(IN)i(size)23 b FC(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h (arra)o(y)e({)h(map)g(size)h(\(in)o(teger\))166 806 y FD(MPI)p 275 806 V 18 w(MAP)p 419 806 V 18 w(SPLIT\(comm,)h(map,)g(k)o (ey)l(,)e(index,)i(newmap\))166 910 y FC(This)22 b(is)g(a)f(collectiv)o (e)j(function)e(that)f(is)h(called)h(b)o(y)f(all)g(pro)q(cesses)g(in)h (the)e(group)h(asso)q(ciated)75 967 y(with)17 b Fp(\(comm,)23 b(map\))p FC(.)g(All)18 b(calling)g(pro)q(cesses)f(pro)o(vide)g(the)g (same)f(v)m(alues)h(for)f(the)h(parameters)f Fp(comm)75 1023 y FC(and)h Fp(map)p FC(.)25 b(A)17 b(separate)f(group)g(of)h(pro)q (cesses)g(is)h(formed)e(for)h(eac)o(h)f(distinct)j(v)m(alue)f(of)e Fp(key)p FC(,)h(with)g(the)75 1079 y(pro)q(cesses)f(ordered)g (according)g(to)f(the)h(v)m(alue)h(of)e Fp(index)p FC(.)21 b(Eac)o(h)15 b(pro)q(cess)h(in)h(suc)o(h)f(group)f(is)i(returned)75 1136 y(in)f Fp(newmap)e FC(the)i(map)f(of)f(this)i(group)f(within)h Fp(comm)p FC(.)75 1237 y FD(IN)h(comm)22 b FC(comm)o(unicator)15 b(ob)s(ject)g(handle)75 1329 y FD(IN)i(map)23 b FC(map)15 b(ob)s(ject)g(handle)75 1421 y FD(IN)i(k)o(ey)22 b FC(\(in)o(teger\))75 1513 y FD(IN)17 b(index)23 b FC(\(in)o(teger\))75 1606 y FD(OUT)18 b(newmap)23 b FC(map)15 b(ob)s(ject)f(handle)166 1707 y(Additional)k(map)f(creation)g(functions)g(ma)o(y)f(b)q(e)h(pro)o (vided.)25 b(F)l(ollo)o(ws)17 b(a)f(p)q(ossible)i(list)g(construc-)75 1763 y(tors.)166 1902 y Fu(Discussion:)d Ft(Do)f(w)o(e)g(w)o(an)o(t)f (to)h(mandate)f(them)g(in)g(MPI?)166 2089 y FD(MPI)p 275 2089 V 18 w(MAP)p 419 2089 V 18 w(UNION\()k(map1,)h(map2,)f (newmap\))166 2192 y Fp(map1)g FC(and)h Fp(map2)f FC(are)h(maps)f(with) i(disjoin)o(t)f(ranges.)27 b(The)18 b(resulting)h(concatenation)f(is)h (a)e(map)75 2249 y(with)d(a)g(range)g(whic)o(h)h(is)f(the)g(union)h(of) f(the)g(ranges)g(of)f Fp(map1)h FC(and)g Fp(map2)p FC(,)f(with)i (elemen)o(ts)f(in)h(the)f(range)75 2305 y(of)h Fp(map1)f FC(listed)i(\014rst,)e(and)h(original)h(order)f(preserv)o(ed)g(within)i (eac)o(h)e(range.)k(The)c(union)h(op)q(eration)f(is)75 2362 y(asso)q(ciativ)o(e)h(but)f(not)g(comm)o(utativ)o(e.)415 2480 y Fp(newmap)o FC(\()p FB(i)p FC(\))d(=)670 2408 y Ff(\()724 2452 y Fp(map1)o FC(\()p FB(i)p FC(\))308 b(if)16 b FB(i)c(<)h(siz)r(e)p FC(\()p Fp(map1)p FC(\))724 2509 y Fp(map2)o FC(\()p FB(i)d Fy(\000)g FB(siz)r(e)p FC(\()p Fp(map1)p FC(\)\))40 b(otherwise)75 2611 y FD(IN)17 b(map1)23 b FC(map)15 b(ob)s(ject)g(handle)75 2704 y FD(IN)i(map2)23 b FC(map)15 b(ob)s(ject)g(handle)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 108 114 108 113 bop 75 -100 a FC(108)693 b Fv(SECTION)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)75 45 y FD(OUT)j(newmap)23 b FC(map)15 b(ob)s(ject)f(handle)166 200 y FD(MPI)p 275 200 16 2 v 18 w(MAP)p 419 200 V 18 w(PR)o(ODUCT\()k(map1,)f(map2,)g (range,)g(newmap\))166 304 y Fp(map1)12 b FC(is)h(a)g(map)f(with)i(v)m (alues)g(in)f(the)g(range)g(0)p FB(;)8 b(:::;)g Fp(r)o(ange)r Fy(\000)d FC(1.)20 b Fp(newmap)11 b FC(is)j(the)f(cartesian)g(pro)q (duct)75 361 y(of)j Fp(map1)f FC(and)h Fp(map2)p FC(,)g(whic)o(h)g (maps)g(the)g(pair)h(\()p FB(i;)8 b(j)s FC(\))13 b(in)o(to)j(the)g (pair)h(\()p Fp(map1)o FC(\()p FB(i)p FC(\))p FB(;)8 b Fp(map2)n FC(\()p FB(j)s FC(\)\).)20 b(P)o(airs)c(in)h(the)75 417 y(domain)f(and)g(in)g(the)f(range)g(of)g(this)h(mapping)g(are)g(n)o (um)o(b)q(ered)g(in)g(ro)o(w)f(ma)s(jor)f(order,)g(i.e.,)i(pair)f(\()p FB(i;)8 b(j)s FC(\))75 473 y(has)15 b(n)o(um)o(b)q(er)h FB(i)9 b Fy(\001)h Fp(range)f FC(+)i FB(j)s FC(.)19 b(The)d(pro)q(duct) f(op)q(eration)h(is)f(asso)q(ciativ)o(e)h(but)f(not)g(comm)o(utativ)o (e.)480 587 y Fp(newmap)o FC(\()p FB(i)10 b Fy(\001)f Fp(range)h FC(+)g FB(j)s FC(\))i(=)h Fp(map1)o FC(\()p FB(i)p FC(\))c Fy(\001)h Fp(range)f FC(+)i Fp(map2)o FC(\()p FB(j)s FC(\))75 695 y FD(IN)17 b(map1)23 b FC(map)15 b(ob)s(ject)g(handle)75 790 y FD(IN)i(map2)23 b FC(map)15 b(ob)s(ject)g(handle)75 885 y FD(OUT)j(newmap)23 b FC(map)15 b(ob)s(ject)f(handle)75 1029 y Fw(9.3)59 b(Context)p 413 1029 18 2 v 20 w(id)75 1131 y FC(A)18 b FD(con)o(text)p 299 1131 16 2 v 19 w(id)g FC(is)h(an)f(in)o(teger.)28 b(The)18 b(range)g(of)f(v)m(alid)j(v)m(alues)f(for)e Fp(context)p 1438 1131 15 2 v 17 w(id)g FC(is)i(implemen)o(tation)75 1188 y(dep)q(enden)o(t,)d(and)g(can)f(b)q(e)h(found)f(b)o(y)h(calling)h (a)d(suitable)j(query)e(function,)h(as)f(describ)q(ed)i(in)f(7.)75 1311 y Fq(9.3.1)49 b(Op)q(erations)16 b(on)h(context)p 681 1311 V 17 w(id's)75 1445 y FD(MPI)p 184 1445 16 2 v 18 w(CONTEXT)p 466 1445 V 19 w(ALLOCA)l(TE\(comm,)g(map,)g(arra)o(y)p 1210 1445 V 18 w(of)p 1270 1445 V 19 w(con)o(textids,)g(len\))166 1549 y FC(Allo)q(cates)i(an)f(arra)o(y)f(of)h(con)o(text)p 759 1549 14 2 v 16 w(id's.)29 b(This)19 b(is)g(a)f(collectiv)o(e)i(op)q (eration)f(that)e(is)i(executed)g(b)o(y)75 1605 y(all)e(pro)q(cesses)g (in)g(the)g(group)f(de\014ned)i(b)o(y)e Fp(\(comm,)23 b(map\))p FC(.)g(The)17 b(con)o(text)p 1367 1605 V 16 w(ids)g(that)e(are)h(returned)h(are)75 1662 y(unique)g(within)g(the)f (group)f(asso)q(ciated)h(with)g Fp(\(comm,)23 b(map\))p FC(.)d(The)c(arra)o(y)e(returned)i(is)g(the)g(same)f(on)75 1718 y(all)j(pro)q(cesses)f(that)f(call)i(the)f(function)h(\(same)e (order,)g(same)h(n)o(um)o(b)q(er)g(of)g(elemen)o(ts\).)25 b(The)17 b(call)h(ma)o(y)75 1775 y(blo)q(c)o(k)e(un)o(til)g(all)g(pro)q (cesses)g(within)g(the)g(executing)g(group)f(ha)o(v)o(e)g(in)o(v)o(ok)o (ed)g(the)g(call.)75 1883 y FD(IN)i(comm)22 b FC(comm)o(unicator)15 b(ob)s(ject)g(handle)75 1978 y FD(IN)i(map)23 b FC(map)15 b(ob)s(ject)g(handle)75 2073 y FD(OUT)j(arra)o(y)p 332 2073 16 2 v 18 w(of)p 392 2073 V 19 w(con)o(text)p 580 2073 V 19 w(ids)23 b FC(\(arra)o(y)14 b(of)g(in)o(tegers\))75 2168 y FD(IN)j(len)23 b FC(n)o(um)o(b)q(er)16 b(of)f(con)o(text)p 611 2168 14 2 v 16 w(id's)g(to)g(allo)q(cate)h(\(in)o(teger\))166 2322 y FD(MPI)p 275 2322 16 2 v 18 w(CONTEXT)p 557 2322 V 19 w(RESER)-6 b(VE\(con)o(text)p 1008 2322 V 21 w(id\))166 2426 y FC(Reserv)o(e)13 b(a)f(con)o(text)p 516 2426 14 2 v 16 w(id)h(v)m(alue.)21 b(This)13 b(is)g(a)f(lo)q(cal)i(call)g(that) d(is)j(in)o(v)o(ok)o(ed)e(b)o(y)h(a)f(single)i(pro)q(cess.)19 b(A)13 b(re-)75 2483 y(serv)o(ed)g(con)o(text)p 362 2483 V 16 w(id)g(v)m(alue)h(will)g(not)e(b)q(e)i(allo)q(cated)f(b)o(y)g(a)f (subsequen)o(t)h(call)h(to)e Fp(MPI)p 1486 2483 15 2 v 17 w(CONTEXT)p 1671 2483 V 16 w(ALLOCATE)75 2539 y FC(b)o(y)j(the)h(same)f(pro)q(cess.)20 b(It)15 b(is)h(erroneous)f(to)g (reserv)o(e)g(a)g(con)o(text)p 1203 2539 14 2 v 16 w(id)h(that)f(has)g (b)q(een)h(already)g(b)q(een)g(al-)75 2596 y(lo)q(cated)g(b)o(y)f Fp(MPI)p 371 2596 15 2 v 17 w(CONTEXT)p 556 2596 V 16 w(ALLOCATE)p FC(.)75 2704 y FD(IN)i(con)o(text)p 325 2704 16 2 v 19 w(id)23 b FC(\(in)o(teger\))1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 109 115 109 114 bop 75 -100 a Fv(9.4.)34 b(COMMUNICA)l(TORS)1201 b FC(109)166 45 y Fu(Discussion:)166 95 y Ft(Ma)o(y)13 b(w)o(an)o(t)h(to)g(reserv)o(e)h(an)f(arra)o(y)g(of)f(v)n(alues,)g (rather)i(than)f(one)g(v)n(alue.)166 151 y(Implemen)o(tations)6 b(ma)o(y)i(ha)o(v)o(e)h(there)h(o)o(wn)f(reserv)o(ed)j(v)n(alues.)k (The)10 b(e\013ect)g(is)g(as)f(if)f(calls)h(to)g Fn(MPI)p 1638 151 14 2 v 15 w(CONTEXT)p 1807 151 V 15 w(RESERVE)75 208 y Ft(where)15 b(executed)h(in)d(a)h(pream)o(ble.)166 394 y FD(MPI)p 275 394 16 2 v 18 w(CONTEXT)p 557 394 V 19 w(FREE\(con)o(text)p 905 394 V 20 w(id\))166 498 y FC(F)l(ree)i(a)g(reserv)o(ed)h(con)o(text)p 634 498 14 2 v 16 w(id)g(v)m(alue.)24 b(The)17 b(v)m(alue)g(b)q(ecomes)g(a)o(v) m(ailable)h(for)e(a)g(subsequen)o(t)h(reser-)75 554 y(v)m(ation)g(b)o (y)g Fp(MPI)p 354 554 15 2 v 16 w(CONTEXT)p 538 554 V 17 w(RESERVE)e FC(or)h(subsequen)o(t)h(allo)q(cation)h(b)o(y)f Fp(MPI)p 1376 554 V 16 w(CONTEXT)p 1560 554 V 17 w(ALLOCATE)p FC(.)e(It)h(is)75 611 y(erroneous)f(to)g(free)g(a)g(v)m(alue)h(that)f (is)g(asso)q(ciated)h(with)f(an)g(activ)o(e)h(comm)o(unicator.)75 702 y FD(IN)h(con)o(text)p 325 702 16 2 v 19 w(id)23 b FC(\(in)o(teger\))166 869 y Fu(Discussion:)166 919 y Ft(Here,)15 b(to)q(o,)e(w)o(e)h(migh)o(t)e(prefer)j(to)e(free)i(an)f (arra)o(y)f(of)h(v)n(alues,)f(rather)h(than)g(one.)166 969 y(Comm)o(uni)o(cators,)c(lik)o(e)h(all)g(opaque)h(ob)r(jects,)h (are)g(freed)g(b)o(y)e(a)h(call)g(to)f Fn(MPI)p 1346 969 14 2 v 16 w(FREE)p Ft(.)f(W)m(e)i(ma)o(y)e(prefer)j(to)f(ha)o(v)o (e)75 1019 y(that)i(same)g(call)f(free)i(also)f(the)h(asso)q(ciated)g (con)o(text)p 915 1019 13 2 v 16 w(id.)j(With)c(the)h(curren)o(t)h (design,)e(if)f(w)o(e)i(w)o(an)o(t)f(to)g(free)h(b)q(oth)75 1069 y(comm)o(unicator)f(and)j(con)o(text)p 568 1069 V 15 w(id)g(w)o(e)g(need)h(to)e(store)i(the)g(con)o(text)p 1165 1069 V 15 w(id)f(v)n(alue,)f(free)i(the)f(comm)o(unicator,)d(next) 75 1119 y(free)h(the)f(con)o(text)p 361 1119 V 16 w(id.)75 1342 y Fw(9.4)59 b(Communicato)n(rs)75 1444 y FC(A)21 b FD(comm)o(unicator)g FC(is)h(an)f(opaque)g(ob)s(ject)g(that)f(iden)o (ti\014es)j(a)e(group)g(of)f(pro)q(cesses)i(and)f(a)g(com-)75 1500 y(m)o(unication)e(con)o(text)e(for)g(that)h(group.)27 b(Lik)o(e)19 b(other)f(opaque)g(ob)s(jects,)f(comm)o(unicators)h (cannot)f(b)q(e)75 1557 y(transfered)e(b)q(et)o(w)o(een)g(pro)q (cesses;)h(con)o(text)p 824 1557 14 2 v 15 w(id's)g(are)f(used)h(to)e (transfer)h(information)g(on)g(con)o(text.)166 1613 y(As)g(a)g (short-hand,)g(w)o(e)g(shall)i(iden)o(ti\014y)g(the)e(group)g(of)g(pro) q(cesses)h(asso)q(ciated)g(with)f(a)g(comm)o(uni-)75 1670 y(cator)h Fp(comm)g FC(with)h(the)g(comm)o(unicator)f(itself.)26 b(Th)o(us,)16 b(\\the)h(pro)q(cess)g(with)g(rank)f FB(i)h FC(in)g Fp(comm)p FC(")f(should)75 1726 y(b)q(e)e(understo)q(o)q(d)g (as)g(meaning)g(\\)f(the)h(pro)q(cess)g(with)g(rank)f FB(i)g FC(in)i(the)f(group)f(asso)q(ciated)h(with)g Fp(comm)p FC(".)k(In)75 1783 y(the)e(same)g(manner,)g(\\the)g(comm)o(unication)h (con)o(text)f Fp(comm)p FC(")f(should)i(b)q(e)g(understo)q(o)q(d)f(to)g (mean)g(\\the)75 1839 y(comm)o(unication)g(con)o(text)f(asso)q(ciated)g (with)g Fp(comm)p FC(".)166 1895 y(An)h(initial)h(comm)o(unicator)f Fp(MPI)p 745 1895 15 2 v 16 w(COMM)p 857 1895 V 17 w(INIT)f FC(is)h(de\014ned)h(when)f(the)g(program)e(starts.)20 b(Its)c(asso-)75 1952 y(ciated)j(group)e(con)o(tains)h(all)h(pro)q (cesses)g(that)e(start)g(the)h(computation.)28 b(Applications)20 b(that)d(do)h(not)75 2008 y(need)e(m)o(ultiple)h(pro)q(cess)f(groups)e (or)h(m)o(ultiple)i(con)o(texts,)d(will)j(only)f(use)f(this)h(comm)o (unicator.)166 2065 y(Let)i Fp(comm)f FC(b)q(e)h(a)g(comm)o(unicator)f (that)g(is)i(asso)q(ciated)f(with)g(a)f(group)h(of)f(size)i FB(n)p FC(,)f(and)g(let)g Fp(map)e FC(:)75 2121 y(0)p FB(::m)8 b Fy(\000)h FC(1)k Fy(!)g FC(0)p FB(::n)8 b Fy(\000)h FC(1)15 b(b)q(e)g(a)f(map.)20 b(Then)15 b(the)g(pair)g Fp(\(comm,)23 b(map\))14 b FC(de\014nes)i(a)e(subgroup,)h(namely)g(the) 75 2178 y(subgroup)g(of)g(pro)q(cesses)h(with)f(ranks)g Fp(map)p FC(\(0\))p FB(;)8 b(:::;)f Fp(map)m FC(\()p FB(m)i Fy(\000)i FC(1\))j(in)i Fp(comm)p FC(.)75 2297 y Fq(9.4.1)49 b(Op)q(erations)16 b(on)h(communicato)o(rs)75 2431 y FD(MPI)p 184 2431 16 2 v 18 w(COMM)p 379 2431 V 18 w(MAP\(comm,)f(sub)q(comm,)g(map\))166 2534 y FC(Returns)g(a)e (map)h(suc)o(h)h(that)e Fp(\(comm,)23 b(map\))15 b FC(is)h(the)f(group) g(asso)q(ciated)g(with)h Fp(subcomm)p FC(;)e(i.e.,)g(if)i Fp(i)75 2591 y FC(is)h(the)f(rank)g(of)g(a)h(pro)q(cess)f(in)h(the)g (group)f(asso)q(ciated)h(with)f Fp(subcomm)p FC(,)g(then)g Fp(map\(i\))g FC(is)h(the)f(rank)g(of)75 2647 y(that)f(same)g(pro)q (cess)g(in)i(the)e(group)g(asso)q(ciated)h(with)g Fp(comm)p FC(.)j(The)d(call)h(is)f(erroneous)f(if)h Fp(subcomm)e FC(has)75 2704 y(a)h(pro)q(cess)g(that)g(is)h(not)e(mem)o(b)q(er)i(of)e Fp(comm)p FC(.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 110 116 110 115 bop 75 -100 a FC(110)693 b Fv(SECTION)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)75 45 y FD(IN)i(comm)22 b FC(comm)o(unicator)15 b(ob)s(ject)g(handle)75 138 y FD(IN)i(sub)q(comm)23 b FC(comm)o(unicator)14 b(ob)s(ject)h(handle)75 231 y FD(OUT)j(map)23 b FC(map)15 b(ob)s(ject)g(handle)166 333 y(The)i Fp(MPI)p 336 333 15 2 v 17 w(COMM)p 449 333 V 17 w(MAP)g FC(function)h(retriev)o(es)f(the)g(group)g(of)g(pro)q (cesses)h(that)e(is)i(asso)q(ciated)g(with)f(a)75 390 y(comm)o(unicator.)h(Ho)o(w)o(ev)o(er,)12 b(since)g(absolute)h(pro)q (cess)f(names)f(are)h(not)f(visible)j(in)f(MPI,)e(the)h(group)g(can)75 446 y(only)j(b)q(e)g(de\014ned)h(relativ)o(e)f(to)f(another)g (encompassing)h(group.)20 b(The)14 b(\\absolute")h(pro)q(cess)g(n)o(um) o(b)q(er)f(is)75 503 y(obtained)i(b)o(y)f(using)h Fp(MPI)p 519 503 V 17 w(COMM)p 632 503 V 16 w(INIT)f FC(as)g(a)g(reference)h (comm)o(unicator.)166 606 y FD(MPI)p 275 606 16 2 v 18 w(COMM)p 470 606 V 18 w(SIZE\(comm,)h(size\))166 710 y FC(Returns)f(the)f(size)h(of)f(the)g(group)g(asso)q(ciated)g(with)h Fp(comm)p FC(.)75 813 y FD(IN)h(comm)22 b FC(handle)17 b(to)d(comm)o(unicator)75 905 y FD(OUT)k(size)23 b FC(group)15 b(size)h(\(in)o(teger\))166 1055 y FD(MPI)p 275 1055 V 18 w(COMM)p 470 1055 V 18 w(CONTEXTID\(comm,)h(con)o(text)p 1170 1055 V 19 w(id\))166 1159 y FC(Returns)f(the)f(con)o(text)p 567 1159 14 2 v 16 w(id)h(asso)q(ciated)f(with)h(the)f(comm)o(unicator) g Fp(comm)p FC(.)75 1261 y FD(IN)i(comm)22 b FC(comm)o(unicator)15 b(ob)s(ject)g(handle)75 1354 y FD(OUT)j(con)o(text)p 380 1354 16 2 v 19 w(id)24 b FC(con)o(text)p 613 1354 14 2 v 15 w(id)75 1475 y Fq(9.4.2)49 b(Communicato)o(r)16 b(constructo)o(rs)75 1608 y FD(MPI)p 184 1608 16 2 v 18 w(COMM)p 379 1608 V 18 w(MAKE\(comm,)g(map,)h(con)o(text)p 1049 1608 V 19 w(id,)h(new)o(comm\))166 1712 y FC(Creates)h(a)h(new)g (comm)o(unicator)g(ob)s(ject,)g(whic)o(h)h(is)f(asso)q(ciated)h(with)f (the)g(group)g(de\014ned)h(b)o(y)75 1769 y Fp(\(comm,)i(map\))p FC(,)14 b(and)h(the)g(con)o(text)p 681 1769 14 2 v 15 w(id)h Fp(context)p 917 1769 15 2 v 16 w(id)p FC(.)j(This)d(is)f(a)f (lo)q(cal)i(call)g(executed)g(b)o(y)e(one)h(pro)q(cess.)75 1825 y(Ho)o(w)o(ev)o(er,)d(the)h(new)h(comm)o(unicator)e(ob)s(ject)h (should)h(not)e(b)q(e)i(used)g(for)e(comm)o(unication)i(b)q(et)o(w)o (een)f(t)o(w)o(o)75 1881 y(pro)q(cesses)j(unless)g(they)f(b)q(oth)h(ha) o(v)o(e)e(called)j(the)e(function.)166 1938 y(It)d(is)h(erroneous)f(to) g(create)g(on)h(the)f(same)g(pro)q(cess)h(t)o(w)o(o)e(distinct)i(comm)o (unicators)f(with)h(the)f(same)75 1994 y(con)o(text)p 225 1994 14 2 v 16 w(id.)75 2097 y FD(IN)17 b(comm)22 b FC(comm)o(unicator)15 b(ob)s(ject)g(handle)75 2190 y FD(IN)i(map)23 b FC(map)15 b(ob)s(ject)g(handle)75 2282 y FD(IN)i(con)o(text)p 325 2282 16 2 v 19 w(id)23 b FC(con)o(text)p 557 2282 14 2 v 16 w(id)75 2375 y FD(OUT)18 b(new)o(comm)k FC(comm)o(unicator)15 b(ob)s(ject)f(handle)166 2478 y(The)23 b(con)o(tect)p 413 2478 V 16 w(id)g(used)g(in)h(this)f (call)g(ma)o(y)f(b)q(e)i(an)e(in)o(teger)h(returned)g(b)o(y)f(a)h (previous)g(call)h(to)75 2534 y Fp(MPI)p 150 2534 15 2 v 17 w(ALLOC)p 287 2534 V 16 w(CONTEXT)p FC(,)17 b(executed)h(within) h(the)f(same)f(group.)27 b(In)18 b(suc)o(h)g(case,)g(it)g(is)g(guaran)o (teed)g(that)75 2591 y(the)f(con)o(text)p 305 2591 14 2 v 16 w(id)h(is)g(unique,)h(and)e(has)g(not)g(b)q(een)h(used)g (already)g(to)e(create)h(a)g(comm)o(unicator.)25 b(Ho)o(w-)75 2647 y(ev)o(er,)17 b(the)g(user)g(is)h(free)f(to)f(manage)g(on)h(it)g (o)o(wn)g(the)g(con)o(text)p 1165 2647 V 16 w(id's,)g(and)g(use)g (other)g(mec)o(hanisms)h(for)75 2704 y(their)e(allo)q(cation.)22 b(F)l(or)14 b(example,)i(one)g(could)g(ha)o(v)o(e)f(a)g(single)i FD(con)o(text)p 1330 2704 16 2 v 19 w(id)h(serv)o(er)13 b FC(generate)i(unique)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 111 117 111 116 bop 75 -100 a Fv(9.5.)34 b(W)o(ORKING)16 b(WITHOUT)f(CONTEXT)p 915 -100 14 2 v 17 w(ID'S)789 b FC(111)75 45 y(con)o(text)p 225 45 V 16 w(id's)14 b(for)g(the)g(en)o(tire)h(system;)f(one)g(can)g (preallo)q(cate)i(statically)f(some)f(con)o(text)p 1609 45 V 15 w(id)h(v)m(alues,)h(for)75 102 y(the)f(use)h(of)f(libraries;)h (etc.)166 234 y Fu(Discussion:)166 284 y Ft(Do)d(w)o(e)h(w)o(an)o(t)f (to)h(prohibit)f(implem)o(en)o(tations)e(that)j(sync)o(hronize)h(the)f (creation)g(of)f(comm)o(unicators)e(\(i.e.,)75 334 y(where)k(a)f(call)f (to)g Fn(MPI)p 425 334 V 16 w(MAKE)p 529 334 V 14 w(COMM)g Ft(blo)q(c)o(ks)h(un)o(til)f(all)g(mem)o(b)q(ers)g(of)g(the)h(group)g (ha)o(v)o(e)g(made)f(the)h(call\)?)166 383 y(Is)g(it)g(preferable)g(to) g(ha)o(v)o(e)g(a)f Fn(array)p 739 383 V 15 w(of)p 798 383 V 15 w(comm)g Ft(as)h(parameter,)f(rather)i(than)f(one)g(comm)o (unicator?)166 440 y(W)m(e)i(don't)g(ha)o(v)o(e)g(no)o(w)g(the)h (abilit)o(y)e(to)h(ha)o(v)o(e)g(t)o(w)o(o)g(di\013eren)o(t)i(groups)e (with)g(the)h(same)f(con)o(text;)i(e.g.,)d(t)o(w)o(o)75 496 y(di\013eren)o(t)d(n)o(um)o(b)q(erings)e(of)g(the)i(pro)q(cesses,)i (used)d(with)g(the)h(same)e(comm)o(unicati)o(on)e(con)o(text.)18 b(The)11 b(alternativ)o(e)g(is)75 553 y(to)h(remo)o(v)o(e)f(the)i (restriction)g(that)f(t)o(w)o(o)g(distinct)g(comm)o(unicators)e(cannot) i(use)h(the)g(same)e(con)o(text)p 1641 553 13 2 v 16 w(id.)17 b(It)12 b(is)g(no)o(w)75 609 y(the)18 b(user)h(resp)q (onsibilit)o(y)e(to)g(disam)o(biguate)e(these)k(t)o(w)o(o)e(comm)o (unicators;)g(the)h(system)f(do)q(es)h(not)f(guaran)o(tee)75 666 y(that)e(a)g(message)g(sen)o(t)h(with)e(one)i(comm)o(unicator)c (can)j(b)q(e)h(receiv)o(ed)g(only)f(with)f(this)h(same)g(comm)o (unicator,)d(if)75 722 y(they)h(use)g(the)g(same)f(con)o(text)p 545 722 V 16 w(id.)17 b(W)m(e)12 b(sa)o(v)o(e)h(on)f(con)o(text)p 972 722 V 16 w(id's)f(but)i(lo)q(ose)g(safet)o(y)f(\(the)h(sender)i (and)d(receiv)o(er)i(ma)o(y)75 779 y(b)q(e)i(using)g(di\013eren)o(t)g (pro)q(cess)i(n)o(um)o(b)q(erings,)c(if)h(t)o(w)o(o)g(di\013eren)o(t)i (comm)o(unicators)c(are)j(used)h({)e(the)i(receiv)o(er)g(ma)o(y)75 835 y(not)d(iden)o(tify)f(correctly)i(the)f(sender\).)75 1060 y Fw(9.5)59 b(W)n(o)n(rking)22 b(without)d(context)p 840 1060 18 2 v 19 w(id's)75 1162 y FC(It)12 b(is)h(p)q(ossible)g(to)f (create)g(comm)o(unicators)f(directly)l(,)j(without)e(using)h(con)o (text)p 1430 1162 14 2 v 16 w(id's.)19 b(A)12 b(single)h(function)75 1218 y(com)o(bines)f(the)f(allo)q(cation)h(of)e(a)h(new)g(con)o(text)p 865 1218 V 16 w(id)h(and)f(the)g(generation)g(of)f(a)h(new)g(comm)o (unicator.)18 b(This)75 1274 y(allo)o(ws)f(a)g(\\naiv)o(e")f(user)h (that)g(do)q(es)g(not)f(need)i(to)e(customize)i(con)o(text)p 1329 1274 V 15 w(id)g(allo)q(cation)g(to)e(ignore)i(this)75 1331 y(MPI)j(feature.)38 b(Direct)21 b(comm)o(unicator)g(creation)h(is) g(safer,)g(since)g(uniqueness)h(of)e(con)o(text)p 1742 1331 V 16 w(id's)g(is)75 1387 y(guaran)o(teed,)14 b(b)o(y)h (construction.)166 1491 y FD(MPI)p 275 1491 16 2 v 18 w(COMM)p 470 1491 V 18 w(SAFEMAKE\(comm,)h(map,)h(new)o(comm\))166 1595 y FC(Creates)c(a)h(new)g(comm)o(unicator)g(\(with)g(an)g(attac)o (hed)f(con)o(text)p 1270 1595 14 2 v 16 w(id\))i(that)e(is)h(asso)q (ciated)h(with)f(the)75 1651 y(group)19 b(de\014ned)i(b)o(y)e(the)g (pair)h Fp(\(comm,)j(map\))p FC(.)31 b(This)20 b(is)g(a)f(collectiv)o (e)i(call)f(that)f(has)g(to)g(b)q(e)h(in)o(v)o(ok)o(ed)75 1708 y(b)o(y)d(all)h(mem)o(b)q(ers)f(in)g(this)h(group.)24 b(All)18 b(of)f(them)g(are)f(pro)o(viding)i(the)f(same)g(input)h (parameters)e Fp(comm)75 1764 y FC(and)j Fp(map)p FC(.)32 b(The)19 b(call)h(ma)o(y)f(blo)q(c)o(k)h(un)o(til)g(all)g(pro)q(cesses) g(in)g(the)f(group)g(ha)o(v)o(e)g(in)o(v)o(ok)o(ed)g(it.)32 b(The)20 b(new)75 1821 y(comm)o(unicator)c(ma)o(y)f(b)q(e)i(safely)g (used)f(for)g(comm)o(unication)h(with)f(an)o(y)g(mem)o(b)q(er)g(of)g (the)g(new)h(group,)75 1877 y(once)f(the)f(call)h(returns.)75 1976 y FD(IN)h(comm)22 b FC(comm)o(unicator)15 b(handle)75 2068 y FD(IN)i(map)23 b FC(map)15 b(handle)75 2160 y FD(OUT)j(new)o(comm)k FC(comm)o(unicator)15 b(handle)166 2259 y Fp(MPI)p 241 2259 15 2 v 17 w(COMM)p 354 2259 V 16 w(SAFEMAKE)f FC(is)75 2359 y Fp(MPI_CONTEXT_ALLOC\(comm,)21 b(map,)i(context_id,)f(1\);)75 2415 y(MPI_COMM_MAKE\(comm,)f(map,)j (context_id,)e(newcomm\);)75 2472 y(``MPI_SYNCH\(comm,map\)'')f(/*)i (by)h(which)f(we)h(mean)f(a)h(synchronization)576 2528 y(operation)f(in)g(the)h(subgroup)f(defined)f(by)i(\(comm,map\))f(*/) 166 2704 y Fu(Discussion:)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 112 118 112 117 bop 75 -100 a FC(112)693 b Fv(SECTION)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)166 45 y Ft(Do)d(w)o(e)g (prefer)i(to)e(create)i(an)e(arra)o(y)g(of)f(comm)o(unicators,)f (rather)j(than)g(one?)k(\(T)m(o)12 b(b)q(e)h(consisten)o(t)g(with)f (the)75 95 y Fn(MPI)p 144 95 14 2 v 15 w(CONTEXT)p 313 95 V 14 w(ALLOC)h Ft(function\).)166 145 y Fn(MPI)p 235 145 V 15 w(SYNCH\(comm,map\))f Ft(is)i(not)h(curren)o(tly)h(de\014ned)g (in)e(the)i(collectiv)o(e)f(comm)o(unicati)o(on)d(library;)i(it)h(can) 75 195 y(b)q(e)g(co)q(ded)f(using)g(p)q(oin)o(t-to-p)q(oin)o(t)e (calls,)h(but)i(seems)f(imp)q(ortan)o(t)e(enough)i(to)f(b)q(e)i (included)f(there.)166 251 y(General)i(though)o(t)h({)f(if)g(w)o(e)g (supp)q(ort)i(collectiv)o(e)e(comm)o(unication)d(calls)k(with)f(a)g (map)f(as)i(an)f(additional)75 307 y(parameter,)g(then)h(w)o(e)g(ha)o (v)o(e)f(disasso)q(ciated)h(the)g(\\con)o(text")g(of)f(a)g(collectiv)o (e)g(comm)o(unication,)e(from)g(its)j(sup-)75 364 y(p)q(orting)c (\\group".)k(W)m(e)12 b(need)i(to)f(do)f(it)h(when)g(w)o(e)g(build)f (new)i(comm)o(unicators,)c(in)i(order)i(to)f(b)q(o)q(otstrap)g({)g(if)f (w)o(e)75 420 y(w)o(an)o(t)h(the)h(call)f(that)h(generates)h(a)f(new)g (comm)o(unicator)d(to)i(in)o(v)o(olv)o(e)f(only)h(the)h(pro)q(cesses)j (in)c(the)h(new)g(group.)75 645 y Fw(9.6)59 b(W)n(o)n(rking)22 b(without)d(maps)g(and)g(context)p 1100 645 18 2 v 19 w(id's)75 747 y FC(It)d(is)g(p)q(ossible)h(to)e(a)o(v)o(oid)g(b)q(oth)h (the)g(use)f(of)h(map)f(ob)s(jects)g(and)h(of)f(con)o(text)p 1384 747 14 2 v 16 w(id's)h(altogether.)k(A)c(single)75 803 y(function)d(com)o(bines)g(the)f(generation)g(of)f(a)h(map,)g(the)g (allo)q(cation)i(of)d(a)h(con)o(text)p 1445 803 V 16 w(id)g(and)h(the)f(generation)75 860 y(of)17 b(a)h(new)g(comm)o (unicator.)28 b(This)18 b(allo)o(ws)g(\\naiv)o(e")g(users)g(to)f(use)h (MPI)g(without)g(b)q(eing)h(exp)q(osed)g(to)75 916 y(maps)c(or)g(con)o (text)p 400 916 V 15 w(id's.)166 1020 y FD(MPI)p 275 1020 16 2 v 18 w(COMM)p 470 1020 V 18 w(BUILD\(comm,)i(arra)o(y)p 966 1020 V 18 w(of)p 1026 1020 V 19 w(ranks,)f(size,)i(new)o(comm\))166 1124 y FC(Build)k(a)f(map)f(from)g(an)g(explicit)j(list)e(represen)o (tation.)36 b(This)21 b(is)g(a)f(collectiv)o(e)i(function)g(that)75 1180 y(is)f(called)i(b)o(y)d(all)i(mem)o(b)q(ers)f(of)g(the)f(new)h (group.)37 b(All)22 b(pro)o(vide)f(the)g(same)g(parameters)f(for)g Fp(comm)p FC(,)75 1236 y Fp(array)p 198 1236 15 2 v 16 w(of)p 262 1236 V 17 w(ranks)15 b FC(and)h Fp(size)p FC(.)j(The)d(call)g(ma)o(y)f(blo)q(c)o(k)h(un)o(til)g(it)g(w)o(as)e(in) o(v)o(ok)o(ed)i(on)f(all)h(the)g(group)f(mem-)75 1293 y(b)q(ers.)40 b(When)22 b(it)f(returns,)i(the)f(new)g(comm)o(unicator)f (ma)o(y)g(b)q(e)h(used)g(to)f(comm)o(unicate)h(with)g(an)o(y)75 1349 y(mem)o(b)q(er)15 b(of)g(the)g(new)h(group.)75 1438 y FD(IN)h(comm)22 b FC(handle)17 b(to)d(comm)o(unicator)75 1530 y FD(IN)j(arra)o(y)p 277 1530 16 2 v 18 w(of)p 337 1530 V 19 w(ranks)22 b FC(list)16 b(of)f(v)m(alues)h(in)g(the)f(range)g (of)g Fp(map)75 1622 y FD(IN)i(size)23 b FC(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(arra)o(y)e({)h(map)g(size)h(\(in)o(teger\))75 1714 y FD(OUT)i(new)o(comm)k FC(handle)16 b(to)f(new)g(comm)o(unicator) 166 1802 y Fp(MPI)p 241 1802 15 2 v 17 w(COMM)p 354 1802 V 16 w(BUILD)g FC(is)75 1902 y Fp(MPI_MAP_BUILD\(comm,)21 b(array_of_ranks,)h(size,)h(newmap\);)75 1958 y (MPI_COMM_SAFEMAKE\(comm,)e(newmap,)i(newcomm\);)166 2105 y FD(MPI)p 275 2105 16 2 v 18 w(COMM)p 470 2105 V 18 w(COPY\(comm,)16 b(new)o(comm\))166 2208 y FC(Creates)h(a)g(new)g (comm)o(unicator)g(with)h(the)g(same)f(group)g(as)g(the)g(old)h(comm)o (unicator.)26 b(This)18 b(is)75 2265 y(a)i(collectiv)o(e)i(function)g (that)e(is)h(called)h(b)o(y)e(all)i(mem)o(b)q(ers)e(of)g(new)h(group.) 36 b(All)22 b(pro)o(vide)f(the)f(same)75 2321 y(parameters)d(for)g Fp(comm)p FC(.)27 b(The)18 b(call)h(ma)o(y)e(blo)q(c)o(k)h(un)o(til)h (it)f(w)o(as)f(in)o(v)o(ok)o(ed)h(on)f(all)i(the)f(group)f(mem)o(b)q (ers.)75 2378 y(When)e(it)f(returns,)g(the)g(new)h(comm)o(unicator)f (ma)o(y)f(b)q(e)i(used)g(to)f(comm)o(unicate)g(with)h(an)o(y)f(mem)o(b) q(er)g(of)75 2434 y(the)h(group.)75 2523 y FD(IN)i(comm)22 b FC(handle)17 b(to)d(comm)o(unicator)75 2615 y FD(OUT)k(new)o(comm)k FC(handle)16 b(to)f(new)g(comm)o(unicator)166 2704 y Fp(MPI)p 241 2704 15 2 v 17 w(COMM)p 354 2704 V 16 w(COPY)g FC(is)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 113 119 113 118 bop 75 -100 a Fv(9.7.)34 b(EXAMPLES)1370 b FC(113)75 45 y Fp(MPI_COMM_SIZE\(comm,)21 b(size\);)75 102 y (MPI_COMM_SAFEMAKE\(comm,)g(MPI_IDENT\(size\),)g(newcomm\);)166 257 y FD(MPI)p 275 257 16 2 v 18 w(COMM)p 470 257 V 18 w(SPLIT\(comm,)c(k)o(ey)l(,)e(index,)j(new)o(comm\))166 361 y FC(Split)c(the)e(group)h(asso)q(ciated)f(with)h Fp(comm)p FC(;)g(creates)f(a)g(new)h(group)f(for)g(eac)o(h)h(distinct)h (v)m(alue)f(of)f Fp(key)75 418 y FC(that)g(con)o(tains)h(the)g(pro)q (cesses)g(that)f(supplied)k(that)c(k)o(ey)g(v)m(alue;)j(the)e(pro)q (cesses)g(are)g(rank)o(ed)f(according)75 474 y(to)g(the)h Fp(index)f FC(v)m(alues)h(they)g(supplied.)22 b(A)12 b(new)h(comm)o(unicator)f(is)i(created)e(for)g(eac)o(h)h(subgroup.)19 b(Eac)o(h)75 531 y(pro)q(cess)c(is)f(returned)h(a)f(handle)i(to)d(the)i (comm)o(unicator)f(for)f(the)i(new)f(subgroup)h(it)f(b)q(elongs)h(to.)k (This)75 587 y(is)f(a)e(collectiv)o(e)j(call)f(executed)g(b)o(y)f(all)h (pro)q(cesses)f(in)h(the)f(group)g(asso)q(ciated)g(with)h Fp(comm)p FC(.)25 b(They)17 b(call)75 644 y(ma)o(y)h(blo)q(c)o(k)i(un)o (til)g(all)h(pro)q(cesses)e(ha)o(v)o(e)g(in)o(v)o(ok)o(ed)g(the)g (function.)33 b(When)19 b(the)g(call)i(returns)e(the)g(new)75 700 y(comm)o(unicator)c(ma)o(y)f(b)q(e)i(safely)g(used)g(for)e(comm)o (unication)i(in)g(the)f(new)h(group.)75 809 y FD(IN)h(comm)22 b FC(handle)17 b(to)d(comm)o(unicator)75 904 y FD(IN)j(k)o(ey)22 b FC(\(in)o(teger\))75 1000 y FD(IN)17 b(index)23 b FC(\(in)o(teger\)) 75 1095 y FD(OUT)18 b(new)o(comm)k FC(handle)16 b(to)f(new)g(comm)o (unicator)166 1204 y Fp(MPI)p 241 1204 15 2 v 17 w(COMM)p 354 1204 V 16 w(SPLIT\(comm,)23 b(key,)g(index,)g(newcomm\))14 b FC(is)75 1313 y Fp(MPI_COMM_SIZE\(comm,)21 b(size\);)75 1369 y(MPI_MAP_SPLIT\(comm,)g(MPI_IDENT\(size\),)h(key,)h(index,)g (newmap\);)75 1426 y(MPI_COMM_SAFEMAKE\(comm,)e(newmap,)i(newcomm\);) 166 1610 y Fu(Discussion:)166 1660 y Ft(Ma)o(y)13 b(w)o(an)o(t)g(a)h (sp)q(ecial)f(DONTCARE)h(k)o(ey)g(v)n(alue)f(that)g(indicates)h(that)g (the)g(caller)g(need)g(not)g(ha)o(v)o(e)f(a)h(new)75 1710 y(comm)o(unicator)75 1939 y Fw(9.7)59 b(Examples)75 2041 y FC(W)l(e)19 b(sa)o(y)f(that)f(a)i(parallel)h(pro)q(cedure)f(is)g Fs(active)g FC(at)f(a)g(pro)q(cess)h(if)g(the)g(pro)q(cess)g(b)q (elongs)g(to)f(a)g(group)75 2097 y(that)e(ma)o(y)h(collectiv)o(ely)i (execute)f(the)f(pro)q(cedure,)h(and)f(some)f(mem)o(b)q(er)h(of)g(that) f(group)h(is)g(curren)o(tly)75 2154 y(executing)c(the)f(pro)q(cedure)g (co)q(de.)20 b(If)12 b(a)f(parallel)j(pro)q(cedure)e(is)h(activ)o(e)e (at)h(a)f(pro)q(cess,)h(then)g(this)h(pro)q(cess)75 2210 y(ma)o(y)e(b)q(e)i(receiving)g(messages)e(p)q(ertaining)i(to)f(this)g (pro)q(cedure,)h(ev)o(en)f(if)g(it)g(do)q(es)h(not)e(curren)o(tly)h (execute)75 2267 y(the)j(co)q(de)h(of)f(this)g(pro)q(cedure.)75 2391 y Fq(9.7.1)49 b(Nonreentrant)15 b(pa)o(rallel)i(p)o(ro)q(cedures) 75 2478 y FC(This)22 b(co)o(v)o(ers)f(the)h(case)g(where,)h(at)e(an)o (y)g(p)q(oin)o(t)h(in)h(time,)g(at)e(most)g(one)h(in)o(v)o(ok)m(ation)g (of)f(a)g(parallel)75 2534 y(pro)q(cedure)e(can)g(b)q(e)g(activ)o(e)g (at)f(an)o(y)g(pro)q(cess.)30 b(I.e.,)19 b(concurren)o(t)g(in)o(v)o(ok) m(ations)f(of)h(the)f(same)g(parallel)75 2591 y(pro)q(cedure)e(ma)o(y)e (o)q(ccur)h(only)g(within)h(disjoin)o(t)f(groups)f(of)h(pro)q(cesses.) 20 b(F)l(or)14 b(example,)h(all)h(in)o(v)o(ok)m(ations)75 2647 y(of)f(parallel)i(pro)q(cedures)f(in)o(v)o(olv)o(e)g(all)h(pro)q (cesses,)e(pro)q(cesses)h(are)f(single-threaded,)i(and)e(there)h(are)f (no)75 2704 y(recursiv)o(e)h(in)o(v)o(ok)m(ations.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 114 120 114 119 bop 75 -100 a FC(114)693 b Fv(SECTION)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)166 45 y FC(In)i(suc)o(h)g(a)g (case,)g(a)f(con)o(text)p 668 45 14 2 v 16 w(id)i(can)f(b)q(e)g (statically)h(allo)q(cated)f(to)g(eac)o(h)g(pro)q(cedure.)25 b(The)17 b(static)75 102 y(allo)q(cation)h(can)g(b)q(e)g(done)g(in)g(a) f(pream)o(ble,)h(as)f(part)g(of)g(initialization)j(co)q(de.)27 b(Or,)17 b(it)h(can)g(b)q(e)g(done)f(a)75 158 y(compile/link)j(time,)d (if)h(the)f(implemen)o(tation)i(has)e(additional)h(mec)o(hanisms)g(to)e (reserv)o(e)h(con)o(text)p 1824 158 V 16 w(id)75 214 y(v)m(alues.)22 b(Comm)o(unicators)14 b(to)h(b)q(e)h(used)g(b)o(y)f (the)h(di\013eren)o(t)f(pro)q(cedures)i(can)e(b)q(e)h(build)h(in)g(a)e (pream)o(ble,)75 271 y(if)i(the)g(executing)h(groups)e(are)h (statically)g(de\014ned;)i(if)e(the)g(executing)h(groups)e(c)o(hange)h (dynamically)l(,)75 327 y(then)f(a)g(new)h(comm)o(unicator)e(has)h(to)g (b)q(e)h(built)g(whenev)o(er)g(the)f(executing)h(group)f(c)o(hanges,)g (but)h(this)75 384 y(new)k(comm)o(unicator)g(can)h(b)q(e)g(built)g (using)g(the)g(same)e(preallo)q(cated)j(con)o(text)p 1491 384 V 16 w(id.)39 b(If)21 b(the)g(parallel)75 440 y(pro)q(cedures)d(can)f(b)q(e)g(organized)g(in)o(to)g(libraries,)h(so)f (that)f(only)h(one)g(pro)q(cedure)h(of)e(eac)o(h)h(library)h(can)75 497 y(b)q(e)i(concurren)o(tly)g(activ)o(e)f(at)f(eac)o(h)i(pro)q (cessor,)f(then)g(it)h(is)g(su\016cien)o(t)f(to)g(allo)q(cate)h(one)f (con)o(text)f(p)q(er)75 553 y(library)l(.)75 688 y Fq(9.7.2)49 b(P)o(a)o(rallel)17 b(p)o(ro)q(cedures)e(that)g(a)o(re)h(nonreentrant)f (within)i(each)f(executing)g(group)75 778 y FC(This)21 b(co)o(v)o(ers)g(the)g(case)f(where,)j(at)d(an)o(y)g(p)q(oin)o(t)i(in)f (time,)h(for)f(eac)o(h)g(pro)q(cess)g(group,)g(there)g(can)g(b)q(e)75 835 y(at)d(most)g(one)h(activ)o(e)g(in)o(v)o(ok)m(ation)g(of)g(a)f (parallel)i(pro)q(cedure)g(b)o(y)f(a)f(pro)q(cess)h(mem)o(b)q(er.)31 b(Ho)o(w)o(ev)o(er,)19 b(it)75 891 y(migh)o(t)i(b)q(e)h(p)q(ossible)i (that)c(the)i(same)f(pro)q(cedure)h(is)g(concurren)o(tly)g(in)o(v)o(ok) o(ed)g(in)g(t)o(w)o(o)e(partially)j(\(or)75 948 y(completely\))17 b(o)o(v)o(erlapping)f(groups.)k(F)l(or)15 b(example,)i(the)e(same)g (collectiv)o(e)j(comm)o(unication)e(function)75 1004 y(ma)o(y)e(b)q(e)i(concurren)o(tly)g(in)o(v)o(ok)o(ed)g(on)f(t)o(w)o(o) f(partially)i(o)o(v)o(erlapping)g(groups.)166 1063 y(In)i(suc)o(h)h(a)e (case,)h(a)g(con)o(text)p 674 1063 V 16 w(id)h(is)f(asso)q(ciated)g (with)g(eac)o(h)g(parallel)i(pro)q(cedure)e(and)g(eac)o(h)g(exe-)75 1119 y(cuting)f(group,)e(so)h(that)f(o)o(v)o(erlapping)i(execution)g (groups)e(ha)o(v)o(e)h(distinct)h(comm)o(unication)g(con)o(texts.)75 1176 y(\(One)c(do)q(es)g(not)g(need)h(a)e(di\013eren)o(t)h(con)o(text)p 831 1176 V 16 w(id)h(from)e(eac)o(h)h(group;)g(one)g(merely)g(needs)h (a)e(\\coloring")h(of)75 1232 y(the)j(groups,)f(so)g(that)g(One)h(can)g (generate)f(the)h(comm)o(unicators)f(for)g(eac)o(h)h(parallel)h(pro)q (cedure)f(when)75 1289 y(the)g(execution)i(groups)e(are)g(de\014ned.)25 b(Here,)17 b(again,)f(one)g(only)h(need)h(one)e(con)o(text)g(for)g(eac) o(h)g(library)l(,)75 1345 y(if)g(no)f(t)o(w)o(o)f(pro)q(cedures)i(from) e(the)h(same)g(library)h(can)g(b)q(e)g(concurren)o(tly)f(activ)o(e)h (in)g(the)f(same)g(group.)166 1404 y(Note)j(that,)h(for)f(collectiv)o (e)j(comm)o(unication)e(libraries,)i(w)o(e)d(do)h(allo)o(w)g(sev)o (eral)g(concurren)o(t)g(in-)75 1460 y(v)o(ok)m(ations)13 b(within)h(the)f(same)g(group:)18 b(a)13 b(broadcast)f(in)i(a)f(group)f (ma)o(y)h(b)q(e)g(started)f(at)h(a)f(pro)q(cess)i(b)q(efore)75 1517 y(the)h(previous)h(broadcast)f(in)h(that)e(group)h(ended)h(at)f (another)g(pro)q(cess.)20 b(In)c(suc)o(h)f(a)g(case,)g(one)g(cannot)75 1573 y(rely)e(on)f(con)o(text)g(mec)o(hanisms)h(to)f(disam)o(biguate)h (successiv)o(e)h(in)o(v)o(ok)m(ations)e(of)g(the)h(same)f(parallel)i (pro-)75 1630 y(cedure)f(within)h(the)f(same)f(group:)18 b(the)13 b(pro)q(cedure)g(need)h(b)q(e)f(implemen)o(ted)h(so)e(as)g(to) g(a)o(v)o(oid)h(confusion.)75 1686 y(E.g.,)h(for)h(broadcast,)f(one)h (ma)o(y)g(need)h(to)f(carry)f(additional)j(information)f(in)g (messages,)e(suc)o(h)i(as)f(the)75 1742 y(broadcast)h(ro)q(ot,)g(to)g (help)i(in)f(suc)o(h)g(disam)o(biguation;)i(one)d(also)h(relies)h(on)f (preserv)m(ation)g(of)f(message)75 1799 y(order)e(b)o(y)g(MPI.)g(With)g (suc)o(h)h(an)f(approac)o(h,)f(w)o(e)h(ma)o(y)g(b)q(e)h(gaining)g(p)q (erformance,)f(but)g(w)o(e)g(lo)q(ose)h(mo)q(d-)75 1855 y(ularit)o(y)l(.)28 b(It)18 b(is)g(not)f(su\016cien)o(t)i(to)e (implemen)o(t)i(the)f(parallel)h(pro)q(cedure)g(so)e(that)g(it)h(w)o (orks)f(correctly)75 1912 y(in)i(isolation,)h(when)g(in)o(v)o(ok)o(ed)e (only)i(once;)g(it)f(needs)g(to)f(b)q(e)i(implemen)o(ted)g(so)e(that)g (an)o(y)g(n)o(um)o(b)q(er)h(of)75 1968 y(successiv)o(e)d(in)o(v)o(ok)m (ations)f(will)h(execute)g(correctly)l(.)k(Of)14 b(course,)h(the)g (same)f(approac)o(h)g(can)h(b)q(e)g(used)h(for)75 2025 y(other)f(parallel)i(libraries.)75 2159 y Fq(9.7.3)49 b(W)o(ell)16 b(nested)f(pa)o(rallel)j(p)o(ro)q(cedures)75 2250 y FC(Calls)d(of)g(parallel)h(pro)q(cedures)g(are)e(w)o(ell)i (nested)f(if)g(a)g(new)g(parallel)h(pro)q(cedure)g(is)f(alw)o(a)o(ys)f (in)o(v)o(ok)o(ed)h(in)75 2306 y(a)j(subset)h(of)e(a)h(group)g (executing)i(the)e(same)g(parallel)i(pro)q(cedure.)30 b(Th)o(us,)19 b(pro)q(cesses)g(that)e(execute)75 2363 y(the)e(same)g(parallel)i(pro)q(cedure)f(ha)o(v)o(e)f(the)g(same)g (execution)h(stac)o(k.)166 2421 y(In)e(suc)o(h)f(a)g(case,)g(a)g(new)g (con)o(text)f(need)i(to)f(b)q(e)h(dynamically)g(allo)q(cated)h(for)d (eac)o(h)h(new)g(in)o(v)o(ok)m(ation)75 2478 y(of)d(a)g(parallel)i(pro) q(cedure.)20 b(Ho)o(w)o(ev)o(er,)10 b(a)g(stac)o(k)g(mec)o(hanism)h (can)f(b)q(e)i(used)f(for)f(allo)q(cating)h(new)g(con)o(texts.)75 2534 y(Th)o(us,)19 b(a)f(p)q(ossible)j(mec)o(hanism)e(is)g(to)f(allo)q (cate)h(\014rst)f(a)h(large)f(n)o(um)o(b)q(er)h(of)f(con)o(text)p 1566 2534 V 16 w(id's)h(\(up)g(to)f(the)75 2591 y(upp)q(er)h(b)q(ound)g (on)f(the)g(depth)h(of)f(nested)g(parallel)i(pro)q(cedure)f(calls\),)g (and)g(then)f(use)h(a)e(lo)q(cal)j(stac)o(k)75 2647 y(managemen)o(t)f (of)h(these)g(con)o(text)p 674 2647 V 15 w(id's)h(on)e(eac)o(h)h(pro)q (cess)h(to)e(create)g(a)h(new)g(comm)o(unicator)g(\(using)75 2704 y Fp(MPI)p 150 2704 15 2 v 17 w(COMM)p 263 2704 V 16 w(MAKE)p FC(\))15 b(for)f(eac)o(h)i(new)f(in)o(v)o(ok)m(ation.) 1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 115 121 115 120 bop 75 -100 a Fv(9.8.)34 b(LEFT)1504 b FC(115)166 45 y Fu(Discussion:)15 b Ft(General)f(case)166 184 y FC(In)h(the)g(general)g(case,)g(there)f(ma)o(y)g(b)q(e)i(m)o(ultiple)g (concurren)o(tly)f(activ)o(e)g(in)o(v)o(ok)m(ations)g(of)g(the)f(same) 75 241 y(parallel)23 b(pro)q(cedure)f(within)g(the)g(same)f(group;)i (in)o(v)o(ok)m(ations)f(ma)o(y)f(not)g(b)q(e)h(w)o(ell)g(nested.)38 b(A)22 b(new)75 297 y(con)o(text)17 b(need)i(to)e(b)q(e)h(created)g (for)f(eac)o(h)g(in)o(v)o(ok)m(ation.)28 b(It)18 b(is)g(the)g(user)g (resp)q(onsibilit)o(y)i(to)d(mak)o(e)g(sure)75 354 y(that,)k(if)h(t)o (w)o(o)d(distinct)j(parallel)h(pro)q(cedures)e(are)g(in)o(v)o(ok)o(ed)g (concurren)o(tly)h(on)e(o)o(v)o(erlapping)i(sets)e(of)75 410 y(pro)q(cesses,)15 b(then)h(con)o(text)p 538 410 14 2 v 16 w(id)g(allo)q(cation)g(or)f(comm)o(unicator)f(creation)i(is)g (prop)q(erly)g(co)q(ordinated.)75 553 y Fw(9.8)59 b(Left)75 655 y FC(Remote)15 b(pro)q(cedure)h(call)h(and)e(C2)g(ob)s(jects.)166 711 y(Ho)o(w)f(to)h(handle)h(gro)o(wing)f(pro)q(cess)h(set.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 116 122 116 121 bop 75 357 a Fx(Section)35 b(10)75 565 y FE(Initial)43 b(Implemen)m(tation)f(Subset)75 806 y Fw(10.1)59 b(Intro)r(duction)75 908 y FC(This)11 b(c)o(hapter)f(de\014nes)h(a)f(minimal)i(subset)f(of)f (MPI)g(for)g(initial)i(implemen)o(tation.)19 b(This)11 b(subset)g(is)g(b)q(eing)75 965 y(de\014ned)i(so)f(that)f(consisten)o (t)h(implemen)o(tations)h(can)f(app)q(ear)g(more)f(rapidly)l(.)21 b(It)12 b(w)o(as)f(recognized)i(early)75 1021 y(in)h(the)e(pro)q(cess)i (that)e(MPI)g(needed)i(to)f(app)q(ear)f(as)h(quic)o(kly)h(as)e(p)q (ossible)j(and)e(practical.)20 b(The)13 b(creation)75 1078 y(of)k(a)g(subset)g(will)i(hop)q(efully)g(allo)o(w)e(users)g (earlier)h(access)g(to)e(the)h(standard)g(and)g(still)i(allo)o(w)e(for) g(the)75 1134 y(writing)f(of)e(p)q(ortable)i(message)f(passing)g(co)q (des.)166 1191 y(This)20 b(subset)g(should)h(not)e(in)i(an)o(y)e(w)o(a) o(y)g(b)q(e)h(construed)g(as)f(a)h(limitation)h(on)e(MPI)h(implemen-)75 1248 y(tations.)35 b(It)20 b(is)h(strictly)g(the)g(minim)o(um)g (necessary)g(to)e(ha)o(v)o(e)h(an)h(initial)h(MPI)e(subset)h (conforming)75 1304 y(implemen)o(tation.)28 b(It)18 b(is)g(hop)q(ed)g (that)f(an)g(o\016cially)i(sanctioned)g(subset)e(will)i(encourage)f (and)g(allo)o(w)75 1361 y(implemen)o(tors)c(to)e(in)o(tro)q(duce)j(MPI) e(in)h(a)e(more)h(timely)h(fashion.)20 b(It)13 b(should)h(b)q(e)g (noted,)f(ho)o(w)o(ev)o(er,)g(that)75 1417 y(implemen)o(tation)20 b(of)e(the)h(subset)g(do)q(es)g(not)f(mak)o(e)h(an)f(implemen)o(tation) i(conform)e(with)h(MPI.)g(The)75 1473 y(subset)g(is)g(only)h(a)e(p)q (oten)o(tial)i(\014rst)e(step)h(in)h(the)f(pro)q(cess.)31 b(All)20 b(implemen)o(tations)g(m)o(ust)f(ultimately)75 1530 y(conform)14 b(to)h(the)g(en)o(tire)g(standard;)g(implemen)o (tations)h(are)f(encouraged)g(to)f(do)h(the)g(full)i(standard)d(as)75 1586 y(rapidly)i(as)f(p)q(ossible.)166 1643 y(The)c(subset)g(presen)o (ted)g(is)g(consisten)o(t)f(with)h(the)g(complete)g(MPI)g(standard.)18 b(This)11 b(w)o(as)f(an)g(imp)q(or-)75 1700 y(tan)o(t)i(goal)g(so)h (the)g(additional)h(MPI)f(features)f(could)i(b)q(e)g(added)f(without)g (c)o(hanging)g(an)o(y)g(functionalit)o(y)75 1756 y(from)19 b(the)i(user's)f(p)q(ersp)q(ectiv)o(e.)36 b(Th)o(us,)21 b(users)f(can)g(b)q(e)h(assured)f(that)g(programs)f(written)h(no)o(w)g (for)75 1813 y(the)c(subset)f(will)i(run)f(without)g(mo)q(di\014cation) g(under)g(the)g(full)h(MPI)e(standard.)21 b(F)l(urthermore,)14 b(using)75 1869 y(additional)j(features)f(of)g(the)g(full)h(MPI)f (standard)f(in)i(the)f(future)g(will)i(not)d(require)i(c)o(hanges)f(to) f(co)q(de)75 1926 y(already)f(written.)19 b(Users)13 b(ma)o(y)g(use)h(MPI)f(features)g(outside)i(this)e(subset)h(that)f(are) g(o\013ered)g(b)o(y)g(v)m(arious)75 1982 y(implemen)o(tors.)33 b(Ho)o(w)o(ev)o(er,)19 b(p)q(eople)i(who)e(require)i(p)q(ortabilit)o(y) f(during)g(the)g(early)f(dev)o(elopmen)o(t)h(of)75 2038 y(MPI)15 b(ma)o(y)g(exp)q(erience)i(some)e(di\016culties)i(un)o(til)g (later)e(in)h(the)f(dev)o(elopmen)o(t)h(pro)q(cess.)75 2184 y Fw(10.2)59 b(Criteria)20 b(and)g(Rational)75 2287 y FC(Ha)o(ving)f(the)g(subset)g(b)q(e)h(consisten)o(t)f(with)g(the)g (en)o(tire)g(MPI)g(standard)g(w)o(as)f(considered)i(critical)h(to)75 2343 y(the)16 b(e\013ort.)22 b(In)16 b(addition)i(to)d(this,)h(there)h (w)o(ere)e(man)o(y)h(p)q(ossible)i(criteria)f(to)e(use)h(in)h (determining)h(the)75 2399 y(elemen)o(ts)d(of)e(MPI)h(to)f(include)j (in)f(the)f(subset.)20 b(The)14 b(main)g(criteria)h(used)f(w)o(ere)g (that)f(the)h(MPI)g(subset)75 2456 y(should)131 2552 y(1.)22 b(con)o(tain)13 b(routines)h(that)e(are)h(as)g(close)h(as)f(p)q (ossible)i(to)e(curren)o(t)g(standard)g(practice)h(to)f(minimize)189 2608 y(the)i(e\013ort)f(to)h(p)q(ort)f(co)q(des.)131 2704 y(2.)22 b(con)o(tain)15 b(as)g(man)o(y)g(new)g(and)g(imp)q(ortan)o (t)g(MPI)g(features)g(as)g(p)q(ossible.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 117 123 117 122 bop 75 -100 a Fv(10.2.)34 b(CRITERIA)16 b(AND)f(RA)l(TIONAL)997 b FC(117)131 45 y(3.)22 b(allo)o(w)17 b(dev)o(elop)q(ers)i(to)e(b)q(e)i (able)f(to)f(ha)o(v)o(e)g(the)h(subset)g(sho)o(w)f(up)h(as)f(rapidly)i (as)e(p)q(ossible)i(while)189 102 y(still)d(meeting)g(the)f(other)g (criteria.)166 373 y(There)i(w)o(ere)g(man)o(y)f(rationales)h(for)g (these)g(criteria.)26 b(It)17 b(w)o(as)f(recognized)i(that)e(it)h(is)h (imp)q(ossible)75 429 y(to)h(come)h(up)g(with)g(an)g(ideal)h(subset)f (and)g(compromises)g(w)o(ere)g(necessary)l(.)34 b(It)19 b(w)o(as)g(felt)i(that)e(cur-)75 486 y(ren)o(t)h(users)g(should)i(b)q (e)f(comfortable)f(in)h(migrating)f(to)g(MPI.)g(Th)o(us,)h(the)g (subset)f(should)i(con)o(tain)75 542 y(the)17 b(message)f(passing)h (elemen)o(ts)g(that)f(represen)o(t)g(common)g(practice)i(to)q(da)o(y)l (.)23 b(F)l(urthermore,)16 b(it)h(w)o(as)75 599 y(felt)f(that)g(the)g (initial)i(\(and)e(later\))g(v)o(ersion)g(of)g(MPI)g(should)h(main)o (tain)g(e\016ciency)l(.)24 b(Otherwise)17 b(users)75 655 y(could)k(su\013er)e(from)g(disenc)o(han)o(tmen)o(t)h(with)g(MPI)f (on)h(their)g(\014rst)f(usage)h(and)f(ma)o(y)g(nev)o(er)h(giv)o(e)g (the)75 711 y(standard)e(a)h(second)g(try)l(.)29 b(Ha)o(ving)19 b(the)g(standard)f(routines)h(in)h(the)e(subset)h(should)h(encourage)e (de-)75 768 y(v)o(elop)q(ers)g(to)f(optimized)i(these)f(routines)g(in)h (the)f(initial)h(v)o(ersion)f(of)f(MPI.)h(Also,)g(though)f(some)h(felt) 75 824 y(that)c(the)g(p)q(ortabilit)o(y)h(o\013ered)f(b)o(y)g(the)h (MPI)f(subset)h(w)o(as)e(su\016cien)o(t)i(to)f(en)o(tice)h(users,)f (man)o(y)g(felt)h(that)75 881 y(the)20 b(subset)g(should)g(con)o(tain)g (some)f(new)h(feature\(s\))f(o)o(v)o(er)g(and)g(ab)q(o)o(v)o(e)h (curren)o(t)f(common)g(practice)75 937 y(as)d(an)g(added)h(incen)o(tiv) o(e.)24 b(The)17 b(additional)h(items)e(selected)i(w)o(ere)e(mean)o(t)f (to)h(represen)o(t)g(some)g(of)g(the)75 994 y(signi\014can)o(t)i(new)f (features)f(of)h(MPI.)f(Balanced)i(against)f(these)g(\014rst)f(t)o(w)o (o)g(goals)h(is)g(the)g(need)h(for)e(the)75 1050 y(subset)e(to)f(sho)o (w)g(up)i(in)f(a)g(timely)h(fashion.)k(Th)o(us,)14 b(eac)o(h)g(feature) g(c)o(hosen)g(for)f(inclusion)j(in)f(the)f(subset)75 1107 y(w)o(as)g(deemed)h(of)f(su\016cien)o(t)h(imp)q(ortance)g(to)f (out)o(w)o(eigh)g(its)h(added)g(complexit)o(y)g(in)h(implemen)o(ting)g (the)75 1163 y(subset.)k(Though)14 b(some)h(functions)g(seem)g(easy)f (to)g(implemen)o(t)i(there)e(are)h(often)f(o)o(v)o(erlo)q(ok)o(ed)g (costs)g(in)75 1220 y(testing,)f(do)q(cumen)o(tation)h(and)g(dev)o (elopmen)o(t)g(that)e(w)o(as)h(considered)h(b)q(efore)g(a)f(feature)g (w)o(as)g(added)h(to)75 1276 y(the)j(MPI)g(subset.)25 b(Inclusion)20 b(of)c(to)q(o)g(man)o(y)h(routines)g(migh)o(t)g(lead)h (to)e(mo)q(derately)i(e\016cien)o(t)f(imple-)75 1332 y(men)o(tation)g(of)f(them)h(all)h(instead)f(of)g(a)f(v)o(ery)h (e\016cien)o(t)h(and)f(p)q(ossibly)h(more)f(useful)h(implemen)o(tation) 75 1389 y(of)d(a)g(smaller)h(subset.)166 1572 y Fu(Discussion:)32 b Ft(Some)11 b(felt)h(that)g(w)o(e)g(should)g(aim)e(the)j(subset)g(at)f (t)o(ypical)f(users.)19 b(Should)12 b(this)g(b)q(e)h(another)75 1629 y(criteria?)18 b(Is)d(the)f(compatibilit)o(y)d(with)i(curren)o(t)j (practice)f(enough?)166 1895 y Fu(Discussion:)36 b Ft(Should)14 b(it)g(b)q(e)h(a)f(criteria)h(that)f(the)h(subset)h(and)e(common)e (practice)j(features)h(b)q(e)f(imple-)75 1952 y(men)o(ted)g(e\016cien)o (tly)m(.)21 b(The)16 b(text)g(touc)o(hes)g(on)f(this)h(but)f(it)g(is)g (not)g(a)g(sp)q(eci\014c)i(item.)k(I)15 b(did)g(not)g(giv)o(e)f(it)h(a) g(bullet)75 2008 y(since)f(it)f(seems)g(lik)o(e)f(a)h(qualit)o(y)f(of)g (implemen)o(tatio)o(n)f(issue.)18 b(P)o(o)q(orly)12 b(implemen)o(ted)f (subsets)k(will)d(b)q(e)h(forced)h(out)75 2064 y(b)o(y)g(the)g(mark)o (et.)166 2331 y Fu(Discussion:)49 b Ft(Should)17 b(it)h(b)q(e)g(a)f (criteria)h(that)g(the)g(rest)h(of)e(MPI)h(b)q(e)h(able)e(to)h(b)q(e)g (la)o(y)o(ered)g(on)f(top)h(so)75 2387 y(users)f(migh)o(t)c(get)j (early)f(\(though)g(lo)o(w)o(er)g(p)q(erformance\))g(access)j(to)d(the) h(complete)e(standard?)23 b(This)15 b(is)h(aimed)75 2443 y(at)g(allo)o(wing)f(easy)i(and)f(e\016cien)o(t)h(la)o(y)o(ering)e(of)h (the)h(rest)h(so)f(that)f(a)g(\\mac)o(hine)f(indep)q(enden)o(t")j(pac)o (k)n(age)e(could)75 2500 y(b)q(e)i(created)g(without)f(ha)o(ving)f(the) h(dev)o(elop)q(er)h(jump)d(through)i(lots)g(of)f(hops.)28 b(This)17 b(migh)o(t)e(happ)q(en)i(if)f(a)h(few)75 2556 y(functions)e(w)o(ere)i(left)e(out)g(that)g(didn't)g(meet)g(the)h (other)g(criteria.)22 b(Ho)o(w)o(ev)o(er,)16 b(some)e(felt)h(it)g(w)o (as)g(not)h(a)f(strong)75 2613 y(enough)f(reason)g(to)g(lea)o(v)o(e)g (it)f(in)h(and)f(m)o(y)g(lead)g(to)h(p)q(o)q(or)g(MPI)g(implemen)o (tatio)o(ns)e(of)h(these)i(la)o(y)o(ered)f(routines.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 118 124 118 123 bop 75 -100 a FC(118)592 b Fv(SECTION)16 b(10.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)75 45 y Fw(10.3)59 b(Subset)19 b(F)n(unctionalit)n(y)75 240 y Fu(Discussion:)45 b Ft(This)17 b(section)g(only)f(discuss)i(the)g (pt-to-pt)e(and)h(collcomm)c(c)o(hapters.)28 b(The)17 b(other)h(c)o(hapters)75 296 y(will)12 b(b)q(e)j(added)f(after)g(they)h (are)f(read)g(at)g(meetings)f(and)h(\014nalized.)75 532 y Fq(10.3.1)49 b(Itemized)16 b(F)o(unctionalit)o(y)75 628 y FC(The)22 b(T)l(able)h(b)q(elo)o(w)g(summarizes)f(the)g(sp)q (eci\014c)i(functionalit)o(y)g(included)g(and)f(excluded)h(from)d(the) 75 684 y(subset.)75 780 y(Section)206 b(Op)q(eration)337 b(Included)404 b(Excluded)p 75 799 1882 2 v 709 871 a(P)o(oin)o(t)15 b(to)f(P)o(oin)o(t)h(Comm)o(unication)p 709 878 614 2 v 100 945 a(1.5.2)151 b(Bu\013er)16 b(Op)q(erations)194 b(\\All")16 b(functions)112 b(Multiple)17 b(app)q(ends,)f(h)f(v)o (ersions\(?\))118 1001 y(1.7)210 b(Comm.)19 b(Mo)q(de)293 b(Standard)287 b(Ready)l(,)16 b(Sync)o(hronous)100 1057 y(1.8.1)86 b(Comm.)19 b(Ob)s(ject)d(Creation)220 b(All\(?\))100 1114 y(1.8.2)197 b(Comm.)19 b(Start)172 b(ST)l(AR)l(T\(if)16 b(ha)o(v)o(e)f(INIT\))100 1170 y(1.8.3)132 b(Comm.)20 b(Completion)295 b(All)100 1227 y(1.8.4)123 b(Multiple)17 b(Completion)286 b(All)118 1283 y(1.9)182 b(Blo)q(c)o(king)16 b(Comm.)235 b(Std.)20 b(v)o(ersion)238 b(Ready)l(,)16 b(Sync)o(h.)k(v)o(ersion)107 1340 y(1.10)133 b(Non)o(blo)q(c)o(king)16 b(Comm.)198 b(Std.)20 b(v)o(ersion)238 b(Ready)l(,)16 b(Sync)o(h.)k(v)o(ersion)107 1396 y(1.11)75 b(Con)o(tiguous)15 b(Bu\013er)g(Comm.)141 b(Std.)20 b(v)o(ersion)238 b(Ready)l(,)16 b(Sync)o(h.)k(v)o(ersion)107 1453 y(1.12)776 b(PR)o(OBE)291 b(CANCEL,GET)p 1726 1453 14 2 v 16 w(LEN)754 1551 y(Collectiv)o(e)16 b(Comm)o(unication)p 754 1558 524 2 v 118 1624 a(2.4)188 b(Sync)o(hronization)334 b(All)118 1680 y(2.5)238 b(Data)14 b(Mo)o(v)o(e)353 b(All\(?\))118 1737 y(2.6)183 b(Global)16 b(Compute)266 b(MPI)15 b(Ops)396 b(User)15 b(Ops)75 1858 y(All)h(other)e(general)h(functionalit)o(y)h(is)f(included)i(in)f(the)e (subset.)20 b(Th)o(us,)14 b(all)i(matc)o(hing)f(criteria,)g(order,)75 1915 y(etc.)20 b(are)15 b(part)f(of)h(the)g(subset.)75 2068 y Fq(10.3.2)49 b(P)o(oint)17 b(to)f(P)o(oint)h(F)o(unctionalit)o (y)75 2164 y FC(The)12 b(op)q(erations)f(in)o(v)o(olving)i(con)o (tiguous)e(bu\013ers)g(are)h(included)i(since)e(they)g(represen)o(t)f (standard)g(prac-)75 2220 y(tice)k(to)q(da)o(y)l(.)20 b(In)15 b(addition,)g(the)g(comm)o(unications)g(bu\013ers)g(are)f(also) h(included)i(b)q(ecause)e(they)g(o\013er)f(an)75 2277 y(imp)q(ortan)o(t)20 b(new)g(feature)g(in)h(MPI.)e(Ho)o(w)o(ev)o(er,)h (to)g(simplify)i(the)e(implemen)o(tation,)i(the)e(comm)o(uni-)75 2333 y(cation)h(bu\013er)g(will)i(b)q(e)e(limited)i(to)d(one)h(comp)q (onen)o(t,)i(e.g.,)e(only)g(one)g(app)q(end)h(is)g(allo)o(w)o(ed.)37 b(This)75 2390 y(limitation)16 b(will)h(still)g(allo)o(w)e(the)g(user)g (access)h(to)e(the)h(strided)h(and)f(indexed)i(cases)e(but)g(require)h (more)75 2446 y(complex)g(cases)f(to)g(b)q(e)h(done)f(b)o(y)g(the)h (user.)166 2591 y Fu(Discussion:)40 b Ft(If)15 b(w)o(e)h(eliminated)d (comm)o(unication)f(bu\013ers)17 b(then)f(the)g(n)o(um)o(b)q(er)f(of)g (routines)h(w)o(ould)e(b)q(e)75 2647 y(greatly)f(reduced.)19 b(W)m(e)12 b(w)o(ould)g(only)g(ha)o(v)o(e)h(the)g(\\C")f(routines)i(in) e(collectiv)o(e)h(comm)o(unicatio)o(ns)e(and)h(con)o(tiguous)75 2704 y(in)h(pt-2-pt.)18 b(Righ)o(t)13 b(no)o(w)g(I)h(think)g(w)o(e)g (should)g(lea)o(v)o(e)f(it)h(in)f(but)h(I)g(p)q(oin)o(t)g(this)f(out) 1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 119 125 119 124 bop 75 -100 a Fv(10.3.)34 b(SUBSET)15 b(FUNCTIONALITY)1014 b FC(119)166 45 y Fu(Discussion:)48 b Ft(The)18 b(heterogeneous)i (cases)e(\(h)o(v)o(ector,)h(hindexed\))f(seem)g(v)o(ery)f(easy)h(to)g (co)q(de)g(but)g(are)75 102 y(another)d(set)g(of)f(functions.)19 b(They)c(allo)o(w)d(elemen)o(ts)j(of)e(a)h(C)h(t)o(yp)q(e)f(structure)j (to)d(b)q(e)h(pulled)f(out)g(in)g(the)h(subset.)75 158 y(It)g(has)g(b)q(een)h(p)q(oin)o(ted)f(out)g(that)g(ev)o(en)h(simple)e (routines)h(ha)o(v)o(e)g(a)g(cost)g(of)g(testing,)g(do)q(cumen)o (tation,)f(etc.)22 b(that)75 214 y(can)14 b(easily)f(out)o(w)o(eigh)h (the)g(co)q(ding)g(cost.)166 354 y FC(Of)j(the)g(three)h(t)o(yp)q(es)f (of)f(comm)o(unication)i(mo)q(des,)f(the)h(ST)l(AND)o(ARD)f(mo)q(de)g (represen)o(ts)g(stan-)75 410 y(dard)12 b(practice)g(and)g(is)g (included.)22 b(The)12 b(other)f(t)o(w)o(o)g(mo)q(des)h(are)f(not)g (used)i(as)e(commonly)h(and)g(therefore)75 467 y(excluded.)22 b(This)16 b(applies)g(to)f(all)h(of)f(the)g(p)q(oin)o(t-to-p)q(oin)o(t) h(comm)o(unications)g(routines.)166 523 y(The)h(common)g(v)o(ersions)h (of)f(blo)q(c)o(king)i(and)e(non)o(blo)q(c)o(king)i(send)f(and)f (receiv)o(e)h(are)f(included.)29 b(In)75 579 y(addition,)16 b(the)f(lo)o(w)g(lev)o(el)i(routines)e(of)g(INIT)h(and)f(ST)l(AR)l(T)h (are)f(included.)23 b(These)15 b(should)h(help)h(users)75 636 y(who)12 b(wish)i(greater)d(con)o(trol)i(o)o(v)o(er)e(comm)o (unications)j(for)e(optimization)h(purp)q(oses.)20 b(F)l(or)12 b(example,)i(this)75 692 y(w)o(ould)i(allo)o(w)f(for)g(the)g(reuse)g (of)g(a)g(p)q(ersisten)o(t)h(handle.)166 831 y Fu(Discussion:)44 b Ft(Using)16 b(INIT,)h(ST)m(AR)m(T,)d(COMPLETE,)j(FREE)g(is)f(not)h (what)f(most)f(p)q(eople)i(do)f(to)q(da)o(y)75 888 y(but)c(do)q(es)h (allo)o(w)d(for)h(greater)i(sp)q(eed.)19 b(It)12 b(seems)g(as)g(if)f(y) o(ou)h(w)o(ould)f(ha)o(v)o(e)g(this)h(in)g(a)f(system)h(an)o(yw)o(a)o (y)f(and)g(it)h(w)o(ould)75 944 y(b)q(e)j(useful)e(to)h(the)h(user.)k (Should)13 b(w)o(e)h(include)g(it?)75 1149 y Fq(10.3.3)49 b(Collective)18 b(Communication)f(F)o(unctionalit)o(y)75 1235 y FC(The)h(collectiv)o(e)i(comm)o(unication)f(routines)g(ha)o(v)o (e)f(the)g(same)g(general)g(restrictions)h(as)f(the)g(p)q(oin)o(t)h(to) 75 1291 y(p)q(oin)o(t)d(routines.)k(Therefore,)15 b(the)g(bu\013er)g (descriptors)h(will)h(b)q(e)e(limited)i(to)e(one)g(en)o(try)l(.)166 1347 y(A)h(collectiv)o(e)h(comm)o(unication)g(op)q(erates)e(on)h(a)f (group.)21 b(This)c(de\014nes)g(the)e(scop)q(e)i(of)e(the)h(op)q(era-) 75 1404 y(tion.)k(Man)o(y)15 b(curren)o(t)g(users)g(only)h(op)q(erate)f (on)h(all)g(the)f(pro)q(cesses.)21 b(This)15 b(is)h(the)g(MPI)p 1589 1404 14 2 v 16 w(ALL)g(group)f(in)75 1460 y(MPI.)g(T)l(o)g (simplify)i(the)e(subset,)g(groups)g(will)i(b)q(e)f(limited)h(to)d(b)q (e)i(MPI)p 1319 1460 V 16 w(ALL.)166 1600 y Fu(Discussion:)32 b Ft(There)13 b(are)g(a)f(lot)g(of)f(data)h(mo)o(v)o(e)f(functions.)17 b(I)12 b(could)g(not)h(think)e(of)h(a)g(w)o(a)o(y)f(to)h(dra)o(w)h(a)e (line)75 1656 y(to)j(exclude)h(some.)i(Do)d(w)o(e)g(w)o(an)o(t)g(all)f (of)g(these)i(in)f(the)h(subset.)20 b(Ho)o(w)13 b(often)h(are)h (ALLCAST,)e(ALLSCA)m(TTER)75 1712 y(used?)166 1934 y Fu(Discussion:)34 b Ft(F)m(or)13 b(the)i(subset)g(do)f(w)o(e)g(w)o(an)o (t)f(b)q(oth)h(the)h(REDUCE)f(and)f(ALLREDUCE)h(v)o(ersions?)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 120 126 120 125 bop 75 377 a FE(Biblio)q(graph)m(y)98 600 y FC([1])21 b(V.)14 b(Bala)h(and)f(S.)h(Kipnis.)20 b(Pro)q(cess)14 b(groups:)19 b(a)14 b(mec)o(hanism)h(for)f(the)g(co)q(ordination)i(of)e (and)g(com-)168 656 y(m)o(unication)i(among)e(pro)q(cesses)i(in)f(the)g (Ven)o(us)h(collectiv)o(e)g(comm)o(unication)g(library)l(.)k(T)l(ec)o (hnical)168 713 y(rep)q(ort,)15 b(IBM)g(T.)g(J.)g(W)l(atson)g(Researc)o (h)g(Cen)o(ter,)g(Octob)q(er)g(1992.)k(Preprin)o(t.)98 806 y([2])i(V.)h(Bala,)h(S.)e(Kipnis,)k(L.)d(Rudolph,)j(and)c(Marc)g (Snir.)40 b(Designing)23 b(e\016cien)o(t,)h(scalable,)g(and)168 863 y(p)q(ortable)17 b(collectiv)o(e)i(comm)o(unication)e(libraries.)26 b(T)l(ec)o(hnical)18 b(rep)q(ort,)e(IBM)h(T.)f(J.)g(W)l(atson)g(Re-)168 919 y(searc)o(h)f(Cen)o(ter,)g(Octob)q(er)h(1992.)i(Preprin)o(t.)98 1013 y([3])j(R.)14 b(Butler)h(and)f(E.)f(Lusk.)18 b(User's)13 b(guide)i(to)e(the)h(P4)g(programming)f(system.)k(T)l(ec)o(hnical)f (Rep)q(ort)168 1069 y(TM-ANL{92/17,)e(Argonne)h(National)h(Lab)q (oratory)l(,)e(1992.)98 1163 y([4])21 b(S.)c(Chittor)g(and)h(R.)f(J.)g (En)o(b)q(o)q(dy)l(.)27 b(P)o(erformance)16 b(ev)m(aluation)j(of)d (mesh{connected)i(w)o(ormhole{)168 1220 y(routed)f(net)o(w)o(orks)e (for)i(in)o(terpro)q(cessor)f(comm)o(unication)i(in)f(m)o (ulticomputers.)25 b(In)17 b Fs(Pr)n(o)n(c)n(e)n(e)n(dings)168 1276 y(of)g(the)f(1990)i(Sup)n(er)n(c)n(omputing)e(Confer)n(enc)n(e)p FC(,)c(pages)j(647{656,)e(1990.)98 1370 y([5])21 b(S.)f(Chittor)g(and)g (R.)g(J.)f(En)o(b)q(o)q(dy)l(.)35 b(Predicting)21 b(the)f(e\013ect)g (of)f(mapping)i(on)e(the)h(comm)o(unica-)168 1426 y(tion)g(p)q (erformance)g(of)f(large)g(m)o(ulticomputers.)34 b(In)20 b Fs(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)k(the)f(1991)h(International)168 1483 y(Confer)n(enc)n(e)15 b(on)h(Par)n(al)r(lel)f(Pr)n(o)n(c)n (essing,)f(vol.)i(II)f(\(Softwar)n(e\))p FC(,)f(pages)h(I)q(I{1)h({)f (I)q(I{4,)g(1991.)98 1577 y([6])21 b(Edin)o(burgh)g(P)o(arallel)f (Computing)g(Cen)o(tre,)g(Univ)o(ersit)o(y)g(of)f(Edin)o(burgh.)34 b Fs(CHIMP)19 b(Conc)n(epts)p FC(,)168 1633 y(June)e(1991.)98 1727 y([7])k(Edin)o(burgh)c(P)o(arallel)g(Computing)f(Cen)o(tre,)f (Univ)o(ersit)o(y)h(of)g(Edin)o(burgh.)22 b Fs(CHIMP)15 b(V)m(ersion)h(1.0)168 1783 y(Interfac)n(e)p FC(,)e(Ma)o(y)g(1992.)98 1877 y([8])21 b(G.)13 b(Geist)f(and)h(V.)g(Sunderam.)k(Net)o(w)o(ork)11 b(based)i(concurren)o(t)g(computing)h(on)e(the)h(PVM)g(system.)168 1933 y(T)l(ec)o(hnical)k(Rep)q(ort)f(TM-11760,)d(Oak)i(Ridge)i (National)e(Lab)q(oratory)l(,)f(June)j(1991.)98 2027 y([9])k(G.)14 b(A.)g(Geist,)g(M.)g(T.)f(Heath,)h(B.)g(W.)g(P)o(eyton,)f (and)i(P)l(.)f(H.)g(W)l(orley)l(.)19 b(A)14 b(user's)g(guide)h(to)f (PICL:)g(a)168 2084 y(p)q(ortable)f(instrumen)o(ted)f(comm)o(unication) h(library)l(.)i(T)l(ec)o(hnical)f(Rep)q(ort)e(TM-11616,)f(Oak)h(Ridge) 168 2140 y(National)k(Lab)q(oratory)l(,)e(Octob)q(er)i(1990.)75 2234 y([10])21 b(R.)15 b(Hemp)q(el.)20 b(The)14 b(ANL/GMD)g(macros)f (\(P)l(ARMA)o(CS\))h(in)i(fortran)d(for)g(p)q(ortable)i(parallel)h (pro-)168 2290 y(gramming)f(using)g(the)g(message)f(passing)h (programming)f(mo)q(del)i({)e(users')h(guide)g(and)g(reference)168 2347 y(man)o(ual.)23 b(T)l(ec)o(hnical)18 b(rep)q(ort,)d(GMD,)g(P)o (ostfac)o(h)f(1316,)h(D-5205)g(Sankt)h(Augustin)g(1,)g(German)o(y)l(,) 168 2403 y(No)o(v)o(em)o(b)q(er)f(1991.)75 2497 y([11])21 b(R.)15 b(Hemp)q(el,)h(H.-C.)e(Hopp)q(e,)h(and)f(A.)h(Supalo)o(v.)k(P)o (armacs{6.0)12 b(library)k(in)o(terface)e(sp)q(eci\014cation.)168 2553 y(T)l(ec)o(hnical)h(rep)q(ort,)d(GMD,)g(P)o(ostfac)o(h)f(1316,)h (D-5205)f(Sankt)h(Augustin)i(1,)e(German)o(y)l(,)h(Decem)o(b)q(er)168 2610 y(1992.)75 2704 y([12])21 b(nCUBE)16 b(Corp)q(oration.)j Fs(nCUBE)c(2)i(Pr)n(o)n(gr)n(ammers)e(Guide,)i(r2.0)p FC(,)f(Decem)o(b)q(er)f(1990.)1967 46 y FA(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 121 127 121 126 bop 75 -100 a Fv(BIBLIOGRAPHY)1371 b FC(121)75 45 y([13])21 b(H.)c(M)q(\177)-24 b(uhlen)o(b)q(ein)19 b(O.)d(Kr\177)-23 b(amer.)24 b(Mapping)16 b(strategies)g(in)i (message{based)e(m)o(ultipro)q(cessor)h(sys-)168 102 y(tems.)j Fs(Par)n(al)r(lel)15 b(Computing)p FC(,)g(9:213{225,)d(1989.) 75 195 y([14])21 b(P)o(arasoft)14 b(Corp)q(oration.)22 b Fs(Expr)n(ess)16 b(V)m(ersion)g(1.0:)22 b(A)17 b(Communic)n(ation)g (Envir)n(onment)e(for)j(Par-)168 252 y(al)r(lel)e(Computers)p FC(,)f(1988.)75 346 y([15])21 b(P)o(aul)c(Pierce.)23 b(The)16 b(NX/2)g(op)q(erating)g(system.)22 b(In)17 b Fs(Pr)n(o)n(c)n(e)n(e)n(dings)e(of)i(the)g(Thir)n(d)g(Confer)n(enc)n(e) e(on)168 402 y(Hyp)n(er)n(cub)n(e)g(Concurr)n(ent)f(Computers)i(and)f (Applic)n(ations)p FC(,)e(pages)h(384{390.)d(A)o(CM)j(Press,)f(1988.)75 496 y([16])21 b(A.)14 b(Skjellum)i(and)e(A.)g(Leung.)19 b(Zip)q(co)q(de:)h(a)14 b(p)q(ortable)h(m)o(ulticomputer)f(comm)o (unication)h(library)168 552 y(atop)g(the)f(reactiv)o(e)h(k)o(ernel.)20 b(In)15 b(D.)f(W.)g(W)l(alk)o(er)h(and)g(Q.)g(F.)f(Stout,)g(editors,)g Fs(Pr)n(o)n(c)n(e)n(e)n(dings)g(of)i(the)168 609 y(Fifth)21 b(Distribute)n(d)h(Memory)f(Concurr)n(ent)g(Computing)g(Confer)n(enc)n (e)p FC(,)e(pages)i(767{776.)d(IEEE)168 665 y(Press,)d(1990.)75 759 y([17])21 b(A.)12 b(Skjellum,)i(S.)e(Smith,)h(C.)e(Still,)j(A.)d (Leung,)i(and)g(M.)e(Morari.)i(The)g(Zip)q(co)q(de)g(message)f(passing) 168 816 y(system.)20 b(T)l(ec)o(hnical)d(rep)q(ort,)d(La)o(wrence)i (Liv)o(ermore)f(National)h(Lab)q(oratory)l(,)e(Septem)o(b)q(er)i(1992.) 75 909 y([18])21 b(V.)e(Sunderam.)30 b(PVM:)18 b(a)g(framew)o(ork)f (for)h(parallel)i(distributed)h(computing.)30 b Fs(Concurr)n(ency:)168 966 y(Pr)n(actic)n(e)16 b(and)g(Exp)n(erienc)n(e)p FC(,)e (2\(4\):315{339)o(,)e(1990.)75 1060 y([19])21 b(D.)12 b(W)l(alk)o(er.)k(Standards)c(for)g(message)g(passing)g(in)i(a)e (distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec)o(h-)168 1116 y(nical)i(Rep)q(ort)f(TM-12147,)d(Oak)i(Ridge)h(National)g(Lab)q (oratory)l(,)e(August)h(1992.)-32 46 y FA(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 122 128 122 127 bop 75 283 a FE(MPI)40 b(F)-10 b(unction)40 b(Index)75 490 y FC(MPI)p 167 490 14 2 v 16 w(ADDRESS,)16 b(46)75 547 y(MPI)p 167 547 V 16 w(ALLCAST,)g(63)75 604 y(MPI)p 167 604 V 16 w(ALLREDUCE,)g(68)75 661 y(MPI)p 167 661 V 16 w(ALL)l(TO)o(ALL,)h(64)75 717 y(MPI)p 167 717 V 16 w(BARRIER,)g(60)75 774 y(MPI)p 167 774 V 16 w(BCAST,)e(62)75 831 y(MPI)p 167 831 V 16 w(CANCEL,)g(33)75 888 y(MPI)p 167 888 V 16 w(CAR)l(T,)h(79)75 944 y(MPI)p 167 944 V 16 w(COMM)p 335 944 V 16 w(FREE,)f(37)75 1001 y(MPI)p 167 1001 V 16 w(COMM)p 335 1001 V 16 w(INIT,)h(56)75 1058 y(MPI)p 167 1058 V 16 w(CREA)l(TE)p 374 1058 V 17 w(GR)o(OUP)l(,)f(76)75 1115 y(MPI)p 167 1115 V 16 w(CREA)l(TE)p 374 1115 V 17 w(RECV,)g(36)75 1171 y(MPI)p 167 1171 V 16 w(CREA)l(TE)p 374 1171 V 17 w(RSEND,)g(35)75 1228 y(MPI)p 167 1228 V 16 w(CREA)l(TE)p 374 1228 V 17 w(SEND,)g(35)75 1285 y(MPI)p 167 1285 V 16 w(CREA)l(TE)p 374 1285 V 17 w(SSEND,)g(36)75 1342 y(MPI)p 167 1342 V 16 w(EX)o(CHANGE,)g(38)75 1398 y(MPI)p 167 1398 V 16 w(GA)l(THER,)g(62)75 1455 y(MPI)p 167 1455 V 16 w(GET)p 283 1455 V 16 w(COUNT,)g(15)75 1512 y(MPI)p 167 1512 V 16 w(GET)p 283 1512 V 16 w(SOUR)o(CE,)h(15)75 1569 y(MPI)p 167 1569 V 16 w(GET)p 283 1569 V 16 w(T)l(A)o(G,)e(15)75 1625 y(MPI)p 167 1625 V 16 w(GRAPH,)h(79)75 1682 y(MPI)p 167 1682 V 16 w(GRAPHDIMS,)g(80)75 1739 y(MPI)p 167 1739 V 16 w(INQCAR)l(T,)h(81)75 1796 y(MPI)p 167 1796 V 16 w(INQDIM,)g(81)75 1852 y(MPI)p 167 1852 V 16 w(INQGRAPH,)g(80)75 1909 y(MPI)p 167 1909 V 16 w(INQLOC,)h(81)75 1966 y(MPI)p 167 1966 V 16 w(INQMAP)l(,)f(80)75 2023 y(MPI)p 167 2023 V 16 w(INQRANK,)h(81)75 2079 y(MPI)p 167 2079 V 16 w(IPR)o(OBE,)f(32)75 2136 y(MPI)p 167 2136 V 16 w(IRECV,)g(26)75 2193 y(MPI)p 167 2193 V 16 w(IRSEND,)g(25)75 2250 y(MPI)p 167 2250 V 16 w(ISEND,)g(25)75 2306 y(MPI)p 167 2306 V 16 w(ISSEND,)g(26)75 2363 y(MPI)p 167 2363 V 16 w(MAKDIM,)f(82)75 2420 y(MPI)p 167 2420 V 16 w(MAP)p 290 2420 V 16 w(CAR)l(T,)h(77)75 2477 y(MPI)p 167 2477 V 16 w(MAP)p 290 2477 V 16 w(GRAPH,)f(77)75 2533 y(MPI)p 167 2533 V 16 w(P)l(AR)l(TC,)h(82)75 2590 y(MPI)p 167 2590 V 16 w(PR)o(OBE,)g(33)75 2647 y(MPI)p 167 2647 V 16 w(PR)o(OBE)p 344 2647 V 17 w(COUNT,)f(32)75 2704 y(MPI)p 167 2704 V 16 w(RECV,)g(14)1048 490 y(MPI)p 1140 490 V 16 w(REDUCE,)g(65)1048 547 y(MPI)p 1140 547 V 16 w(RSEND,)g(23)1048 603 y(MPI)p 1140 603 V 16 w(SCAN,)g(69)1048 660 y(MPI)p 1140 660 V 16 w(SCA)l(TTER,)g(63)1048 716 y(MPI)p 1140 716 V 16 w(SEND,)g(12)1048 773 y(MPI)p 1140 773 V 16 w(SENDRECV,)g(38)1048 829 y(MPI)p 1140 829 V 16 w(SHIFT)p 1294 829 V 17 w(ID,)g(82)1048 886 y(MPI)p 1140 886 V 16 w(SSEND,)g(23)1048 942 y(MPI)p 1140 942 V 16 w(ST)l(AR)l(T,)h(36)1048 999 y(MPI)p 1140 999 V 16 w(TEST,)f(27)1048 1055 y(MPI)p 1140 1055 V 16 w(TEST)p 1278 1055 V 16 w(CANCELLED,)g(34)1048 1111 y(MPI)p 1140 1111 V 16 w(TEST)l(ALL,)h(30)1048 1168 y(MPI)p 1140 1168 V 16 w(TEST)l(ANY,)f(29)1048 1224 y(MPI)p 1140 1224 V 16 w(TYPE)p 1285 1224 V 16 w(COMMIT,)g(47)1048 1281 y(MPI)p 1140 1281 V 16 w(TYPE)p 1285 1281 V 16 w(CONTIGUOUS,)h(41)1048 1337 y(MPI)p 1140 1337 V 16 w(TYPE)p 1285 1337 V 16 w(EXTENT,)f(46)1048 1394 y(MPI)p 1140 1394 V 16 w(TYPE)p 1285 1394 V 16 w(FREE,)g(48)1048 1450 y(MPI)p 1140 1450 V 16 w(TYPE)p 1285 1450 V 16 w(HINDEXED,)g(45) 1048 1507 y(MPI)p 1140 1507 V 16 w(TYPE)p 1285 1507 V 16 w(HVECTOR,)h(43)1048 1563 y(MPI)p 1140 1563 V 16 w(TYPE)p 1285 1563 V 16 w(INDEXED,)f(44)1048 1620 y(MPI)p 1140 1620 V 16 w(TYPE)p 1285 1620 V 16 w(STR)o(UCT,)g(45)1048 1676 y(MPI)p 1140 1676 V 16 w(TYPE)p 1285 1676 V 16 w(VECTOR,)g(42)1048 1732 y(MPI)p 1140 1732 V 16 w(USER)p 1279 1732 V 17 w(ALLREDUCE,)h(69) 1048 1789 y(MPI)p 1140 1789 V 16 w(USER)p 1279 1789 V 17 w(ALLREDUCEA,)g(69)1048 1845 y(MPI)p 1140 1845 V 16 w(USER)p 1279 1845 V 17 w(REDUCE,)f(67)1048 1902 y(MPI)p 1140 1902 V 16 w(USER)p 1279 1902 V 17 w(REDUCEA,)g(67)1048 1958 y(MPI)p 1140 1958 V 16 w(USER)p 1279 1958 V 17 w(SCAN,)g(70)1048 2015 y(MPI)p 1140 2015 V 16 w(USER)p 1279 2015 V 17 w(SCANA,)g(70)1048 2071 y(MPI)p 1140 2071 V 16 w(W)-5 b(AIT,)15 b(27)1048 2128 y(MPI)p 1140 2128 V 16 w(W)-5 b(AIT)l(ALL,)16 b(30)1048 2184 y(MPI)p 1140 2184 V 16 w(W)-5 b(AIT)l(ANY,)15 b(29)941 2828 y(122)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .